From a43185870857adea59c7b30c27f4d6eec0124adc Mon Sep 17 00:00:00 2001
From: Amir MOHAMMADI <amir.mohammadi@idiap.ch>
Date: Mon, 15 Jan 2018 16:52:05 +0100
Subject: [PATCH] Add migrate script
---
conda/migrate.py | 190 ++++++++++++++++++++++++++++++++++++++++
conda/migrate.sh | 64 ++++++++++++++
gitlab/deploy.sh | 5 +-
templates/gitlab-ci.yml | 14 ---
4 files changed, 256 insertions(+), 17 deletions(-)
create mode 100755 conda/migrate.py
create mode 100755 conda/migrate.sh
diff --git a/conda/migrate.py b/conda/migrate.py
new file mode 100755
index 0000000..c504f3a
--- /dev/null
+++ b/conda/migrate.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+'''Migrates a Bob package recipe from bob.conda into its own repository
+
+Usage: {prog} <template_recipe> <old_recipe> <conda_build_variant> <package_folder>
+'''
+
+from __future__ import absolute_import, division, print_function
+import re
+from conda_build.api import render, Config
+from conda_build.variants import parse_config_file
+from conda_build.conda_interface import PY3
+import os
+
+
+def extract_requirements_text(path):
+ text = ""
+ if path:
+ with open(path) as f:
+ recipe_text = f.read()
+ if PY3 and hasattr(recipe_text, 'decode'):
+ recipe_text = recipe_text.decode()
+ match = re.search(
+ r'(^requirements:.*?)(^test:|^extra:|^about:|^outputs:|\Z)',
+ recipe_text, flags=re.MULTILINE | re.DOTALL)
+ text = match.group(1) if match else ""
+ return text
+
+
+def extract_test_text(path):
+ text = ""
+ if path:
+ with open(path) as f:
+ recipe_text = f.read()
+ if PY3 and hasattr(recipe_text, 'decode'):
+ recipe_text = recipe_text.decode()
+ match = re.search(
+ r'(^test:.*?)(^extra:|^about:|^outputs:|\Z)',
+ recipe_text, flags=re.MULTILINE | re.DOTALL)
+ text = match.group(1) if match else ""
+ i1 = text.find('requires:\n') + 10
+ i2 = text.find('imports:', i1)
+ i3 = text.find('commands:', i1)
+ i2 = i2 if i2 > -1 else len(text)
+ i3 = i3 if i3 > -1 else len(text)
+ i2 = min(i2, i3)
+ return text[i1:i2]
+
+
+def add_variant(doc, star=True):
+ lines = doc.split('\n')
+ for i, line in enumerate(lines):
+ if line.find('{{') == -1:
+ if line.find(' - ') == -1:
+ continue
+ variant = line.split('-', 1)[1].strip()
+ if variant.startswith('bob.') or variant == 'bob':
+ continue
+ variant = variant.replace('-', '_').replace('.', '_')
+ variant = ' {{ ' + variant.split('#')[0].strip() + ' }}'
+ line = line.split('#')
+ line = [line[0] + variant] + line[1:]
+ lines[i] = '#'.join(line)
+ doc = '\n'.join(lines)
+ if star:
+ doc = doc.replace('}}.*', '}}').replace('}}', '}}.*')
+ doc = doc.replace("compiler('c') }}.*", "compiler('c') }}")
+ doc = doc.replace("compiler('cxx') }}.*", "compiler('cxx') }}")
+ return doc
+
+
+def migrate(template_recipe_path, old_recipe_path, build_variant_path,
+ output_folder):
+ config = Config()
+ variants = parse_config_file(build_variant_path, config)
+ recipe = render(old_recipe_path, variants=variants)[0][0]
+ with open(template_recipe_path) as f:
+ final_recipe = f.read()
+ final_recipe = final_recipe.replace('<PACKAGE>', recipe.name())
+ entry_points, entry_points_test = '', ''
+ if 'entry_points' in recipe.meta['build']:
+ entry_points = ' entry_points:\n - ' + \
+ '\n - '.join(recipe.meta['build']['entry_points']) + '\n'
+ entry_points_test = ' - ' + '\n - '.join(
+ l.split('=')[0].strip() + ' --help' for l in
+ recipe.meta['build']['entry_points']) + '\n'
+ final_recipe = final_recipe.replace(' <ENTRY_POINTS>\n', entry_points)
+ final_recipe = final_recipe.replace(
+ ' <ENTRY_POINTS_TEST>\n', entry_points_test)
+ requirements_text = extract_requirements_text(old_recipe_path)
+ already_build = ['- python\n',
+ '- setuptools\n',
+ '- python {{ python }}\n',
+ '- setuptools {{ setuptools }}\n',
+ '- toolchain {{ toolchain }}\n']
+ i1 = requirements_text.find('build:\n') + 7
+ i2 = requirements_text.find('run:\n')
+ build_text = requirements_text[i1:i2]
+ requires_compilers = 'toolchain' in build_text
+ for line in already_build:
+ build_text = build_text.replace(line, '')
+ build_text = build_text.strip()
+ if build_text:
+ build_text = ' ' + build_text + '\n'
+ build_text = add_variant(build_text, star=False)
+ final_recipe = final_recipe.replace(' <HOST_DEPS>\n', build_text)
+ if requires_compilers:
+ final_recipe = final_recipe.replace(' <BUILD_DEPS>\n', ''' build:
+ - {{ compiler('c')}}
+ - {{ compiler('cxx')}}
+''')
+ else:
+ final_recipe = final_recipe.replace(' <BUILD_DEPS>\n', '')
+
+ run_text = requirements_text[i2 + 5:]
+ for line in already_build[:2]:
+ run_text = run_text.replace(line, '')
+ run_text = run_text.strip()
+ run_text = ' ' + run_text + '\n' if run_text else run_text
+ final_recipe = final_recipe.replace(' <RUN_DEPS>\n', run_text)
+
+ already_test = ['nose', 'coverage', 'sphinx', 'sphinx_rtd_theme']
+ already_test += [l + ' {{ ' + l + ' }}' for l in already_test]
+ already_test = [' - ' + l + '\n' for l in already_test]
+ test_requires = extract_test_text(old_recipe_path)
+ for line in already_test:
+ test_requires = test_requires.replace(line, '')
+ test_requires = test_requires.strip()
+ if test_requires:
+ test_requires = ' ' + test_requires + '\n'
+ test_requires = add_variant(test_requires)
+ final_recipe = final_recipe.replace(
+ ' <TEST_DEPS>\n', test_requires)
+ final_recipe = final_recipe.replace(
+ '<LICENSE>', recipe.meta['about']['license'])
+ final_recipe = final_recipe.replace(
+ '<SHORT_DESCRIPTION>', recipe.meta['about']['summary'])
+ license_family = recipe.meta['about']['license']
+ license_family = 'BSD' if 'BSD' in license_family else 'GPL'
+ final_recipe = final_recipe.replace('<LICENSE_FAMILY>', license_family)
+
+ output_folder = os.path.join(output_folder, 'conda')
+ os.makedirs(output_folder, exist_ok=True)
+ out_path = os.path.join(output_folder, 'meta.yaml')
+
+ # custom modifications:
+ if recipe.name() == 'bob.extension':
+ final_recipe = final_recipe.replace(
+ '-sv {{ name }}', '-sv {{ name }} --exclude=test_extensions')
+
+ if recipe.name() in ('bob.ip.gabor', 'bob.learn.em'):
+ final_recipe = final_recipe.replace(
+ '{{ sphinx_rtd_theme }}\n', '{{ sphinx_rtd_theme }}\n' +
+ ' - matplotlib {{ matplotlib }}\n')
+
+ with open(out_path, 'w') as f:
+ f.write(final_recipe)
+
+ # fix the doc/conf.py
+ latex_conf = '''import sphinx
+if sphinx.__version__ >= "1.4.1":
+ extensions.append('sphinx.ext.imgmath')
+ imgmath_image_format = 'svg'
+else:
+ extensions.append('sphinx.ext.pngmath')
+
+'''
+ place_to_insert = ''' 'sphinx.ext.viewcode',
+'''
+ replace = place_to_insert + place_to_insert.replace('viewcode', 'mathjax')
+ conf = os.path.join(output_folder, '..', 'doc', 'conf.py')
+ with open(conf) as f:
+ lines = f.read()
+ lines = lines.replace(latex_conf, '')
+ lines = lines.replace(place_to_insert, replace)
+ with open(conf, 'w') as f:
+ f.write(lines)
+
+
+def main(argv=None):
+ from docopt import docopt
+ import sys
+ doc = __doc__.format(prog=sys.argv[0])
+ args = docopt(doc, argv=argv)
+
+ migrate(args['<template_recipe>'], args['<old_recipe>'],
+ args['<conda_build_variant>'], args['<package_folder>'])
+
+
+if __name__ == '__main__':
+ main()
diff --git a/conda/migrate.sh b/conda/migrate.sh
new file mode 100755
index 0000000..66512dd
--- /dev/null
+++ b/conda/migrate.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/bash
+
+set -ex
+
+for pkg in \
+bob.buildout \
+bob.extension \
+bob.blitz \
+bob.core \
+bob.io.base \
+bob.sp \
+bob.ap \
+bob.math \
+bob.measure \
+bob.db.base \
+bob.io.audio \
+bob.io.image \
+bob.io.video \
+bob.io.matlab \
+bob.ip.base \
+bob.ip.color \
+bob.ip.draw \
+bob.ip.gabor \
+bob.learn.activation \
+bob.learn.libsvm \
+bob.learn.linear \
+bob.learn.mlp \
+bob.learn.em \
+bob.learn.boosting \
+bob.db.iris \
+bob.db.wine \
+bob.db.mnist \
+bob.db.atnt \
+bob.ip.facedetect \
+bob.ip.optflow.hornschunck \
+bob.ip.optflow.liu \
+bob.ip.flandmark \
+; do
+ git clone git@gitlab.idiap.ch:bob/$pkg || true
+ cd $pkg
+ git co -b condapackage || git co -- . && git clean -ffdx && git co master && git pull --ff-only && git branch -D condapackage && git co -b condapackage
+
+ ../migrate.py ../../templates/meta.yaml ../../../bob.conda/recipes/$pkg/meta.yaml \
+ ../../gitlab/conda_build_config.yaml .
+ cp ../../templates/gitlab-ci.yml .gitlab-ci.yml
+ echo "record.txt" >> .gitignore
+
+ subl conda/meta.yaml
+ conda build -m ../../gitlab/conda_build_config.yaml --python=2.7 conda
+
+ git add -A
+ git commit -m "Migrate to conda based CI"
+
+ while true; do
+ read -p "Do you wish to continue?" yn
+ case $yn in
+ [Yy]* ) break;;
+ [Nn]* ) exit;;
+ * ) echo "Please answer yes or no.";;
+ esac
+ done
+ git push -u --force origin condapackage
+ cd ..
+done
diff --git a/gitlab/deploy.sh b/gitlab/deploy.sh
index c002df8..ca90a56 100755
--- a/gitlab/deploy.sh
+++ b/gitlab/deploy.sh
@@ -8,11 +8,10 @@ for os in "osx-64" "noarch" "linux-64"; do
for f in ${CONDA_ENVS_PATH}/${os}/*.tar.bz2; do
if [[ -f $f ]]; then
if [ -z "${CI_COMMIT_TAG}" ]; then #beta
- url="private"
+ dav_upload "${f}" "private-upload/conda/${os}/"
else
- url="public"
+ dav_upload "${f}" "public-upload/conda/label/main/${os}/"
fi
- dav_upload "${f}" "${url}-upload/conda/${os}/"
fi
done
done
diff --git a/templates/gitlab-ci.yml b/templates/gitlab-ci.yml
index 7120af5..1e2d7cd 100644
--- a/templates/gitlab-ci.yml
+++ b/templates/gitlab-ci.yml
@@ -72,12 +72,6 @@ build_linux_27:
<<: *linux_variables
PYTHON_VERSION: "2.7"
-# build_linux_35:
-# <<: *linux_build_job
-# variables:
-# <<: *linux_variables
-# PYTHON_VERSION: "3.5"
-
build_linux_36:
<<: *linux_build_job
variables:
@@ -98,12 +92,6 @@ build_macosx_27:
<<: *macosx_variables
PYTHON_VERSION: "2.7"
-# build_macosx_35:
-# <<: *macosx_build_job
-# variables:
-# <<: *macosx_variables
-# PYTHON_VERSION: "3.5"
-
build_macosx_36:
<<: *macosx_build_job
variables:
@@ -120,10 +108,8 @@ build_macosx_36:
- ./_ci/deploy.sh
dependencies:
- build_linux_27
- # - build_linux_35
- build_linux_36
- build_macosx_27
- # - build_macosx_35
- build_macosx_36
tags:
- deployer
--
GitLab