Add migrate script

conda/migrate.py

+#!/usr/bin/env python
+'''Migrates a Bob package recipe from bob.conda into its own repository
+
+Usage: {prog} <template_recipe> <old_recipe> <conda_build_variant> <package_folder>
+'''
+
+from __future__ import absolute_import, division, print_function
+import re
+from conda_build.api import render, Config
+from conda_build.variants import parse_config_file
+from conda_build.conda_interface import PY3
+import os
+
+
+def extract_requirements_text(path):
+  text = ""
+  if path:
+    with open(path) as f:
+      recipe_text = f.read()
+    if PY3 and hasattr(recipe_text, 'decode'):
+      recipe_text = recipe_text.decode()
+    match = re.search(
+        r'(^requirements:.*?)(^test:|^extra:|^about:|^outputs:|\Z)',
+        recipe_text, flags=re.MULTILINE | re.DOTALL)
+    text = match.group(1) if match else ""
+  return text
+
+
+def extract_test_text(path):
+  text = ""
+  if path:
+    with open(path) as f:
+      recipe_text = f.read()
+    if PY3 and hasattr(recipe_text, 'decode'):
+      recipe_text = recipe_text.decode()
+    match = re.search(
+        r'(^test:.*?)(^extra:|^about:|^outputs:|\Z)',
+        recipe_text, flags=re.MULTILINE | re.DOTALL)
+    text = match.group(1) if match else ""
+  i1 = text.find('requires:\n') + 10
+  i2 = text.find('imports:', i1)
+  i3 = text.find('commands:', i1)
+  i2 = i2 if i2 > -1 else len(text)
+  i3 = i3 if i3 > -1 else len(text)
+  i2 = min(i2, i3)
+  return text[i1:i2]
+
+
+def add_variant(doc, star=True):
+  lines = doc.split('\n')
+  for i, line in enumerate(lines):
+    if line.find('{{') == -1:
+      if line.find(' - ') == -1:
+        continue
+      variant = line.split('-', 1)[1].strip()
+      if variant.startswith('bob.') or variant == 'bob':
+        continue
+      variant = variant.replace('-', '_').replace('.', '_')
+      variant = ' {{ ' + variant.split('#')[0].strip() + ' }}'
+      line = line.split('#')
+      line = [line[0] + variant] + line[1:]
+      lines[i] = '#'.join(line)
+  doc = '\n'.join(lines)
+  if star:
+    doc = doc.replace('}}.*', '}}').replace('}}', '}}.*')
+    doc = doc.replace("compiler('c') }}.*", "compiler('c') }}")
+    doc = doc.replace("compiler('cxx') }}.*", "compiler('cxx') }}")
+  return doc
+
+
+def migrate(template_recipe_path, old_recipe_path, build_variant_path,
+            output_folder):
+  config = Config()
+  variants = parse_config_file(build_variant_path, config)
+  recipe = render(old_recipe_path, variants=variants)[0][0]
+  with open(template_recipe_path) as f:
+    final_recipe = f.read()
+  final_recipe = final_recipe.replace('<PACKAGE>', recipe.name())
+  entry_points, entry_points_test = '', ''
+  if 'entry_points' in recipe.meta['build']:
+    entry_points = '  entry_points:\n    - ' + \
+        '\n    - '.join(recipe.meta['build']['entry_points']) + '\n'
+    entry_points_test = '    - ' + '\n    - '.join(
+        l.split('=')[0].strip() + ' --help' for l in
+        recipe.meta['build']['entry_points']) + '\n'
+  final_recipe = final_recipe.replace('  <ENTRY_POINTS>\n', entry_points)
+  final_recipe = final_recipe.replace(
+      '    <ENTRY_POINTS_TEST>\n', entry_points_test)
+  requirements_text = extract_requirements_text(old_recipe_path)
+  already_build = ['- python\n',
+                   '- setuptools\n',
+                   '- python {{ python }}\n',
+                   '- setuptools {{ setuptools }}\n',
+                   '- toolchain {{ toolchain }}\n']
+  i1 = requirements_text.find('build:\n') + 7
+  i2 = requirements_text.find('run:\n')
+  build_text = requirements_text[i1:i2]
+  requires_compilers = 'toolchain' in build_text
+  for line in already_build:
+    build_text = build_text.replace(line, '')
+  build_text = build_text.strip()
+  if build_text:
+    build_text = '    ' + build_text + '\n'
+    build_text = add_variant(build_text, star=False)
+  final_recipe = final_recipe.replace('    <HOST_DEPS>\n', build_text)
+  if requires_compilers:
+    final_recipe = final_recipe.replace('  <BUILD_DEPS>\n', '''  build:
+  - {{ compiler('c')}}
+  - {{ compiler('cxx')}}
+''')
+  else:
+    final_recipe = final_recipe.replace('  <BUILD_DEPS>\n', '')
+
+  run_text = requirements_text[i2 + 5:]
+  for line in already_build[:2]:
+    run_text = run_text.replace(line, '')
+  run_text = run_text.strip()
+  run_text = '    ' + run_text + '\n' if run_text else run_text
+  final_recipe = final_recipe.replace('    <RUN_DEPS>\n', run_text)
+
+  already_test = ['nose', 'coverage', 'sphinx', 'sphinx_rtd_theme']
+  already_test += [l + ' {{ ' + l + ' }}' for l in already_test]
+  already_test = ['    - ' + l + '\n' for l in already_test]
+  test_requires = extract_test_text(old_recipe_path)
+  for line in already_test:
+    test_requires = test_requires.replace(line, '')
+  test_requires = test_requires.strip()
+  if test_requires:
+    test_requires = '    ' + test_requires + '\n'
+    test_requires = add_variant(test_requires)
+  final_recipe = final_recipe.replace(
+      '    <TEST_DEPS>\n', test_requires)
+  final_recipe = final_recipe.replace(
+      '<LICENSE>', recipe.meta['about']['license'])
+  final_recipe = final_recipe.replace(
+      '<SHORT_DESCRIPTION>', recipe.meta['about']['summary'])
+  license_family = recipe.meta['about']['license']
+  license_family = 'BSD' if 'BSD' in license_family else 'GPL'
+  final_recipe = final_recipe.replace('<LICENSE_FAMILY>', license_family)
+
+  output_folder = os.path.join(output_folder, 'conda')
+  os.makedirs(output_folder, exist_ok=True)
+  out_path = os.path.join(output_folder, 'meta.yaml')
+
+  # custom modifications:
+  if recipe.name() == 'bob.extension':
+    final_recipe = final_recipe.replace(
+        '-sv {{ name }}', '-sv {{ name }} --exclude=test_extensions')
+
+  if recipe.name() in ('bob.ip.gabor', 'bob.learn.em'):
+    final_recipe = final_recipe.replace(
+        '{{ sphinx_rtd_theme }}\n', '{{ sphinx_rtd_theme }}\n' +
+        '    - matplotlib {{ matplotlib }}\n')
+
+  with open(out_path, 'w') as f:
+    f.write(final_recipe)
+
+  # fix the doc/conf.py
+  latex_conf = '''import sphinx
+if sphinx.__version__ >= "1.4.1":
+    extensions.append('sphinx.ext.imgmath')
+    imgmath_image_format = 'svg'
+else:
+    extensions.append('sphinx.ext.pngmath')
+
+'''
+  place_to_insert = '''    'sphinx.ext.viewcode',
+'''
+  replace = place_to_insert + place_to_insert.replace('viewcode', 'mathjax')
+  conf = os.path.join(output_folder, '..', 'doc', 'conf.py')
+  with open(conf) as f:
+    lines = f.read()
+  lines = lines.replace(latex_conf, '')
+  lines = lines.replace(place_to_insert, replace)
+  with open(conf, 'w') as f:
+    f.write(lines)
+
+
+def main(argv=None):
+  from docopt import docopt
+  import sys
+  doc = __doc__.format(prog=sys.argv[0])
+  args = docopt(doc, argv=argv)
+
+  migrate(args['<template_recipe>'], args['<old_recipe>'],
+          args['<conda_build_variant>'], args['<package_folder>'])
+
+
+if __name__ == '__main__':
+  main()

conda/migrate.sh

+#!/usr/bin/bash
+
+set -ex
+
+for pkg in \
+bob.buildout \
+bob.extension \
+bob.blitz \
+bob.core \
+bob.io.base \
+bob.sp \
+bob.ap \
+bob.math \
+bob.measure \
+bob.db.base \
+bob.io.audio \
+bob.io.image \
+bob.io.video \
+bob.io.matlab \
+bob.ip.base \
+bob.ip.color \
+bob.ip.draw \
+bob.ip.gabor \
+bob.learn.activation \
+bob.learn.libsvm \
+bob.learn.linear \
+bob.learn.mlp \
+bob.learn.em \
+bob.learn.boosting \
+bob.db.iris \
+bob.db.wine \
+bob.db.mnist \
+bob.db.atnt \
+bob.ip.facedetect \
+bob.ip.optflow.hornschunck \
+bob.ip.optflow.liu \
+bob.ip.flandmark \
+; do
+    git clone git@gitlab.idiap.ch:bob/$pkg || true
+    cd $pkg
+    git co -b condapackage || git co -- . && git clean -ffdx && git co master && git pull --ff-only && git branch -D condapackage && git co -b condapackage
+
+    ../migrate.py ../../templates/meta.yaml ../../../bob.conda/recipes/$pkg/meta.yaml \
+        ../../gitlab/conda_build_config.yaml .
+    cp ../../templates/gitlab-ci.yml .gitlab-ci.yml
+    echo "record.txt" >> .gitignore
+
+    subl conda/meta.yaml
+    conda build -m ../../gitlab/conda_build_config.yaml --python=2.7 conda
+
+    git add -A
+    git commit -m "Migrate to conda based CI"
+
+    while true; do
+        read -p "Do you wish to continue?" yn
+        case $yn in
+            [Yy]* ) break;;
+            [Nn]* ) exit;;
+            * ) echo "Please answer yes or no.";;
+        esac
+    done
+    git push -u --force origin condapackage
+    cd ..
+done

gitlab/deploy.sh

@@ -8,11 +8,10 @@ for os in "osx-64" "noarch" "linux-64"; do
   for f in ${CONDA_ENVS_PATH}/${os}/*.tar.bz2; do
     if [[ -f $f ]]; then
       if [ -z "${CI_COMMIT_TAG}" ]; then #beta
-        url="private"
+        dav_upload "${f}" "private-upload/conda/${os}/"
       else
-        url="public"
+        dav_upload "${f}" "public-upload/conda/label/main/${os}/"
       fi
-      dav_upload "${f}" "${url}-upload/conda/${os}/"
     fi
   done
 done

templates/gitlab-ci.yml

@@ -72,12 +72,6 @@ build_linux_27:
     <<: *linux_variables
     PYTHON_VERSION: "2.7"
 
-# build_linux_35:
-#   <<: *linux_build_job
-#   variables:
-#     <<: *linux_variables
-#     PYTHON_VERSION: "3.5"
-
 build_linux_36:
   <<: *linux_build_job
   variables:
@@ -98,12 +92,6 @@ build_macosx_27:
     <<: *macosx_variables
     PYTHON_VERSION: "2.7"
 
-# build_macosx_35:
-#   <<: *macosx_build_job
-#   variables:
-#     <<: *macosx_variables
-#     PYTHON_VERSION: "3.5"
-
 build_macosx_36:
   <<: *macosx_build_job
   variables:
@@ -120,10 +108,8 @@ build_macosx_36:
     - ./_ci/deploy.sh
   dependencies:
     - build_linux_27
-    # - build_linux_35
     - build_linux_36
     - build_macosx_27
-    # - build_macosx_35
     - build_macosx_36
   tags:
     - deployer