Conda package based CI

Fixes #35 (closed) and #48 (closed)

conda/migrate.py

+#!/usr/bin/env python
+'''Migrates a Bob package recipe from bob.conda into its own repository
+
+Usage: {prog} <template_recipe> <old_recipe> <conda_build_variant> <package_folder>
+'''
+
+from __future__ import absolute_import, division, print_function
+import re
+from conda_build.api import render, Config
+from conda_build.variants import parse_config_file
+from conda_build.conda_interface import PY3
+import os
+
+
+def extract_requirements_text(path):
+  text = ""
+  if path:
+    with open(path) as f:
+      recipe_text = f.read()
+    if PY3 and hasattr(recipe_text, 'decode'):
+      recipe_text = recipe_text.decode()
+    match = re.search(
+        r'(^requirements:.*?)(^test:|^extra:|^about:|^outputs:|\Z)',
+        recipe_text, flags=re.MULTILINE | re.DOTALL)
+    text = match.group(1) if match else ""
+  return text
+
+
+def extract_test_text(path):
+  text = ""
+  if path:
+    with open(path) as f:
+      recipe_text = f.read()
+    if PY3 and hasattr(recipe_text, 'decode'):
+      recipe_text = recipe_text.decode()
+    match = re.search(
+        r'(^test:.*?)(^extra:|^about:|^outputs:|\Z)',
+        recipe_text, flags=re.MULTILINE | re.DOTALL)
+    text = match.group(1) if match else ""
+  i1 = text.find('requires:\n') + 10
+  i2 = text.find('imports:', i1)
+  i3 = text.find('commands:', i1)
+  i2 = i2 if i2 > -1 else len(text)
+  i3 = i3 if i3 > -1 else len(text)
+  i2 = min(i2, i3)
+  return text[i1:i2]
+
+
+def add_variant(doc, star=True):
+  lines = doc.split('\n')
+  for i, line in enumerate(lines):
+    if line.find('{{') == -1:
+      if line.find(' - ') == -1:
+        continue
+      variant = line.split('-', 1)[1].strip()
+      if variant.startswith('bob.') or variant == 'bob':
+        continue
+      variant = variant.replace('-', '_').replace('.', '_')
+      variant = ' {{ ' + variant.split('#')[0].strip() + ' }}'
+      line = line.split('#')
+      line = [line[0] + variant] + line[1:]
+      lines[i] = '#'.join(line)
+  doc = '\n'.join(lines)
+  if star:
+    doc = doc.replace('}}.*', '}}').replace('}}', '}}.*')
+    doc = doc.replace("compiler('c') }}.*", "compiler('c') }}")
+    doc = doc.replace("compiler('cxx') }}.*", "compiler('cxx') }}")
+  return doc
+
+
+def migrate(template_recipe_path, old_recipe_path, build_variant_path,
+            output_folder):
+  config = Config()
+  variants = parse_config_file(build_variant_path, config)
+  recipe = render(old_recipe_path, variants=variants)[0][0]
+  with open(template_recipe_path) as f:
+    final_recipe = f.read()
+  final_recipe = final_recipe.replace('<PACKAGE>', recipe.name())
+  entry_points, entry_points_test = '', ''
+  if 'entry_points' in recipe.meta['build']:
+    entry_points = '  entry_points:\n    - ' + \
+        '\n    - '.join(recipe.meta['build']['entry_points']) + '\n'
+    entry_points_test = '    - ' + '\n    - '.join(
+        l.split('=')[0].strip() + ' --help' for l in
+        recipe.meta['build']['entry_points']) + '\n'
+  final_recipe = final_recipe.replace('  <ENTRY_POINTS>\n', entry_points)
+  final_recipe = final_recipe.replace(
+      '    <ENTRY_POINTS_TEST>\n', entry_points_test)
+  requirements_text = extract_requirements_text(old_recipe_path)
+  already_build = ['- python\n',
+                   '- setuptools\n',
+                   '- python {{ python }}\n',
+                   '- setuptools {{ setuptools }}\n',
+                   '- toolchain {{ toolchain }}\n']
+  i1 = requirements_text.find('build:\n') + 7
+  i2 = requirements_text.find('run:\n')
+  build_text = requirements_text[i1:i2]
+  requires_compilers = 'toolchain' in build_text
+  for line in already_build:
+    build_text = build_text.replace(line, '')
+  build_text = build_text.strip()
+  if build_text:
+    build_text = '    ' + build_text + '\n'
+    build_text = add_variant(build_text, star=False)
+  final_recipe = final_recipe.replace('    <HOST_DEPS>\n', build_text)
+  if requires_compilers:
+    final_recipe = final_recipe.replace('  <BUILD_DEPS>\n', '''  build:
+  - {{ compiler('c')}}
+  - {{ compiler('cxx')}}
+''')
+  else:
+    final_recipe = final_recipe.replace('  <BUILD_DEPS>\n', '')
+
+  run_text = requirements_text[i2 + 5:]
+  for line in already_build[:2]:
+    run_text = run_text.replace(line, '')
+  run_text = run_text.strip()
+  run_text = '    ' + run_text + '\n' if run_text else run_text
+  final_recipe = final_recipe.replace('    <RUN_DEPS>\n', run_text)
+
+  already_test = ['nose', 'coverage', 'sphinx', 'sphinx_rtd_theme']
+  already_test += [l + ' {{ ' + l + ' }}' for l in already_test]
+  already_test = ['    - ' + l + '\n' for l in already_test]
+  test_requires = extract_test_text(old_recipe_path)
+  for line in already_test:
+    test_requires = test_requires.replace(line, '')
+  test_requires = test_requires.strip()
+  if test_requires:
+    test_requires = '    ' + test_requires + '\n'
+    test_requires = add_variant(test_requires)
+  final_recipe = final_recipe.replace(
+      '    <TEST_DEPS>\n', test_requires)
+  final_recipe = final_recipe.replace(
+      '<LICENSE>', recipe.meta['about']['license'])
+  final_recipe = final_recipe.replace(
+      '<SHORT_DESCRIPTION>', recipe.meta['about']['summary'])
+  license_family = recipe.meta['about']['license']
+  license_family = 'BSD' if 'BSD' in license_family else 'GPL'
+  final_recipe = final_recipe.replace('<LICENSE_FAMILY>', license_family)
+
+  output_folder = os.path.join(output_folder, 'conda')
+  os.makedirs(output_folder, exist_ok=True)
+  out_path = os.path.join(output_folder, 'meta.yaml')
+
+  # custom modifications:
+  if recipe.name() == 'bob.extension':
+    final_recipe = final_recipe.replace(
+        '-sv {{ name }}', '-sv {{ name }} --exclude=test_extensions')
+
+  if recipe.name() in ('bob.ip.gabor', 'bob.learn.em'):
+    final_recipe = final_recipe.replace(
+        '{{ sphinx_rtd_theme }}\n', '{{ sphinx_rtd_theme }}\n' +
+        '    - matplotlib {{ matplotlib }}\n')
+
+  with open(out_path, 'w') as f:
+    f.write(final_recipe)
+
+  # fix the doc/conf.py
+  latex_conf = '''import sphinx
+if sphinx.__version__ >= "1.4.1":
+    extensions.append('sphinx.ext.imgmath')
+    imgmath_image_format = 'svg'
+else:
+    extensions.append('sphinx.ext.pngmath')
+
+'''
+  place_to_insert = '''    'sphinx.ext.viewcode',
+'''
+  replace = place_to_insert + place_to_insert.replace('viewcode', 'mathjax')
+  conf = os.path.join(output_folder, '..', 'doc', 'conf.py')
+  with open(conf) as f:
+    lines = f.read()
+  lines = lines.replace(latex_conf, '')
+  lines = lines.replace(place_to_insert, replace)
+  with open(conf, 'w') as f:
+    f.write(lines)
+
+
+def main(argv=None):
+  import sys
+  migrate(*sys.argv[1:])
+
+
+if __name__ == '__main__':
+  main()