Merge branch 'conda-forge-deploy' into 'master'

Add conda-forge deploy script



See merge request !15

conda/batch-update-bob.sh → conda/batch-update-conda-forge.sh

@@ -27,11 +27,14 @@ case $1 in
 	7)
 		packages=("bob.learn.em" "bob.db.iris")
 		;;
+	8)
+		packages=("bob")
+		;;
 esac
 
 
 for pkg in "${packages[@]}"
 do
 	gh re --fork $pkg-feedstock --user conda-forge --organization bioidiap || true
-	python update_feedstock.py $pkg
+	python ../gitlab/update_feedstock.py $pkg
 done

gitlab/deploy.sh

@@ -20,3 +20,40 @@ if [ -d sphinx ]; then
 fi
 
 unlock_pypirc
+
+condaforge_packages=("bob" \
+"bob.extension" \
+"bob.blitz" \
+"bob.core" \
+"bob.ip.draw" \
+"bob.io.base" \
+"bob.sp" \
+"bob.math" \
+"bob.ap" \
+"bob.measure" \
+"bob.db.base" \
+"bob.io.image" \
+"bob.io.video" \
+"bob.io.matlab" \
+"bob.ip.base" \
+"bob.ip.color" \
+"bob.ip.gabor" \
+"bob.learn.activation" \
+"bob.learn.libsvm" \
+"bob.learn.boosting" \
+"bob.io.audio" \
+"bob.learn.linear" \
+"bob.learn.mlp" \
+"bob.db.wine" \
+"bob.db.mnist" \
+"bob.db.atnt" \
+"bob.ip.flandmark" \
+"bob.ip.facedetect" \
+"bob.ip.optflow.hornschunck" \
+"bob.ip.optflow.liu" \
+"bob.learn.em" \
+"bob.db.iris")
+
+if contains_element ${CI_PROJECT_NAME}  "${condaforge_packages[@]}"; then
+  run_cmd ${CONDA_FOLDER}/bin/python _ci/update_feedstock.py ${CI_PROJECT_NAME}
+fi

gitlab/functions.sh

@@ -246,3 +246,10 @@ if [ -z "${CI_BUILD_TAG}" ]; then
 else
   log_info "Building tag, not setting BOB_DOCUMENTATION_SERVER"
 fi
+
+# taken from here: https://stackoverflow.com/questions/3685970/check-if-an-array-contains-a-value
+contains_element () {
+  local e
+  for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
+  return 1
+}

gitlab/install.sh

@@ -57,3 +57,4 @@ for stage in "build" "test" "docs" "wheels" "deploy"; do
   get_exec ${1} ${stage}.sh
   get_exec ${1} after_${stage}.sh
 done
+get_exec ${1} update_feedstock.py

conda/update_feedstock.py → gitlab/update_feedstock.py

 #!/usr/bin/env python
 
 import hashlib
-import urllib.request
+try:
+  from urllib2 import urlopen
+except ImportError:
+  from urllib.request import urlopen
 import requests
 import json
 try:
@@ -21,22 +24,18 @@ def run_commands(*calls):
   """runs the given commands."""
   # get all calls
   for call in calls:
-    if True or False:
-      print(' - ' + ' '.join(call))
-    if not False:
-      # execute call
-      if subprocess.call(call):
-        # call failed (has non-zero exit status)
-        raise ValueError("Command '%s' failed; stopping" % ' '.join(call))
+    print(' - ' + ' '.join(call))
+    # execute call
+    if subprocess.call(call):
+      # call failed (has non-zero exit status)
+      raise ValueError("Command '%s' failed; stopping" % ' '.join(call))
 
 
 def get_version(package, url_pattern=URL_PATTERN):
   """Return version of package on pypi.python.org using json."""
-  # reader = codecs.getreader("utf-8")
   req = requests.get(url_pattern.format(package=package))
   version = parse('0')
   if req.status_code == requests.codes.ok:
-    # j = json.loads(req.text.encode(req.encoding))
     j = json.loads(req.text)
     if 'releases' in j:
       releases = j['releases']
@@ -48,7 +47,7 @@ def get_version(package, url_pattern=URL_PATTERN):
 
 
 def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024):
-  remote = urllib.request.urlopen(url)
+  remote = urlopen(url)
   hash = hashlib.md5()
 
   total_read = 0
@@ -58,16 +57,16 @@ def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024):
 
     if not data or total_read > max_file_size:
       break
-
     hash.update(data)
 
   return hash.hexdigest()
 
 
-def main(package):
+def main(package, direct_push=False):
   stable_version = get_version(package)
   print('latest stable version for {} is {}'.format(package, stable_version))
-  url = 'https://pypi.io/packages/source/{0}/{1}/{1}-{2}.zip'.format(package[0], package, stable_version)
+  url = 'https://pypi.io/packages/source/{0}/{1}/{1}-{2}.zip'.format(
+    package[0], package, stable_version)
   try:
     md5 = get_remote_md5_sum(url)
   except Exception:
@@ -77,14 +76,19 @@ def main(package):
     print("\nClonning the feedstock")
     feedstock = os.path.join(temp_dir, '{}-feedstock'.format(package))
     try:
-      run_commands(['git', 'clone', 'git@github.com:conda-forge/{}-feedstock.git'.format(package), feedstock])
+      run_commands(
+        ['git', 'clone',
+         'git@github.com:conda-forge/{}-feedstock.git'.format(package),
+         feedstock])
     except ValueError:
       print("\nThe feedstock does not exist on conda-forge. Exiting ...")
       raise
     os.chdir(feedstock)
-    run_commands(['git', 'remote', 'add', 'bioidiap', 'git@github.com:bioidiap/{}-feedstock.git'.format(package)],
-                 # ['git', 'fetch', '--all'],
-                 ['git', 'checkout', '-b', stable_version])
+    if not direct_push:
+      run_commands(
+        ['git', 'remote', 'add', 'bioidiap',
+         'git@github.com:bioidiap/{}-feedstock.git'.format(package)],
+        ['git', 'checkout', '-b', stable_version])
     # update meta.yaml
     with open('recipe/meta.yaml') as f:
       doc = f.read()
@@ -92,25 +96,74 @@ def main(package):
       build_number = '200'
     else:
       build_number = '0'
-    doc = re.sub(r'\{\s?%\s?set\s?version\s?=\s?".*"\s?%\s?\}', '{% set version = "' + str(stable_version) + '" %}', doc, count=1)
-    doc = re.sub(r'\s+number\:\s?[0-9]+', '\n  number: ' + build_number, doc, count=1)
-    doc = re.sub(r'\{\s?%\s?set\s?build_number\s?=\s?"[0-9]+"\s?%\s?\}', '{% set build_number = "' + build_number + '" %}', doc, count=1)
+    doc = re.sub(r'\{\s?%\s?set\s?version\s?=\s?".*"\s?%\s?\}',
+                 '{% set version = "' + str(stable_version) + '" %}',
+                 doc, count=1)
+    doc = re.sub(r'\s+number\:\s?[0-9]+', '\n  number: ' + build_number, doc,
+                 count=1)
+    doc = re.sub(r'\{\s?%\s?set\s?build_number\s?=\s?"[0-9]+"\s?%\s?\}',
+                 '{% set build_number = "' + build_number + '" %}',
+                 doc, count=1)
     doc = re.sub(r'\s+md5\:.*', '\n  md5: {}'.format(md5), doc, count=1)
-    doc = re.sub(r'\s+url\:.*', '\n  url: {}'.format(url.replace(stable_version, '{{ version }}')), doc, count=1)
-    doc = re.sub(r'\s+home\:.*', '\n  home: https://www.idiap.ch/software/bob/', doc, count=1)
+    doc = re.sub(r'\s+url\:.*',
+                 '\n  url: {}'.format(
+                  url.replace(stable_version, '{{ version }}')),
+                 doc, count=1)
+    doc = re.sub(r'\s+home\:.*',
+                 '\n  home: https://www.idiap.ch/software/bob/',
+                 doc, count=1)
+
+    if package == 'bob':
+      requrl = 'https://gitlab.idiap.ch/bob/bob/raw/master/requirements.txt'
+      remote = requests.get(requrl)
+      req = remote.content.decode()
+      req = '\n    - '.join(req.replace('== ', '==').strip().split('\n'))
+      be_id = doc.find('bob.extension')
+      te_id = doc.find('test:\n', be_id)
+      template = '''{req}
+
+  run:
+    - python
+    - {req}
+
+'''.format(req=req)
+      doc = doc[:be_id] + template + doc[te_id:]
+
     with open('recipe/meta.yaml', 'w') as f:
       f.write(doc)
+
     run_commands(['conda-smithy', 'rerender'],
                  ['git', '--no-pager', 'diff'],
-                 ['git', 'add', '-A'],
-                 ['git', 'commit', '-am', 'Update to version {}'.format(stable_version)],
-                 ['git', 'push', '--force', '--set-upstream', 'bioidiap', stable_version],
-                 ['hub', 'pull-request', '--browse', '-b', 'conda-forge:master', '-h', 'bioidiap:{}'.format(stable_version), '-m', 'Update to version {}'.format(stable_version)])
-    print('\nPlease create the pull request in the webpage that was openned.')
+                 ['git', 'add', '-A'])
+    try:
+      run_commands(['git', 'commit', '-am',
+                    'Fix badges [skip ci]'])
+                    # 'Update to version {}'.format(stable_version)])
+    except ValueError:
+      print('Feedstock is already uptodate, skipping.')
+      return
+    if direct_push:
+      print(feedstock)
+      try:
+        answer = raw_input('Would you like to push directly to master?').lower()
+      except Exception:
+        answer = input('Would you like to push directly to master?').lower()
+      if answer.startswith('y') or answer == '':
+        run_commands(['git', 'push'])
+        print('See the changes at:\n'
+              'https://github.com/conda-forge/'
+              '{}-feedstock/commits/master\n\n'.format(package))
+    else:
+      run_commands(['git', 'push', '--force', '--set-upstream',
+                    'bioidiap', stable_version],
+                   ['hub', 'pull-request', '-b', 'conda-forge:master',
+                    '-h', 'bioidiap:{}'.format(stable_version),
+                    '-m', 'Update to version {}'.format(stable_version)])
   finally:
     shutil.rmtree(temp_dir)
 
 
 if __name__ == '__main__':
   import sys
-  main(*sys.argv[1:])
+  pkg = sys.argv[1]
+  main(pkg)

install/install-conda.sh

@@ -34,3 +34,8 @@ fi
 
 echo "[>>] Updating conda in the root environment..."
 ${CONDA} update --yes -n root conda
+
+echo "[>>] Installing conda-smithy..."
+${CONDA} install --yes -n root conda-smithy
+
+echo "You need to manually install https://hub.github.com/ too ..."