diff --git a/conda/batch-update-bob.sh b/conda/batch-update-conda-forge.sh
similarity index 92%
rename from conda/batch-update-bob.sh
rename to conda/batch-update-conda-forge.sh
index 6a2261a150bb37ad997e2671368b81b11cc2bf00..6e9c349cf05b3c547bd21a17c96612acffd6d9bf 100755
--- a/conda/batch-update-bob.sh
+++ b/conda/batch-update-conda-forge.sh
@@ -27,11 +27,14 @@ case $1 in
7)
packages=("bob.learn.em" "bob.db.iris")
;;
+ 8)
+ packages=("bob")
+ ;;
esac
for pkg in "${packages[@]}"
do
gh re --fork $pkg-feedstock --user conda-forge --organization bioidiap || true
- python update_feedstock.py $pkg
+ python ../gitlab/update_feedstock.py $pkg
done
diff --git a/conda/update_feedstock.py b/conda/update_feedstock.py
deleted file mode 100755
index 1357ba390afa24573f9be367407c580ab4b2c2ce..0000000000000000000000000000000000000000
--- a/conda/update_feedstock.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env python
-
-import hashlib
-import urllib.request
-import requests
-import json
-try:
- from packaging.version import parse
-except ImportError:
- from pip._vendor.packaging.version import parse
-import re
-import tempfile
-import shutil
-import os
-import subprocess
-
-URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json'
-
-
-def run_commands(*calls):
- """runs the given commands."""
- # get all calls
- for call in calls:
- if True or False:
- print(' - ' + ' '.join(call))
- if not False:
- # execute call
- if subprocess.call(call):
- # call failed (has non-zero exit status)
- raise ValueError("Command '%s' failed; stopping" % ' '.join(call))
-
-
-def get_version(package, url_pattern=URL_PATTERN):
- """Return version of package on pypi.python.org using json."""
- # reader = codecs.getreader("utf-8")
- req = requests.get(url_pattern.format(package=package))
- version = parse('0')
- if req.status_code == requests.codes.ok:
- # j = json.loads(req.text.encode(req.encoding))
- j = json.loads(req.text)
- if 'releases' in j:
- releases = j['releases']
- for release in releases:
- ver = parse(release)
- if not ver.is_prerelease:
- version = max(version, ver)
- return str(version)
-
-
-def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024):
- remote = urllib.request.urlopen(url)
- hash = hashlib.md5()
-
- total_read = 0
- while True:
- data = remote.read(4096)
- total_read += 4096
-
- if not data or total_read > max_file_size:
- break
-
- hash.update(data)
-
- return hash.hexdigest()
-
-
-def main(package):
- stable_version = get_version(package)
- print('latest stable version for {} is {}'.format(package, stable_version))
- url = 'https://pypi.io/packages/source/{0}/{1}/{1}-{2}.zip'.format(package[0], package, stable_version)
- try:
- md5 = get_remote_md5_sum(url)
- except Exception:
- raise
- temp_dir = tempfile.mkdtemp()
- try:
- print("\nClonning the feedstock")
- feedstock = os.path.join(temp_dir, '{}-feedstock'.format(package))
- try:
- run_commands(['git', 'clone', 'git@github.com:conda-forge/{}-feedstock.git'.format(package), feedstock])
- except ValueError:
- print("\nThe feedstock does not exist on conda-forge. Exiting ...")
- raise
- os.chdir(feedstock)
- run_commands(['git', 'remote', 'add', 'bioidiap', 'git@github.com:bioidiap/{}-feedstock.git'.format(package)],
- # ['git', 'fetch', '--all'],
- ['git', 'checkout', '-b', stable_version])
- # update meta.yaml
- with open('recipe/meta.yaml') as f:
- doc = f.read()
- if package == 'bob.math':
- build_number = '200'
- else:
- build_number = '0'
- doc = re.sub(r'\{\s?%\s?set\s?version\s?=\s?".*"\s?%\s?\}', '{% set version = "' + str(stable_version) + '" %}', doc, count=1)
- doc = re.sub(r'\s+number\:\s?[0-9]+', '\n number: ' + build_number, doc, count=1)
- doc = re.sub(r'\{\s?%\s?set\s?build_number\s?=\s?"[0-9]+"\s?%\s?\}', '{% set build_number = "' + build_number + '" %}', doc, count=1)
- doc = re.sub(r'\s+md5\:.*', '\n md5: {}'.format(md5), doc, count=1)
- doc = re.sub(r'\s+url\:.*', '\n url: {}'.format(url.replace(stable_version, '{{ version }}')), doc, count=1)
- doc = re.sub(r'\s+home\:.*', '\n home: https://www.idiap.ch/software/bob/', doc, count=1)
- with open('recipe/meta.yaml', 'w') as f:
- f.write(doc)
- run_commands(['conda-smithy', 'rerender'],
- ['git', '--no-pager', 'diff'],
- ['git', 'add', '-A'],
- ['git', 'commit', '-am', 'Update to version {}'.format(stable_version)],
- ['git', 'push', '--force', '--set-upstream', 'bioidiap', stable_version],
- ['hub', 'pull-request', '--browse', '-b', 'conda-forge:master', '-h', 'bioidiap:{}'.format(stable_version), '-m', 'Update to version {}'.format(stable_version)])
- print('\nPlease create the pull request in the webpage that was openned.')
- finally:
- shutil.rmtree(temp_dir)
-
-
-if __name__ == '__main__':
- import sys
- main(*sys.argv[1:])
diff --git a/gitlab/deploy.sh b/gitlab/deploy.sh
index 9ad085cfd97b999ab6e4692d50a82b2db69cce1d..34b1ca2372e35333c200e5188de5c57fd133f73c 100755
--- a/gitlab/deploy.sh
+++ b/gitlab/deploy.sh
@@ -20,3 +20,40 @@ if [ -d sphinx ]; then
fi
unlock_pypirc
+
+condaforge_packages=("bob" \
+"bob.extension" \
+"bob.blitz" \
+"bob.core" \
+"bob.ip.draw" \
+"bob.io.base" \
+"bob.sp" \
+"bob.math" \
+"bob.ap" \
+"bob.measure" \
+"bob.db.base" \
+"bob.io.image" \
+"bob.io.video" \
+"bob.io.matlab" \
+"bob.ip.base" \
+"bob.ip.color" \
+"bob.ip.gabor" \
+"bob.learn.activation" \
+"bob.learn.libsvm" \
+"bob.learn.boosting" \
+"bob.io.audio" \
+"bob.learn.linear" \
+"bob.learn.mlp" \
+"bob.db.wine" \
+"bob.db.mnist" \
+"bob.db.atnt" \
+"bob.ip.flandmark" \
+"bob.ip.facedetect" \
+"bob.ip.optflow.hornschunck" \
+"bob.ip.optflow.liu" \
+"bob.learn.em" \
+"bob.db.iris")
+
+if contains_element ${CI_PROJECT_NAME} "${condaforge_packages[@]}"; then
+ run_cmd ${CONDA_FOLDER}/bin/python _ci/update_feedstock.py ${CI_PROJECT_NAME}
+fi
diff --git a/gitlab/functions.sh b/gitlab/functions.sh
index ea4640d903728b71eb493dd16ae4d0c6f55ca758..e82893c3b1e9213ea7a833b1d4e9be1cf14325d4 100644
--- a/gitlab/functions.sh
+++ b/gitlab/functions.sh
@@ -246,3 +246,10 @@ if [ -z "${CI_BUILD_TAG}" ]; then
else
log_info "Building tag, not setting BOB_DOCUMENTATION_SERVER"
fi
+
+# taken from here: https://stackoverflow.com/questions/3685970/check-if-an-array-contains-a-value
+contains_element () {
+ local e
+ for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
+ return 1
+}
diff --git a/gitlab/install.sh b/gitlab/install.sh
index 07c0cb52c931836d4800d3a402adf71dbf25c8d8..8adb34b7f3cebf8ad4ab118feaefe90726231282 100755
--- a/gitlab/install.sh
+++ b/gitlab/install.sh
@@ -57,3 +57,4 @@ for stage in "build" "test" "docs" "wheels" "deploy"; do
get_exec ${1} ${stage}.sh
get_exec ${1} after_${stage}.sh
done
+get_exec ${1} update_feedstock.py
diff --git a/gitlab/update_feedstock.py b/gitlab/update_feedstock.py
new file mode 100755
index 0000000000000000000000000000000000000000..a0c1df579858dcae6ab0521d0a8ebc2d08ebfeae
--- /dev/null
+++ b/gitlab/update_feedstock.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+
+import hashlib
+try:
+ from urllib2 import urlopen
+except ImportError:
+ from urllib.request import urlopen
+import requests
+import json
+try:
+ from packaging.version import parse
+except ImportError:
+ from pip._vendor.packaging.version import parse
+import re
+import tempfile
+import shutil
+import os
+import subprocess
+
+URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json'
+
+
+def run_commands(*calls):
+ """runs the given commands."""
+ # get all calls
+ for call in calls:
+ print(' - ' + ' '.join(call))
+ # execute call
+ if subprocess.call(call):
+ # call failed (has non-zero exit status)
+ raise ValueError("Command '%s' failed; stopping" % ' '.join(call))
+
+
+def get_version(package, url_pattern=URL_PATTERN):
+ """Return version of package on pypi.python.org using json."""
+ req = requests.get(url_pattern.format(package=package))
+ version = parse('0')
+ if req.status_code == requests.codes.ok:
+ j = json.loads(req.text)
+ if 'releases' in j:
+ releases = j['releases']
+ for release in releases:
+ ver = parse(release)
+ if not ver.is_prerelease:
+ version = max(version, ver)
+ return str(version)
+
+
+def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024):
+ remote = urlopen(url)
+ hash = hashlib.md5()
+
+ total_read = 0
+ while True:
+ data = remote.read(4096)
+ total_read += 4096
+
+ if not data or total_read > max_file_size:
+ break
+ hash.update(data)
+
+ return hash.hexdigest()
+
+
+def main(package, direct_push=False):
+ stable_version = get_version(package)
+ print('latest stable version for {} is {}'.format(package, stable_version))
+ url = 'https://pypi.io/packages/source/{0}/{1}/{1}-{2}.zip'.format(
+ package[0], package, stable_version)
+ try:
+ md5 = get_remote_md5_sum(url)
+ except Exception:
+ raise
+ temp_dir = tempfile.mkdtemp()
+ try:
+ print("\nClonning the feedstock")
+ feedstock = os.path.join(temp_dir, '{}-feedstock'.format(package))
+ try:
+ run_commands(
+ ['git', 'clone',
+ 'git@github.com:conda-forge/{}-feedstock.git'.format(package),
+ feedstock])
+ except ValueError:
+ print("\nThe feedstock does not exist on conda-forge. Exiting ...")
+ raise
+ os.chdir(feedstock)
+ if not direct_push:
+ run_commands(
+ ['git', 'remote', 'add', 'bioidiap',
+ 'git@github.com:bioidiap/{}-feedstock.git'.format(package)],
+ ['git', 'checkout', '-b', stable_version])
+ # update meta.yaml
+ with open('recipe/meta.yaml') as f:
+ doc = f.read()
+ if package == 'bob.math':
+ build_number = '200'
+ else:
+ build_number = '0'
+ doc = re.sub(r'\{\s?%\s?set\s?version\s?=\s?".*"\s?%\s?\}',
+ '{% set version = "' + str(stable_version) + '" %}',
+ doc, count=1)
+ doc = re.sub(r'\s+number\:\s?[0-9]+', '\n number: ' + build_number, doc,
+ count=1)
+ doc = re.sub(r'\{\s?%\s?set\s?build_number\s?=\s?"[0-9]+"\s?%\s?\}',
+ '{% set build_number = "' + build_number + '" %}',
+ doc, count=1)
+ doc = re.sub(r'\s+md5\:.*', '\n md5: {}'.format(md5), doc, count=1)
+ doc = re.sub(r'\s+url\:.*',
+ '\n url: {}'.format(
+ url.replace(stable_version, '{{ version }}')),
+ doc, count=1)
+ doc = re.sub(r'\s+home\:.*',
+ '\n home: https://www.idiap.ch/software/bob/',
+ doc, count=1)
+
+ if package == 'bob':
+ requrl = 'https://gitlab.idiap.ch/bob/bob/raw/master/requirements.txt'
+ remote = requests.get(requrl)
+ req = remote.content.decode()
+ req = '\n - '.join(req.replace('== ', '==').strip().split('\n'))
+ be_id = doc.find('bob.extension')
+ te_id = doc.find('test:\n', be_id)
+ template = '''{req}
+
+ run:
+ - python
+ - {req}
+
+'''.format(req=req)
+ doc = doc[:be_id] + template + doc[te_id:]
+
+ with open('recipe/meta.yaml', 'w') as f:
+ f.write(doc)
+
+ run_commands(['conda-smithy', 'rerender'],
+ ['git', '--no-pager', 'diff'],
+ ['git', 'add', '-A'])
+ try:
+ run_commands(['git', 'commit', '-am',
+ 'Fix badges [skip ci]'])
+ # 'Update to version {}'.format(stable_version)])
+ except ValueError:
+ print('Feedstock is already uptodate, skipping.')
+ return
+ if direct_push:
+ print(feedstock)
+ try:
+ answer = raw_input('Would you like to push directly to master?').lower()
+ except Exception:
+ answer = input('Would you like to push directly to master?').lower()
+ if answer.startswith('y') or answer == '':
+ run_commands(['git', 'push'])
+ print('See the changes at:\n'
+ 'https://github.com/conda-forge/'
+ '{}-feedstock/commits/master\n\n'.format(package))
+ else:
+ run_commands(['git', 'push', '--force', '--set-upstream',
+ 'bioidiap', stable_version],
+ ['hub', 'pull-request', '-b', 'conda-forge:master',
+ '-h', 'bioidiap:{}'.format(stable_version),
+ '-m', 'Update to version {}'.format(stable_version)])
+ finally:
+ shutil.rmtree(temp_dir)
+
+
+if __name__ == '__main__':
+ import sys
+ pkg = sys.argv[1]
+ main(pkg)
diff --git a/install/install-conda.sh b/install/install-conda.sh
index 80d6fb064fd47ad9f44ac1ba49aeb8c2e0143dbe..f87a79302be69274c281de0afc80eab1ba7f055e 100755
--- a/install/install-conda.sh
+++ b/install/install-conda.sh
@@ -34,3 +34,8 @@ fi
echo "[>>] Updating conda in the root environment..."
${CONDA} update --yes -n root conda
+
+echo "[>>] Installing conda-smithy..."
+${CONDA} install --yes -n root conda-smithy
+
+echo "You need to manually install https://hub.github.com/ too ..."