Merge branch 'newchannel' into 'master'

Adapt our scripts to use our channel

See merge request !34

conda/create_new_env.sh

@@ -4,13 +4,27 @@ set -xe
 # usage create_new_env.sh (-n env_name|-p env_folder) pyver [bobver]
 # This is intented to be used only with our conda installation at idiap/group/torch5spro
 # example create_new_env.sh -n bob-2.3.4-py27_0 2.7
-# example create_new_env.sh -n bob-2.3.4-py27_0 2.7 =2.3.4
-# example create_new_env.sh -n bob-2.3.4-py27_0 2.7 =2.3.4=py27_0
+# example create_new_env.sh -n bob-2.3.4-py27_0 2.7 =4.2.0
+# example create_new_env.sh -n bob-2.3.4-py27_0 2.7 =4.2.0 =2.3.4
+# example create_new_env.sh -n bob-2.3.4-py27_0 2.7 =4.2.0 =2.3.4=py27_0
 # example create_new_env.sh -p /path/to/bob-2.2.0-py27_0 2.7
 
-conda create --yes $1 $2 --channel defaults --override-channels python=$3 anaconda
+conda create --yes $1 $2 \
+	--channel defaults \
+	--channel https://www.idiap.ch/software/bob/conda \
+	--override-channels \
+	python=$3 \
+	anaconda$4 \
+	bob$5 \
+	caffe \
+	tensorflow \
+	docopt \
+	sphinx_rtd_theme \
+	gcc \
+	coverage
+
 source activate $2
-conda install --yes --no-update-deps bob$4 caffe tensorflow ipdb docopt sphinx_rtd_theme gcc
+
 pip install bob.db.arface \
 	bob.db.atvskeystroke \
 	bob.db.avspoof \
@@ -29,7 +43,6 @@ pip install bob.db.arface \
 	bob.db.replay \
 	bob.db.scface \
 	bob.db.utfvp \
-	bob.db.voxforge \
 	bob.db.xm2vts \
 	bob.db.youtube \
 	bob.bio.base \
@@ -37,8 +50,9 @@ pip install bob.db.arface \
 	bob.bio.spear \
 	bob.bio.face \
 	bob.bio.video \
+	bob.db.voxforge \
 	gridtk \
-	coveralls
+	ipdb
 
 echo "You can now activate your environment like this:"
 echo "source activate $2"

gitlab/update_feedstock.py

@@ -62,6 +62,55 @@ def get_remote_md5_sum(url, max_file_size=100 * 1024 * 1024):
   return hash.hexdigest()
 
 
+class Gitlab(object):
+    """A class that wraps Gitlab API using curl"""
+
+    def __init__(self, token):
+        super(Gitlab, self).__init__()
+        self.token = token
+        self.base_url = 'https://gitlab.idiap.ch/api/v3/'
+
+    def get_project(self, project_name, namespace='bob'):
+        cmd = ["curl", "--header",
+               "PRIVATE-TOKEN: {}".format(self.token),
+               self.base_url + "projects/{}%2F{}".format(
+                   namespace, project_name)]
+        pipeline = subprocess.check_output(cmd)
+        return json.loads(pipeline.decode())
+
+    def create_pipeline(self, project_id):
+        cmd = ["curl", "--request", "POST", "--header",
+               "PRIVATE-TOKEN: {}".format(self.token),
+               self.base_url + "projects/{}/pipeline?ref=master".format(
+                   project_id)]
+        pipeline = subprocess.check_output(cmd)
+        return json.loads(pipeline.decode())
+
+    def get_pipeline(self, project_id, pipeline_id):
+        cmd = ["curl", "--header",
+               "PRIVATE-TOKEN: {}".format(self.token),
+               self.base_url + "projects/{}/pipelines/{}".format(
+                   project_id, pipeline_id)]
+        pipeline = subprocess.check_output(cmd)
+        return json.loads(pipeline.decode())
+
+    def create_merge_request(self, project_id, source_branch, target_branch, title,
+                             assignee_id='', description='', target_project_id='',
+                             labels='', milestone_id='', remove_source_branch=''):
+        url = "projects/{}/merge_requests?"
+        url += "&".join(['source_branch={}', 'target_branch={}', 'title={}'])
+        url += "&".join(['assignee_id={}', 'description={}', 'target_project_id={}',
+                         'labels={}', 'milestone_id={}', 'remove_source_branch={}'])
+        url = url.format(project_id, source_branch, target_branch, title,
+                         assignee_id, description, target_project_id, labels,
+                         milestone_id, remove_source_branch)
+        cmd = ["curl", "--request", "POST", "--header",
+               "PRIVATE-TOKEN: {}".format(self.token),
+               self.base_url + url]
+        pipeline = subprocess.check_output(cmd)
+        return json.loads(pipeline.decode())
+
+
 def main(package, direct_push=False):
   stable_version = get_version(package)
   print('latest stable version for {} is {}'.format(package, stable_version))
@@ -73,29 +122,27 @@ def main(package, direct_push=False):
       raise
   temp_dir = tempfile.mkdtemp()
   try:
-    print("\nClonning the feedstock")
-    feedstock = os.path.join(temp_dir, '{}-feedstock'.format(package))
+    print("\nClonning bob.conda")
+    root = os.path.join(temp_dir, 'bob.conda')
+    feedstock = os.path.join(root, 'recipes', package)
     try:
       run_commands(
         ['git', 'clone',
-         'git@github.com:conda-forge/{}-feedstock.git'.format(package),
-         feedstock])
+         'git@gitlab.idiap.ch:bob/bob.conda.git',
+         root])
     except ValueError:
-      print("\nThe feedstock does not exist on conda-forge. Exiting ...")
+      print("\nFailed to clone `bob.conda`, Exiting ...")
       raise
+    branch_name = '{}-{}'.format(package, stable_version)
     os.chdir(feedstock)
     if not direct_push:
       run_commands(
-        ['git', 'remote', 'add', 'bioidiap',
-         'git@github.com:bioidiap/{}-feedstock.git'.format(package)],
-        ['git', 'checkout', '-b', stable_version])
+        ['git', 'checkout', '-b', branch_name])
     # update meta.yaml
-    with open('recipe/meta.yaml') as f:
+    meta_path = 'meta.yaml'
+    with open(meta_path) as f:
       doc = f.read()
-    if package == 'bob.math':
-      build_number = '200'
-    else:
-      build_number = '0'
+    build_number = '0'
     doc = re.sub(r'\{\s?%\s?set\s?version\s?=\s?".*"\s?%\s?\}',
                  '{% set version = "' + str(stable_version) + '" %}',
                  doc, count=1)
@@ -130,14 +177,10 @@ def main(package, direct_push=False):
 '''.format(req=req)
       doc = doc[:be_id] + template + doc[te_id:]
 
-    with open('recipe/meta.yaml', 'w') as f:
+    with open(meta_path, 'w') as f:
       f.write(doc)
 
-    conda_smithy_path = ''
-    if os.path.isdir('/local/conda/bin/'):
-        conda_smithy_path = '/local/conda/bin/'
-    run_commands([conda_smithy_path+'conda-smithy', 'rerender'],
-                 ['git', '--no-pager', 'diff'],
+    run_commands(['git', '--no-pager', 'diff'],
                  ['git', 'add', '-A'])
     try:
       run_commands(['git', 'commit', '-am',
@@ -158,10 +201,12 @@ def main(package, direct_push=False):
               '{}-feedstock/commits/master\n\n'.format(package))
     else:
       run_commands(['git', 'push', '--force', '--set-upstream',
-                    'bioidiap', stable_version],
-                   ['hub', 'pull-request', '-b', 'conda-forge:master',
-                    '-h', 'bioidiap:{}'.format(stable_version),
-                    '-m', 'Update to version {}'.format(stable_version)])
+                    'origin', branch_name])
+      gitlab = Gitlab(os.environ.get('GITLAB_API_TOKEN'))
+      project_id = gitlab.get_project(package)['id']
+      title = 'Update to version {}'.format(branch_name)
+      gitlab.create_merge_request(project_id, branch_name, 'master', title,
+                                  remove_source_branch='true')
   finally:
     shutil.rmtree(temp_dir)
 

nightlies/trigger_pipelines.py

@@ -91,6 +91,22 @@ class Gitlab(object):
         pipeline = subprocess.check_output(cmd)
         return json.loads(pipeline.decode())
 
+    def create_merge_request(self, project_id, source_branch, target_branch, title,
+                             assignee_id='', description='', target_project_id='',
+                             labels='', milestone_id='', remove_source_branch=''):
+        url = "projects/{}/merge_requests?"
+        url += "&".join(['source_branch={}', 'target_branch={}', 'title={}'])
+        url += "&".join(['assignee_id={}', 'description={}', 'target_project_id={}',
+                         'labels={}', 'milestone_id={}', 'remove_source_branch={}'])
+        url = url.format(project_id, source_branch, target_branch, title,
+                         assignee_id, description, target_project_id, labels,
+                         milestone_id, remove_source_branch)
+        cmd = ["curl", "--request", "POST", "--header",
+               "PRIVATE-TOKEN: {}".format(self.token),
+               self.base_url + url]
+        pipeline = subprocess.check_output(cmd)
+        return json.loads(pipeline.decode())
+
 
 def main(packages_list):
     gitlab = Gitlab(os.environ.get('GITLAB_API_TOKEN'))