diff --git a/release/changelog_since_last_release.rst b/release/changelog_since_last_release.rst
index 73834874f532291d0bc045e469b975de7aa121cd..50cde5c2a26a04337934362db5f0c6f93e1e0c1b 100644
--- a/release/changelog_since_last_release.rst
+++ b/release/changelog_since_last_release.rst
@@ -178,12 +178,14 @@
* minor
* Migrate to conda based CI
* bob.db.asvspoof
- * v1.0.3 (Sep 22, 2017 14:20)
+ * v1.1.7 (Sep 22, 2017 14:20)
+ * Docs updates
* minor
* Migrate to conda based CI
- * Removed redundant debug_asvspoof2017 script
+ * Removed redundant debug_asvspoof script
* bob.db.asvspoof2017
* v1.0.3 (Sep 22, 2017 14:20)
+ * Docs updates
* minor
* Migrate to conda based CI
* Removed redundant debug_asvspoof2017 script
@@ -361,18 +363,40 @@
* Created methods modality_separator and modalities
* Included test case for objects.modality
* bob.bio.base
- * v3.3.0 (Sep 22, 2017 13:41)
- * bob.bio.base!20 Docs url fix
- * bob.bio.base!19 Python36 & docs fixes
- * bob.bio.base!18 Removed `Algorithm.read_probe`
- * major
- * Add as_array to frame containers
- * Make the original_directory and annotation_directory a property
- * Fixed tuple indexing bug in youtube db load function and added tests
- * Updated docs and tests
- * Fixed bob.bio.base#11
- * Added video annotator, updated documentation accordingly
+ * v3.2.0 (Sep 22, 2017 10:55)
+ * bob.bio.base!89 [ci] Fixed issue with matplotlib
+ * bob.bio.base!95 Removed class client from FileListDatabase
+ * bob.bio.base!94 Implemented the plots for the Detection Identification Rate (open-set identification)
+ * bob.bio.base!93 Corrections in the documentation
+ * bob.bio.base!92 Documented the configuration file input for the `verify.py` script
+ * bob.bio.base!90 Implemented a way to use, at the same time, 4 and 5 columns score file in the `evaluate.py` script
+ * bob.bio.base!88 Improved ROC and CMC plots
+ * bob.bio.base!87 Fixed * ZT files are processed even when no ZT processing is wanted
+ * bob.bio.base!81 Droped dependency on Latex
+ * bob.bio.base!86 Implemented five-column score file -* only during concatenation
+ * bob.bio.base!83 Improve suggestion on variable naming convention to match python's standard
+ * bob.bio.base!78 Removed `Algorithm.read_probe` method, since this is already solved via `bob.bio.base`
+ * bob.bio.base!82 Add a function to read features with generators
+ * v3.2.1 (Sep 26, 2017 15:04)
+ * Changed backend in `bob/bio/base/scripts/evaluate.py` to `'pdf'` instead of `'agg'`
+ * v4.2.1 (Mar 31, 2018 )
* Migrate to conda based CI
+ * Updated docs and tests
+ * Added allow_missing_files option and added tests
+ * Removed write_commands function in grid_search (closes bob.bio.base#71)
+ * Added annotator, updated documentation accordingly
+ * Allow for comment lines in file-lists
+ * Improves verbosity for preprocessing, extraction and enrollment
+ * bob.db.base.Database is deprecated.
+ * Using config file loading mechanism from bob.extension
+ * Fix debug message (closes bob.bio.base#103)
+ * Fixed the exception that is raised when score file is not found
+ * Mentions bob.bio.vein on bob.bio.base docs (closes bob.bio.base#104)
+ * Added metadata in preprocessing, feature extraction, and algorithm
+ * Removed write_commands function in grid_search (closes bob.bio.base#71)
+ * Improve replace directories
+ * patch
+ * small docs building fix
* bob.bio.gmm
* v3.1.0 (Sep 22, 2017 14:05)
* bob.bio.gmm!10 Removed Algorithm.read_probe method, since this is already solved via bob.bio.base
diff --git a/release/release_bob.py b/release/release_bob.py
new file mode 100755
index 0000000000000000000000000000000000000000..56068bd3c51e93d811a3a0d2016a2bed7e819afb
--- /dev/null
+++ b/release/release_bob.py
@@ -0,0 +1,318 @@
+#!/usr/bin/env python
+
+"""
+By using changelog file as an input (can be generated with 'generate_changelog.py' script),
+this script goes through all packages in changelog file (in order listed), tags them correctly
+as per the file, and releases them one by one. A script can also be used to release a single package.
+This script uses python-gitlab package for accessing GitLab's API.
+
+Usage:
+ {0} [-v...] [options] [--] <private_token>
+ {0} -h | --help
+ {0} --version
+
+Arguments:
+ <private_token> Private token used to access GitLab.
+
+Options:
+ -h --help Show this screen.
+ --version Show version.
+ -c, --changelog-file STR A changelog file with all packages to release with their tags, listed in order.
+ [default: changelog_since_last_release.rst].
+ -g, --group-name STR Group name where we are assuming that all packages are located.
+ [default: bob].
+ -p, --package STR If the name of a package is provided, then this package will be found
+ in the changelog file and the release will resume from it (if option --resume is set)
+ or only this package will be released.
+ -r, --resume The overall release will resume from the provided package name.
+ -q, --dry-run Only print the actions, but do not execute them.
+
+"""
+
+import sys
+import os
+from docopt import docopt
+import gitlab
+import datetime
+import re
+from distutils.version import StrictVersion as Version
+import numpy
+import time
+
+
+def _insure_correct_package(candidates, group_name, pkg_name):
+ for pkg in candidates:
+ # make sure the name and the group name match exactly
+ if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
+ return pkg
+ raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
+
+
+# adapted from the same-name function in new_version.py script of bob.extension package
+def _update_readme(readme, version=None):
+ # replace the travis badge in the README.rst with the given version
+ DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
+ BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
+
+ new_readme = []
+ for line in readme.splitlines():
+ if BRANCH_RE.search(line) is not None:
+ if "gitlab" in line: # gitlab links
+ replacement = "/v%s" % version if version is not None else "/master"
+ line = BRANCH_RE.sub(replacement, line)
+ if "software/bob" in line: # our doc server
+ if 'master' not in line: # don't replace 'latest' pointer
+ replacement = "/v%s" % version if version is not None \
+ else "/stable"
+ line = BRANCH_RE.sub(replacement, line)
+ if DOC_IMAGE.search(line) is not None:
+ replacement = '-v%s-' % version if version is not None else '-stable-'
+ line = DOC_IMAGE.sub(replacement, line)
+ new_readme.append(line)
+ return '\n'.join(new_readme)
+
+
+def get_parsed_tag(gitpkg, tag):
+ """
+ An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3
+ The latest tag is either patch, minor, major, or none
+ """
+ m = re.search(r"(v\d.\d.\d)", tag)
+ if m:
+ return m.group(0)
+ # tag = Version(tag)
+
+ # if we bump the version, we need to find the latest released version for this package
+ if 'patch' == tag or 'minor' == tag or 'major' == tag:
+ latest_tag = gitpkg.tags.list(per_page=1, page=1)
+ # if there were no tags yet, assume the first version
+ if not latest_tag:
+ return 'v1.0.0'
+ latest_tag = latest_tag[0]
+ latest_tag_name = latest_tag.name
+ # check that it has expected format v#.#.#
+ # latest_tag_name = Version(latest_tag_name)
+ m = re.match(r"(v\d.\d.\d)", latest_tag_name)
+ if not m:
+ raise ValueError(
+ 'The latest tag name {0} in package {1} has unknown format'.format(latest_tag_name, gitpkg.name))
+ # increase the version accordingly
+ if 'major' == tag: # increment the first number in 'v#.#.#'
+ return latest_tag_name[0] + str(int(latest_tag_name[1]) + 1) + latest_tag_name[2:]
+ if 'minor' == tag: # increment the second number in 'v#.#.#'
+ return latest_tag_name[:3] + str(int(latest_tag_name[3]) + 1) + latest_tag_name[4:]
+ if 'patch' == tag: # increment the last number in 'v#.#.#'
+ return latest_tag_name[:-1] + str(int(latest_tag_name[-1]) + 1)
+
+ if 'none' == tag: # we do nothing in this case
+ return tag
+
+ raise ValueError('Cannot parse changelog tag {0} of the package {1}'.format(tag, gitpkg.name))
+
+
+def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
+ # get tag and update its description
+ tag = gitpkg.tags.get(tag_name)
+ print('Found tag {1}, updating its comments with:'.format(gitpkg.name, tag.name))
+ print(tag_comments_list)
+ if not dry_run:
+ tag.set_release_description('\n'.join(tag_comments_list))
+ return tag
+
+
+def commit_files(gitpkg, files_list, message='Updated files', dry_run=False):
+ data = {
+ 'branch': 'master', # v4
+ 'commit_message': message,
+ 'actions': []
+ }
+ # add files to update
+ for filename in files_list.keys():
+ update_action = dict(action='update', file_path=filename)
+ # with open(filename, 'r') as f:
+ # update_action['content'] = f.read()
+ update_action['content'] = files_list[filename]
+ data['actions'].append(update_action)
+
+ print("Committing changes in files: {0}".format(str(files_list.keys())))
+ if not dry_run:
+ gitpkg.commits.create(data)
+
+
+def get_last_nonskip_pipeline(gitpkg, before_last=False):
+ # sleep for 10 seconds to ensure that if a pipeline was just submitted,
+ # we can retrieve it
+ time.sleep(10)
+ if before_last:
+ # take the pipeline before the last
+ return gitpkg.pipelines.list(per_page=2, page=1)[1]
+ else:
+ # otherwise take the last pipeline
+ return gitpkg.pipelines.list(per_page=1, page=1)[0]
+
+
+def just_build_package(gitpkg, dry_run=False):
+ # we assume the last pipeline is with commit [skip ci]
+ # so, we take the pipeline that can be re-built, which the previous to the last one
+ last_pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True)
+
+ # check that the chosen pipeline is the one we are looking for
+ latest_tag_name = gitpkg.tags.list(per_page=1, page=1)[0].name
+ # the pipeline should be the one built for the latest tag, so check if it is the correct choice
+ if last_pipeline.ref != latest_tag_name:
+ raise ValueError('While deploying {0}, found pipeline {1} but it does not match '
+ 'the latest tag {2}'.format(gitpkg.name, last_pipeline.id, latest_tag_name))
+ # the pipeline should have succeeded, otherwise we cannot release
+ if last_pipeline.status != 'success':
+ raise ValueError('While deploying {0}, found pipeline {1} but its status is "{2}" instead '
+ 'of the expected "sucess"'.format(gitpkg.name, last_pipeline.id, last_pipeline.status))
+
+ print("Retrying pipeline {0}".format(last_pipeline.id))
+ if not dry_run:
+ last_pipeline.retry()
+
+
+def wait_for_pipeline_to_finish(gitpkg, tag, dry_run=False):
+ sleep_step = 30
+ max_sleep = 60 * 60 # one hour
+ pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True)
+
+ pipeline_id = pipeline.id
+
+ print('Waiting for the pipeline {0} of package {1} to finish. Do not interrupt.'.format(pipeline_id, gitpkg.name))
+
+ if dry_run:
+ return
+
+ # probe and wait for the pipeline to finish
+ slept_so_far = 0
+ while pipeline.status == 'running' or pipeline.status == 'pending':
+ time.sleep(sleep_step)
+ slept_so_far += sleep_step
+ if slept_so_far > max_sleep:
+ raise ValueError('I cannot wait longer than {0} seconds for '
+ 'pipeline {1} to finish running!'.format(max_sleep, pipeline_id))
+ # probe gitlab to update the status of the pipeline
+ pipeline = gitpkg.pipelines.get(pipeline_id)
+
+ # finished running, now check if it succeeded
+ if pipeline.status != 'success':
+ raise ValueError('Pipeline {0} of project {1} exited with undesired status "{2}". '
+ 'Release is not possible.'.format(pipeline_id, gitpkg.name, pipeline.status))
+
+ print('Pipeline {0} of package {1} succeeded. Continue processing.'.format(pipeline_id, gitpkg.name))
+
+
+def cancel_last_pipeline(gitpkg):
+ pipeline = get_last_nonskip_pipeline(gitpkg)
+ print('Cancelling the last pipeline {0} of project {1}'.format(pipeline.id, gitpkg.name))
+ pipeline.cancel()
+
+
+def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False):
+ # if there is nothing to release, just rebuild the package
+ if tag_name == 'none':
+ print("Since the tag is 'none', we just re-build the last pipeline")
+ return just_build_package(gitpkg)
+
+ # 1. Replace branch tag in Readme to new tag, change version file to new version tag. Add and commit to gitlab
+ version_number = tag_name[1:] # remove 'v' in front
+ readme_file = gitpkg.files.get(file_path='README.rst', ref='master')
+ readme_content = readme_file.decode().decode()
+ readme_content = _update_readme(readme_content, version=version_number)
+ # commit and push changes
+ commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
+ 'Increased stable version to %s' % version_number, dry_run)
+ if not dry_run:
+ # cancel running the pipeline triggered by the last commit
+ cancel_last_pipeline(gitpkg)
+
+ # 2. Tag package with new tag and push
+ print("Creating tag {}".format(tag_name))
+ print("updating tag's comments with:".format(gitpkg.name, tag_name))
+ print(tag_comments_list)
+ if not dry_run:
+ tag = gitpkg.tags.create({'tag_name': tag_name, 'ref': 'master'})
+ # update tag with comments
+ tag.set_release_description('\n'.join(tag_comments_list))
+
+ # 3. Replace branch tag in Readme to master, change version file to beta version tag. Git add, commit, and push.
+ readme_content = _update_readme(readme_content)
+ version_number += 'b0'
+ # commit and push changes
+ commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
+ 'Increased latest version to %s [skip ci]' % version_number, dry_run)
+
+
+def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changelog, dry_run=False):
+ cur_tag = None
+ cur_tag_comments = []
+
+ grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), bob_group.name, pkg_name)
+ # so, we need to retrieve the full info from GitLab using correct project id
+ gitpkg = gl.projects.get(id=grpkg.id)
+
+ # we assume that changelog is formatted as structured text
+ # first line is the name of the package
+ for line in package_changelog:
+ if ' *' == line[:3]: # a tag level
+ # write the comments collected for the previous tag
+ if cur_tag:
+ update_tag_comments(gitpkg, cur_tag, cur_tag_comments, dry_run)
+ cur_tag_comments = [] # reset comments
+
+ # parse the current tag name
+ cur_tag = get_parsed_tag(gitpkg, line[3:].strip())
+
+ else: # all other lines are assumed to be comments
+ cur_tag_comments.append(line.strip())
+
+ # return the last tag and comments for release
+ return gitpkg, cur_tag, cur_tag_comments
+
+
+def main(private_token, group_name='bob', changelog_file='changelog.rst', dry_run=False, package=None, resume=False):
+ gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
+ bob_group = gl.groups.list(search=group_name)[0]
+
+ # traverse all packages in the changelog, edit older tags with updated comments,
+ # tag them with a suggested version, then try to release, and
+ # wait until done to proceed to the next package
+ with open(changelog_file) as f:
+ changelog = f.readlines()
+
+ # find the starts of each package's description in the changelog
+ pkgs = numpy.asarray([i for i, line in enumerate(changelog) if line[0] == '*'])
+ pkgs = numpy.concatenate([pkgs, [len(changelog)]])
+ start_idx = 0
+ if package:
+ # get the index where the package first appears in the list
+ start_idx = [i for i, line in enumerate(changelog) if line[1:].strip() == package]
+ if not start_idx:
+ print('Package {0} was not found in the changelog'.format(package))
+ return
+ start_idx = pkgs.tolist().index(start_idx[0])
+
+ for i in range(start_idx, pkgs.shape[0] - 1):
+ cur_package_name = changelog[pkgs[i]][1:].strip()
+ print('\nProcessing package {0}'.format(changelog[pkgs[i]]))
+ gitpkg, tag, tag_comments = parse_and_process_package_changelog(gl, bob_group, cur_package_name,
+ changelog[pkgs[i] + 1: pkgs[i + 1]], dry_run)
+ # release the package with the found tag and its comments
+ if gitpkg:
+ release_package(gitpkg, tag, tag_comments, dry_run)
+ # now, wait for the pipeline to finish, before we can release the next package
+ wait_for_pipeline_to_finish(gitpkg, tag, dry_run)
+
+ # if package name is provided and resume is not set, process only this package
+ if package == cur_package_name and not resume:
+ break
+
+ print('\nFinished processing changelog')
+
+
+if __name__ == '__main__':
+ arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1')
+ main(arguments['<private_token>'], group_name=arguments['--group-name'],
+ changelog_file=arguments['--changelog-file'], dry_run=arguments['--dry-run'],
+ package=arguments['--package'], resume=arguments['--resume'])