diff --git a/release/release_bob.py b/release/release_bob.py index 56068bd3c51e93d811a3a0d2016a2bed7e819afb..c41d3d5e652d3d91dfa5d2e18b1a4f0f27fe27e7 100755 --- a/release/release_bob.py +++ b/release/release_bob.py @@ -3,9 +3,52 @@ """ By using changelog file as an input (can be generated with 'generate_changelog.py' script), this script goes through all packages in changelog file (in order listed), tags them correctly -as per the file, and releases them one by one. A script can also be used to release a single package. +as per the file, and releases them one by one. This script uses python-gitlab package for accessing GitLab's API. +This script uses the provided changelog file to release a package. +The changelog is expected to have the following structure: + + * package name + * tag1 name (date of the tag). + * tag description. Each line of the tag description starts with `*` character. + - commits (from earliest to latest). Each line of the commit starts with `-` character. + * tag2 name (date of the tag). + * tag description. Each line of the tag description starts with `*` character. + - commits (from earliest to latest). Each line of the commit starts with `-` character. + * patch + - leftover not-tagged commits (from earliest to latest) + +This script can also be used to release a single package. +`IMPORTANT`: there are some considerations that needs to be taken +into account **before** you release a new version of a package: + + * In the changelog file: + - write the name of this package and write the last tag. + For the tag name, you can either indicate `patch`, `minor` or `major`, + and the package will be then released with either patch, minor, or major version bump. + - Alternatively, you can specify the version name directly but be careful that it is higher + than the last release tag of this package. + Make sure that the version that you are trying to release is not already released. + Also, make sure you follow semantic versions: http://semver.org. + - Then, under the desired new tag version of the package, please write down the changes that are applied + to the package between the last released version and this version. This + changes are written to release tags of packages in the Gitlab interface. + For an example look at: https://gitlab.idiap.ch/bob/bob.extension/tags + * Make sure all the tests for the package are passing. + * Make sure the documentation is building with the following command: + ``sphinx-build -aEWn doc sphinx`` + * Make sure all changes are committed to the git repository and pushed. + * Make sure the documentation badges in README.rst are pointing to: + https://www.idiap.ch/software/bob/docs/bob/... + * For database packages, make sure that the '.sql3' file or other + metadata files have been generated (if any). + * Make sure bob.nightlies is green after the changes are submitted if the + package is a part of the nightlies. + * If your package depends on an unreleased version of another Bob package, + you need to release that package first. + + Usage: {0} [-v...] [options] [--] <private_token> {0} -h | --help @@ -18,7 +61,7 @@ Options: -h --help Show this screen. --version Show version. -c, --changelog-file STR A changelog file with all packages to release with their tags, listed in order. - [default: changelog_since_last_release.rst]. + [default: changelog_since_last_release.md]. -g, --group-name STR Group name where we are assuming that all packages are located. [default: bob]. -p, --package STR If the name of a package is provided, then this package will be found @@ -35,7 +78,7 @@ from docopt import docopt import gitlab import datetime import re -from distutils.version import StrictVersion as Version +from distutils.version import StrictVersion import numpy import time @@ -50,7 +93,17 @@ def _insure_correct_package(candidates, group_name, pkg_name): # adapted from the same-name function in new_version.py script of bob.extension package def _update_readme(readme, version=None): - # replace the travis badge in the README.rst with the given version + """ + Inside text of the readme, replaces parts of the links to the provided version. + If version is not provided, replace to `stable` or `master`. + Args: + readme: Text of the README.rst file from a bob package + version: Format of the version string is '#.#.#' + + Returns: New text of readme with all replaces done + + """ + # replace the badge in the readme's text with the given version DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-') BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))') @@ -72,6 +125,28 @@ def _update_readme(readme, version=None): return '\n'.join(new_readme) +def get_latest_tag_name(gitpkg): + """ + Find the name of the latest tag for a given package in the format '#.#.#' + Args: + gitpkg: gitlab package object + + Returns: The name of the latest tag in format '#.#.#'. None if no tags for the package were found. + + """ + # get 50 latest tags as a list + latest_tags = gitpkg.tags.list(all=True) + if not latest_tags: + return None + # create list of tags' names but ignore the first 'v' character in each name + tag_names = [tag.name[1:] for tag in latest_tags] + # sort them correctly according to each subversion number + tag_names.sort(key=StrictVersion) + # take the last one, as it is the latest tag in the sorted tags + latest_tag_name = tag_names[-1] + return latest_tag_name + + def get_parsed_tag(gitpkg, tag): """ An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3 @@ -84,25 +159,26 @@ def get_parsed_tag(gitpkg, tag): # if we bump the version, we need to find the latest released version for this package if 'patch' == tag or 'minor' == tag or 'major' == tag: - latest_tag = gitpkg.tags.list(per_page=1, page=1) - # if there were no tags yet, assume the first version - if not latest_tag: - return 'v1.0.0' - latest_tag = latest_tag[0] - latest_tag_name = latest_tag.name - # check that it has expected format v#.#.# + # find the correct latest tag of this package (without 'v' in front), + # None if there are no tags yet + latest_tag_name = get_latest_tag_name(gitpkg) + # if there were no tags yet, assume the very first version + if not latest_tag_name: + return 'v0.0.1' + # check that it has expected format #.#.# # latest_tag_name = Version(latest_tag_name) - m = re.match(r"(v\d.\d.\d)", latest_tag_name) + m = re.match(r"(\d.\d.\d)", latest_tag_name) if not m: raise ValueError( - 'The latest tag name {0} in package {1} has unknown format'.format(latest_tag_name, gitpkg.name)) + 'The latest tag name {0} in package {1} has unknown format'.format('v' + latest_tag_name, gitpkg.name)) # increase the version accordingly - if 'major' == tag: # increment the first number in 'v#.#.#' - return latest_tag_name[0] + str(int(latest_tag_name[1]) + 1) + latest_tag_name[2:] - if 'minor' == tag: # increment the second number in 'v#.#.#' - return latest_tag_name[:3] + str(int(latest_tag_name[3]) + 1) + latest_tag_name[4:] + major, minor, patch = latest_tag_name.split('.') + if 'major' == tag: # increment the first number in 'v#.#.#' but make minor and patch to be 0 + return 'v' + str(int(major) + 1) + '.0.0' + if 'minor' == tag: # increment the second number in 'v#.#.#' but make patch to be 0 + return 'v' + major + '.' + str(int(minor) + 1) + '.0' if 'patch' == tag: # increment the last number in 'v#.#.#' - return latest_tag_name[:-1] + str(int(latest_tag_name[-1]) + 1) + return 'v' + major + '.' + minor + '.' + str(int(patch) + 1) if 'none' == tag: # we do nothing in this case return tag @@ -111,6 +187,17 @@ def get_parsed_tag(gitpkg, tag): def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False): + """ + Write annotations inside the provided tag of a given package. + Args: + gitpkg: gitlab package object + tag_name: The name of the tag to update + tag_comments_list: New annotations for this tag in a form of list + dry_run: If True, nothing will be committed or pushed to GitLab + + Returns: The gitlab object for the tag that was updated + + """ # get tag and update its description tag = gitpkg.tags.get(tag_name) print('Found tag {1}, updating its comments with:'.format(gitpkg.name, tag.name)) @@ -120,26 +207,42 @@ def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False): return tag -def commit_files(gitpkg, files_list, message='Updated files', dry_run=False): +def commit_files(gitpkg, files_dict, message='Updated files', dry_run=False): + """ + Commit files of a given GitLab package. + Args: + gitpkg: gitlab package object + files_dict: Dictionary of file names and their contents (as text) + message: Commit message + dry_run: If True, nothing will be committed or pushed to GitLab + + """ data = { 'branch': 'master', # v4 'commit_message': message, 'actions': [] } # add files to update - for filename in files_list.keys(): + for filename in files_dict.keys(): update_action = dict(action='update', file_path=filename) - # with open(filename, 'r') as f: - # update_action['content'] = f.read() - update_action['content'] = files_list[filename] + update_action['content'] = files_dict[filename] data['actions'].append(update_action) - print("Committing changes in files: {0}".format(str(files_list.keys()))) + print("Committing changes in files: {0}".format(str(files_dict.keys()))) if not dry_run: gitpkg.commits.create(data) def get_last_nonskip_pipeline(gitpkg, before_last=False): + """ + Returns the last running pipeline or the one before the last. + Args: + gitpkg: gitlab package object + before_last: If True, the pipeline before the last is returned + + Returns: The gtilab object of the pipeline + + """ # sleep for 10 seconds to ensure that if a pipeline was just submitted, # we can retrieve it time.sleep(10) @@ -152,12 +255,21 @@ def get_last_nonskip_pipeline(gitpkg, before_last=False): def just_build_package(gitpkg, dry_run=False): + """ + Restrt the last runnable pipeline of the package + Args: + gitpkg: gitlab package object + dry_run: If True, the pipeline will not be actually restarted on GitLab + + Returns: + + """ # we assume the last pipeline is with commit [skip ci] # so, we take the pipeline that can be re-built, which the previous to the last one last_pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True) # check that the chosen pipeline is the one we are looking for - latest_tag_name = gitpkg.tags.list(per_page=1, page=1)[0].name + latest_tag_name = get_latest_tag_name(gitpkg) # the pipeline should be the one built for the latest tag, so check if it is the correct choice if last_pipeline.ref != latest_tag_name: raise ValueError('While deploying {0}, found pipeline {1} but it does not match ' @@ -172,7 +284,15 @@ def just_build_package(gitpkg, dry_run=False): last_pipeline.retry() -def wait_for_pipeline_to_finish(gitpkg, tag, dry_run=False): +def wait_for_pipeline_to_finish(gitpkg, dry_run=False): + """ + Using sleep function, wait for the latest pipeline to finish building. + This function pauses the script until pipeline completes either successfully or with error. + Args: + gitpkg: gitlab package object + dry_run: If True, print log message and exit. There wil be no waiting. + + """ sleep_step = 30 max_sleep = 60 * 60 # one hour pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True) @@ -204,12 +324,28 @@ def wait_for_pipeline_to_finish(gitpkg, tag, dry_run=False): def cancel_last_pipeline(gitpkg): + """ + Cancel the last started pipeline of a package + Args: + gitpkg: gitlab package object + + """ pipeline = get_last_nonskip_pipeline(gitpkg) print('Cancelling the last pipeline {0} of project {1}'.format(pipeline.id, gitpkg.name)) pipeline.cancel() def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False): + """ + Release package. The provided tag will be annotated with a given list of comments. + README.rst and version.txt files will also be updated according to the release procedures. + Args: + gitpkg: gitlab package object + tag_name: The name of the release tag + tag_comments_list: New annotations for this tag in a form of list + dry_run: If True, nothing will be committed or pushed to GitLab + + """ # if there is nothing to release, just rebuild the package if tag_name == 'none': print("Since the tag is 'none', we just re-build the last pipeline") @@ -245,6 +381,19 @@ def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False): def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changelog, dry_run=False): + """ + Process the changelog of a single package. Parse the log following specific format. + Update annotations of the provided older tags and release the package by following the last tag description. + Args: + gl: Gitlab API object + bob_group: gitlab object for the group + pkg_name: name of the package + package_changelog: the changelog corresponding to the provided package + dry_run: If True, nothing will be committed or pushed to GitLab + + Returns: gitlab handle for the package, name of the latest tag, and tag's comments + + """ cur_tag = None cur_tag_comments = [] @@ -272,6 +421,17 @@ def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changel def main(private_token, group_name='bob', changelog_file='changelog.rst', dry_run=False, package=None, resume=False): + """ + Main function that updates and releases packages according to the provided changelog file. + Args: + private_token: GitLab token with the access rights to update and release Bob's packages + group_name: Name of the group in GitLab. By default, the name of the group is 'bob' + changelog_file: The changelog file that defines which packages will be processed + dry_run: If True, nothing will be committed or pushed to GitLab + package: If provided, only this package will be processed + resume: If True, the processing of changelog will be resumed starting with the provided package + + """ gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4) bob_group = gl.groups.list(search=group_name)[0] @@ -302,7 +462,7 @@ def main(private_token, group_name='bob', changelog_file='changelog.rst', dry_ru if gitpkg: release_package(gitpkg, tag, tag_comments, dry_run) # now, wait for the pipeline to finish, before we can release the next package - wait_for_pipeline_to_finish(gitpkg, tag, dry_run) + wait_for_pipeline_to_finish(gitpkg, dry_run) # if package name is provided and resume is not set, process only this package if package == cur_package_name and not resume: