Merge branch 'updaterelease' into 'master'

Correcting release script: sorting tags, corrected version bump, updated help docs

See merge request !85

release/release_bob.py

@@ -3,9 +3,52 @@
 """
 By using changelog file as an input (can be generated with 'generate_changelog.py' script),
 this script goes through all packages in changelog file (in order listed), tags them correctly
-as per the file, and releases them one by one. A script can also be used to release a single package.
+as per the file, and releases them one by one.
 This script uses python-gitlab package for accessing GitLab's API.
 
+This script uses the provided changelog file to release a package.
+The changelog is expected to have the following structure:
+
+    * package name
+      * tag1 name (date of the tag).
+        * tag description. Each line of the tag description starts with `*` character.
+        - commits (from earliest to latest). Each line of the commit starts with `-` character.
+      * tag2 name (date of the tag).
+        * tag description. Each line of the tag description starts with `*` character.
+        - commits (from earliest to latest). Each line of the commit starts with `-` character.
+      * patch
+        - leftover not-tagged commits (from earliest to latest)
+
+This script can also be used to release a single package.
+`IMPORTANT`: there are some considerations that needs to be taken
+into account **before** you release a new version of a package:
+
+  * In the changelog file:
+    - write the name of this package and write the last tag.
+      For the tag name, you can either indicate `patch`, `minor` or `major`,
+      and the package will be then released with either patch, minor, or major version bump.
+    - Alternatively, you can specify the version name directly but be careful that it is higher
+      than the last release tag of this package.
+      Make sure that the version that you are trying to release is not already released.
+      Also, make sure you follow semantic versions: http://semver.org.
+    - Then, under the desired new tag version of the package, please write down the changes that are applied
+      to the package between the last released version and this version. This
+      changes are written to release tags of packages in the Gitlab interface.
+      For an example look at: https://gitlab.idiap.ch/bob/bob.extension/tags
+  * Make sure all the tests for the package are passing.
+  * Make sure the documentation is building with the following command:
+    ``sphinx-build -aEWn doc sphinx``
+  * Make sure all changes are committed to the git repository and pushed.
+  * Make sure the documentation badges in README.rst are pointing to:
+    https://www.idiap.ch/software/bob/docs/bob/...
+  * For database packages, make sure that the '.sql3' file or other
+    metadata files have been generated (if any).
+  * Make sure bob.nightlies is green after the changes are submitted if the
+    package is a part of the nightlies.
+  * If your package depends on an unreleased version of another Bob package,
+    you need to release that package first.
+
+
 Usage:
     {0} [-v...] [options] [--] <private_token>
     {0} -h | --help
@@ -18,7 +61,7 @@ Options:
     -h --help                     Show this screen.
     --version                     Show version.
     -c, --changelog-file STR      A changelog file with all packages to release with their tags, listed in order.
-                                  [default: changelog_since_last_release.rst].
+                                  [default: changelog_since_last_release.md].
     -g, --group-name STR          Group name where we are assuming that all packages are located.
                                   [default: bob].
     -p, --package STR             If the name of a package is provided, then this package will be found
@@ -35,7 +78,7 @@ from docopt import docopt
 import gitlab
 import datetime
 import re
-from distutils.version import StrictVersion as Version
+from distutils.version import StrictVersion
 import numpy
 import time
 
@@ -50,7 +93,17 @@ def _insure_correct_package(candidates, group_name, pkg_name):
 
 # adapted from the same-name function in new_version.py script of bob.extension package
 def _update_readme(readme, version=None):
-    # replace the travis badge in the README.rst with the given version
+    """
+    Inside text of the readme, replaces parts of the links to the provided version.
+    If version is not provided, replace to `stable` or `master`.
+    Args:
+        readme: Text of the README.rst file from a bob package
+        version: Format of the version string is '#.#.#'
+
+    Returns: New text of readme with all replaces done
+
+    """
+    # replace the badge in the readme's text with the given version
     DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
     BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
 
@@ -72,6 +125,28 @@ def _update_readme(readme, version=None):
     return '\n'.join(new_readme)
 
 
+def get_latest_tag_name(gitpkg):
+    """
+    Find the name of the latest tag for a given package in the format '#.#.#'
+    Args:
+        gitpkg: gitlab package object
+
+    Returns: The name of the latest tag in format '#.#.#'. None if no tags for the package were found.
+
+    """
+    # get 50 latest tags as a list
+    latest_tags = gitpkg.tags.list(all=True)
+    if not latest_tags:
+        return None
+    # create list of tags' names but ignore the first 'v' character in each name
+    tag_names = [tag.name[1:] for tag in latest_tags]
+    # sort them correctly according to each subversion number
+    tag_names.sort(key=StrictVersion)
+    # take the last one, as it is the latest tag in the sorted tags
+    latest_tag_name = tag_names[-1]
+    return latest_tag_name
+
+
 def get_parsed_tag(gitpkg, tag):
     """
         An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3
@@ -84,25 +159,26 @@ def get_parsed_tag(gitpkg, tag):
 
     # if we bump the version, we need to find the latest released version for this package
     if 'patch' == tag or 'minor' == tag or 'major' == tag:
-        latest_tag = gitpkg.tags.list(per_page=1, page=1)
-        # if there were no tags yet, assume the first version
-        if not latest_tag:
-            return 'v1.0.0'
-        latest_tag = latest_tag[0]
-        latest_tag_name = latest_tag.name
-        # check that it has expected format v#.#.#
+        # find the correct latest tag of this package (without 'v' in front),
+        # None if there are no tags yet
+        latest_tag_name = get_latest_tag_name(gitpkg)
+        # if there were no tags yet, assume the very first version
+        if not latest_tag_name:
+            return 'v0.0.1'
+        # check that it has expected format #.#.#
         # latest_tag_name = Version(latest_tag_name)
-        m = re.match(r"(v\d.\d.\d)", latest_tag_name)
+        m = re.match(r"(\d.\d.\d)", latest_tag_name)
         if not m:
             raise ValueError(
-                'The latest tag name {0} in package {1} has unknown format'.format(latest_tag_name, gitpkg.name))
+                'The latest tag name {0} in package {1} has unknown format'.format('v' + latest_tag_name, gitpkg.name))
         # increase the version accordingly
-        if 'major' == tag:  # increment the first number in 'v#.#.#'
-            return latest_tag_name[0] + str(int(latest_tag_name[1]) + 1) + latest_tag_name[2:]
-        if 'minor' == tag:  # increment the second number in 'v#.#.#'
-            return latest_tag_name[:3] + str(int(latest_tag_name[3]) + 1) + latest_tag_name[4:]
+        major, minor, patch = latest_tag_name.split('.')
+        if 'major' == tag:  # increment the first number in 'v#.#.#' but make minor and patch to be 0
+            return 'v' + str(int(major) + 1) + '.0.0'
+        if 'minor' == tag:  # increment the second number in 'v#.#.#' but make patch to be 0
+            return 'v' + major + '.' + str(int(minor) + 1) + '.0'
         if 'patch' == tag:  # increment the last number in 'v#.#.#'
-            return latest_tag_name[:-1] + str(int(latest_tag_name[-1]) + 1)
+            return 'v' + major + '.' + minor + '.' + str(int(patch) + 1)
 
     if 'none' == tag:  # we do nothing in this case
         return tag
@@ -111,6 +187,17 @@ def get_parsed_tag(gitpkg, tag):
 
 
 def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
+    """
+    Write annotations inside the provided tag of a given package.
+    Args:
+        gitpkg: gitlab package object
+        tag_name: The name of the tag to update
+        tag_comments_list: New annotations for this tag in a form of list
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    Returns: The gitlab object for the tag that was updated
+
+    """
     # get tag and update its description
     tag = gitpkg.tags.get(tag_name)
     print('Found tag {1}, updating its comments with:'.format(gitpkg.name, tag.name))
@@ -120,26 +207,42 @@ def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
     return tag
 
 
-def commit_files(gitpkg, files_list, message='Updated files', dry_run=False):
+def commit_files(gitpkg, files_dict, message='Updated files', dry_run=False):
+    """
+    Commit files of a given GitLab package.
+    Args:
+        gitpkg: gitlab package object
+        files_dict: Dictionary of file names and their contents (as text)
+        message: Commit message
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    """
     data = {
         'branch': 'master',  # v4
         'commit_message': message,
         'actions': []
     }
     # add files to update
-    for filename in files_list.keys():
+    for filename in files_dict.keys():
         update_action = dict(action='update', file_path=filename)
-        # with open(filename, 'r') as f:
-        #     update_action['content'] = f.read()
-        update_action['content'] = files_list[filename]
+        update_action['content'] = files_dict[filename]
         data['actions'].append(update_action)
 
-    print("Committing changes in files: {0}".format(str(files_list.keys())))
+    print("Committing changes in files: {0}".format(str(files_dict.keys())))
     if not dry_run:
         gitpkg.commits.create(data)
 
 
 def get_last_nonskip_pipeline(gitpkg, before_last=False):
+    """
+    Returns the last running pipeline or the one before the last.
+    Args:
+        gitpkg: gitlab package object
+        before_last: If True, the pipeline before the last is returned
+
+    Returns: The gtilab object of the pipeline
+
+    """
     # sleep for 10 seconds to ensure that if a pipeline was just submitted,
     # we can retrieve it
     time.sleep(10)
@@ -152,12 +255,21 @@ def get_last_nonskip_pipeline(gitpkg, before_last=False):
 
 
 def just_build_package(gitpkg, dry_run=False):
+    """
+    Restrt the last runnable pipeline of the package
+    Args:
+        gitpkg: gitlab package object
+        dry_run: If True, the pipeline will not be actually restarted on GitLab
+
+    Returns:
+
+    """
     # we assume the last pipeline is with commit [skip ci]
     # so, we take the pipeline that can be re-built, which the previous to the last one
     last_pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True)
 
     # check that the chosen pipeline is the one we are looking for
-    latest_tag_name = gitpkg.tags.list(per_page=1, page=1)[0].name
+    latest_tag_name = get_latest_tag_name(gitpkg)
     # the pipeline should be the one built for the latest tag, so check if it is the correct choice
     if last_pipeline.ref != latest_tag_name:
         raise ValueError('While deploying {0}, found pipeline {1} but it does not match '
@@ -172,7 +284,15 @@ def just_build_package(gitpkg, dry_run=False):
         last_pipeline.retry()
 
 
-def wait_for_pipeline_to_finish(gitpkg, tag, dry_run=False):
+def wait_for_pipeline_to_finish(gitpkg, dry_run=False):
+    """
+    Using sleep function, wait for the latest pipeline to finish building.
+    This function pauses the script until pipeline completes either successfully or with error.
+    Args:
+        gitpkg: gitlab package object
+        dry_run: If True, print log message and exit. There wil be no waiting.
+
+    """
     sleep_step = 30
     max_sleep = 60 * 60  # one hour
     pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True)
@@ -204,12 +324,28 @@ def wait_for_pipeline_to_finish(gitpkg, tag, dry_run=False):
 
 
 def cancel_last_pipeline(gitpkg):
+    """
+    Cancel the last started pipeline of a package
+    Args:
+        gitpkg: gitlab package object
+
+    """
     pipeline = get_last_nonskip_pipeline(gitpkg)
     print('Cancelling the last pipeline {0} of project {1}'.format(pipeline.id, gitpkg.name))
     pipeline.cancel()
 
 
 def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False):
+    """
+    Release package. The provided tag will be annotated with a given list of comments.
+    README.rst and version.txt files will also be updated according to the release procedures.
+    Args:
+        gitpkg: gitlab package object
+        tag_name: The name of the release tag
+        tag_comments_list: New annotations for this tag in a form of list
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    """
     # if there is nothing to release, just rebuild the package
     if tag_name == 'none':
         print("Since the tag is 'none', we just re-build the last pipeline")
@@ -245,6 +381,19 @@ def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False):
 
 
 def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changelog, dry_run=False):
+    """
+    Process the changelog of a single package. Parse the log following specific format. 
+    Update annotations of the provided older tags and release the package by following the last tag description.
+    Args:
+        gl: Gitlab API object
+        bob_group: gitlab object for the group
+        pkg_name: name of the package
+        package_changelog: the changelog corresponding to the provided package
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    Returns: gitlab handle for the package, name of the latest tag, and tag's comments
+
+    """
     cur_tag = None
     cur_tag_comments = []
 
@@ -272,6 +421,17 @@ def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changel
 
 
 def main(private_token, group_name='bob', changelog_file='changelog.rst', dry_run=False, package=None, resume=False):
+    """
+    Main function that updates and releases packages according to the provided changelog file.
+    Args:
+        private_token: GitLab token with the access rights to update and release Bob's packages
+        group_name: Name of the group in GitLab. By default, the name of the group is 'bob'
+        changelog_file: The changelog file that defines which packages will be processed
+        dry_run: If True, nothing will be committed or pushed to GitLab
+        package: If provided, only this package will be processed
+        resume: If True, the processing of changelog will be resumed starting with the provided package
+
+    """
     gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
     bob_group = gl.groups.list(search=group_name)[0]
 
@@ -302,7 +462,7 @@ def main(private_token, group_name='bob', changelog_file='changelog.rst', dry_ru
         if gitpkg:
             release_package(gitpkg, tag, tag_comments, dry_run)
             # now, wait for the pipeline to finish, before we can release the next package
-            wait_for_pipeline_to_finish(gitpkg, tag, dry_run)
+            wait_for_pipeline_to_finish(gitpkg, dry_run)
 
         # if package name is provided and resume is not set, process only this package
         if package == cur_package_name and not resume: