first draft of the release script for bob

release/release_bob.py

+#!/usr/bin/env python
+
+"""
+By using changelog file as an input (can be generated with 'generate_changelog.py' script), this script goes through all packages in changelog file (in order listed), tags them correctly as per the file, an releases them one by one. This script uses python-gitlab package for accessing GitLab's API.
+
+Usage:
+  {0} [-v...] [options] [--] <private_token>
+  {0} -h | --help
+  {0} --version
+
+Arguments:
+  <private_token>  Private token used to access GitLab.
+  
+Options:
+  -h --help                     Show this screen.
+  --version                     Show version.
+  -c, --changelog-file STR      A changelog file with all packages to release with their tags, listed in order.
+                                [default: changelog_since_last_release.rst].
+  -g, --group-name STR          Group name where we are assuming that all packages are located.
+                                [default: bob].
+"""
+
+import sys
+from docopt import docopt
+import gitlab
+import datetime
+import re
+
+
+def _insure_correct_package(candidates, group_name, pkg_name):
+    for pkg in candidates:
+        # make sure the name and the group name match exactly
+        if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
+            return pkg
+    raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
+
+
+def get_packages_list(gl, gl_group=None):
+    if gl_group:
+        grp_nightlies = gl_group.projects.list(search='bob.nightlies')[0]
+        nightlies = gl.projects.get(id=grp_nightlies.id)
+    else:
+        nightlies = gl.projects.list(search='bob.nightlies')[0]
+    nightlies_order = nightlies.files.get(file_path='order.txt', ref='master')
+    pkg_list_ordered = nightlies_order.decode().decode().split('\n')
+    pkg_list_ordered = [line for line in pkg_list_ordered if (line.strip() and not line.startswith('#'))]
+    return pkg_list_ordered
+
+
+# release date of last Bob release + 1 day
+def bob_last_release(gl):
+    bobpkg = gl.projects.get(id=1535)  # 1535 is id of 'bob' meta-package
+    last_bob_tag = bobpkg.tags.list()[0]  # get the last tag
+    return last_bob_tag.commit['committed_date']
+
+
+def get_datetime_from_gitdate(gitdate):
+    return datetime.datetime.strptime(gitdate[:-6], '%Y-%m-%dT%H:%M:%S.%f')
+
+
+def sort_commits(commits):
+    return sorted(commits, key=lambda x: get_datetime_from_gitdate(x.committed_date))
+
+
+def sort_tags(tags):
+    return sorted(tags, key=lambda x: get_datetime_from_gitdate(x.commit['committed_date']))
+
+
+def get_tag_changelog(tag):
+    try:
+        return tag.release['description']
+    except Exception:
+        return ''
+
+
+def print_tags(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
+    since = get_datetime_from_gitdate(since)
+    tags = gitpkg.tags.list()
+    # sort tags by date
+    tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= since, tags)
+    tags = sort_tags(tags)
+    print('* ' + pkg_name)
+    for tag in tags:
+        print_one_tag(pkg_name, tag)
+
+
+def print_one_tag(pkg_name, tag):
+    print('  * ' + tag.name + ' (' + get_datetime_from_gitdate(tag.commit['committed_date']).strftime(
+        '%b %d, %Y %H:%M') + ')')
+    for line in get_tag_changelog(tag).split('\r\n'):
+        line = line.strip()
+        if line.startswith('* ') or line.startswith('- '):
+            line = line[2:]
+        line = line.replace('!', pkg_name + '!')
+        line = line.replace('#', pkg_name + '#')
+        if not line:
+            continue
+        print(' ' * 5 + '* ' + line)
+
+
+def print_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
+    #    import ipdb; ipdb.set_trace()
+    commits = gitpkg.commits.list(since=since, all=True)
+    # sort commits by date
+    commits = sort_commits(commits)
+    print('* ' + pkg_name)
+    print_commits_range(pkg_name, commits)
+
+
+def print_commits_range(pkg_name, commits):
+    for commit in commits:
+        commit_title = commit.title
+        # skip commits that do not carry much useful information
+        if '[skip ci]' in commit_title or \
+                        'Merge branch' in commit_title or \
+                        'Increased stable' in commit_title:
+            continue
+        commit_title = commit_title.strip()
+        commit_title = commit_title.replace('!', pkg_name + '!')
+        commit_title = commit_title.replace('#', pkg_name + '#')
+        # print('  - ' + get_datetime_from_gitdate(x.committed_date).strftime('%Y-%m-%d %H:%M:%S') + ":" + commit_title)
+        print(' ' * 5 + '- ' + commit_title)
+
+
+def print_tags_with_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
+    # get tags since release and sort them
+    datetime_since = get_datetime_from_gitdate(since)
+    tags = gitpkg.tags.list()
+    # sort tags by date
+    tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= datetime_since, tags)
+    tags = sort_tags(tags)
+    # get commits since release date and sort them too
+    commits = gitpkg.commits.list(since=since, all=True)
+    # sort commits by date
+    commits = sort_commits(commits)
+    print('* ' + pkg_name)
+    # go through tags and print each with its message and corresponding commits
+    start_date = datetime_since
+    for tag in tags:
+        # print tag name and its text
+        print_one_tag(pkg_name, tag)
+        # print commits from the previous tag up to this one
+        end_date = get_datetime_from_gitdate(tag.commit['committed_date'])
+        commits4tag = filter(lambda x: (
+            get_datetime_from_gitdate(x.committed_date) > start_date and get_datetime_from_gitdate(
+                x.committed_date) <= end_date), commits)
+        print_commits_range(pkg_name, commits4tag)
+        start_date = end_date
+    # print the tentative patch version bump for the future tag
+    print('  * patch')
+    # print leftover commits that were not tagged yet
+    leftover_commits = filter(lambda x: get_datetime_from_gitdate(x.committed_date) > start_date, commits)
+    print_commits_range(pkg_name, leftover_commits)
+
+
+def correct_tag(gitpkg, tag):
+    """
+        An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3
+        The latest tag is either patch, minor, major, or none
+    """
+    m = re.search(r"(v\d.\d.\d)", tag)
+    if m:
+        return m.group(0)
+
+    # if we bump the version, we need to find the latest released version for this package
+    if 'patch' == tag or 'minor' == tag or 'major' == tag:
+        latest_tags = gitpkg.tags.list()
+        latest_tags = sort_tags(latest_tags)
+        latest_tag_name = latest_tags[-1].name
+        # check that it has expected format v#.#.#
+        m = re.match(r"(v\d.\d.\d)", latest_tag_name)
+        if not m:
+            raise ValueError(
+                'The latest tag name {0} in package {1} has unknown format'.format(latest_tag_name, gitpkg.name))
+        # increase the version accordingly
+        if 'major' == tag:  # increment the first number in 'v#.#.#'
+            return latest_tag_name[0] + str(int(latest_tag_name[1]) + 1) + latest_tag_name[2:]
+        if 'minor' == tag:  # increment the second number in 'v#.#.#'
+            return latest_tag_name[:3] + str(int(latest_tag_name[3]) + 1) + latest_tag_name[4:]
+        if 'patch' == tag:  # increment the last number in 'v#.#.#'
+            return latest_tag_name[:-1] + str(int(latest_tag_name[-1]) + 1)
+
+    if 'none' == tag:  # we do nothing in this case
+        return tag
+
+    raise ValueError('Cannot parse changelog tag {0} of the package {1}'.format(tag, gitpkg.name))
+
+
+def main(private_token, group_name='bob', changelog='changelog.rst'):
+    gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
+    bob_group = gl.groups.list(search=group_name)[0]
+
+    # traverse all packages in the changelog, edit older tags with updated comments,
+    # tag them with a suggested version, then try to release, and
+    # wait until done to proceed to the next package
+    with open(changelog) as f:
+        changelog_insides = f.readlines()
+        pkg_name = cur_gitpkg = None
+        cur_tag = None
+        cur_tag_comments = []
+        # we assume that changelog is formatted as structured text
+        for line in changelog_insides:
+            if '*' == line[0]:  # name of the package
+
+                if pkg_name and cur_gitpkg:  # release previous package
+                    release_package(cur_gitpkg, cur_tag, cur_tag_comments)
+
+                pkg_name = line[1:].strip()
+                # find the correct package in GitLab
+                # group returns a simplified description of the project
+                grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), group_name, pkg_name)
+                # so, we need to retrieve the full info from GitLab using correct project id
+                cur_gitpkg = gl.projects.get(id=grpkg.id)
+
+            elif '  *' == line[:3]:  # a tag level
+                if not cur_gitpkg:  # it better be not None, as we are in the middle of the tags for the package
+                    raise ValueError('How come package for {0} is empty?'.format(pkg_name))
+
+                # write the collected comments in the previous tag
+                if cur_tag:
+                    update_tag_comments(cur_gitpkg, cur_tag, cur_tag_comments)
+                    cur_tag_comments = []  # reset comments
+
+                # parse the current tag name
+                cur_tag = correct_tag(cur_gitpkg, line[3:].strip())
+
+                if 'none' == cur_tag:  # no tagging, just deploy the package
+                    deploy_package(cur_gitpkg)
+                    continue
+
+            else:  # all other lines are assumed to be comments
+                cur_tag_comments += line.strip()
+
+        if pkg_name and cur_gitpkg:  # release the last package
+            release_package(cur_gitpkg, cur_tag, cur_tag_comments)
+
+
+if __name__ == '__main__':
+    arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1')
+    main(arguments['<private_token>'],, group_name = arguments['--group-name'], changelog = arguments['--changelog'])