Merge branch 'changelog' into 'master'

Added script to generate changelog since last release and generated changelog itself

See merge request !72

release/README.md

+
+Generate changelog
+------------------
+
+To generate a changelog that shows changes (tags and commits) in all packages since last release, please run the following script:
+
+    $ python generate_changelog.py <Gitlab Private Access Token> > changelog_since_last_release.txt
+
+The script will go through all packages listed in https://gitlab.idiap.ch/bob/bob.nightlies/blob/master/order.txt file. The date of the latest tag of `bob` package, will be taken as the last release date. All changes including tags and commits will be logged for each package. The resulted log file should be manually updated, so the changes descriptions become more concise. Once updated, the changelog should be put in https://gitlab.idiap.ch/bob/bob/tags.
+
+Manually update changelog
+-------------------------
+
+The changelog since the last release can be found in `changelog_since_last_release.txt`. The structure is the following:
+
+    * package name
+      * tag1 name (date of the tag). 
+        * tag description. Each line of the tag description starts with `*` character.
+        - commits (from earliest to latest). Each line of the commit starts with `-` character.
+      * tag2 name (date of the tag). 
+        * tag description. Each line of the tag description starts with `*` character.
+        - commits (from earliest to latest). Each line of the commit starts with `-` character.
+      * patch
+        - leftover not-tagged commits (from earliest to latest)
+
+To manually update the changelog, follow these steps:
+
+    1. For each tag, summarize the commits into several descriptive lines. These summaries become
+    tag descriptions and they should extend/update the existing tag description. Therefore, each line 
+    of the summary should also start with `*` character like the tag descriptions.
+    2. The last tag name is called `patch`. This indicates that a patch version of this package will 
+    be automatically updated during the next tag/release. You can change `patch` to `minor` or `major`, 
+    and the package will be then tagged/released with either minor or major version bump. 
+    3. Once all commits were changed to corresponding tag descriptions (no more lines start with `-` characters), 
+    this package is ready for release and you can continue to another package in the changelog. 
+
+
+Release Bob
+-----------
+
 Here are some instructions to do a massive Bob release and scripts to help you
 release Bob:
 
@@ -141,8 +181,7 @@ Run the pipeline for master of bob.conda once and fix till every conda package
 is released.
 
 
-Release `bob`. Here is some code to get the changelog generated automatically
-for bob:
+Release `bob`. 
 
 
     import gitlab
@@ -189,51 +228,3 @@ for bob:
         if max_version != version:
             print(pkg, versions)
 
-Generate the changelog:
-
-    import gitlab
-    import distutils.version
-    import datetime
-    from collections import OrderedDict
-    gl = gitlab.Gitlab.from_config()
-    bob_group = gl.groups.search('bob')[0]
-    path = '.../git/bobs/bob/requirements.txt'
-    pkgs = OrderedDict()
-    for line in open(path):
-        pkg, version = line.split('==')
-        pkgs[pkg.strip()] = version.strip()
-
-    # release date of last Bob release + 1 day
-    last_release = datetime.datetime(2017, 2, 10)
-
-    def get_datetime_from_tag(tag):
-        return datetime.datetime.strptime(tag.commit.committed_date[:-6], '%Y-%m-%dT%H:%M:%S.%f')
-
-    def sort_tags(tags):
-        return sorted(tags, key=lambda x: get_datetime_from_tag(x))
-
-    def get_tag_changelog(tag):
-        try:
-            return tag.release.description
-        except Exception:
-            return ''
-
-    for pkg, version in pkgs.items():
-        projects = bob_group.projects.search(pkg)
-        project = [p for p in projects if p.name == pkg][0]
-        tags = project.tags.list(all=True)
-        # sort tags by date
-        tags = filter(lambda x: get_datetime_from_tag(x) >= last_release, tags)
-        tags = sort_tags(tags)
-        print('* ' + pkg)
-        for tag in tags:
-            print('  * ' + tag.name)
-            for line in get_tag_changelog(tag).split('\r\n'):
-                line = line.strip()
-                line = line.replace('!', pkg+'!')
-                line = line.replace('#', pkg+'#')
-                if not line:
-                    continue
-                print(' '*5 + '* ' + line)
-
-Put this awesome changelog in https://gitlab.idiap.ch/bob/bob/tags

release/changelog_environment.yml

+# run: conda env create --file environment.yml
+name: changelog
+dependencies:
+- python>=3.5
+- docopt
+- pip
+- pip:
+  - python-gitlab

release/generate_changelog.py

+#!/usr/bin/env python
+
+"""
+Generate changelog file for all packages before release. This script uses python-gitlab package for access to Gitlab's API.
+
+Usage:
+  {0} <private_token>
+  {0} -h | --help
+  {0} --version
+
+Arguments:
+  <private_token>  Private token used to access GitLab.
+  
+Options:
+  -h --help                    Show this screen.
+  --version                    Show version.
+
+"""
+
+import sys
+from docopt import docopt
+import gitlab
+import datetime
+
+
+def get_packages_list(gl):
+    nightlies = gl.projects.list(search='bob.nightlies')[0]
+    nightlies_order = nightlies.files.get(file_path='order.txt', ref='master')
+    pkg_list_ordered = nightlies_order.decode().decode().split('\n')
+    pkg_list_ordered = [line for line in pkg_list_ordered if (line.strip() and not line.startswith('#'))]
+    return pkg_list_ordered
+
+
+# release date of last Bob release + 1 day
+def bob_last_release(gl):
+    bobpkg = gl.projects.get(id=1535)  # 1535 is id of 'bob' meta-package
+    last_bob_tag = bobpkg.tags.list()[0]  # get the last tag
+    return last_bob_tag.commit['committed_date']
+
+
+def get_datetime_from_gitdate(gitdate):
+    #    import ipdb; ipdb.set_trace()
+    return datetime.datetime.strptime(gitdate[:-6], '%Y-%m-%dT%H:%M:%S.%f')
+
+
+def sort_commits(commits):
+    return sorted(commits, key=lambda x: get_datetime_from_gitdate(x.committed_date))
+
+
+def sort_tags(tags):
+    return sorted(tags, key=lambda x: get_datetime_from_gitdate(x.commit['committed_date']))
+
+
+def get_tag_changelog(tag):
+    try:
+        return tag.release['description']
+    except Exception:
+        return ''
+
+
+def print_tags(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
+    since = get_datetime_from_gitdate(since)
+    tags = gitpkg.tags.list()
+    # sort tags by date
+    tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= since, tags)
+    tags = sort_tags(tags)
+    print('* ' + pkg_name)
+    for tag in tags:
+        print_one_tag(pkg_name, tag)
+
+
+def print_one_tag(pkg_name, tag):
+    print('  * ' + tag.name + ' (' + get_datetime_from_gitdate(tag.commit['committed_date']).strftime(
+        '%b %d, %Y %H:%M') + ')')
+    for line in get_tag_changelog(tag).split('\r\n'):
+        line = line.strip()
+        if line.startswith('* ') or line.startswith('- '):
+            line = line[2:]
+        line = line.replace('!', pkg_name + '!')
+        line = line.replace('#', pkg_name + '#')
+        if not line:
+            continue
+        print(' ' * 5 + '* ' + line)
+
+
+def print_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
+    #    import ipdb; ipdb.set_trace()
+    commits = gitpkg.commits.list(since=since, all=True)
+    # sort commits by date
+    commits = sort_commits(commits)
+    print('* ' + pkg_name)
+    print_commits_range(pkg_name, commits)
+
+
+def print_commits_range(pkg_name, commits):
+    for commit in commits:
+        commit_title = commit.title
+        # skip commits that do not carry much useful information
+        if '[skip ci]' in commit_title or \
+                        'Merge branch' in commit_title or \
+                        'Increased stable' in commit_title:
+            continue
+        commit_title = commit_title.strip()
+        commit_title = commit_title.replace('!', pkg_name + '!')
+        commit_title = commit_title.replace('#', pkg_name + '#')
+        # print('  - ' + get_datetime_from_gitdate(x.committed_date).strftime('%Y-%m-%d %H:%M:%S') + ":" + commit_title)
+        print(' ' * 5 + '- ' + commit_title)
+
+
+def print_tags_with_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
+    # get tags since release and sort them
+    datetime_since = get_datetime_from_gitdate(since)
+    tags = gitpkg.tags.list()
+    # sort tags by date
+    tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= datetime_since, tags)
+    tags = sort_tags(tags)
+    # get commits since release date and sort them too
+    commits = gitpkg.commits.list(since=since, all=True)
+    # sort commits by date
+    commits = sort_commits(commits)
+    print('* ' + pkg_name)
+    # go through tags and print each with its message and corresponding commits
+    start_date = datetime_since
+    for tag in tags:
+        # print tag name and its text
+        print_one_tag(pkg_name, tag)
+        # print commits from the previous tag up to this one
+        end_date = get_datetime_from_gitdate(tag.commit['committed_date'])
+        commits4tag = filter(lambda x: (
+            get_datetime_from_gitdate(x.committed_date) > start_date and get_datetime_from_gitdate(
+                x.committed_date) <= end_date), commits)
+        print_commits_range(pkg_name, commits4tag)
+        start_date = end_date
+    # print the tentative patch version bump for the future tag
+    print('  * patch')
+    # print leftover commits that were not tagged yet
+    leftover_commits = filter(lambda x: get_datetime_from_gitdate(x.committed_date) > start_date, commits)
+    print_commits_range(pkg_name, leftover_commits)
+
+
+def main(private_token):
+    gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
+    pkg_list_ordered = get_packages_list(gl)
+    #    print(pkg_list_ordered)
+    last_release_date = bob_last_release(gl)
+    #    print(last_release_date)
+    # print name of the package, its version and tags
+    for pkg_name in pkg_list_ordered:
+        gitpkg = gl.projects.list(search=pkg_name)[0]
+        #        print_tags(pkg_name, gitpkg, since=last_release_date))
+        #        print_commits(pkg_name, gitpkg, since=last_release_date)
+        print_tags_with_commits(pkg_name, gitpkg, since=last_release_date)
+
+
+if __name__ == '__main__':
+    arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1')
+    main(arguments['<private_token>'])