Major commit to refactor this package as a hub for more functionality

- Start moving bob.admin scripts here
- Add baseline documentation system
- Add logging support
- Rename it as bob.devtools
- Prepare for conda-packaging
- Name main tool `bdt`

MANIFEST.in

 include LICENSE README.rst buildout.cfg version.txt
 recursive-include doc conf.py *.rst
-recursive-include bob *.cpp *.h
-recursive-include bob/extension/examples *
-recursive-include bob/extension/data *
+recursive-include bob/devtools/data *.md

README.rst

@@ -5,18 +5,24 @@
 ===============================================
 
 This package is part of the signal-processing and machine learning toolbox
-Bob_. It provides some tools to help maintain Bob_.
+Bob_. It provides tools to help maintain Bob_.
+
 
 Installation
 ------------
 
-This package needs to be installed in your base conda environment. To install
+This package needs to be installed in a conda environment. To install
 this package, run::
 
-  $ conda activate base
-  # the dependency list below matches the ones in setup.py
-  $ conda install pip click click-plugins conda-build
-  $ pip install -e .
+  $ conda env create -f env.yml
+  $ conda activate bdt
+  (bdt) $ buildout
+  (bdt) $ ./bin/bdt --help
+  ...
+
+To build the documentation, just do::
+
+  (bdt) $ ./bin/sphinx-build doc sphinx
 
 
 Contact

bob/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

bob/devtools/changelog.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import io
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+
+import pytz
+import dateutil.parser
+
+from .release import ensure_correct_package
+
+
+def parse_date(d):
+    '''Parses any date supported by :py:func:`dateutil.parser.parse`'''
+
+    return dateutil.parser.parse(d, ignoretz=True).replace(
+                tzinfo=pytz.timezone("Europe/Zurich"))
+
+
+def _sort_commits(commits, reverse):
+    '''Sorts gitlab commit objects using their ``committed_date`` attribute'''
+
+    return sorted(commits,
+        key=lambda x: parse_date(x.committed_date),
+        reverse=reverse,
+        )
+
+
+def _sort_tags(tags, reverse):
+    '''Sorts gitlab tag objects using their ``committed_date`` attribute'''
+
+    return sorted(tags,
+        key=lambda x: parse_date(x.commit['committed_date']),
+        reverse=reverse,
+        )
+
+
+def get_file_from_gitlab(gitpkg, path, ref='master'):
+    '''Retrieves a file from a Gitlab repository, returns a (StringIO) file'''
+
+    return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
+
+
+def get_last_tag(package):
+    '''Returns the last (gitlab object) tag for the given package
+
+    Args:
+
+        package: The gitlab project object from where to fetch the last release
+                 date information
+
+
+    Returns: a tag object
+    '''
+
+    # according to the Gitlab API documentation, tags are sorted from the last
+    # updated to the first, by default - no need to do further sorting!
+    tag_list = package.tags.list()
+
+    if tag_list:
+        # there are tags, use these
+        return tag_list[0]
+
+
+def get_last_tag_date(package):
+    '''Returns the last release date for the given package
+
+    Falls back to the first commit date if the package has not yet been tagged
+
+
+    Args:
+
+        package: The gitlab project object from where to fetch the last release
+                 date information
+
+
+    Returns: a datetime object that refers to the last date the package was
+             released.  If the package was never released, then returns the
+             date just before the first commit.
+    '''
+
+    # according to the Gitlab API documentation, tags are sorted from the last
+    # updated to the first, by default - no need to do further sorting!
+    tag_list = package.tags.list()
+
+    if tag_list:
+        # there are tags, use these
+        last = tag_list[0]
+        logger.debug('Last tag for package %s (id=%d) is %s', package.name,
+            package.id, last.name)
+        return parse_date(last.commit['committed_date']) + \
+            datetime.timedelta(milliseconds=500)
+
+    else:
+        commit_list = package.commits.list(all=True)
+
+        if commit_list:
+            # there are commits, use these
+            first = _sort_commits(commit_list, reverse=False)[0]
+            logger.debug('First commit for package %s (id=%d) is from %s',
+                package.name, package.id, first.committed_date)
+            return parse_date(first.committed_date) - \
+                datetime.timedelta(milliseconds=500)
+
+        else:
+            # there are no commits nor tags - abort
+            raise RuntimeError('package %s (id=%d) does not have commits ' \
+                'or tags so I cannot devise a good starting date' % \
+                (package.name, package.id))
+
+
+def _get_tag_changelog(tag):
+
+    try:
+        return tag.release['description']
+    except Exception:
+        return ''
+
+
+def _write_one_tag(f, pkg_name, tag):
+    '''Prints commit information for a single tag of a given package
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        pkg_name: The name of the package we are writing tags of
+        tag: The tag value
+
+    '''
+
+    git_date = parse_date(tag.commit['committed_date'])
+    f.write('  * %s (%s)\n' % (tag.name,  git_date.strftime('%b %d, %Y %H:%M')))
+
+    for line in _get_tag_changelog(tag).replace('\r\n', '\n').split('\n'):
+
+        line = line.strip()
+        if line.startswith('* ') or line.startswith('- '):
+            line = line[2:]
+
+        line = line.replace('!', pkg_name + '!').replace(pkg_name + \
+            pkg_name, pkg_name)
+        line = line.replace('#', pkg_name + '#')
+        if not line:
+            continue
+        f.write('%s* %s' % (5*' ', line))
+
+
+def _write_commits_range(f, pkg_name, commits):
+    '''Writes all commits of a given package within a range, to the output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        pkg_name: The name of the package we are writing tags of
+        commits: List of commits to be written
+
+    '''
+
+
+    for commit in commits:
+        commit_title = commit.title
+
+        # skip commits that do not carry much useful information
+        if '[skip ci]' in commit_title or \
+                        'Merge branch' in commit_title or \
+                        'Increased stable' in commit_title:
+            continue
+
+        commit_title = commit_title.strip()
+        commit_title = commit_title.replace('!', pkg_name + '!').replace(pkg_name + pkg_name, pkg_name)
+        commit_title = commit_title.replace('#', pkg_name + '#')
+        f.write('%s- %s' % (' ' * 5, commit_title))
+
+
+def _write_mergerequests_range(f, pkg_name, mrs):
+    '''Writes all merge-requests of a given package, with a range, to the
+    output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        pkg_name: The name of the package we are writing tags of
+        mrs: The list of merge requests to write
+
+    '''
+
+    for mr in mrs:
+        title = mr.title.strip().replace('\r','').replace('\n', '  ')
+        title = title.replace(' !', ' ' + pkg_name + '!')
+        title = title.replace(' #', ' ' + pkg_name + '#')
+        description = mr.description.strip().replace('\r','').replace('\n', '  ')
+        description = description.replace(' !', ' ' + pkg_name + '!')
+        description = description.replace(' #', ' ' + pkg_name + '#')
+        space = ': ' if description else ''
+        log = '''     - {pkg}!{iid} {title}{space}{description}'''
+        f.write(log.format(pkg=pkg_name, iid=mr.iid, title=title, space=space, description=description))
+        f.write('\n')
+
+
+def write_tags_with_commits(f, gitpkg, since, mode):
+    '''Writes all tags and commits of a given package to the output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        gitpkg: A pointer to the gitlab package object
+        since: Starting date (as a datetime object)
+        mode: One of mrs (merge-requests), commits or tags indicating how to
+              list entries in the changelog for this package
+
+    '''
+
+    # get tags since release and sort them
+    tags = gitpkg.tags.list()
+
+    # sort tags by date
+    tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
+    tags = _sort_tags(tags, reverse=False)
+
+    # get commits since release date and sort them too
+    commits = gitpkg.commits.list(since=since, all=True)
+
+    # sort commits by date
+    commits = _sort_commits(commits, reverse=False)
+
+    # get merge requests since the release data
+    mrs = list(reversed(gitpkg.mergerequests.list(state='merged', updated_after=since, order_by='updated_at', all=True)))
+    f.write('* %s\n' % (gitpkg.name,))
+
+    # go through tags and writes each with its message and corresponding
+    # commits
+    start_date = since
+    for tag in tags:
+
+        # write tag name and its text
+        _write_one_tag(f, gitpkg.name, tag)
+        end_date = parse_date(tag.commit['committed_date'])
+
+        if mode == 'commits':
+            # write commits from the previous tag up to this one
+            commits4tag = [k for k in commits \
+                if (start_date < parse_date(k.committed_date) <= end_date)]
+            _write_commits_range(f, gitpkg.name, commits4tag)
+
+        elif mode == 'mrs':
+            # write merge requests from the previous tag up to this one
+            # the attribute 'merged_at' is not available in GitLab API as of 27
+            # June 2018
+            mrs4tag = [k for k in mrs \
+                if (start_date < parse_date(k.updated_at) <= end_date)]
+            _write_mergerequests_range(f, gitpkg.name, mrs4tag)
+
+        start_date = end_date
+
+    if mode != 'tags':
+
+        # write the tentative patch version bump for the future tag
+        f.write('  * patch\n')
+
+        if mode == 'mrs':
+            # write leftover merge requests
+            # the attribute 'merged_at' is not available in GitLab API as of 27
+            # June 2018
+            leftover_mrs = [k for k in mrs \
+                if parse_date(k.updated_at) > start_date]
+            _write_mergerequests_range(f, gitpkg.name, leftover_mrs)
+
+        else:
+            # write leftover commits that were not tagged yet
+            leftover_commits = [k for k in commits \
+                if parse_date(k.committed_date) > start_date]
+            _write_commits_range(f, gitpkg.name, leftover_commits)
+
+
+def write_tags(f, gitpkg, since):
+    '''Writes all tags of a given package to the output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        gitpkg: A pointer to the gitlab package object
+        since: Starting date as a datetime object
+
+    '''
+
+    tags = gitpkg.tags.list()
+    # sort tags by date
+    tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
+    tags = _sort_tags(tags, reverse=False)
+    f.write('* %s\n')
+
+    for tag in tags:
+        _write_one_tag(gitpkg.name, tag)

bob/devtools/release.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import time
+import gitlab
+
+import logging
+logger = logging.getLogger(__name__)
+
+from distutils.version import StrictVersion
+
+
+def get_gitlab_instance():
+    '''Returns an instance of the gitlab object for remote operations'''
+
+    # tries to figure if we can authenticate using a global configuration
+    cfgs = ['~/.python-gitlab.cfg', '/etc/python-gitlab.cfg']
+    cfgs = [os.path.expanduser(k) for k in cfgs]
+    if any([os.path.exists(k) for k in cfgs]):
+        gl = gitlab.Gitlab.from_config('idiap', cfgs)
+    else: #ask the user for a token
+        server = "https://gitlab.idiap.ch"
+        token = input("%s token: " % server)
+        gl = gitlab.Gitlab(server, private_token=token, api_version=4)
+
+    return gl
+
+
+def ensure_correct_package(candidates, group_name, pkg_name):
+
+  for pkg in candidates:
+    # make sure the name and the group name match exactly
+    if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
+      return pkg
+
+  raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
+
+
+def _update_readme(readme, version):
+    """
+    Inside text of the readme, replaces parts of the links to the provided
+    version. If version is not provided, replace to `stable` or `master`.
+
+    Args:
+
+        readme: Text of the README.rst file from a bob package
+        version: Format of the version string is '#.#.#'
+
+    Returns: New text of readme with all replaces done
+    """
+
+    # replace the badge in the readme's text with the given version
+    DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
+    BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
+
+    new_readme = []
+    for line in readme.splitlines():
+        if BRANCH_RE.search(line) is not None:
+            if "gitlab" in line:  # gitlab links
+                replacement = "/v%s" % version if version is not None \
+                    else "/master"
+                line = BRANCH_RE.sub(replacement, line)
+            if ("software/bob" in line) or \
+                ("software/beat" in line):  # our doc server
+                if 'master' not in line:  # don't replace 'latest' pointer
+                    replacement = "/v%s" % version if version is not None \
+                        else "/stable"
+                    line = BRANCH_RE.sub(replacement, line)
+        if DOC_IMAGE.search(line) is not None:
+            replacement = '-v%s-' % version if version is not None \
+                else '-stable-'
+            line = DOC_IMAGE.sub(replacement, line)
+        new_readme.append(line)
+    return '\n'.join(new_readme) + '\n'
+
+
+def get_latest_tag_name(gitpkg):
+    """Find the name of the latest tag for a given package in the format '#.#.#'
+
+    Args:
+        gitpkg: gitlab package object
+
+    Returns: The name of the latest tag in format '#.#.#'. None if no tags for
+    the package were found.
+    """
+
+    # get 50 latest tags as a list
+    latest_tags = gitpkg.tags.list(all=True)
+    if not latest_tags:
+        return None
+    # create list of tags' names but ignore the first 'v' character in each name
+    # also filter out non version tags
+    tag_names = [tag.name[1:] for tag in latest_tags \
+        if StrictVersion.version_re.match(tag.name[1:])]
+    # sort them correctly according to each subversion number
+    tag_names.sort(key=StrictVersion)
+    # take the last one, as it is the latest tag in the sorted tags
+    latest_tag_name = tag_names[-1]
+    return latest_tag_name
+
+
+def get_parsed_tag(gitpkg, tag):
+    """
+    An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we
+    need only v2.1.3
+
+    The latest tag is either patch, minor, major, or none
+    """
+
+    m = re.search(r"(v\d+.\d+.\d+)", tag)
+    if m:
+        return m.group(0)
+    # tag = Version(tag)
+
+    # if we bump the version, we need to find the latest released version for
+    # this package
+    if 'patch' == tag or 'minor' == tag or 'major' == tag:
+
+        # find the correct latest tag of this package (without 'v' in front),
+        # None if there are no tags yet
+        latest_tag_name = get_latest_tag_name(gitpkg)
+
+        # if there were no tags yet, assume the very first version
+        if not latest_tag_name: return 'v0.0.1'
+
+        # check that it has expected format #.#.#
+        # latest_tag_name = Version(latest_tag_name)
+        m = re.match(r"(\d.\d.\d)", latest_tag_name)
+        if not m:
+            raise ValueError('The latest tag name {0} in package {1} has ' \
+                'unknown format'.format('v' + latest_tag_name, gitpkg.name))
+
+        # increase the version accordingly
+        major, minor, patch = latest_tag_name.split('.')
+
+        if 'major' == tag:
+            # increment the first number in 'v#.#.#' but make minor and patch
+            # to be 0
+            return 'v' + str(int(major) + 1) + '.0.0'
+
+        if 'minor' == tag:
+            # increment the second number in 'v#.#.#' but make patch to be 0
+            return 'v' + major + '.' + str(int(minor) + 1) + '.0'
+
+        if 'patch' == tag:
+            # increment the last number in 'v#.#.#'
+            return 'v' + major + '.' + minor + '.' + str(int(patch) + 1)
+
+    if 'none' == tag:
+        # we do nothing in this case
+        return tag
+
+    raise ValueError('Cannot parse changelog tag {0} of the ' \
+        'package {1}'.format(tag, gitpkg.name))
+
+
+def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
+    """Write annotations inside the provided tag of a given package.
+
+    Args:
+
+        gitpkg: gitlab package object
+        tag_name: The name of the tag to update
+        tag_comments_list: New annotations for this tag in a form of list
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    Returns: The gitlab object for the tag that was updated
+    """
+
+    # get tag and update its description
+    logger.info(tag_name)
+    tag = gitpkg.tags.get(tag_name)
+    tag_comments = '\n'.join(tag_comments_list)
+    logger.info('Found tag %s, updating its comments with:\n%s', tag.name,
+        tag_comments)
+    if not dry_run: tag.set_release_description(tag_comments)
+    return tag
+
+
+def commit_files(gitpkg, files_dict, message='Updated files', dry_run=False):
+    """Commit files of a given GitLab package.
+
+    Args:
+
+        gitpkg: gitlab package object
+        files_dict: Dictionary of file names and their contents (as text)
+        message: Commit message
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    """
+
+    data = {
+        'branch': 'master',  # v4
+        'commit_message': message,
+        'actions': []
+    }
+
+    # add files to update
+    for filename in files_dict.keys():
+        update_action = dict(action='update', file_path=filename)
+        update_action['content'] = files_dict[filename]
+        data['actions'].append(update_action)
+
+    logger.info("Committing changes in files: %s", str(files_dict.keys()))
+    if not dry_run:
+        gitpkg.commits.create(data)
+
+
+def get_last_pipeline(gitpkg):
+    """Returns the last pipeline of the project
+
+    Args:
+
+        gitpkg: gitlab package object
+
+    Returns: The gtilab object of the pipeline
+    """
+
+    # wait for 10 seconds to ensure that if a pipeline was just submitted,
+    # we can retrieve it
+    time.sleep(10)
+
+    # get the last pipeline
+    return gitpkg.pipelines.list(per_page=1, page=1)[0]
+
+
+def just_build_package(gitpkg, dry_run=False):
+    """Creates the pipeline with the latest tag and starts it
+
+    Args:
+
+        gitpkg: gitlab package object
+        dry_run: If True, the pipeline will not be created on GitLab
+
+    Returns:
+
+    """
+
+    # get the latest tag
+    latest_tag_name = 'v' + get_latest_tag_name(gitpkg)
+
+    # create the pipeline with this tag and start it
+    logger.info("Creating and starting pipeline for tag %s", latest_tag_name)
+
+    if not dry_run:
+        new_pipeline = gitpkg.pipelines.create({'ref': latest_tag_name})
+        return new_pipeline.id
+
+    return None
+
+
+def wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run=False):
+    """Using sleep function, wait for the latest pipeline to finish building.
+
+    This function pauses the script until pipeline completes either
+    successfully or with error.
+
+    Args:
+
+        gitpkg: gitlab package object
+        pipeline_id: id of the pipeline for which we are waiting to finish
+        dry_run: If True, outputs log message and exit. There wil be no
+                 waiting.
+
+    """
+
+    sleep_step = 30
+    max_sleep = 120 * 60  # two hours
+    # pipeline = get_last_pipeline(gitpkg, before_last=before_last)
+
+    logger.info('Waiting for the pipeline %s of package %s to finish. ' \
+        'Do not interrupt.', pipeline_id, gitpkg.name)
+
+    if dry_run: return
+
+    # retrieve the pipeline we are waiting for
+    pipeline = gitpkg.pipelines.get(pipeline_id)
+
+    # probe and wait for the pipeline to finish
+    slept_so_far = 0
+
+    while pipeline.status == 'running' or pipeline.status == 'pending':
+
+        time.sleep(sleep_step)
+        slept_so_far += sleep_step
+        if slept_so_far > max_sleep:
+            raise ValueError('I cannot wait longer than {0} seconds for '
+                'pipeline {1} to finish running!'.format(max_sleep, pipeline_id))
+        # probe gitlab to update the status of the pipeline
+        pipeline = gitpkg.pipelines.get(pipeline_id)
+
+    # finished running, now check if it succeeded
+    if pipeline.status != 'success':
+        raise ValueError('Pipeline {0} of project {1} exited with ' \
+            'undesired status "{2}". Release is not possible.' \
+            .format(pipeline_id, gitpkg.name, pipeline.status))
+
+    logger.info('Pipeline %s of package %s SUCCEEDED. Continue processing.',
+        pipeline_id, gitpkg.name)
+
+
+def cancel_last_pipeline(gitpkg):
+    """ Cancel the last started pipeline of a package
+
+    Args:
+
+        gitpkg: gitlab package object
+
+    """
+
+    pipeline = get_last_pipeline(gitpkg)
+    logger.info('Cancelling the last pipeline %s of project %s', pipeline.id,
+      gitpkg.name)
+    pipeline.cancel()
+
+
+def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False):
+    """Release package
+
+    The provided tag will be annotated with a given list of comments.
+    README.rst and version.txt files will also be updated according to the
+    release procedures.
+
+    Args:
+
+        gitpkg: gitlab package object
+        tag_name: The name of the release tag
+        tag_comments_list: New annotations for this tag in a form of list
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    """
+
+    # if there is nothing to release, just rebuild the package
+    latest_tag = get_latest_tag_name(gitpkg)
+
+    if tag_name == 'none' or (latest_tag and ('v' + latest_tag) == tag_name):
+        logger.warn("Since the tag is 'none' or already exists, we just " \
+            "re-build the last pipeline")
+        return just_build_package(gitpkg, dry_run)
+
+    # 1. Replace branch tag in Readme to new tag, change version file to new
+    # version tag. Add and commit to gitlab
+    version_number = tag_name[1:]  # remove 'v' in front
+    readme_file = gitpkg.files.get(file_path='README.rst', ref='master')
+    readme_content = readme_file.decode().decode()
+    readme_content = _update_readme(readme_content, version_number)
+    # commit and push changes
+    commit_files(gitpkg,
+        {
+          'README.rst': readme_content,
+          'version.txt': version_number
+          },
+        'Increased stable version to %s' % version_number, dry_run)
+
+    if not dry_run:
+        # cancel running the pipeline triggered by the last commit
+        cancel_last_pipeline(gitpkg)
+
+    # 2. Tag package with new tag and push
+    logger.info("Creating tag %s", tag_name)
+    tag_comments = '\n'.join(tag_comments_list)
+    logger.info("Updating tag comments with:\n%s", tag_comments)
+    if not dry_run:
+        tag = gitpkg.tags.create({'tag_name': tag_name, 'ref': 'master'})
+        # update tag with comments
+        tag.set_release_description(tag_comments)
+
+    # get the pipeline that is actually running with no skips
+    running_pipeline = get_last_pipeline(gitpkg)
+
+    # 3. Replace branch tag in Readme to master, change version file to beta
+    # version tag. Git add, commit, and push.
+    readme_content = _update_readme(readme_content, None)
+    major, minor, patch = version_number.split('.')
+    version_number = '{}.{}.{}b0'.format(major, minor, int(patch)+1)
+    # commit and push changes
+    commit_files(gitpkg, {
+      'README.rst': readme_content,
+      'version.txt': version_number,
+      },
+      'Increased latest version to %s [skip ci]' % version_number, dry_run)
+
+    return running_pipeline.id
+
+
+def parse_and_process_package_changelog(gl, bob_group, pkg_name,
+    package_changelog, dry_run):
+    """Process the changelog of a single package
+
+    Parse the log following specific format.  Update annotations of the
+    provided older tags and release the package by following the last tag
+    description.
+
+    Args:
+
+        gl: Gitlab API object
+        bob_group: gitlab object for the group
+        pkg_name: name of the package
+        package_changelog: the changelog corresponding to the provided package
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    Returns: gitlab handle for the package, name of the latest tag, and tag's
+    comments
+
+    """
+
+    cur_tag = None
+    cur_tag_comments = []
+
+    grpkg = ensure_correct_package(bob_group.projects.list(search=pkg_name),
+        bob_group.name, pkg_name)
+
+    # so, we need to retrieve the full info from GitLab using correct project id
+    gitpkg = gl.projects.get(id=grpkg.id)
+
+    # we assume that changelog is formatted as structured text
+    # first line is the name of the package
+    for line in package_changelog:
+        if '  *' == line[:3]:  # a tag level
+            # write the comments collected for the previous tag
+            if cur_tag:
+                update_tag_comments(gitpkg, cur_tag, cur_tag_comments, dry_run)
+                cur_tag_comments = []  # reset comments
+
+            # parse the current tag name
+            cur_tag = get_parsed_tag(gitpkg, line[3:].strip())
+
+        else:  # all other lines are assumed to be comments
+            cur_tag_comments.append(line.strip())
+
+    # return the last tag and comments for release
+    return gitpkg, cur_tag, cur_tag_comments
+
+
+def release_bob(changelog_file):
+    """Process the changelog and releases the ``bob`` metapackage"""
+
+    logger.info('Read the section "Releasing the Bob meta package" ' \
+        'on the documentation')
+
+    # get the list of bob's dependencies.
+    # Get their latest tags (since bob's last release) and the tag's changelog
+    saw_a_new_package = True
+    latest_tag = None
+    latest_pkg = None
+    for line in changelog_file:
+        # if saw_a_new_package:
+        if line.startswith('*'):
+            pkg = line[2:].strip()
+            saw_a_new_package = True
+            logger.info('%s == %s', latest_pkg, latest_tag)
+            latest_pkg = pkg
+            latest_tag = None
+            continue
+        if line.startswith('  *'):
+            latest_tag = line.split()[1][1:]
+        saw_a_new_package = False
+    logger.info('%s == %s', latest_pkg, latest_tag)
+    readme = open('../../bob/README.rst').read()
+    readme = _update_readme(readme, bob_version)
+    open('../../bob/README.rst', 'wt').write(readme)
+    open('../../bob/version.txt', 'wt').write(bob_version)

bob/devtools/scripts/bdt.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Main entry point for bdt
+"""
+
+import pkg_resources
+
+import click
+from click_plugins import with_plugins
+
+import logging
+logger = logging.getLogger('bdt')
+
+
+def set_verbosity_level(logger, level):
+    """Sets the log level for the given logger.
+
+    Parameters
+    ----------
+    logger : :py:class:`logging.Logger` or str
+        The logger to generate logs for, or the name  of the module to generate
+        logs for.
+    level : int
+        Possible log levels are: 0: Error; 1: Warning; 2: Info; 3: Debug.
+    Raises
+    ------
+    ValueError
+        If the level is not in range(0, 4).
+    """
+    if level not in range(0, 4):
+        raise ValueError(
+            "The verbosity level %d does not exist. Please reduce the number "
+            "of '--verbose' parameters in your command line" % level
+        )
+    # set up the verbosity level of the logging system
+    log_level = {
+        0: logging.ERROR,
+        1: logging.WARNING,
+        2: logging.INFO,
+        3: logging.DEBUG
+    }[level]
+
+    # set this log level to the logger with the specified name
+    if isinstance(logger, str):
+        logger = logging.getLogger(logger)
+    logger.setLevel(log_level)
+
+
+def verbosity_option(**kwargs):
+    """Adds a -v/--verbose option to a click command.
+
+    Parameters
+    ----------
+    **kwargs
+        All kwargs are passed to click.option.
+
+    Returns
+    -------
+    callable
+        A decorator to be used for adding this option.
+    """
+    def custom_verbosity_option(f):
+        def callback(ctx, param, value):
+            ctx.meta['verbosity'] = value
+            set_verbosity_level(logger, value)
+            logger.debug("`bdt' logging level set to %d", value)
+            return value
+        return click.option(
+            '-v', '--verbose', count=True,
+            expose_value=False, default=2,
+            help="Increase the verbosity level from 0 (only error messages) "
+            "to 1 (warnings), 2 (info messages), 3 (debug information) by "
+            "adding the --verbose option as often as desired "
+            "(e.g. '-vvv' for debug).",
+            callback=callback, **kwargs)(f)
+    return custom_verbosity_option
+
+
+class AliasedGroup(click.Group):
+  ''' Class that handles prefix aliasing for commands '''
+  def get_command(self, ctx, cmd_name):
+    rv = click.Group.get_command(self, ctx, cmd_name)
+    if rv is not None:
+      return rv
+    matches = [x for x in self.list_commands(ctx)
+               if x.startswith(cmd_name)]
+    if not matches:
+      return None
+    elif len(matches) == 1:
+      return click.Group.get_command(self, ctx, matches[0])
+    ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
+
+
+def raise_on_error(view_func):
+    """Raise a click exception if returned value is not zero.
+
+    Click exits successfully if anything is returned, in order to exit properly
+    when something went wrong an exception must be raised.
+    """
+
+    from functools import wraps
+
+    def _decorator(*args, **kwargs):
+        value = view_func(*args, **kwargs)
+        if value not in [None, 0]:
+            exception = click.ClickException("Error occured")
+            exception.exit_code = value
+            raise exception
+        return value
+    return wraps(view_func)(_decorator)
+
+
+@with_plugins(pkg_resources.iter_entry_points('bdt.cli'))
+@click.group(cls=AliasedGroup,
+             context_settings=dict(help_option_names=['-?', '-h', '--help']))
+@verbosity_option()
+def main():
+    """Bob Development Tools - see available commands below"""

bob/devtools/scripts/cb_output.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import click
+from click.testing import CliRunner
+import conda_build.api as cb
+
+from . import bdt
+
+from ..conda import should_skip_build
+
+
+@click.command(context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True,
+    ),
+    epilog='''\b
+Examples:
+$ bdt cb-output conda_recipe_dir
+$ bdt cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
+'''
+)
+@click.argument('recipe_path')
+@click.option('-m', '--variant-config-files', help='see conda build --help')
+@click.option('--python', help='see conda build --help')
+@bdt.raise_on_error
+def cb_output(recipe_path, variant_config_files, python):
+  """Outputs name(s) of package(s) that would be generated by conda build.
+
+  This command accepts extra unknown arguments so you can give it the same
+  arguments that you would give to conda build.
+
+  As of now, it only parses -m/--variant_config_files and --python and other
+  arguments are ignored.
+  """
+  clirunner = CliRunner()
+  with clirunner.isolation():
+    # render
+    config = cb.get_or_merge_config(
+        None, variant_config_files=variant_config_files, python=python)
+    metadata_tuples = cb.render(recipe_path, config=config)
+
+    # check if build(s) should be skipped
+    if should_skip_build(metadata_tuples):
+      return 0
+
+    paths = cb.get_output_file_paths(metadata_tuples, config=config)
+  click.echo('\n'.join(sorted(paths)))

bob/devtools/scripts/changelog.py

+#!/usr/bin/env python
+
+import os
+import sys
+import datetime
+
+import click
+
+from . import bdt
+from ..changelog import get_last_tag_date, write_tags_with_commits
+from ..changelog import parse_date
+from ..release import get_gitlab_instance
+
+
+@click.command(context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True,
+    ),
+    epilog='''
+Examples:
+
+  1. Generates the changelog for a single package using merge requests:
+
+     $ bdt -vvv changelog group/package.xyz changelog.md
+
+
+  2. The same as above, but dumps the changelog to stdout instead of a file
+
+     $ bdt -vvv changelog group/package.xyz -
+
+
+  3. Generates the changelog for a single package looking at commits
+     (not merge requests):
+
+     $ bdt -vvv changelog --mode=commits group/package.xyz changelog.md
+
+
+  4. Generates the changelog for a single package looking at merge requests starting from a given date of January 1, 2016:
+
+\b
+     $ bdt -vvv changelog --mode=mrs --since=2016-01-01 group/package.xyz changelog.md
+
+
+  5. Generates a complete list of changelogs for a list of packages (one per line:
+
+\b
+     $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
+     $ bdt lasttag bob/bob
+     # copy and paste date to next command
+     $ bdt -vvv changelog --since="2018-07-17 10:23:40" order.txt changelog.md
+''')
+@click.argument('target')
+@click.argument('changelog', type=click.Path(exists=False, dir_okay=False,
+  file_okay=True, writable=True))
+@click.option('-g', '--group', default='bob', show_default=True,
+    help='Gitlab default group name where packages are located (if not ' \
+        'specified using a "/" on the package name - e.g. ' \
+        '"bob/bob.extension")')
+@click.option('-m', '--mode', type=click.Choice(['mrs', 'tags', 'commits']),
+    default='mrs', show_default=True,
+    help='Changes the way we produce the changelog.  By default, uses the ' \
+        'text in every merge request (mode "mrs"). To use tag annotations, ' \
+        'use mode "tags". If you use "commits" as mode, we use the text ' \
+        'in commits to produce the changelog')
+@click.option('-s', '--since',
+    help='A starting date in any format accepted by dateutil.parser.parse() ' \
+    '(see https://dateutil.readthedocs.io/en/stable/parser.html) from ' \
+    'which you want to generate the changelog.  If not set, the package\'s' \
+    'last release date will be used')
+@bdt.raise_on_error
+def changelog(target, changelog, group, mode, since):
+    """Generates changelog file for package(s) from the Gitlab server.
+
+    This script generates changelogs for either a single package or multiple
+    packages, depending on the value of TARGET.  The changelog (in markdown
+    format) is written to the output file CHANGELOG.
+
+    There are two modes of operation: you may provide the package name in the
+    format ``<gitlab-group>/<package-name>`` (or simply ``<package-name>``, in
+    which case the value of ``--group`` will be used). Or, optionally, provide
+    an existing file containing a list of packages that will be iterated on.
+
+    For each package, we will contact the Gitlab server and create a changelog
+		using merge-requests (default), tags or commits since a given date.  If a
+		starting date is not passed, we'll use the date of the last tagged value or
+		the date of the first commit, if no tags are available in the package.
+    """
+
+    gl = get_gitlab_instance()
+
+    # reads package list or considers name to be a package name
+    if os.path.exists(target) and os.path.isfile(target):
+        bdt.logger.info('Reading package names from file %s...', target)
+        with open(target, 'rb') as f:
+            packages = [k.strip() for k in f.readlines() if k and not \
+                k.strip().startswith('#')]
+    else:
+        bdt.logger.info('Assuming %s is a package name (file does not ' \
+            'exist)...', target)
+        packages = [target]
+
+    # if the user passed a date, convert it
+    if since: since = parse_date(since)
+
+    # iterates over the packages and dumps required information
+    for package in packages:
+
+        if '/' not in package:
+            package = '/'.join(group, package)
+
+        # retrieves the gitlab package object
+        use_package = gl.projects.get(package)
+        bdt.logger.info('Found gitlab project %s (id=%d)',
+            use_package.attributes['path_with_namespace'], use_package.id)
+
+        last_release_date = since or get_last_tag_date(use_package)
+        bdt.logger.info('Retrieving data (mode=%s) since %s', mode,
+            last_release_date.strftime('%b %d, %Y %H:%M'))
+
+        # add 1s to avoid us retrieving previous release data
+        last_release_date += datetime.timedelta(seconds=1)
+
+        if mode == 'tags':
+            visibility = ('public',)
+        else:
+            visibility = ('public', 'private', 'internal')
+
+        if use_package.attributes['namespace'] == use_package.name:
+            # skip system meta-package
+            bdt.logger.warn('Skipping meta package %s...',
+                use_package.attributes['path_with_namespace'])
+            continue
+
+        if use_package.attributes['visibility'] not in visibility:
+            bdt.logger.warn('Skipping package %s (visibility not in ' \
+                '"%s")...', use_package.attributes['path_with_namespace'],
+                '|'.join(visibility))
+            continue
+
+        if changelog == '-':
+          changelog_file = sys.stdout
+        else:
+          changelog_file = open(changelog, 'at')
+
+        # write_tags(f, use_package, last_release_date)
+        write_tags_with_commits(changelog_file, use_package, last_release_date,
+          mode)
+        changelog_file.flush()

bob/devtools/scripts/lasttag.py

+#!/usr/bin/env python
+
+import os
+
+import click
+
+from . import bdt
+from ..changelog import get_last_tag, parse_date
+from ..release import get_gitlab_instance
+
+
+@click.command(context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True,
+    ),
+    epilog='''
+Examples:
+
+  1. Get the last tag information of the bob/bob package
+
+     $ bdt lasttag bob/bob
+
+
+  2. Get the last tag information of the beat/beat.core package
+
+     $ bdt lasttag beat/beat.core
+
+''')
+@click.argument('package')
+@bdt.raise_on_error
+def lasttag(package):
+    """Returns the last tag information on a given PACKAGE
+    """
+
+    if '/' not in package:
+        raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+    gl = get_gitlab_instance()
+
+    # we lookup the gitlab group once
+    use_package = gl.projects.get(package)
+    bdt.logger.info('Found gitlab project %s (id=%d)',
+        use_package.attributes['path_with_namespace'], use_package.id)
+
+    tag = get_last_tag(use_package)
+    date = parse_date(tag.commit['committed_date'])
+    click.echo('Lastest tag for %s is %s (%s)' % \
+        (package, tag.name, date.strftime('%Y-%m-%d %H:%M:%S')))

bob/devtools/scripts/release.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import os
+
+import click
+
+from . import bdt
+from ..release import release_bob, parse_and_process_package_changelog
+from ..release import release_package, wait_for_pipeline_to_finish
+from ..release import get_gitlab_instance
+
+@click.command(context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True,
+    ),
+    epilog='''
+Examples:
+
+  1. Releases a single package:
+
+     $ bdt -vvv release --package=bob.package.xyz changelog.md
+
+
+  2. If there is a single package in the ``changelog.md`` file, the flag
+     ``--package`` is not required:
+
+     $ bdt -vvv release changelog.md
+
+
+  2. Releases the whole of bob using `changelog_since_last_release.md`:
+
+     $ bdt -vvv release bob/devtools/data/changelog_since_last_release.md
+
+
+  3. In case of errors, resume the release of the whole of Bob:
+
+     $ bdt -vvv release --resume bob/devtools/data/changelog_since_last_release.md
+
+
+  4. The option `-dry-run` can be used to let the script print what it would do instead of actually doing it:
+
+     $ bdt -vvv release --dry-run changelog_since_last_release.md
+'''
+)
+@click.argument('changelog', type=click.File('rb', lazy=False))
+@click.option('-g', '--group', default='bob', show_default=True,
+    help='Group name where all packages are located')
+@click.option('-p', '--package',
+    help='If the name of a package is provided, then this package will be ' \
+        'found in the changelog file and the release will resume from it ' \
+        '(if option ``--resume`` is set) or only this package will be ' \
+        'released.  If there is only a single package in the changelog, ' \
+        'then you do NOT need to set this flag')
+@click.option('-r', '--resume/--no-resume', default=False,
+    help='The overall release will resume from the provided package name')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@bdt.raise_on_error
+def release(changelog, group, package, resume, dry_run):
+    """\b
+    Tags packages on gitlab from an input CHANGELOG in markdown formatting
+
+    By using a CHANGELOG file as an input (that can be generated with the ``bdt
+    changelog`` command), this script goes through all packages in CHANGELOG
+    file (in order listed), tags them correctly as per the file, and pushes
+    this tag to gitlab them one by one.  Tagged releases are treated specially
+    by the CI and are auto-deployed to our stable conda channels and PyPI.
+
+    This script uses the provided CHANGELOG file to release one or more
+    package.  The CHANGELOG is expected to have the following structure:
+
+    \b
+    * package name
+      * tag1 name (date of the tag).
+        * tag description. Each line of the tag description starts with `*`
+          character.
+        - commits (from earliest to latest). Each line of the commit starts
+          with `-` character.
+      * tag2 name (date of the tag).
+        * tag description. Each line of the tag description starts with `*`
+          character.
+        - commits (from earliest to latest). Each line of the commit starts
+          with `-` character.
+      * patch
+        - leftover not-tagged commits (from earliest to latest)
+
+    This script can also be used to release a single package.
+
+    IMPORTANT: There are some considerations that needs to be taken into
+    account **before** you release a new version of a package:
+
+    \b
+    * In the changelog file:
+      - write the name of this package and write (at least) the next tag value.
+        For the next tag value, you can either indicate one of the special
+        values: ``patch``, ``minor`` or ``major``, and the package will be then
+        released with either patch, minor, or major version **bump**.
+      - Alternatively, you can specify the tag value directly (using
+        a ``vX.Y.Z`` format), but be careful that it is higher than the last
+        release tag of this package.  Make sure that the version that you are
+        trying to release is not already released.  You must follow semantic
+        versioning: http://semver.org.
+      - Then, under the desired new tag version of the package, please write
+        down the changes that are applied to the package between the last
+        released version and this version. This changes are written to
+        release tags of packages in the Gitlab interface.  For an example
+        look at: https://gitlab.idiap.ch/bob/bob.extension/tags
+    * Make sure all the tests for the package are passing.
+    * Make sure the documentation is building with the following command:
+      ``sphinx-build -aEWn doc sphinx``
+    * Ensure all changes are committed to the git repository and pushed.
+    * Ensure the documentation badges in README.rst are pointing to:
+      https://www.idiap.ch/software/bob/docs/bob/...
+    * For database packages, ensure that the '.sql3' file or other metadata
+      files have been generated (if any).
+    * Ensure the nightlies build is green after the changes are submitted if
+      the package is a part of the nightlies.
+    * If your package depends on an unreleased version of another package,
+      you need to release that package first.
+    """
+
+    gl = get_gitlab_instance()
+
+    use_group = gl.groups.list(search='"%s"' % group)[0]
+
+    # if we are releasing 'bob' metapackage, it's a simple thing, no GitLab
+    # API
+    if package == 'bob':
+        release_bob(changelog)
+        return
+
+    # traverse all packages in the changelog, edit older tags with updated
+    # comments, tag them with a suggested version, then try to release, and
+    # wait until done to proceed to the next package
+    changelogs = changelog.readlines()
+
+    # find the starts of each package's description in the changelog
+    pkgs = [i for i, line in enumerate(changelogs) if line[0] == '*']
+    pkgs.append(len(changelogs)) #the end
+    start_idx = 0
+
+    if package:
+        # get the index where the package first appears in the list
+        start_idx = [i for i, line in enumerate(changelogs) \
+            if line[1:].strip() == package]
+
+        if not start_idx:
+            bdt.logger.error('Package %s was not found in the changelog',
+                package)
+            return
+
+        start_idx = pkgs.index(start_idx[0])
+
+    # if we are in a dry-run mode, let's let it be known
+    if dry_run:
+        bdt.logger.warn('!!!! DRY RUN MODE !!!!')
+        bdt.logger.warn('Nothing is being committed to Gitlab')
+
+    # go through the list of packages and release them starting from the
+    # start_idx
+    for i in range(start_idx, len(pkgs) - 1):
+        cur_package_name = changelogs[pkgs[i]][1:].strip()
+        bdt.logger.info('Processing package %s', changelogs[pkgs[i]])
+        gitpkg, tag, tag_comments = parse_and_process_package_changelog(gl,
+            use_group, cur_package_name,
+            changelogs[pkgs[i] + 1: pkgs[i + 1]], dry_run)
+
+        # release the package with the found tag and its comments
+        if gitpkg:
+            pipeline_id = release_package(gitpkg, tag, tag_comments, dry_run)
+            # now, wait for the pipeline to finish, before we can release the
+            # next package
+            wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run)
+
+        # if package name is provided and resume is not set, process only
+        # this package
+        if package == cur_package_name and not resume:
+            break
+
+    bdt.logger.info('Finished processing %s', changelog)

bob_tools/scripts/cb_output.py

-from ..utils.click import raise_on_error
-from ..utils.conda import should_skip_build
-from click.testing import CliRunner
-import click
-import conda_build.api as cb
-
-
-@click.command(context_settings=dict(
-    ignore_unknown_options=True, allow_extra_args=True),
-    epilog='''\b
-Examples:
-$ bob-tools cb-output conda_recipe_dir
-$ bob-tools cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
-'''
-)
-@click.argument('recipe_path')
-@click.option('-m', '--variant-config-files', help='see conda build --help')
-@click.option('--python', help='see conda build --help')
-@raise_on_error
-def cb_output(recipe_path, variant_config_files, python):
-    """Outputs name(s) of package(s) that would be generated by conda build.
-
-    This command accepts extra unknown arguments so you can give it the same
-    arguments that you would give to conda build.
-
-    As of now, it only parses -m/--variant_config_files and --python and other
-    arguments are ignored.
-    """
-    clirunner = CliRunner()
-    with clirunner.isolation():
-        # render
-        config = cb.get_or_merge_config(
-            None, variant_config_files=variant_config_files, python=python)
-        metadata_tuples = cb.render(recipe_path, config=config)
-
-        # check if build(s) should be skipped
-        if should_skip_build(metadata_tuples):
-            return 0
-
-        paths = cb.get_output_file_paths(metadata_tuples, config=config)
-    click.echo('\n'.join(sorted(paths)))

bob_tools/scripts/main.py

-"""This is the main entry to bob_tools's scripts.
-"""
-import pkg_resources
-import click
-from click_plugins import with_plugins
-
-
-@with_plugins(pkg_resources.iter_entry_points('bob_tools.cli'))
-@click.group(context_settings=dict(help_option_names=['-?', '-h', '--help']))
-def main():
-    """The main command line interface for bob tools. Look below for available
-    commands."""
-    pass

bob_tools/utils/click.py

-from functools import wraps
-
-import click
-
-
-def raise_on_error(view_func):
-    """Raise a click exception if returned value is not zero.
-
-    Click exits successfully if anything is returned, in order to exit properly
-    when something went wrong an exception must be raised.
-    """
-
-    def _decorator(*args, **kwargs):
-        value = view_func(*args, **kwargs)
-        if value not in [None, 0]:
-            exception = click.ClickException("Error occurred")
-            exception.exit_code = value
-            raise exception
-        return value
-    return wraps(view_func)(_decorator)

buildout.cfg

@@ -4,8 +4,7 @@
 [buildout]
 parts = scripts
 develop = .
-eggs = bob_tools
-
+eggs = bob.devtools
 newest = false
 
 [scripts]

doc/api.rst

+.. vim: set fileencoding=utf-8 :
+
+
+============
+ Python API
+============
+
+.. autosummary::
+   bob.devtools.conda
+   bob.devtools.release
+   bob.devtools.changelog
+
+
+Detailed Information
+--------------------
+
+.. automodule:: bob.devtools.conda
+
+.. automodule:: bob.devtools.release
+
+.. automodule:: bob.devtools.changelog