Major commit to refactor this package as a hub for more functionality

- Start moving bob.admin scripts here
- Add baseline documentation system
- Add logging support
- Rename it as bob.devtools
- Prepare for conda-packaging
- Name main tool `bdt`

MANIFEST.in

 include LICENSE README.rst buildout.cfg version.txt
 recursive-include doc conf.py *.rst
-recursive-include bob *.cpp *.h
-recursive-include bob/extension/examples *
-recursive-include bob/extension/data *
+recursive-include bob/devtools/data *.md

README.rst

@@ -5,18 +5,24 @@
 ===============================================
 
 This package is part of the signal-processing and machine learning toolbox
-Bob_. It provides some tools to help maintain Bob_.
+Bob_. It provides tools to help maintain Bob_.
+
 
 Installation
 ------------
 
-This package needs to be installed in your base conda environment. To install
+This package needs to be installed in a conda environment. To install
 this package, run::
 
-  $ conda activate base
-  # the dependency list below matches the ones in setup.py
-  $ conda install pip click click-plugins conda-build
-  $ pip install -e .
+  $ conda env create -f env.yml
+  $ conda activate bdt
+  (bdt) $ buildout
+  (bdt) $ ./bin/bdt --help
+  ...
+
+To build the documentation, just do::
+
+  (bdt) $ ./bin/sphinx-build doc sphinx
 
 
 Contact

bob/__init__.py

+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

bob/devtools/changelog.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import io
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+
+import pytz
+import dateutil.parser
+
+from .release import ensure_correct_package
+
+
+def parse_date(d):
+    '''Parses any date supported by :py:func:`dateutil.parser.parse`'''
+
+    return dateutil.parser.parse(d, ignoretz=True).replace(
+                tzinfo=pytz.timezone("Europe/Zurich"))
+
+
+def _sort_commits(commits, reverse):
+    '''Sorts gitlab commit objects using their ``committed_date`` attribute'''
+
+    return sorted(commits,
+        key=lambda x: parse_date(x.committed_date),
+        reverse=reverse,
+        )
+
+
+def _sort_tags(tags, reverse):
+    '''Sorts gitlab tag objects using their ``committed_date`` attribute'''
+
+    return sorted(tags,
+        key=lambda x: parse_date(x.commit['committed_date']),
+        reverse=reverse,
+        )
+
+
+def get_file_from_gitlab(gitpkg, path, ref='master'):
+    '''Retrieves a file from a Gitlab repository, returns a (StringIO) file'''
+
+    return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
+
+
+def get_last_tag(package):
+    '''Returns the last (gitlab object) tag for the given package
+
+    Args:
+
+        package: The gitlab project object from where to fetch the last release
+                 date information
+
+
+    Returns: a tag object
+    '''
+
+    # according to the Gitlab API documentation, tags are sorted from the last
+    # updated to the first, by default - no need to do further sorting!
+    tag_list = package.tags.list()
+
+    if tag_list:
+        # there are tags, use these
+        return tag_list[0]
+
+
+def get_last_tag_date(package):
+    '''Returns the last release date for the given package
+
+    Falls back to the first commit date if the package has not yet been tagged
+
+
+    Args:
+
+        package: The gitlab project object from where to fetch the last release
+                 date information
+
+
+    Returns: a datetime object that refers to the last date the package was
+             released.  If the package was never released, then returns the
+             date just before the first commit.
+    '''
+
+    # according to the Gitlab API documentation, tags are sorted from the last
+    # updated to the first, by default - no need to do further sorting!
+    tag_list = package.tags.list()
+
+    if tag_list:
+        # there are tags, use these
+        last = tag_list[0]
+        logger.debug('Last tag for package %s (id=%d) is %s', package.name,
+            package.id, last.name)
+        return parse_date(last.commit['committed_date']) + \
+            datetime.timedelta(milliseconds=500)
+
+    else:
+        commit_list = package.commits.list(all=True)
+
+        if commit_list:
+            # there are commits, use these
+            first = _sort_commits(commit_list, reverse=False)[0]
+            logger.debug('First commit for package %s (id=%d) is from %s',
+                package.name, package.id, first.committed_date)
+            return parse_date(first.committed_date) - \
+                datetime.timedelta(milliseconds=500)
+
+        else:
+            # there are no commits nor tags - abort
+            raise RuntimeError('package %s (id=%d) does not have commits ' \
+                'or tags so I cannot devise a good starting date' % \
+                (package.name, package.id))
+
+
+def _get_tag_changelog(tag):
+
+    try:
+        return tag.release['description']
+    except Exception:
+        return ''
+
+
+def _write_one_tag(f, pkg_name, tag):
+    '''Prints commit information for a single tag of a given package
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        pkg_name: The name of the package we are writing tags of
+        tag: The tag value
+
+    '''
+
+    git_date = parse_date(tag.commit['committed_date'])
+    f.write('  * %s (%s)\n' % (tag.name,  git_date.strftime('%b %d, %Y %H:%M')))
+
+    for line in _get_tag_changelog(tag).replace('\r\n', '\n').split('\n'):
+
+        line = line.strip()
+        if line.startswith('* ') or line.startswith('- '):
+            line = line[2:]
+
+        line = line.replace('!', pkg_name + '!').replace(pkg_name + \
+            pkg_name, pkg_name)
+        line = line.replace('#', pkg_name + '#')
+        if not line:
+            continue
+        f.write('%s* %s' % (5*' ', line))
+
+
+def _write_commits_range(f, pkg_name, commits):
+    '''Writes all commits of a given package within a range, to the output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        pkg_name: The name of the package we are writing tags of
+        commits: List of commits to be written
+
+    '''
+
+
+    for commit in commits:
+        commit_title = commit.title
+
+        # skip commits that do not carry much useful information
+        if '[skip ci]' in commit_title or \
+                        'Merge branch' in commit_title or \
+                        'Increased stable' in commit_title:
+            continue
+
+        commit_title = commit_title.strip()
+        commit_title = commit_title.replace('!', pkg_name + '!').replace(pkg_name + pkg_name, pkg_name)
+        commit_title = commit_title.replace('#', pkg_name + '#')
+        f.write('%s- %s' % (' ' * 5, commit_title))
+
+
+def _write_mergerequests_range(f, pkg_name, mrs):
+    '''Writes all merge-requests of a given package, with a range, to the
+    output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        pkg_name: The name of the package we are writing tags of
+        mrs: The list of merge requests to write
+
+    '''
+
+    for mr in mrs:
+        title = mr.title.strip().replace('\r','').replace('\n', '  ')
+        title = title.replace(' !', ' ' + pkg_name + '!')
+        title = title.replace(' #', ' ' + pkg_name + '#')
+        description = mr.description.strip().replace('\r','').replace('\n', '  ')
+        description = description.replace(' !', ' ' + pkg_name + '!')
+        description = description.replace(' #', ' ' + pkg_name + '#')
+        space = ': ' if description else ''
+        log = '''     - {pkg}!{iid} {title}{space}{description}'''
+        f.write(log.format(pkg=pkg_name, iid=mr.iid, title=title, space=space, description=description))
+        f.write('\n')
+
+
+def write_tags_with_commits(f, gitpkg, since, mode):
+    '''Writes all tags and commits of a given package to the output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        gitpkg: A pointer to the gitlab package object
+        since: Starting date (as a datetime object)
+        mode: One of mrs (merge-requests), commits or tags indicating how to
+              list entries in the changelog for this package
+
+    '''
+
+    # get tags since release and sort them
+    tags = gitpkg.tags.list()
+
+    # sort tags by date
+    tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
+    tags = _sort_tags(tags, reverse=False)
+
+    # get commits since release date and sort them too
+    commits = gitpkg.commits.list(since=since, all=True)
+
+    # sort commits by date
+    commits = _sort_commits(commits, reverse=False)
+
+    # get merge requests since the release data
+    mrs = list(reversed(gitpkg.mergerequests.list(state='merged', updated_after=since, order_by='updated_at', all=True)))
+    f.write('* %s\n' % (gitpkg.name,))
+
+    # go through tags and writes each with its message and corresponding
+    # commits
+    start_date = since
+    for tag in tags:
+
+        # write tag name and its text
+        _write_one_tag(f, gitpkg.name, tag)
+        end_date = parse_date(tag.commit['committed_date'])
+
+        if mode == 'commits':
+            # write commits from the previous tag up to this one
+            commits4tag = [k for k in commits \
+                if (start_date < parse_date(k.committed_date) <= end_date)]
+            _write_commits_range(f, gitpkg.name, commits4tag)
+
+        elif mode == 'mrs':
+            # write merge requests from the previous tag up to this one
+            # the attribute 'merged_at' is not available in GitLab API as of 27
+            # June 2018
+            mrs4tag = [k for k in mrs \
+                if (start_date < parse_date(k.updated_at) <= end_date)]
+            _write_mergerequests_range(f, gitpkg.name, mrs4tag)
+
+        start_date = end_date
+
+    if mode != 'tags':
+
+        # write the tentative patch version bump for the future tag
+        f.write('  * patch\n')
+
+        if mode == 'mrs':
+            # write leftover merge requests
+            # the attribute 'merged_at' is not available in GitLab API as of 27
+            # June 2018
+            leftover_mrs = [k for k in mrs \
+                if parse_date(k.updated_at) > start_date]
+            _write_mergerequests_range(f, gitpkg.name, leftover_mrs)
+
+        else:
+            # write leftover commits that were not tagged yet
+            leftover_commits = [k for k in commits \
+                if parse_date(k.committed_date) > start_date]
+            _write_commits_range(f, gitpkg.name, leftover_commits)
+
+
+def write_tags(f, gitpkg, since):
+    '''Writes all tags of a given package to the output file
+
+    Args:
+
+        f: A :py:class:`File` ready to be written at
+        gitpkg: A pointer to the gitlab package object
+        since: Starting date as a datetime object
+
+    '''
+
+    tags = gitpkg.tags.list()
+    # sort tags by date
+    tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
+    tags = _sort_tags(tags, reverse=False)
+    f.write('* %s\n')
+
+    for tag in tags:
+        _write_one_tag(gitpkg.name, tag)

bob/devtools/release.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import time
+import gitlab
+
+import logging
+logger = logging.getLogger(__name__)
+
+from distutils.version import StrictVersion
+
+
+def get_gitlab_instance():
+    '''Returns an instance of the gitlab object for remote operations'''
+
+    # tries to figure if we can authenticate using a global configuration
+    cfgs = ['~/.python-gitlab.cfg', '/etc/python-gitlab.cfg']
+    cfgs = [os.path.expanduser(k) for k in cfgs]
+    if any([os.path.exists(k) for k in cfgs]):
+        gl = gitlab.Gitlab.from_config('idiap', cfgs)
+    else: #ask the user for a token
+        server = "https://gitlab.idiap.ch"
+        token = input("%s token: " % server)
+        gl = gitlab.Gitlab(server, private_token=token, api_version=4)
+
+    return gl
+
+
+def ensure_correct_package(candidates, group_name, pkg_name):
+
+  for pkg in candidates:
+    # make sure the name and the group name match exactly
+    if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
+      return pkg
+
+  raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
+
+
+def _update_readme(readme, version):
+    """
+    Inside text of the readme, replaces parts of the links to the provided
+    version. If version is not provided, replace to `stable` or `master`.
+
+    Args:
+
+        readme: Text of the README.rst file from a bob package
+        version: Format of the version string is '#.#.#'
+
+    Returns: New text of readme with all replaces done
+    """
+
+    # replace the badge in the readme's text with the given version
+    DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
+    BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
+
+    new_readme = []
+    for line in readme.splitlines():
+        if BRANCH_RE.search(line) is not None:
+            if "gitlab" in line:  # gitlab links
+                replacement = "/v%s" % version if version is not None \
+                    else "/master"
+                line = BRANCH_RE.sub(replacement, line)
+            if ("software/bob" in line) or \
+                ("software/beat" in line):  # our doc server
+                if 'master' not in line:  # don't replace 'latest' pointer
+                    replacement = "/v%s" % version if version is not None \
+                        else "/stable"
+                    line = BRANCH_RE.sub(replacement, line)
+        if DOC_IMAGE.search(line) is not None:
+            replacement = '-v%s-' % version if version is not None \
+                else '-stable-'
+            line = DOC_IMAGE.sub(replacement, line)
+        new_readme.append(line)
+    return '\n'.join(new_readme) + '\n'
+
+
+def get_latest_tag_name(gitpkg):
+    """Find the name of the latest tag for a given package in the format '#.#.#'
+
+    Args:
+        gitpkg: gitlab package object
+
+    Returns: The name of the latest tag in format '#.#.#'. None if no tags for
+    the package were found.
+    """
+
+    # get 50 latest tags as a list
+    latest_tags = gitpkg.tags.list(all=True)
+    if not latest_tags:
+        return None
+    # create list of tags' names but ignore the first 'v' character in each name
+    # also filter out non version tags
+    tag_names = [tag.name[1:] for tag in latest_tags \
+        if StrictVersion.version_re.match(tag.name[1:])]
+    # sort them correctly according to each subversion number
+    tag_names.sort(key=StrictVersion)
+    # take the last one, as it is the latest tag in the sorted tags
+    latest_tag_name = tag_names[-1]
+    return latest_tag_name
+
+
+def get_parsed_tag(gitpkg, tag):
+    """
+    An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we
+    need only v2.1.3
+
+    The latest tag is either patch, minor, major, or none
+    """
+
+    m = re.search(r"(v\d+.\d+.\d+)", tag)
+    if m:
+        return m.group(0)
+    # tag = Version(tag)
+
+    # if we bump the version, we need to find the latest released version for
+    # this package
+    if 'patch' == tag or 'minor' == tag or 'major' == tag:
+
+        # find the correct latest tag of this package (without 'v' in front),
+        # None if there are no tags yet
+        latest_tag_name = get_latest_tag_name(gitpkg)
+
+        # if there were no tags yet, assume the very first version
+        if not latest_tag_name: return 'v0.0.1'
+
+        # check that it has expected format #.#.#
+        # latest_tag_name = Version(latest_tag_name)
+        m = re.match(r"(\d.\d.\d)", latest_tag_name)
+        if not m:
+            raise ValueError('The latest tag name {0} in package {1} has ' \
+                'unknown format'.format('v' + latest_tag_name, gitpkg.name))
+
+        # increase the version accordingly
+        major, minor, patch = latest_tag_name.split('.')
+
+        if 'major' == tag:
+            # increment the first number in 'v#.#.#' but make minor and patch
+            # to be 0
+            return 'v' + str(int(major) + 1) + '.0.0'
+
+        if 'minor' == tag:
+            # increment the second number in 'v#.#.#' but make patch to be 0
+            return 'v' + major + '.' + str(int(minor) + 1) + '.0'
+
+        if 'patch' == tag:
+            # increment the last number in 'v#.#.#'
+            return 'v' + major + '.' + minor + '.' + str(int(patch) + 1)
+
+    if 'none' == tag:
+        # we do nothing in this case
+        return tag
+
+    raise ValueError('Cannot parse changelog tag {0} of the ' \
+        'package {1}'.format(tag, gitpkg.name))
+
+
+def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
+    """Write annotations inside the provided tag of a given package.
+
+    Args:
+
+        gitpkg: gitlab package object
+        tag_name: The name of the tag to update
+        tag_comments_list: New annotations for this tag in a form of list
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    Returns: The gitlab object for the tag that was updated
+    """
+
+    # get tag and update its description
+    logger.info(tag_name)
+    tag = gitpkg.tags.get(tag_name)
+    tag_comments = '\n'.join(tag_comments_list)
+    logger.info('Found tag %s, updating its comments with:\n%s', tag.name,
+        tag_comments)
+    if not dry_run: tag.set_release_description(tag_comments)
+    return tag
+
+
+def commit_files(gitpkg, files_dict, message='Updated files', dry_run=False):
+    """Commit files of a given GitLab package.
+
+    Args:
+
+        gitpkg: gitlab package object
+        files_dict: Dictionary of file names and their contents (as text)
+        message: Commit message
+        dry_run: If True, nothing will be committed or pushed to GitLab
+
+    """
+
+    data = {
+        'branch': 'master',  # v4
+        'commit_message': message,
+        'actions': []
+    }
+
+    # add files to update
+    for filename in files_dict.keys():
+        update_action = dict(action='update', file_path=filename)
+        update_action['content'] = files_dict[filename]
+        data['actions'].append(update_action)
+
+    logger.info("Committing changes in files: %s", str(files_dict.keys()))
+    if not dry_run:
+        gitpkg.commits.create(data)
+
+
+def get_last_pipeline(gitpkg):
+    """Returns the last pipeline of the project
+
+    Args:
+
+        gitpkg: gitlab package object
+
+    Returns: The gtilab object of the pipeline
+    """
+
+    # wait for 10 seconds to ensure that if a pipeline was just submitted,
+    # we can retrieve it
+    time.sleep(10)
+
+    # get the last pipeline
+    return gitpkg.pipelines.list(per_page=1, page=1)[0]
+
+
+def just_build_package(gitpkg, dry_run=False):
+    """Creates the pipeline with the latest tag and starts it
+
+    Args:
+
+        gitpkg: gitlab package object
+        dry_run: If True, the pipeline will not be created on GitLab
+
+    Returns:
+
+    """
+
+    # get the latest tag
+    latest_tag_name = 'v' + get_latest_tag_name(gitpkg)
+
+    # create the pipeline with this tag and start it
+    logger.info("Creating and starting pipeline for tag %s", latest_tag_name)
+
+    if not dry_run:
+        new_pipeline = gitpkg.pipelines.create({'ref': latest_tag_name})
+        return new_pipeline.id
+
+    return None
+
+
+def wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run=False):
+    """Using sleep function, wait for the latest pipeline to finish building.
+
+    This function pauses the script until pipeline completes either
+    successfully or with error.
+
+    Args:
+
+        gitpkg: gitlab package object
+        pipeline_id: id of the pipeline for which we are waiting to finish
+        dry_run: If True, outputs log message and exit. There wil be no
+                 waiting.
+
+    """
+
+    sleep_step = 30
+    max_sleep = 120 * 60  # two hours
+    # pipeline = get_last_pipeline(gitpkg, before_last=before_last)
+
+    logger.info('Waiting for the pipeline %s of package %s to finish. ' \
+        'Do not interrupt.', pipeline_id, gitpkg.name)