Commit d0f789a0 authored by André Anjos's avatar André Anjos 💬
Browse files

Major commit to refactor this package as a hub for more functionality

- Start moving bob.admin scripts here
- Add baseline documentation system
- Add logging support
- Rename it as bob.devtools
- Prepare for conda-packaging
- Name main tool `bdt`
parent e9f4ed82
include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst
recursive-include bob *.cpp *.h
recursive-include bob/extension/examples *
recursive-include bob/extension/data *
recursive-include bob/devtools/data *.md
......@@ -5,18 +5,24 @@
===============================================
This package is part of the signal-processing and machine learning toolbox
Bob_. It provides some tools to help maintain Bob_.
Bob_. It provides tools to help maintain Bob_.
Installation
------------
This package needs to be installed in your base conda environment. To install
This package needs to be installed in a conda environment. To install
this package, run::
$ conda activate base
# the dependency list below matches the ones in setup.py
$ conda install pip click click-plugins conda-build
$ pip install -e .
$ conda env create -f env.yml
$ conda activate bdt
(bdt) $ buildout
(bdt) $ ./bin/bdt --help
...
To build the documentation, just do::
(bdt) $ ./bin/sphinx-build doc sphinx
Contact
......
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import io
import datetime
import logging
logger = logging.getLogger(__name__)
import pytz
import dateutil.parser
from .release import ensure_correct_package
def parse_date(d):
'''Parses any date supported by :py:func:`dateutil.parser.parse`'''
return dateutil.parser.parse(d, ignoretz=True).replace(
tzinfo=pytz.timezone("Europe/Zurich"))
def _sort_commits(commits, reverse):
'''Sorts gitlab commit objects using their ``committed_date`` attribute'''
return sorted(commits,
key=lambda x: parse_date(x.committed_date),
reverse=reverse,
)
def _sort_tags(tags, reverse):
'''Sorts gitlab tag objects using their ``committed_date`` attribute'''
return sorted(tags,
key=lambda x: parse_date(x.commit['committed_date']),
reverse=reverse,
)
def get_file_from_gitlab(gitpkg, path, ref='master'):
'''Retrieves a file from a Gitlab repository, returns a (StringIO) file'''
return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
def get_last_tag(package):
'''Returns the last (gitlab object) tag for the given package
Args:
package: The gitlab project object from where to fetch the last release
date information
Returns: a tag object
'''
# according to the Gitlab API documentation, tags are sorted from the last
# updated to the first, by default - no need to do further sorting!
tag_list = package.tags.list()
if tag_list:
# there are tags, use these
return tag_list[0]
def get_last_tag_date(package):
'''Returns the last release date for the given package
Falls back to the first commit date if the package has not yet been tagged
Args:
package: The gitlab project object from where to fetch the last release
date information
Returns: a datetime object that refers to the last date the package was
released. If the package was never released, then returns the
date just before the first commit.
'''
# according to the Gitlab API documentation, tags are sorted from the last
# updated to the first, by default - no need to do further sorting!
tag_list = package.tags.list()
if tag_list:
# there are tags, use these
last = tag_list[0]
logger.debug('Last tag for package %s (id=%d) is %s', package.name,
package.id, last.name)
return parse_date(last.commit['committed_date']) + \
datetime.timedelta(milliseconds=500)
else:
commit_list = package.commits.list(all=True)
if commit_list:
# there are commits, use these
first = _sort_commits(commit_list, reverse=False)[0]
logger.debug('First commit for package %s (id=%d) is from %s',
package.name, package.id, first.committed_date)
return parse_date(first.committed_date) - \
datetime.timedelta(milliseconds=500)
else:
# there are no commits nor tags - abort
raise RuntimeError('package %s (id=%d) does not have commits ' \
'or tags so I cannot devise a good starting date' % \
(package.name, package.id))
def _get_tag_changelog(tag):
try:
return tag.release['description']
except Exception:
return ''
def _write_one_tag(f, pkg_name, tag):
'''Prints commit information for a single tag of a given package
Args:
f: A :py:class:`File` ready to be written at
pkg_name: The name of the package we are writing tags of
tag: The tag value
'''
git_date = parse_date(tag.commit['committed_date'])
f.write(' * %s (%s)\n' % (tag.name, git_date.strftime('%b %d, %Y %H:%M')))
for line in _get_tag_changelog(tag).replace('\r\n', '\n').split('\n'):
line = line.strip()
if line.startswith('* ') or line.startswith('- '):
line = line[2:]
line = line.replace('!', pkg_name + '!').replace(pkg_name + \
pkg_name, pkg_name)
line = line.replace('#', pkg_name + '#')
if not line:
continue
f.write('%s* %s' % (5*' ', line))
def _write_commits_range(f, pkg_name, commits):
'''Writes all commits of a given package within a range, to the output file
Args:
f: A :py:class:`File` ready to be written at
pkg_name: The name of the package we are writing tags of
commits: List of commits to be written
'''
for commit in commits:
commit_title = commit.title
# skip commits that do not carry much useful information
if '[skip ci]' in commit_title or \
'Merge branch' in commit_title or \
'Increased stable' in commit_title:
continue
commit_title = commit_title.strip()
commit_title = commit_title.replace('!', pkg_name + '!').replace(pkg_name + pkg_name, pkg_name)
commit_title = commit_title.replace('#', pkg_name + '#')
f.write('%s- %s' % (' ' * 5, commit_title))
def _write_mergerequests_range(f, pkg_name, mrs):
'''Writes all merge-requests of a given package, with a range, to the
output file
Args:
f: A :py:class:`File` ready to be written at
pkg_name: The name of the package we are writing tags of
mrs: The list of merge requests to write
'''
for mr in mrs:
title = mr.title.strip().replace('\r','').replace('\n', ' ')
title = title.replace(' !', ' ' + pkg_name + '!')
title = title.replace(' #', ' ' + pkg_name + '#')
description = mr.description.strip().replace('\r','').replace('\n', ' ')
description = description.replace(' !', ' ' + pkg_name + '!')
description = description.replace(' #', ' ' + pkg_name + '#')
space = ': ' if description else ''
log = ''' - {pkg}!{iid} {title}{space}{description}'''
f.write(log.format(pkg=pkg_name, iid=mr.iid, title=title, space=space, description=description))
f.write('\n')
def write_tags_with_commits(f, gitpkg, since, mode):
'''Writes all tags and commits of a given package to the output file
Args:
f: A :py:class:`File` ready to be written at
gitpkg: A pointer to the gitlab package object
since: Starting date (as a datetime object)
mode: One of mrs (merge-requests), commits or tags indicating how to
list entries in the changelog for this package
'''
# get tags since release and sort them
tags = gitpkg.tags.list()
# sort tags by date
tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
tags = _sort_tags(tags, reverse=False)
# get commits since release date and sort them too
commits = gitpkg.commits.list(since=since, all=True)
# sort commits by date
commits = _sort_commits(commits, reverse=False)
# get merge requests since the release data
mrs = list(reversed(gitpkg.mergerequests.list(state='merged', updated_after=since, order_by='updated_at', all=True)))
f.write('* %s\n' % (gitpkg.name,))
# go through tags and writes each with its message and corresponding
# commits
start_date = since
for tag in tags:
# write tag name and its text
_write_one_tag(f, gitpkg.name, tag)
end_date = parse_date(tag.commit['committed_date'])
if mode == 'commits':
# write commits from the previous tag up to this one
commits4tag = [k for k in commits \
if (start_date < parse_date(k.committed_date) <= end_date)]
_write_commits_range(f, gitpkg.name, commits4tag)
elif mode == 'mrs':
# write merge requests from the previous tag up to this one
# the attribute 'merged_at' is not available in GitLab API as of 27
# June 2018
mrs4tag = [k for k in mrs \
if (start_date < parse_date(k.updated_at) <= end_date)]
_write_mergerequests_range(f, gitpkg.name, mrs4tag)
start_date = end_date
if mode != 'tags':
# write the tentative patch version bump for the future tag
f.write(' * patch\n')
if mode == 'mrs':
# write leftover merge requests
# the attribute 'merged_at' is not available in GitLab API as of 27
# June 2018
leftover_mrs = [k for k in mrs \
if parse_date(k.updated_at) > start_date]
_write_mergerequests_range(f, gitpkg.name, leftover_mrs)
else:
# write leftover commits that were not tagged yet
leftover_commits = [k for k in commits \
if parse_date(k.committed_date) > start_date]
_write_commits_range(f, gitpkg.name, leftover_commits)
def write_tags(f, gitpkg, since):
'''Writes all tags of a given package to the output file
Args:
f: A :py:class:`File` ready to be written at
gitpkg: A pointer to the gitlab package object
since: Starting date as a datetime object
'''
tags = gitpkg.tags.list()
# sort tags by date
tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
tags = _sort_tags(tags, reverse=False)
f.write('* %s\n')
for tag in tags:
_write_one_tag(gitpkg.name, tag)
This diff is collapsed.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import re
import time
import gitlab
import logging
logger = logging.getLogger(__name__)
from distutils.version import StrictVersion
def get_gitlab_instance():
'''Returns an instance of the gitlab object for remote operations'''
# tries to figure if we can authenticate using a global configuration
cfgs = ['~/.python-gitlab.cfg', '/etc/python-gitlab.cfg']
cfgs = [os.path.expanduser(k) for k in cfgs]
if any([os.path.exists(k) for k in cfgs]):
gl = gitlab.Gitlab.from_config('idiap', cfgs)
else: #ask the user for a token
server = "https://gitlab.idiap.ch"
token = input("%s token: " % server)
gl = gitlab.Gitlab(server, private_token=token, api_version=4)
return gl
def ensure_correct_package(candidates, group_name, pkg_name):
for pkg in candidates:
# make sure the name and the group name match exactly
if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
return pkg
raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
def _update_readme(readme, version):
"""
Inside text of the readme, replaces parts of the links to the provided
version. If version is not provided, replace to `stable` or `master`.
Args:
readme: Text of the README.rst file from a bob package
version: Format of the version string is '#.#.#'
Returns: New text of readme with all replaces done
"""
# replace the badge in the readme's text with the given version
DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
new_readme = []
for line in readme.splitlines():
if BRANCH_RE.search(line) is not None:
if "gitlab" in line: # gitlab links
replacement = "/v%s" % version if version is not None \
else "/master"
line = BRANCH_RE.sub(replacement, line)
if ("software/bob" in line) or \
("software/beat" in line): # our doc server
if 'master' not in line: # don't replace 'latest' pointer
replacement = "/v%s" % version if version is not None \
else "/stable"
line = BRANCH_RE.sub(replacement, line)
if DOC_IMAGE.search(line) is not None:
replacement = '-v%s-' % version if version is not None \
else '-stable-'
line = DOC_IMAGE.sub(replacement, line)
new_readme.append(line)
return '\n'.join(new_readme) + '\n'
def get_latest_tag_name(gitpkg):
"""Find the name of the latest tag for a given package in the format '#.#.#'
Args:
gitpkg: gitlab package object
Returns: The name of the latest tag in format '#.#.#'. None if no tags for
the package were found.
"""
# get 50 latest tags as a list
latest_tags = gitpkg.tags.list(all=True)
if not latest_tags:
return None
# create list of tags' names but ignore the first 'v' character in each name
# also filter out non version tags
tag_names = [tag.name[1:] for tag in latest_tags \
if StrictVersion.version_re.match(tag.name[1:])]
# sort them correctly according to each subversion number
tag_names.sort(key=StrictVersion)
# take the last one, as it is the latest tag in the sorted tags
latest_tag_name = tag_names[-1]
return latest_tag_name
def get_parsed_tag(gitpkg, tag):
"""
An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we
need only v2.1.3
The latest tag is either patch, minor, major, or none
"""
m = re.search(r"(v\d+.\d+.\d+)", tag)
if m:
return m.group(0)
# tag = Version(tag)
# if we bump the version, we need to find the latest released version for
# this package
if 'patch' == tag or 'minor' == tag or 'major' == tag:
# find the correct latest tag of this package (without 'v' in front),
# None if there are no tags yet
latest_tag_name = get_latest_tag_name(gitpkg)
# if there were no tags yet, assume the very first version
if not latest_tag_name: return 'v0.0.1'
# check that it has expected format #.#.#
# latest_tag_name = Version(latest_tag_name)
m = re.match(r"(\d.\d.\d)", latest_tag_name)
if not m:
raise ValueError('The latest tag name {0} in package {1} has ' \
'unknown format'.format('v' + latest_tag_name, gitpkg.name))
# increase the version accordingly
major, minor, patch = latest_tag_name.split('.')
if 'major' == tag:
# increment the first number in 'v#.#.#' but make minor and patch
# to be 0
return 'v' + str(int(major) + 1) + '.0.0'
if 'minor' == tag:
# increment the second number in 'v#.#.#' but make patch to be 0
return 'v' + major + '.' + str(int(minor) + 1) + '.0'
if 'patch' == tag:
# increment the last number in 'v#.#.#'
return 'v' + major + '.' + minor + '.' + str(int(patch) + 1)
if 'none' == tag:
# we do nothing in this case
return tag
raise ValueError('Cannot parse changelog tag {0} of the ' \
'package {1}'.format(tag, gitpkg.name))
def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
"""Write annotations inside the provided tag of a given package.
Args:
gitpkg: gitlab package object
tag_name: The name of the tag to update
tag_comments_list: New annotations for this tag in a form of list
dry_run: If True, nothing will be committed or pushed to GitLab
Returns: The gitlab object for the tag that was updated
"""
# get tag and update its description
logger.info(tag_name)
tag = gitpkg.tags.get(tag_name)
tag_comments = '\n'.join(tag_comments_list)
logger.info('Found tag %s, updating its comments with:\n%s', tag.name,
tag_comments)
if not dry_run: tag.set_release_description(tag_comments)
return tag
def commit_files(gitpkg, files_dict, message='Updated files', dry_run=False):
"""Commit files of a given GitLab package.
Args:
gitpkg: gitlab package object
files_dict: Dictionary of file names and their contents (as text)
message: Commit message
dry_run: If True, nothing will be committed or pushed to GitLab
"""
data = {
'branch': 'master', # v4
'commit_message': message,
'actions': []
}
# add files to update
for filename in files_dict.keys():
update_action = dict(action='update', file_path=filename)
update_action['content'] = files_dict[filename]
data['actions'].append(update_action)
logger.info("Committing changes in files: %s", str(files_dict.keys()))
if not dry_run:
gitpkg.commits.create(data)
def get_last_pipeline(gitpkg):
"""Returns the last pipeline of the project
Args:
gitpkg: gitlab package object
Returns: The gtilab object of the pipeline
"""
# wait for 10 seconds to ensure that if a pipeline was just submitted,
# we can retrieve it
time.sleep(10)
# get the last pipeline
return gitpkg.pipelines.list(per_page=1, page=1)[0]
def just_build_package(gitpkg, dry_run=False):
"""Creates the pipeline with the latest tag and starts it
Args:
gitpkg: gitlab package object
dry_run: If True, the pipeline will not be created on GitLab
Returns:
"""
# get the latest tag
latest_tag_name = 'v' + get_latest_tag_name(gitpkg)
# create the pipeline with this tag and start it
logger.info("Creating and starting pipeline for tag %s", latest_tag_name)
if not dry_run:
new_pipeline = gitpkg.pipelines.create({'ref': latest_tag_name})
return new_pipeline.id
return None
def wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run=False):
"""Using sleep function, wait for the latest pipeline to finish building.
This function pauses the script until pipeline completes either
successfully or with error.
Args:
gitpkg: gitlab package object
pipeline_id: id of the pipeline for which we are waiting to finish
dry_run: If True, outputs log message and exit. There wil be no
waiting.
"""
sleep_step = 30
max_sleep = 120 * 60 # two hours
# pipeline = get_last_pipeline(gitpkg, before_last=before_last)
logger.info('Waiting for the pipeline %s of package %s to finish. ' \
'Do not interrupt.', pipeline_id, gitpkg.name)