#!/usr/bin/env python
"""
By using changelog file as an input (can be generated with 'generate_changelog.py' script),
this script goes through all packages in changelog file (in order listed), tags them correctly
as per the file, and releases them one by one. A script can also be used to release a single package.
This script uses python-gitlab package for accessing GitLab's API.
Usage:
{0} [-v...] [options] [--] <private_token>
{0} -h | --help
{0} --version
Arguments:
<private_token> Private token used to access GitLab.
Options:
-h --help Show this screen.
--version Show version.
-c, --changelog-file STR A changelog file with all packages to release with their tags, listed in order.
[default: changelog_since_last_release.md].
-g, --group-name STR Group name where we are assuming that all packages are located.
[default: bob].
-p, --package STR If the name of a package is provided, then this package will be found
in the changelog file and the release will resume from it (if option --resume is set)
or only this package will be released.
-r, --resume The overall release will resume from the provided package name.
-q, --dry-run Only print the actions, but do not execute them.
"""
import sys
import os
from docopt import docopt
import gitlab
import datetime
import re
from distutils.version import StrictVersion as Version
import numpy
import time
def _insure_correct_package(candidates, group_name, pkg_name):
for pkg in candidates:
# make sure the name and the group name match exactly
if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
return pkg
raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
# adapted from the same-name function in new_version.py script of bob.extension package
def _update_readme(readme, version=None):
# replace the travis badge in the README.rst with the given version
DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
new_readme = []
for line in readme.splitlines():
if BRANCH_RE.search(line) is not None:
if "gitlab" in line: # gitlab links
replacement = "/v%s" % version if version is not None else "/master"
line = BRANCH_RE.sub(replacement, line)
if "software/bob" in line: # our doc server
if 'master' not in line: # don't replace 'latest' pointer
replacement = "/v%s" % version if version is not None \
else "/stable"
line = BRANCH_RE.sub(replacement, line)
if DOC_IMAGE.search(line) is not None:
replacement = '-v%s-' % version if version is not None else '-stable-'
line = DOC_IMAGE.sub(replacement, line)
new_readme.append(line)
return '\n'.join(new_readme)
def get_parsed_tag(gitpkg, tag):
"""
An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3
The latest tag is either patch, minor, major, or none
"""
m = re.search(r"(v\d.\d.\d)", tag)
if m:
return m.group(0)
# tag = Version(tag)
# if we bump the version, we need to find the latest released version for this package
if 'patch' == tag or 'minor' == tag or 'major' == tag:
latest_tag = gitpkg.tags.list(per_page=1, page=1)
# if there were no tags yet, assume the first version
if not latest_tag:
return 'v1.0.0'
latest_tag = latest_tag[0]
latest_tag_name = latest_tag.name
# check that it has expected format v#.#.#
# latest_tag_name = Version(latest_tag_name)
m = re.match(r"(v\d.\d.\d)", latest_tag_name)
if not m:
raise ValueError(
'The latest tag name {0} in package {1} has unknown format'.format(latest_tag_name, gitpkg.name))
# increase the version accordingly
if 'major' == tag: # increment the first number in 'v#.#.#'
return latest_tag_name[0] + str(int(latest_tag_name[1]) + 1) + latest_tag_name[2:]
if 'minor' == tag: # increment the second number in 'v#.#.#'
return latest_tag_name[:3] + str(int(latest_tag_name[3]) + 1) + latest_tag_name[4:]
if 'patch' == tag: # increment the last number in 'v#.#.#'
return latest_tag_name[:-1] + str(int(latest_tag_name[-1]) + 1)
if 'none' == tag: # we do nothing in this case
return tag
raise ValueError('Cannot parse changelog tag {0} of the package {1}'.format(tag, gitpkg.name))
def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
# get tag and update its description
tag = gitpkg.tags.get(tag_name)
print('Found tag {1}, updating its comments with:'.format(gitpkg.name, tag.name))
print(tag_comments_list)
if not dry_run:
tag.set_release_description('\n'.join(tag_comments_list))
return tag
def commit_files(gitpkg, files_list, message='Updated files', dry_run=False):
data = {
'branch': 'master', # v4
'commit_message': message,
'actions': []
}
# add files to update
for filename in files_list.keys():
update_action = dict(action='update', file_path=filename)
# with open(filename, 'r') as f:
# update_action['content'] = f.read()
update_action['content'] = files_list[filename]
data['actions'].append(update_action)
print("Committing changes in files: {0}".format(str(files_list.keys())))
if not dry_run:
gitpkg.commits.create(data)
def get_last_nonskip_pipeline(gitpkg, before_last=False):
# sleep for 10 seconds to ensure that if a pipeline was just submitted,
# we can retrieve it
time.sleep(10)
if before_last:
# take the pipeline before the last
return gitpkg.pipelines.list(per_page=2, page=1)[1]
else:
# otherwise take the last pipeline
return gitpkg.pipelines.list(per_page=1, page=1)[0]
def just_build_package(gitpkg, dry_run=False):
# we assume the last pipeline is with commit [skip ci]
# so, we take the pipeline that can be re-built, which the previous to the last one
last_pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True)
# check that the chosen pipeline is the one we are looking for
latest_tag_name = gitpkg.tags.list(per_page=1, page=1)[0].name
# the pipeline should be the one built for the latest tag, so check if it is the correct choice
if last_pipeline.ref != latest_tag_name:
raise ValueError('While deploying {0}, found pipeline {1} but it does not match '
'the latest tag {2}'.format(gitpkg.name, last_pipeline.id, latest_tag_name))
# the pipeline should have succeeded, otherwise we cannot release
if last_pipeline.status != 'success':
raise ValueError('While deploying {0}, found pipeline {1} but its status is "{2}" instead '
'of the expected "sucess"'.format(gitpkg.name, last_pipeline.id, last_pipeline.status))
print("Retrying pipeline {0}".format(last_pipeline.id))
if not dry_run:
last_pipeline.retry()
def wait_for_pipeline_to_finish(gitpkg, tag, dry_run=False):
sleep_step = 30
max_sleep = 60 * 60 # one hour
pipeline = get_last_nonskip_pipeline(gitpkg, before_last=True)
pipeline_id = pipeline.id
print('Waiting for the pipeline {0} of package {1} to finish. Do not interrupt.'.format(pipeline_id, gitpkg.name))
if dry_run:
return
# probe and wait for the pipeline to finish
slept_so_far = 0
while pipeline.status == 'running' or pipeline.status == 'pending':
time.sleep(sleep_step)
slept_so_far += sleep_step
if slept_so_far > max_sleep:
raise ValueError('I cannot wait longer than {0} seconds for '
'pipeline {1} to finish running!'.format(max_sleep, pipeline_id))
# probe gitlab to update the status of the pipeline
pipeline = gitpkg.pipelines.get(pipeline_id)
# finished running, now check if it succeeded
if pipeline.status != 'success':
raise ValueError('Pipeline {0} of project {1} exited with undesired status "{2}". '
'Release is not possible.'.format(pipeline_id, gitpkg.name, pipeline.status))
print('Pipeline {0} of package {1} succeeded. Continue processing.'.format(pipeline_id, gitpkg.name))
def cancel_last_pipeline(gitpkg):
pipeline = get_last_nonskip_pipeline(gitpkg)
print('Cancelling the last pipeline {0} of project {1}'.format(pipeline.id, gitpkg.name))
pipeline.cancel()
def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False):
# if there is nothing to release, just rebuild the package
if tag_name == 'none':
print("Since the tag is 'none', we just re-build the last pipeline")
return just_build_package(gitpkg)
# 1. Replace branch tag in Readme to new tag, change version file to new version tag. Add and commit to gitlab
version_number = tag_name[1:] # remove 'v' in front
readme_file = gitpkg.files.get(file_path='README.rst', ref='master')
readme_content = readme_file.decode().decode()
readme_content = _update_readme(readme_content, version=version_number)
# commit and push changes
commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
'Increased stable version to %s' % version_number, dry_run)
if not dry_run:
# cancel running the pipeline triggered by the last commit
cancel_last_pipeline(gitpkg)
# 2. Tag package with new tag and push
print("Creating tag {}".format(tag_name))
print("updating tag's comments with:".format(gitpkg.name, tag_name))
print(tag_comments_list)
if not dry_run:
tag = gitpkg.tags.create({'tag_name': tag_name, 'ref': 'master'})
# update tag with comments
tag.set_release_description('\n'.join(tag_comments_list))
# 3. Replace branch tag in Readme to master, change version file to beta version tag. Git add, commit, and push.
readme_content = _update_readme(readme_content)
version_number += 'b0'
# commit and push changes
commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
'Increased latest version to %s [skip ci]' % version_number, dry_run)
def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changelog, dry_run=False):
cur_tag = None
cur_tag_comments = []
grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), bob_group.name, pkg_name)
# so, we need to retrieve the full info from GitLab using correct project id
gitpkg = gl.projects.get(id=grpkg.id)
# we assume that changelog is formatted as structured text
# first line is the name of the package
for line in package_changelog:
if ' *' == line[:3]: # a tag level
# write the comments collected for the previous tag
if cur_tag:
update_tag_comments(gitpkg, cur_tag, cur_tag_comments, dry_run)
cur_tag_comments = [] # reset comments
# parse the current tag name
cur_tag = get_parsed_tag(gitpkg, line[3:].strip())
else: # all other lines are assumed to be comments
cur_tag_comments.append(line.strip())
# return the last tag and comments for release
return gitpkg, cur_tag, cur_tag_comments
def main(private_token, group_name='bob', changelog_file='changelog.rst', dry_run=False, package=None, resume=False):
gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
bob_group = gl.groups.list(search=group_name)[0]
# traverse all packages in the changelog, edit older tags with updated comments,
# tag them with a suggested version, then try to release, and
# wait until done to proceed to the next package
with open(changelog_file) as f:
changelog = f.readlines()
# find the starts of each package's description in the changelog
pkgs = numpy.asarray([i for i, line in enumerate(changelog) if line[0] == '*'])
pkgs = numpy.concatenate([pkgs, [len(changelog)]])
start_idx = 0
if package:
# get the index where the package first appears in the list
start_idx = [i for i, line in enumerate(changelog) if line[1:].strip() == package]
if not start_idx:
print('Package {0} was not found in the changelog'.format(package))
return
start_idx = pkgs.tolist().index(start_idx[0])
for i in range(start_idx, pkgs.shape[0] - 1):
cur_package_name = changelog[pkgs[i]][1:].strip()
print('\nProcessing package {0}'.format(changelog[pkgs[i]]))
gitpkg, tag, tag_comments = parse_and_process_package_changelog(gl, bob_group, cur_package_name,
changelog[pkgs[i] + 1: pkgs[i + 1]], dry_run)
# release the package with the found tag and its comments
if gitpkg:
release_package(gitpkg, tag, tag_comments, dry_run)
# now, wait for the pipeline to finish, before we can release the next package
wait_for_pipeline_to_finish(gitpkg, tag, dry_run)
# if package name is provided and resume is not set, process only this package
if package == cur_package_name and not resume:
break
print('\nFinished processing changelog')
if __name__ == '__main__':
arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1')
main(arguments['<private_token>'], group_name=arguments['--group-name'],
changelog_file=arguments['--changelog-file'], dry_run=arguments['--dry-run'],
package=arguments['--package'], resume=arguments['--resume'])