Skip to content
Snippets Groups Projects
Commit 92058658 authored by Pavel KORSHUNOV's avatar Pavel KORSHUNOV
Browse files

can tag and realease packages based on changelog

parent 69293d55
No related branches found
No related tags found
1 merge request!75The release script for bob that uses changelog as a guidance for release
Hide whitespace changes
Inline Side-by-side
Showing
with 186 additions and 164 deletions
release/release_bob.py
+ 186
− 164
View file @ 92058658
......@@ -21,10 +21,14 @@ Options:
"""
import sys
import os
from docopt import docopt
import gitlab
import datetime
import re
from distutils.version import StrictVersion as Version
import numpy
import time
def _insure_correct_package(candidates, group_name, pkg_name):
......@@ -35,125 +39,31 @@ def _insure_correct_package(candidates, group_name, pkg_name):
raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
def get_packages_list(gl, gl_group=None):
if gl_group:
grp_nightlies = gl_group.projects.list(search='bob.nightlies')[0]
nightlies = gl.projects.get(id=grp_nightlies.id)
else:
nightlies = gl.projects.list(search='bob.nightlies')[0]
nightlies_order = nightlies.files.get(file_path='order.txt', ref='master')
pkg_list_ordered = nightlies_order.decode().decode().split('\n')
pkg_list_ordered = [line for line in pkg_list_ordered if (line.strip() and not line.startswith('#'))]
return pkg_list_ordered
# release date of last Bob release + 1 day
def bob_last_release(gl):
bobpkg = gl.projects.get(id=1535) # 1535 is id of 'bob' meta-package
last_bob_tag = bobpkg.tags.list()[0] # get the last tag
return last_bob_tag.commit['committed_date']
def get_datetime_from_gitdate(gitdate):
return datetime.datetime.strptime(gitdate[:-6], '%Y-%m-%dT%H:%M:%S.%f')
def sort_commits(commits):
return sorted(commits, key=lambda x: get_datetime_from_gitdate(x.committed_date))
def sort_tags(tags):
return sorted(tags, key=lambda x: get_datetime_from_gitdate(x.commit['committed_date']))
def get_tag_changelog(tag):
try:
return tag.release['description']
except Exception:
return ''
def print_tags(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
since = get_datetime_from_gitdate(since)
tags = gitpkg.tags.list()
# sort tags by date
tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= since, tags)
tags = sort_tags(tags)
print('* ' + pkg_name)
for tag in tags:
print_one_tag(pkg_name, tag)
def print_one_tag(pkg_name, tag):
print(' * ' + tag.name + ' (' + get_datetime_from_gitdate(tag.commit['committed_date']).strftime(
'%b %d, %Y %H:%M') + ')')
for line in get_tag_changelog(tag).split('\r\n'):
line = line.strip()
if line.startswith('* ') or line.startswith('- '):
line = line[2:]
line = line.replace('!', pkg_name + '!')
line = line.replace('#', pkg_name + '#')
if not line:
continue
print(' ' * 5 + '* ' + line)
def print_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
# import ipdb; ipdb.set_trace()
commits = gitpkg.commits.list(since=since, all=True)
# sort commits by date
commits = sort_commits(commits)
print('* ' + pkg_name)
print_commits_range(pkg_name, commits)
def print_commits_range(pkg_name, commits):
for commit in commits:
commit_title = commit.title
# skip commits that do not carry much useful information
if '[skip ci]' in commit_title or \
'Merge branch' in commit_title or \
'Increased stable' in commit_title:
continue
commit_title = commit_title.strip()
commit_title = commit_title.replace('!', pkg_name + '!')
commit_title = commit_title.replace('#', pkg_name + '#')
# print(' - ' + get_datetime_from_gitdate(x.committed_date).strftime('%Y-%m-%d %H:%M:%S') + ":" + commit_title)
print(' ' * 5 + '- ' + commit_title)
def print_tags_with_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
# get tags since release and sort them
datetime_since = get_datetime_from_gitdate(since)
tags = gitpkg.tags.list()
# sort tags by date
tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= datetime_since, tags)
tags = sort_tags(tags)
# get commits since release date and sort them too
commits = gitpkg.commits.list(since=since, all=True)
# sort commits by date
commits = sort_commits(commits)
print('* ' + pkg_name)
# go through tags and print each with its message and corresponding commits
start_date = datetime_since
for tag in tags:
# print tag name and its text
print_one_tag(pkg_name, tag)
# print commits from the previous tag up to this one
end_date = get_datetime_from_gitdate(tag.commit['committed_date'])
commits4tag = filter(lambda x: (
get_datetime_from_gitdate(x.committed_date) > start_date and get_datetime_from_gitdate(
x.committed_date) <= end_date), commits)
print_commits_range(pkg_name, commits4tag)
start_date = end_date
# print the tentative patch version bump for the future tag
print(' * patch')
# print leftover commits that were not tagged yet
leftover_commits = filter(lambda x: get_datetime_from_gitdate(x.committed_date) > start_date, commits)
print_commits_range(pkg_name, leftover_commits)
def correct_tag(gitpkg, tag):
# adapted from the same-name function in new_version.py script of bob.extension package
def _update_readme(readme, version=None):
# replace the travis badge in the README.rst with the given version
DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
new_readme = []
for line in readme.splitlines():
if BRANCH_RE.search(line) is not None:
if "gitlab" in line: # gitlab links
replacement = "/v%s" % version if version is not None else "/master"
line = BRANCH_RE.sub(replacement, line)
if "software/bob" in line: # our doc server
if 'master' not in line: # don't replace 'latest' pointer
replacement = "/v%s" % version if version is not None \
else "/stable"
line = BRANCH_RE.sub(replacement, line)
if DOC_IMAGE.search(line) is not None:
replacement = '-v%s-' % version if version is not None else '-stable-'
line = DOC_IMAGE.sub(replacement, line)
new_readme.append(line)
return '\n'.join(new_readme)
def get_parsed_tag(gitpkg, tag):
"""
An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3
The latest tag is either patch, minor, major, or none
......@@ -161,13 +71,14 @@ def correct_tag(gitpkg, tag):
m = re.search(r"(v\d.\d.\d)", tag)
if m:
return m.group(0)
# tag = Version(tag)
# if we bump the version, we need to find the latest released version for this package
if 'patch' == tag or 'minor' == tag or 'major' == tag:
latest_tags = gitpkg.tags.list()
latest_tags = sort_tags(latest_tags)
latest_tag_name = latest_tags[-1].name
latest_tag = gitpkg.tags.list(per_page=1, page=1)[0]
latest_tag_name = latest_tag.name
# check that it has expected format v#.#.#
# latest_tag_name = Version(latest_tag_name)
m = re.match(r"(v\d.\d.\d)", latest_tag_name)
if not m:
raise ValueError(
......@@ -186,55 +97,166 @@ def correct_tag(gitpkg, tag):
raise ValueError('Cannot parse changelog tag {0} of the package {1}'.format(tag, gitpkg.name))
def main(private_token, group_name='bob', changelog='changelog.rst'):
def update_tag_comments(gitpkg, tag_name, tag_comments_list):
# get tag and update its description
tag = gitpkg.tags.get(tag_name)
print('package {0}, tag {1}, updating comments with:'.format(gitpkg.name, tag.name))
print(tag_comments_list)
tag.set_release_description('\n'.join(tag_comments_list))
return tag
def commit_files(gitpkg, files_list, message='Updated files'):
data = {
'branch': 'master', # v4
'commit_message': message,
'actions': []
}
# add files to update
for filename in files_list.keys():
update_action = dict(action='update', file_path=filename)
# with open(filename, 'r') as f:
# update_action['content'] = f.read()
update_action['content'] = files_list[filename]
data['actions'].append(update_action)
print("Committing changes")
gitpkg.commits.create(data)
def just_build_package(gitpkg):
# we assume the last pipeline is with commit [skip ci]
# so, we take the pipeline that can be re-built, which the previous to the last one
last_pipeline = gitpkg.pipelines.list(per_page=2, page=1)[1]
# check that the chosen pipeline is the one we are looking for
latest_tag_name = gitpkg.tags.list(per_page=1, page=1)[0].name
# the pipeline should be the one built for the latest tag, so check if it is the correct choice
if last_pipeline.ref != latest_tag_name:
raise ValueError('While deploying package, found pipeline {0} but it does not match '
'the latest tag {1}'.format(last_pipeline.id, latest_tag_name))
# the pipeline should have succeeded, otherwise we cannot release
if last_pipeline.status != 'success':
raise ValueError('While deploying package, found pipeline {0} but its status '
'is "{1}" instead of the expected "sucess"'.format(last_pipeline.id, last_pipeline.status))
print("Retrying pipeline {0}".format(last_pipeline.id))
print(last_pipeline)
last_pipeline.retry()
def wait_for_pipeline_to_finish(gitpkg, tag):
sleep_step = 30
max_sleep = 60 * 60 # one hour
if tag == 'none':
# take the pipeline before the last
pipeline = gitpkg.pipelines.list(per_page=2, page=1)[1]
else:
# otherwise just take the last pipeline
pipeline = gitpkg.pipelines.list(per_page=1, page=1)[0]
# probe and wait for the pipeline to finish
pipeline_id = pipeline.id
slept_so_far = 0
while pipeline.status == 'running' or pipeline.status == 'pending':
time.sleep(sleep_step)
slept_so_far += sleep_step
if slept_so_far > max_sleep:
raise ValueError('I cannot wait longer than {0} seconds for '
'pipeline {1} to finish running!'.format(max_sleep, pipeline_id))
# probe gitlab to update the status of the pipeline
pipeline = gitpkg.pipelines.get(pipeline_id)
# finished running, now check if it succeeded
if pipeline.status != 'success':
raise ValueError('Pipeline {0} of project {1} exited with undesired status "{2}". '
'Release is not possible.'.format(pipeline_id, gitpkg.name, pipeline.status))
def release_package(gitpkg, tag_name, tag_comments_list):
# if there is nothing to release, just rebuild the package
if tag_name == 'none':
print("Since the tag is 'none', we just re-build the last pipeline")
return just_build_package(gitpkg)
# 1. Replace branch tag in Readme to new tag, change version file to new version tag. Add and commit to gitlab
version_number = tag_name[1:] # remove 'v' in front
readme_file = gitpkg.files.get(file_path='README.rst', ref='master')
readme_content = readme_file.decode().decode()
readme_content = _update_readme(readme_content, version=version_number)
# commit and push changes
commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
'Increased stable version to %s [skip ci]' % version_number)
# 2. Tag package with new tag and push
print("creating tag {}".format(tag_name))
print('package {0}, tag {1}, updating comments with:'.format(gitpkg.name, tag_name))
print(tag_comments_list)
tag = gitpkg.tags.create({'tag_name': tag_name, 'ref': 'master'})
# update tag with comments
tag.set_release_description('\n'.join(tag_comments_list))
# 3. Replace branch tag in Readme to master, change version file to beta version tag. Git add, commit, and push.
readme_content = _update_readme(readme_content)
version_number += 'b0'
# commit and push changes
commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
'Increased latest version to %s [skip ci]' % version_number)
def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changelog):
cur_tag = None
cur_tag_comments = []
grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), bob_group.name, pkg_name)
# so, we need to retrieve the full info from GitLab using correct project id
gitpkg = gl.projects.get(id=grpkg.id)
# we assume that changelog is formatted as structured text
# first line is the name of the package
for line in package_changelog:
if ' *' == line[:3]: # a tag level
# write the comments collected for the previous tag
if cur_tag:
update_tag_comments(gitpkg, cur_tag, cur_tag_comments)
cur_tag_comments = [] # reset comments
# parse the current tag name
cur_tag = get_parsed_tag(gitpkg, line[3:].strip())
else: # all other lines are assumed to be comments
cur_tag_comments.append(line.strip())
# return the last tag and comments for release
return gitpkg, cur_tag, cur_tag_comments
def main(private_token, group_name='bob', changelog_file='changelog.rst'):
gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
bob_group = gl.groups.list(search=group_name)[0]
# traverse all packages in the changelog, edit older tags with updated comments,
# tag them with a suggested version, then try to release, and
# wait until done to proceed to the next package
with open(changelog) as f:
changelog_insides = f.readlines()
pkg_name = cur_gitpkg = None
cur_tag = None
cur_tag_comments = []
# we assume that changelog is formatted as structured text
for line in changelog_insides:
if '*' == line[0]: # name of the package
if pkg_name and cur_gitpkg: # release previous package
release_package(cur_gitpkg, cur_tag, cur_tag_comments)
pkg_name = line[1:].strip()
# find the correct package in GitLab
# group returns a simplified description of the project
grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), group_name, pkg_name)
# so, we need to retrieve the full info from GitLab using correct project id
cur_gitpkg = gl.projects.get(id=grpkg.id)
elif ' *' == line[:3]: # a tag level
if not cur_gitpkg: # it better be not None, as we are in the middle of the tags for the package
raise ValueError('How come package for {0} is empty?'.format(pkg_name))
# write the collected comments in the previous tag
if cur_tag:
update_tag_comments(cur_gitpkg, cur_tag, cur_tag_comments)
cur_tag_comments = [] # reset comments
# parse the current tag name
cur_tag = correct_tag(cur_gitpkg, line[3:].strip())
if 'none' == cur_tag: # no tagging, just deploy the package
deploy_package(cur_gitpkg)
continue
else: # all other lines are assumed to be comments
cur_tag_comments += line.strip()
if pkg_name and cur_gitpkg: # release the last package
release_package(cur_gitpkg, cur_tag, cur_tag_comments)
with open(changelog_file) as f:
changelog = f.readlines()
# find the starts of each package's description in the changelog
pkgs = numpy.asarray([i for i, line in enumerate(changelog) if line[0] == '*'])
pkgs = numpy.concatenate([pkgs, [len(changelog)]])
for i in range(pkgs.shape[0] - 1):
print('Processing package {0}'.format(changelog[pkgs[i]]))
# print(changelog[pkgs[i] + 1: pkgs[i + 1]])
gitpkg, tag, tag_comments = parse_and_process_package_changelog(gl, bob_group, changelog[pkgs[i]][1:].strip(),
changelog[pkgs[i] + 1: pkgs[i + 1]])
# release the package with the found tag and its comments
if gitpkg:
release_package(gitpkg, tag, tag_comments)
# now, wait for the pipeline to finish, before we can release the next package
wait_for_pipeline_to_finish(gitpkg, tag)
if __name__ == '__main__':
arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1')
main(arguments['<private_token>'],, group_name = arguments['--group-name'], changelog = arguments['--changelog'])
main(arguments['<private_token>'], group_name=arguments['--group-name'],
changelog_file=arguments['--changelog-file'])
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment