diff --git a/release/release_bob.py b/release/release_bob.py
index 27d8ce686424fc45040cf69b37eca50c10aadd11..fbf7690557295b2c6feec443089da0ef44a96a7b 100755
--- a/release/release_bob.py
+++ b/release/release_bob.py
@@ -21,10 +21,14 @@ Options:
"""
import sys
+import os
from docopt import docopt
import gitlab
import datetime
import re
+from distutils.version import StrictVersion as Version
+import numpy
+import time
def _insure_correct_package(candidates, group_name, pkg_name):
@@ -35,125 +39,31 @@ def _insure_correct_package(candidates, group_name, pkg_name):
raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
-def get_packages_list(gl, gl_group=None):
- if gl_group:
- grp_nightlies = gl_group.projects.list(search='bob.nightlies')[0]
- nightlies = gl.projects.get(id=grp_nightlies.id)
- else:
- nightlies = gl.projects.list(search='bob.nightlies')[0]
- nightlies_order = nightlies.files.get(file_path='order.txt', ref='master')
- pkg_list_ordered = nightlies_order.decode().decode().split('\n')
- pkg_list_ordered = [line for line in pkg_list_ordered if (line.strip() and not line.startswith('#'))]
- return pkg_list_ordered
-
-
-# release date of last Bob release + 1 day
-def bob_last_release(gl):
- bobpkg = gl.projects.get(id=1535) # 1535 is id of 'bob' meta-package
- last_bob_tag = bobpkg.tags.list()[0] # get the last tag
- return last_bob_tag.commit['committed_date']
-
-
-def get_datetime_from_gitdate(gitdate):
- return datetime.datetime.strptime(gitdate[:-6], '%Y-%m-%dT%H:%M:%S.%f')
-
-
-def sort_commits(commits):
- return sorted(commits, key=lambda x: get_datetime_from_gitdate(x.committed_date))
-
-
-def sort_tags(tags):
- return sorted(tags, key=lambda x: get_datetime_from_gitdate(x.commit['committed_date']))
-
-
-def get_tag_changelog(tag):
- try:
- return tag.release['description']
- except Exception:
- return ''
-
-
-def print_tags(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
- since = get_datetime_from_gitdate(since)
- tags = gitpkg.tags.list()
- # sort tags by date
- tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= since, tags)
- tags = sort_tags(tags)
- print('* ' + pkg_name)
- for tag in tags:
- print_one_tag(pkg_name, tag)
-
-
-def print_one_tag(pkg_name, tag):
- print(' * ' + tag.name + ' (' + get_datetime_from_gitdate(tag.commit['committed_date']).strftime(
- '%b %d, %Y %H:%M') + ')')
- for line in get_tag_changelog(tag).split('\r\n'):
- line = line.strip()
- if line.startswith('* ') or line.startswith('- '):
- line = line[2:]
- line = line.replace('!', pkg_name + '!')
- line = line.replace('#', pkg_name + '#')
- if not line:
- continue
- print(' ' * 5 + '* ' + line)
-
-
-def print_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
- # import ipdb; ipdb.set_trace()
- commits = gitpkg.commits.list(since=since, all=True)
- # sort commits by date
- commits = sort_commits(commits)
- print('* ' + pkg_name)
- print_commits_range(pkg_name, commits)
-
-
-def print_commits_range(pkg_name, commits):
- for commit in commits:
- commit_title = commit.title
- # skip commits that do not carry much useful information
- if '[skip ci]' in commit_title or \
- 'Merge branch' in commit_title or \
- 'Increased stable' in commit_title:
- continue
- commit_title = commit_title.strip()
- commit_title = commit_title.replace('!', pkg_name + '!')
- commit_title = commit_title.replace('#', pkg_name + '#')
- # print(' - ' + get_datetime_from_gitdate(x.committed_date).strftime('%Y-%m-%d %H:%M:%S') + ":" + commit_title)
- print(' ' * 5 + '- ' + commit_title)
-
-
-def print_tags_with_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'):
- # get tags since release and sort them
- datetime_since = get_datetime_from_gitdate(since)
- tags = gitpkg.tags.list()
- # sort tags by date
- tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= datetime_since, tags)
- tags = sort_tags(tags)
- # get commits since release date and sort them too
- commits = gitpkg.commits.list(since=since, all=True)
- # sort commits by date
- commits = sort_commits(commits)
- print('* ' + pkg_name)
- # go through tags and print each with its message and corresponding commits
- start_date = datetime_since
- for tag in tags:
- # print tag name and its text
- print_one_tag(pkg_name, tag)
- # print commits from the previous tag up to this one
- end_date = get_datetime_from_gitdate(tag.commit['committed_date'])
- commits4tag = filter(lambda x: (
- get_datetime_from_gitdate(x.committed_date) > start_date and get_datetime_from_gitdate(
- x.committed_date) <= end_date), commits)
- print_commits_range(pkg_name, commits4tag)
- start_date = end_date
- # print the tentative patch version bump for the future tag
- print(' * patch')
- # print leftover commits that were not tagged yet
- leftover_commits = filter(lambda x: get_datetime_from_gitdate(x.committed_date) > start_date, commits)
- print_commits_range(pkg_name, leftover_commits)
-
-
-def correct_tag(gitpkg, tag):
+# adapted from the same-name function in new_version.py script of bob.extension package
+def _update_readme(readme, version=None):
+ # replace the travis badge in the README.rst with the given version
+ DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
+ BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
+
+ new_readme = []
+ for line in readme.splitlines():
+ if BRANCH_RE.search(line) is not None:
+ if "gitlab" in line: # gitlab links
+ replacement = "/v%s" % version if version is not None else "/master"
+ line = BRANCH_RE.sub(replacement, line)
+ if "software/bob" in line: # our doc server
+ if 'master' not in line: # don't replace 'latest' pointer
+ replacement = "/v%s" % version if version is not None \
+ else "/stable"
+ line = BRANCH_RE.sub(replacement, line)
+ if DOC_IMAGE.search(line) is not None:
+ replacement = '-v%s-' % version if version is not None else '-stable-'
+ line = DOC_IMAGE.sub(replacement, line)
+ new_readme.append(line)
+ return '\n'.join(new_readme)
+
+
+def get_parsed_tag(gitpkg, tag):
"""
An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3
The latest tag is either patch, minor, major, or none
@@ -161,13 +71,14 @@ def correct_tag(gitpkg, tag):
m = re.search(r"(v\d.\d.\d)", tag)
if m:
return m.group(0)
+ # tag = Version(tag)
# if we bump the version, we need to find the latest released version for this package
if 'patch' == tag or 'minor' == tag or 'major' == tag:
- latest_tags = gitpkg.tags.list()
- latest_tags = sort_tags(latest_tags)
- latest_tag_name = latest_tags[-1].name
+ latest_tag = gitpkg.tags.list(per_page=1, page=1)[0]
+ latest_tag_name = latest_tag.name
# check that it has expected format v#.#.#
+ # latest_tag_name = Version(latest_tag_name)
m = re.match(r"(v\d.\d.\d)", latest_tag_name)
if not m:
raise ValueError(
@@ -186,55 +97,166 @@ def correct_tag(gitpkg, tag):
raise ValueError('Cannot parse changelog tag {0} of the package {1}'.format(tag, gitpkg.name))
-def main(private_token, group_name='bob', changelog='changelog.rst'):
+def update_tag_comments(gitpkg, tag_name, tag_comments_list):
+ # get tag and update its description
+ tag = gitpkg.tags.get(tag_name)
+ print('package {0}, tag {1}, updating comments with:'.format(gitpkg.name, tag.name))
+ print(tag_comments_list)
+ tag.set_release_description('\n'.join(tag_comments_list))
+ return tag
+
+
+def commit_files(gitpkg, files_list, message='Updated files'):
+ data = {
+ 'branch': 'master', # v4
+ 'commit_message': message,
+ 'actions': []
+ }
+ # add files to update
+ for filename in files_list.keys():
+ update_action = dict(action='update', file_path=filename)
+ # with open(filename, 'r') as f:
+ # update_action['content'] = f.read()
+ update_action['content'] = files_list[filename]
+ data['actions'].append(update_action)
+
+ print("Committing changes")
+ gitpkg.commits.create(data)
+
+
+def just_build_package(gitpkg):
+ # we assume the last pipeline is with commit [skip ci]
+ # so, we take the pipeline that can be re-built, which the previous to the last one
+ last_pipeline = gitpkg.pipelines.list(per_page=2, page=1)[1]
+
+ # check that the chosen pipeline is the one we are looking for
+ latest_tag_name = gitpkg.tags.list(per_page=1, page=1)[0].name
+ # the pipeline should be the one built for the latest tag, so check if it is the correct choice
+ if last_pipeline.ref != latest_tag_name:
+ raise ValueError('While deploying package, found pipeline {0} but it does not match '
+ 'the latest tag {1}'.format(last_pipeline.id, latest_tag_name))
+ # the pipeline should have succeeded, otherwise we cannot release
+ if last_pipeline.status != 'success':
+ raise ValueError('While deploying package, found pipeline {0} but its status '
+ 'is "{1}" instead of the expected "sucess"'.format(last_pipeline.id, last_pipeline.status))
+
+ print("Retrying pipeline {0}".format(last_pipeline.id))
+ print(last_pipeline)
+ last_pipeline.retry()
+
+
+def wait_for_pipeline_to_finish(gitpkg, tag):
+ sleep_step = 30
+ max_sleep = 60 * 60 # one hour
+ if tag == 'none':
+ # take the pipeline before the last
+ pipeline = gitpkg.pipelines.list(per_page=2, page=1)[1]
+ else:
+ # otherwise just take the last pipeline
+ pipeline = gitpkg.pipelines.list(per_page=1, page=1)[0]
+
+ # probe and wait for the pipeline to finish
+ pipeline_id = pipeline.id
+ slept_so_far = 0
+ while pipeline.status == 'running' or pipeline.status == 'pending':
+ time.sleep(sleep_step)
+ slept_so_far += sleep_step
+ if slept_so_far > max_sleep:
+ raise ValueError('I cannot wait longer than {0} seconds for '
+ 'pipeline {1} to finish running!'.format(max_sleep, pipeline_id))
+ # probe gitlab to update the status of the pipeline
+ pipeline = gitpkg.pipelines.get(pipeline_id)
+
+ # finished running, now check if it succeeded
+ if pipeline.status != 'success':
+ raise ValueError('Pipeline {0} of project {1} exited with undesired status "{2}". '
+ 'Release is not possible.'.format(pipeline_id, gitpkg.name, pipeline.status))
+
+
+def release_package(gitpkg, tag_name, tag_comments_list):
+ # if there is nothing to release, just rebuild the package
+ if tag_name == 'none':
+ print("Since the tag is 'none', we just re-build the last pipeline")
+ return just_build_package(gitpkg)
+
+ # 1. Replace branch tag in Readme to new tag, change version file to new version tag. Add and commit to gitlab
+ version_number = tag_name[1:] # remove 'v' in front
+ readme_file = gitpkg.files.get(file_path='README.rst', ref='master')
+ readme_content = readme_file.decode().decode()
+ readme_content = _update_readme(readme_content, version=version_number)
+ # commit and push changes
+ commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
+ 'Increased stable version to %s [skip ci]' % version_number)
+
+ # 2. Tag package with new tag and push
+ print("creating tag {}".format(tag_name))
+ print('package {0}, tag {1}, updating comments with:'.format(gitpkg.name, tag_name))
+ print(tag_comments_list)
+ tag = gitpkg.tags.create({'tag_name': tag_name, 'ref': 'master'})
+ # update tag with comments
+ tag.set_release_description('\n'.join(tag_comments_list))
+
+ # 3. Replace branch tag in Readme to master, change version file to beta version tag. Git add, commit, and push.
+ readme_content = _update_readme(readme_content)
+ version_number += 'b0'
+ # commit and push changes
+ commit_files(gitpkg, {'README.rst': readme_content, 'version.txt': version_number},
+ 'Increased latest version to %s [skip ci]' % version_number)
+
+
+def parse_and_process_package_changelog(gl, bob_group, pkg_name, package_changelog):
+ cur_tag = None
+ cur_tag_comments = []
+
+ grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), bob_group.name, pkg_name)
+ # so, we need to retrieve the full info from GitLab using correct project id
+ gitpkg = gl.projects.get(id=grpkg.id)
+
+ # we assume that changelog is formatted as structured text
+ # first line is the name of the package
+ for line in package_changelog:
+ if ' *' == line[:3]: # a tag level
+ # write the comments collected for the previous tag
+ if cur_tag:
+ update_tag_comments(gitpkg, cur_tag, cur_tag_comments)
+ cur_tag_comments = [] # reset comments
+
+ # parse the current tag name
+ cur_tag = get_parsed_tag(gitpkg, line[3:].strip())
+
+ else: # all other lines are assumed to be comments
+ cur_tag_comments.append(line.strip())
+
+ # return the last tag and comments for release
+ return gitpkg, cur_tag, cur_tag_comments
+
+
+def main(private_token, group_name='bob', changelog_file='changelog.rst'):
gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4)
bob_group = gl.groups.list(search=group_name)[0]
# traverse all packages in the changelog, edit older tags with updated comments,
# tag them with a suggested version, then try to release, and
# wait until done to proceed to the next package
- with open(changelog) as f:
- changelog_insides = f.readlines()
- pkg_name = cur_gitpkg = None
- cur_tag = None
- cur_tag_comments = []
- # we assume that changelog is formatted as structured text
- for line in changelog_insides:
- if '*' == line[0]: # name of the package
-
- if pkg_name and cur_gitpkg: # release previous package
- release_package(cur_gitpkg, cur_tag, cur_tag_comments)
-
- pkg_name = line[1:].strip()
- # find the correct package in GitLab
- # group returns a simplified description of the project
- grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), group_name, pkg_name)
- # so, we need to retrieve the full info from GitLab using correct project id
- cur_gitpkg = gl.projects.get(id=grpkg.id)
-
- elif ' *' == line[:3]: # a tag level
- if not cur_gitpkg: # it better be not None, as we are in the middle of the tags for the package
- raise ValueError('How come package for {0} is empty?'.format(pkg_name))
-
- # write the collected comments in the previous tag
- if cur_tag:
- update_tag_comments(cur_gitpkg, cur_tag, cur_tag_comments)
- cur_tag_comments = [] # reset comments
-
- # parse the current tag name
- cur_tag = correct_tag(cur_gitpkg, line[3:].strip())
-
- if 'none' == cur_tag: # no tagging, just deploy the package
- deploy_package(cur_gitpkg)
- continue
-
- else: # all other lines are assumed to be comments
- cur_tag_comments += line.strip()
-
- if pkg_name and cur_gitpkg: # release the last package
- release_package(cur_gitpkg, cur_tag, cur_tag_comments)
+ with open(changelog_file) as f:
+ changelog = f.readlines()
+
+ # find the starts of each package's description in the changelog
+ pkgs = numpy.asarray([i for i, line in enumerate(changelog) if line[0] == '*'])
+ pkgs = numpy.concatenate([pkgs, [len(changelog)]])
+ for i in range(pkgs.shape[0] - 1):
+ print('Processing package {0}'.format(changelog[pkgs[i]]))
+ # print(changelog[pkgs[i] + 1: pkgs[i + 1]])
+ gitpkg, tag, tag_comments = parse_and_process_package_changelog(gl, bob_group, changelog[pkgs[i]][1:].strip(),
+ changelog[pkgs[i] + 1: pkgs[i + 1]])
+ # release the package with the found tag and its comments
+ if gitpkg:
+ release_package(gitpkg, tag, tag_comments)
+ # now, wait for the pipeline to finish, before we can release the next package
+ wait_for_pipeline_to_finish(gitpkg, tag)
if __name__ == '__main__':
arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1')
- main(arguments['<private_token>'],, group_name = arguments['--group-name'], changelog = arguments['--changelog'])
+ main(arguments['<private_token>'], group_name=arguments['--group-name'],
+ changelog_file=arguments['--changelog-file'])