diff --git a/release/release_bob.py b/release/release_bob.py new file mode 100755 index 0000000000000000000000000000000000000000..27d8ce686424fc45040cf69b37eca50c10aadd11 --- /dev/null +++ b/release/release_bob.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python + +""" +By using changelog file as an input (can be generated with 'generate_changelog.py' script), this script goes through all packages in changelog file (in order listed), tags them correctly as per the file, an releases them one by one. This script uses python-gitlab package for accessing GitLab's API. + +Usage: + {0} [-v...] [options] [--] <private_token> + {0} -h | --help + {0} --version + +Arguments: + <private_token> Private token used to access GitLab. + +Options: + -h --help Show this screen. + --version Show version. + -c, --changelog-file STR A changelog file with all packages to release with their tags, listed in order. + [default: changelog_since_last_release.rst]. + -g, --group-name STR Group name where we are assuming that all packages are located. + [default: bob]. +""" + +import sys +from docopt import docopt +import gitlab +import datetime +import re + + +def _insure_correct_package(candidates, group_name, pkg_name): + for pkg in candidates: + # make sure the name and the group name match exactly + if pkg.name == pkg_name and pkg.namespace['name'] == group_name: + return pkg + raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name)) + + +def get_packages_list(gl, gl_group=None): + if gl_group: + grp_nightlies = gl_group.projects.list(search='bob.nightlies')[0] + nightlies = gl.projects.get(id=grp_nightlies.id) + else: + nightlies = gl.projects.list(search='bob.nightlies')[0] + nightlies_order = nightlies.files.get(file_path='order.txt', ref='master') + pkg_list_ordered = nightlies_order.decode().decode().split('\n') + pkg_list_ordered = [line for line in pkg_list_ordered if (line.strip() and not line.startswith('#'))] + return pkg_list_ordered + + +# release date of last Bob release + 1 day +def bob_last_release(gl): + bobpkg = gl.projects.get(id=1535) # 1535 is id of 'bob' meta-package + last_bob_tag = bobpkg.tags.list()[0] # get the last tag + return last_bob_tag.commit['committed_date'] + + +def get_datetime_from_gitdate(gitdate): + return datetime.datetime.strptime(gitdate[:-6], '%Y-%m-%dT%H:%M:%S.%f') + + +def sort_commits(commits): + return sorted(commits, key=lambda x: get_datetime_from_gitdate(x.committed_date)) + + +def sort_tags(tags): + return sorted(tags, key=lambda x: get_datetime_from_gitdate(x.commit['committed_date'])) + + +def get_tag_changelog(tag): + try: + return tag.release['description'] + except Exception: + return '' + + +def print_tags(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'): + since = get_datetime_from_gitdate(since) + tags = gitpkg.tags.list() + # sort tags by date + tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= since, tags) + tags = sort_tags(tags) + print('* ' + pkg_name) + for tag in tags: + print_one_tag(pkg_name, tag) + + +def print_one_tag(pkg_name, tag): + print(' * ' + tag.name + ' (' + get_datetime_from_gitdate(tag.commit['committed_date']).strftime( + '%b %d, %Y %H:%M') + ')') + for line in get_tag_changelog(tag).split('\r\n'): + line = line.strip() + if line.startswith('* ') or line.startswith('- '): + line = line[2:] + line = line.replace('!', pkg_name + '!') + line = line.replace('#', pkg_name + '#') + if not line: + continue + print(' ' * 5 + '* ' + line) + + +def print_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'): + # import ipdb; ipdb.set_trace() + commits = gitpkg.commits.list(since=since, all=True) + # sort commits by date + commits = sort_commits(commits) + print('* ' + pkg_name) + print_commits_range(pkg_name, commits) + + +def print_commits_range(pkg_name, commits): + for commit in commits: + commit_title = commit.title + # skip commits that do not carry much useful information + if '[skip ci]' in commit_title or \ + 'Merge branch' in commit_title or \ + 'Increased stable' in commit_title: + continue + commit_title = commit_title.strip() + commit_title = commit_title.replace('!', pkg_name + '!') + commit_title = commit_title.replace('#', pkg_name + '#') + # print(' - ' + get_datetime_from_gitdate(x.committed_date).strftime('%Y-%m-%d %H:%M:%S') + ":" + commit_title) + print(' ' * 5 + '- ' + commit_title) + + +def print_tags_with_commits(pkg_name, gitpkg, since='2017-01-01T00:00:00Z'): + # get tags since release and sort them + datetime_since = get_datetime_from_gitdate(since) + tags = gitpkg.tags.list() + # sort tags by date + tags = filter(lambda x: get_datetime_from_gitdate(x.commit['committed_date']) >= datetime_since, tags) + tags = sort_tags(tags) + # get commits since release date and sort them too + commits = gitpkg.commits.list(since=since, all=True) + # sort commits by date + commits = sort_commits(commits) + print('* ' + pkg_name) + # go through tags and print each with its message and corresponding commits + start_date = datetime_since + for tag in tags: + # print tag name and its text + print_one_tag(pkg_name, tag) + # print commits from the previous tag up to this one + end_date = get_datetime_from_gitdate(tag.commit['committed_date']) + commits4tag = filter(lambda x: ( + get_datetime_from_gitdate(x.committed_date) > start_date and get_datetime_from_gitdate( + x.committed_date) <= end_date), commits) + print_commits_range(pkg_name, commits4tag) + start_date = end_date + # print the tentative patch version bump for the future tag + print(' * patch') + # print leftover commits that were not tagged yet + leftover_commits = filter(lambda x: get_datetime_from_gitdate(x.committed_date) > start_date, commits) + print_commits_range(pkg_name, leftover_commits) + + +def correct_tag(gitpkg, tag): + """ + An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we need only v2.1.3 + The latest tag is either patch, minor, major, or none + """ + m = re.search(r"(v\d.\d.\d)", tag) + if m: + return m.group(0) + + # if we bump the version, we need to find the latest released version for this package + if 'patch' == tag or 'minor' == tag or 'major' == tag: + latest_tags = gitpkg.tags.list() + latest_tags = sort_tags(latest_tags) + latest_tag_name = latest_tags[-1].name + # check that it has expected format v#.#.# + m = re.match(r"(v\d.\d.\d)", latest_tag_name) + if not m: + raise ValueError( + 'The latest tag name {0} in package {1} has unknown format'.format(latest_tag_name, gitpkg.name)) + # increase the version accordingly + if 'major' == tag: # increment the first number in 'v#.#.#' + return latest_tag_name[0] + str(int(latest_tag_name[1]) + 1) + latest_tag_name[2:] + if 'minor' == tag: # increment the second number in 'v#.#.#' + return latest_tag_name[:3] + str(int(latest_tag_name[3]) + 1) + latest_tag_name[4:] + if 'patch' == tag: # increment the last number in 'v#.#.#' + return latest_tag_name[:-1] + str(int(latest_tag_name[-1]) + 1) + + if 'none' == tag: # we do nothing in this case + return tag + + raise ValueError('Cannot parse changelog tag {0} of the package {1}'.format(tag, gitpkg.name)) + + +def main(private_token, group_name='bob', changelog='changelog.rst'): + gl = gitlab.Gitlab('https://gitlab.idiap.ch', private_token=private_token, api_version=4) + bob_group = gl.groups.list(search=group_name)[0] + + # traverse all packages in the changelog, edit older tags with updated comments, + # tag them with a suggested version, then try to release, and + # wait until done to proceed to the next package + with open(changelog) as f: + changelog_insides = f.readlines() + pkg_name = cur_gitpkg = None + cur_tag = None + cur_tag_comments = [] + # we assume that changelog is formatted as structured text + for line in changelog_insides: + if '*' == line[0]: # name of the package + + if pkg_name and cur_gitpkg: # release previous package + release_package(cur_gitpkg, cur_tag, cur_tag_comments) + + pkg_name = line[1:].strip() + # find the correct package in GitLab + # group returns a simplified description of the project + grpkg = _insure_correct_package(bob_group.projects.list(search=pkg_name), group_name, pkg_name) + # so, we need to retrieve the full info from GitLab using correct project id + cur_gitpkg = gl.projects.get(id=grpkg.id) + + elif ' *' == line[:3]: # a tag level + if not cur_gitpkg: # it better be not None, as we are in the middle of the tags for the package + raise ValueError('How come package for {0} is empty?'.format(pkg_name)) + + # write the collected comments in the previous tag + if cur_tag: + update_tag_comments(cur_gitpkg, cur_tag, cur_tag_comments) + cur_tag_comments = [] # reset comments + + # parse the current tag name + cur_tag = correct_tag(cur_gitpkg, line[3:].strip()) + + if 'none' == cur_tag: # no tagging, just deploy the package + deploy_package(cur_gitpkg) + continue + + else: # all other lines are assumed to be comments + cur_tag_comments += line.strip() + + if pkg_name and cur_gitpkg: # release the last package + release_package(cur_gitpkg, cur_tag, cur_tag_comments) + + +if __name__ == '__main__': + arguments = docopt(__doc__.format(sys.argv[0]), version='Changelog 0.0.1') + main(arguments['<private_token>'],, group_name = arguments['--group-name'], changelog = arguments['--changelog'])