Skip to content
GitLab
Commit 75882c0c authored by André Anjos's avatar André Anjos 💬
Browse files

Passed black over module

parent 167315df
Expand all Hide whitespace changes
Inline Side-by-side
bob/__init__.py
View file @ 75882c0c
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
bob/devtools/bootstrap.py
View file @ 75882c0c
This diff is collapsed.
bob/devtools/build.py
View file @ 75882c0c
This diff is collapsed.
bob/devtools/changelog.py
View file @ 75882c0c
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Utilities for retrieving, parsing and auto-generating changelogs'''
"""Utilities for retrieving, parsing and auto-generating changelogs"""
import io
import datetime
......@@ -10,42 +10,44 @@ import pytz
import dateutil.parser
from .log import get_logger
logger = get_logger(__name__)
def parse_date(d):
'''Parses any date supported by :py:func:`dateutil.parser.parse`'''
"""Parses any date supported by :py:func:`dateutil.parser.parse`"""
return dateutil.parser.parse(d, ignoretz=True).replace(
tzinfo=pytz.timezone("Europe/Zurich"))
tzinfo=pytz.timezone("Europe/Zurich")
)
def _sort_commits(commits, reverse):
'''Sorts gitlab commit objects using their ``committed_date`` attribute'''
"""Sorts gitlab commit objects using their ``committed_date`` attribute"""
return sorted(commits,
key=lambda x: parse_date(x.committed_date),
reverse=reverse,
)
return sorted(
commits, key=lambda x: parse_date(x.committed_date), reverse=reverse
)
def _sort_tags(tags, reverse):
'''Sorts gitlab tag objects using their ``committed_date`` attribute'''
"""Sorts gitlab tag objects using their ``committed_date`` attribute"""
return sorted(tags,
key=lambda x: parse_date(x.commit['committed_date']),
return sorted(
tags,
key=lambda x: parse_date(x.commit["committed_date"]),
reverse=reverse,
)
)
def get_file_from_gitlab(gitpkg, path, ref='master'):
'''Retrieves a file from a Gitlab repository, returns a (StringIO) file'''
def get_file_from_gitlab(gitpkg, path, ref="master"):
"""Retrieves a file from a Gitlab repository, returns a (StringIO) file"""
return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
def get_last_tag(package):
'''Returns the last (gitlab object) tag for the given package
"""Returns the last (gitlab object) tag for the given package
Args:
......@@ -54,7 +56,7 @@ def get_last_tag(package):
Returns: a tag object
'''
"""
# according to the Gitlab API documentation, tags are sorted from the last
# updated to the first, by default - no need to do further sorting!
......@@ -66,7 +68,7 @@ def get_last_tag(package):
def get_last_tag_date(package):
'''Returns the last release date for the given package
"""Returns the last release date for the given package
Falls back to the first commit date if the package has not yet been tagged
......@@ -80,7 +82,7 @@ def get_last_tag_date(package):
Returns: a datetime object that refers to the last date the package was
released. If the package was never released, then returns the
date just before the first commit.
'''
"""
# according to the Gitlab API documentation, tags are sorted from the last
# updated to the first, by default - no need to do further sorting!
......@@ -89,10 +91,15 @@ def get_last_tag_date(package):
if tag_list:
# there are tags, use these
last = tag_list[0]
logger.debug('Last tag for package %s (id=%d) is %s', package.name,
package.id, last.name)
return parse_date(last.commit['committed_date']) + \
datetime.timedelta(milliseconds=500)
logger.debug(
"Last tag for package %s (id=%d) is %s",
package.name,
package.id,
last.name,
)
return parse_date(last.commit["committed_date"]) + datetime.timedelta(
milliseconds=500
)
else:
commit_list = package.commits.list(all=True)
......@@ -100,28 +107,35 @@ def get_last_tag_date(package):
if commit_list:
# there are commits, use these
first = _sort_commits(commit_list, reverse=False)[0]
logger.debug('First commit for package %s (id=%d) is from %s',
package.name, package.id, first.committed_date)
return parse_date(first.committed_date) - \
datetime.timedelta(milliseconds=500)
logger.debug(
"First commit for package %s (id=%d) is from %s",
package.name,
package.id,
first.committed_date,
)
return parse_date(first.committed_date) - datetime.timedelta(
milliseconds=500
)
else:
# there are no commits nor tags - abort
raise RuntimeError('package %s (id=%d) does not have commits ' \
'or tags so I cannot devise a good starting date' % \
(package.name, package.id))
raise RuntimeError(
"package %s (id=%d) does not have commits "
"or tags so I cannot devise a good starting date"
% (package.name, package.id)
)
def _get_tag_changelog(tag):
try:
return tag.release['description']
return tag.release["description"]
except Exception:
return ''
return ""
def _write_one_tag(f, pkg_name, tag):
'''Prints commit information for a single tag of a given package
"""Prints commit information for a single tag of a given package
Args:
......@@ -129,27 +143,28 @@ def _write_one_tag(f, pkg_name, tag):
pkg_name: The name of the package we are writing tags of
tag: The tag value
'''
"""
git_date = parse_date(tag.commit['committed_date'])
f.write(' * %s (%s)\n' % (tag.name, git_date.strftime('%b %d, %Y %H:%M')))
git_date = parse_date(tag.commit["committed_date"])
f.write(" * %s (%s)\n" % (tag.name, git_date.strftime("%b %d, %Y %H:%M")))
for line in _get_tag_changelog(tag).replace('\r\n', '\n').split('\n'):
for line in _get_tag_changelog(tag).replace("\r\n", "\n").split("\n"):
line = line.strip()
if line.startswith('* ') or line.startswith('- '):
if line.startswith("* ") or line.startswith("- "):
line = line[2:]
line = line.replace('!', pkg_name + '!').replace(pkg_name + \
pkg_name, pkg_name)
line = line.replace('#', pkg_name + '#')
line = line.replace("!", pkg_name + "!").replace(
pkg_name + pkg_name, pkg_name
)
line = line.replace("#", pkg_name + "#")
if not line:
continue
f.write('%s* %s' % (5*' ', line))
f.write("%s* %s" % (5 * " ", line))
def _write_commits_range(f, pkg_name, commits):
'''Writes all commits of a given package within a range, to the output file
"""Writes all commits of a given package within a range, to the output file
Args:
......@@ -157,26 +172,29 @@ def _write_commits_range(f, pkg_name, commits):
pkg_name: The name of the package we are writing tags of
commits: List of commits to be written
'''
"""
for commit in commits:
commit_title = commit.title
# skip commits that do not carry much useful information
if '[skip ci]' in commit_title or \
'Merge branch' in commit_title or \
'Increased stable' in commit_title:
if (
"[skip ci]" in commit_title
or "Merge branch" in commit_title
or "Increased stable" in commit_title
):
continue
commit_title = commit_title.strip()
commit_title = commit_title.replace('!', pkg_name + '!').replace(pkg_name + pkg_name, pkg_name)
commit_title = commit_title.replace('#', pkg_name + '#')
f.write('%s- %s\n' % (' ' * 5, commit_title))
commit_title = commit_title.replace("!", pkg_name + "!").replace(
pkg_name + pkg_name, pkg_name
)
commit_title = commit_title.replace("#", pkg_name + "#")
f.write("%s- %s\n" % (" " * 5, commit_title))
def _write_mergerequests_range(f, pkg_name, mrs):
'''Writes all merge-requests of a given package, with a range, to the
"""Writes all merge-requests of a given package, with a range, to the
output file
Args:
......@@ -185,27 +203,36 @@ def _write_mergerequests_range(f, pkg_name, mrs):
pkg_name: The name of the package we are writing tags of
mrs: The list of merge requests to write
'''
"""
for mr in mrs:
title = mr.title.strip().replace('\r','').replace('\n', ' ')
title = title.replace(' !', ' ' + pkg_name + '!')
title = title.replace(' #', ' ' + pkg_name + '#')
title = mr.title.strip().replace("\r", "").replace("\n", " ")
title = title.replace(" !", " " + pkg_name + "!")
title = title.replace(" #", " " + pkg_name + "#")
if mr.description is not None:
description = \
mr.description.strip().replace('\r','').replace('\n', ' ')
description = description.replace(' !', ' ' + pkg_name + '!')
description = description.replace(' #', ' ' + pkg_name + '#')
description = (
mr.description.strip().replace("\r", "").replace("\n", " ")
)
description = description.replace(" !", " " + pkg_name + "!")
description = description.replace(" #", " " + pkg_name + "#")
else:
description = 'No description for this MR'
space = ': ' if description else ''
log = ''' - {pkg}!{iid} {title}{space}{description}'''
f.write(log.format(pkg=pkg_name, iid=mr.iid, title=title, space=space, description=description))
f.write('\n')
description = "No description for this MR"
space = ": " if description else ""
log = """ - {pkg}!{iid} {title}{space}{description}"""
f.write(
log.format(
pkg=pkg_name,
iid=mr.iid,
title=title,
space=space,
description=description,
)
)
f.write("\n")
def write_tags_with_commits(f, gitpkg, since, mode):
'''Writes all tags and commits of a given package to the output file
"""Writes all tags and commits of a given package to the output file
Args:
......@@ -215,13 +242,13 @@ def write_tags_with_commits(f, gitpkg, since, mode):
mode: One of mrs (merge-requests), commits or tags indicating how to
list entries in the changelog for this package
'''
"""
# get tags since release and sort them
tags = gitpkg.tags.list()
# sort tags by date
tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
tags = [k for k in tags if parse_date(k.commit["committed_date"]) >= since]
tags = _sort_tags(tags, reverse=False)
# get commits since release date and sort them too
......@@ -231,8 +258,17 @@ def write_tags_with_commits(f, gitpkg, since, mode):
commits = _sort_commits(commits, reverse=False)
# get merge requests since the release data
mrs = list(reversed(gitpkg.mergerequests.list(state='merged', updated_after=since, order_by='updated_at', all=True)))
f.write('* %s\n' % (gitpkg.attributes['path_with_namespace'],))
mrs = list(
reversed(
gitpkg.mergerequests.list(
state="merged",
updated_after=since,
order_by="updated_at",
all=True,
)
)
)
f.write("* %s\n" % (gitpkg.attributes["path_with_namespace"],))
# go through tags and writes each with its message and corresponding
# commits
......@@ -240,51 +276,63 @@ def write_tags_with_commits(f, gitpkg, since, mode):
for tag in tags:
# write tag name and its text
_write_one_tag(f, gitpkg.attributes['path_with_namespace'], tag)
end_date = parse_date(tag.commit['committed_date'])
_write_one_tag(f, gitpkg.attributes["path_with_namespace"], tag)
end_date = parse_date(tag.commit["committed_date"])
if mode == 'commits':
if mode == "commits":
# write commits from the previous tag up to this one
commits4tag = [k for k in commits \
if (start_date < parse_date(k.committed_date) <= end_date)]
_write_commits_range(f, gitpkg.attributes['path_with_namespace'],
commits4tag)
elif mode == 'mrs':
commits4tag = [
k
for k in commits
if (start_date < parse_date(k.committed_date) <= end_date)
]
_write_commits_range(
f, gitpkg.attributes["path_with_namespace"], commits4tag
)
elif mode == "mrs":
# write merge requests from the previous tag up to this one
# the attribute 'merged_at' is not available in GitLab API as of 27
# June 2018
mrs4tag = [k for k in mrs \
if (start_date < parse_date(k.updated_at) <= end_date)]
_write_mergerequests_range(f,
gitpkg.attributes['path_with_namespace'], mrs4tag)
mrs4tag = [
k
for k in mrs
if (start_date < parse_date(k.updated_at) <= end_date)
]
_write_mergerequests_range(
f, gitpkg.attributes["path_with_namespace"], mrs4tag
)
start_date = end_date
if mode != 'tags':
if mode != "tags":
# write the tentative patch version bump for the future tag
f.write(' * patch\n')
f.write(" * patch\n")
if mode == 'mrs':
if mode == "mrs":
# write leftover merge requests
# the attribute 'merged_at' is not available in GitLab API as of 27
# June 2018
leftover_mrs = [k for k in mrs \
if parse_date(k.updated_at) > start_date]
_write_mergerequests_range(f,
gitpkg.attributes['path_with_namespace'], leftover_mrs)
leftover_mrs = [
k for k in mrs if parse_date(k.updated_at) > start_date
]
_write_mergerequests_range(
f, gitpkg.attributes["path_with_namespace"], leftover_mrs
)
else:
# write leftover commits that were not tagged yet
leftover_commits = [k for k in commits \
if parse_date(k.committed_date) > start_date]
_write_commits_range(f, gitpkg.attributes['path_with_namespace'],
leftover_commits)
leftover_commits = [
k for k in commits if parse_date(k.committed_date) > start_date
]
_write_commits_range(
f, gitpkg.attributes["path_with_namespace"], leftover_commits
)
def write_tags(f, gitpkg, since):
'''Writes all tags of a given package to the output file
"""Writes all tags of a given package to the output file
Args:
......@@ -292,13 +340,13 @@ def write_tags(f, gitpkg, since):
gitpkg: A pointer to the gitlab package object
since: Starting date as a datetime object
'''
"""
tags = gitpkg.tags.list()
# sort tags by date
tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
tags = [k for k in tags if parse_date(k.commit["committed_date"]) >= since]
tags = _sort_tags(tags, reverse=False)
f.write('* %s\n')
f.write("* %s\n")
for tag in tags:
_write_one_tag(gitpkg.attributes['path_with_namespace'], tag)
_write_one_tag(gitpkg.attributes["path_with_namespace"], tag)
bob/devtools/ci.py
View file @ 75882c0c
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Tools to help CI-based builds and artifact deployment'''
"""Tools to help CI-based builds and artifact deployment"""
import git
import distutils.version
from .log import get_logger
logger = get_logger(__name__)
def is_master(refname, tag, repodir):
'''Tells if we're on the master branch via ref_name or tag
"""Tells if we're on the master branch via ref_name or tag
This function checks if the name of the branch being built is "master". If a
tag is set, then it checks if the tag is on the master branch. If so, then
......@@ -26,18 +27,18 @@ def is_master(refname, tag, repodir):
Returns: a boolean, indicating we're building the master branch **or** that
the tag being built was issued on the master branch.
'''
"""
if tag is not None:
repo = git.Repo(repodir)
_tag = repo.tag('refs/tags/%s' % tag)
return _tag.commit in repo.iter_commits(rev='master')
if tag is not None:
repo = git.Repo(repodir)
_tag = repo.tag("refs/tags/%s" % tag)
return _tag.commit in repo.iter_commits(rev="master")
return refname == 'master'
return refname == "master"
def is_stable(package, refname, tag, repodir):
'''Determines if the package being published is stable
"""Determines if the package being published is stable
This is done by checking if a tag was set for the package. If that is the
case, we still cross-check the tag is on the "master" branch. If everything
......@@ -52,77 +53,85 @@ def is_stable(package, refname, tag, repodir):
repodir: The directory that contains the clone of the git repository
Returns: a boolean, indicating if the current build is for a stable release
'''
"""
if tag is not None:
logger.info('Project %s tag is "%s"', package, tag)
parsed_tag = distutils.version.LooseVersion(tag[1:]).version #remove 'v'
is_prerelease = any([isinstance(k, str) for k in parsed_tag])
if tag is not None:
logger.info('Project %s tag is "%s"', package, tag)
parsed_tag = distutils.version.LooseVersion(
tag[1:]
).version # remove 'v'
is_prerelease = any([isinstance(k, str) for k in parsed_tag])
if is_prerelease:
logger.warn('Pre-release detected - not publishing to stable channels')
return False
if is_prerelease:
logger.warn(
"Pre-release detected - not publishing to stable channels"
)
return False
if is_master(refname, tag, repodir):
return True
else:
logger.warn('Tag %s in non-master branch will be ignored', tag)
return False
if is_master(refname, tag, repodir):
return True
else:
logger.warn("Tag %s in non-master branch will be ignored", tag)
return False
logger.info('No tag information available at build')
logger.info('Considering this to be a pre-release build')
return False
logger.info("No tag information available at build")
logger.info("Considering this to be a pre-release build")
return False
def comment_cleanup(lines):
"""Cleans-up comments and empty lines from textual data read from files"""
"""Cleans-up comments and empty lines from textual data read from files"""
no_comments = [k.partition('#')[0].strip() for k in lines]
return [k for k in no_comments if k]
no_comments = [k.partition("#")[0].strip() for k in lines]
return [k for k in no_comments if k]
def read_packages(filename):
"""
"""
Return a python list of tuples (repository, branch), given a file containing
one package (and branch) per line. Comments are excluded
"""
# loads dirnames from order file (accepts # comments and empty lines)
with open(filename, 'rt') as f:
lines = comment_cleanup(f.readlines())
# loads dirnames from order file (accepts # comments and empty lines)
with open(filename, "rt") as f:
lines = comment_cleanup(f.readlines())
packages = []
for line in lines:
if ',' in line: #user specified a branch
path, branch = [k.strip() for k in line.split(',', 1)]
packages.append((path, branch))
else:
packages.append((line, 'master'))
packages = []
for line in lines:
if "," in line: # user specified a branch
path, branch = [k.strip() for k in line.split(",", 1)]
packages.append((path, branch))
else:
packages.append((line, "master"))
return packages
return packages
def uniq(seq, idfun=None):
"""Very fast, order preserving uniq function"""
# order preserving
if idfun is None:
def idfun(x): return x
seen = {}
result = []
for item in seq:
marker = idfun(item)
# in old Python versions:
# if seen.has_key(marker)
# but in new ones:
if marker in seen: continue
seen[marker] = 1
result.append(item)
return result
"""Very fast, order preserving uniq function"""
# order preserving
if idfun is None:
def idfun(x):
return x
seen = {}
result = []
for item in seq:
marker = idfun(item)
# in old Python versions:
# if seen.has_key(marker)
# but in new ones:
if marker in seen:
continue
seen[marker] = 1
result.append(item)
return result
def select_build_file(basename, paths, branch):
'''Selects the file to use for a build
"""Selects the file to use for a build