Skip to content
GitLab
Commit 1ff56b93 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'docformatter' into 'master' 

Docformatter

See merge request !89
parents 167315df 1729c472
Pipeline #32386 canceled with stages
in 4 minutes and 14 seconds
Expand all Hide whitespace changes
Inline Side-by-side
bob/__init__.py
View file @ 1ff56b93
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
bob/devtools/bootstrap.py
View file @ 1ff56b93
This diff is collapsed.
bob/devtools/build.py
View file @ 1ff56b93
This diff is collapsed.
bob/devtools/changelog.py
View file @ 1ff56b93
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Utilities for retrieving, parsing and auto-generating changelogs'''
"""Utilities for retrieving, parsing and auto-generating changelogs."""
import io
import datetime
......@@ -10,42 +10,44 @@ import pytz
import dateutil.parser
from .log import get_logger
logger = get_logger(__name__)
def parse_date(d):
'''Parses any date supported by :py:func:`dateutil.parser.parse`'''
"""Parses any date supported by :py:func:`dateutil.parser.parse`"""
return dateutil.parser.parse(d, ignoretz=True).replace(
tzinfo=pytz.timezone("Europe/Zurich"))
tzinfo=pytz.timezone("Europe/Zurich")
)
def _sort_commits(commits, reverse):
'''Sorts gitlab commit objects using their ``committed_date`` attribute'''
"""Sorts gitlab commit objects using their ``committed_date`` attribute."""
return sorted(commits,
key=lambda x: parse_date(x.committed_date),
reverse=reverse,
)
return sorted(
commits, key=lambda x: parse_date(x.committed_date), reverse=reverse
)
def _sort_tags(tags, reverse):
'''Sorts gitlab tag objects using their ``committed_date`` attribute'''
"""Sorts gitlab tag objects using their ``committed_date`` attribute."""
return sorted(tags,
key=lambda x: parse_date(x.commit['committed_date']),
return sorted(
tags,
key=lambda x: parse_date(x.commit["committed_date"]),
reverse=reverse,
)
)
def get_file_from_gitlab(gitpkg, path, ref='master'):
'''Retrieves a file from a Gitlab repository, returns a (StringIO) file'''
def get_file_from_gitlab(gitpkg, path, ref="master"):
"""Retrieves a file from a Gitlab repository, returns a (StringIO) file."""
return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
def get_last_tag(package):
'''Returns the last (gitlab object) tag for the given package
"""Returns the last (gitlab object) tag for the given package.
Args:
......@@ -54,7 +56,7 @@ def get_last_tag(package):
Returns: a tag object
'''
"""
# according to the Gitlab API documentation, tags are sorted from the last
# updated to the first, by default - no need to do further sorting!
......@@ -66,7 +68,7 @@ def get_last_tag(package):
def get_last_tag_date(package):
'''Returns the last release date for the given package
"""Returns the last release date for the given package.
Falls back to the first commit date if the package has not yet been tagged
......@@ -80,7 +82,7 @@ def get_last_tag_date(package):
Returns: a datetime object that refers to the last date the package was
released. If the package was never released, then returns the
date just before the first commit.
'''
"""
# according to the Gitlab API documentation, tags are sorted from the last
# updated to the first, by default - no need to do further sorting!
......@@ -89,10 +91,15 @@ def get_last_tag_date(package):
if tag_list:
# there are tags, use these
last = tag_list[0]
logger.debug('Last tag for package %s (id=%d) is %s', package.name,
package.id, last.name)
return parse_date(last.commit['committed_date']) + \
datetime.timedelta(milliseconds=500)
logger.debug(
"Last tag for package %s (id=%d) is %s",
package.name,
package.id,
last.name,
)
return parse_date(last.commit["committed_date"]) + datetime.timedelta(
milliseconds=500
)
else:
commit_list = package.commits.list(all=True)
......@@ -100,112 +107,130 @@ def get_last_tag_date(package):
if commit_list:
# there are commits, use these
first = _sort_commits(commit_list, reverse=False)[0]
logger.debug('First commit for package %s (id=%d) is from %s',
package.name, package.id, first.committed_date)
return parse_date(first.committed_date) - \
datetime.timedelta(milliseconds=500)
logger.debug(
"First commit for package %s (id=%d) is from %s",
package.name,
package.id,
first.committed_date,
)
return parse_date(first.committed_date) - datetime.timedelta(
milliseconds=500
)
else:
# there are no commits nor tags - abort
raise RuntimeError('package %s (id=%d) does not have commits ' \
'or tags so I cannot devise a good starting date' % \
(package.name, package.id))
raise RuntimeError(
"package %s (id=%d) does not have commits "
"or tags so I cannot devise a good starting date"
% (package.name, package.id)
)
def _get_tag_changelog(tag):
try:
return tag.release['description']
return tag.release["description"]
except Exception:
return ''
return ""
def _write_one_tag(f, pkg_name, tag):
'''Prints commit information for a single tag of a given package
"""Prints commit information for a single tag of a given package.
Args:
f: A :py:class:`File` ready to be written at
pkg_name: The name of the package we are writing tags of
tag: The tag value
"""
'''
git_date = parse_date(tag.commit['committed_date'])
f.write(' * %s (%s)\n' % (tag.name, git_date.strftime('%b %d, %Y %H:%M')))
git_date = parse_date(tag.commit["committed_date"])
f.write(" * %s (%s)\n" % (tag.name, git_date.strftime("%b %d, %Y %H:%M")))
for line in _get_tag_changelog(tag).replace('\r\n', '\n').split('\n'):
for line in _get_tag_changelog(tag).replace("\r\n", "\n").split("\n"):
line = line.strip()
if line.startswith('* ') or line.startswith('- '):
if line.startswith("* ") or line.startswith("- "):
line = line[2:]
line = line.replace('!', pkg_name + '!').replace(pkg_name + \
pkg_name, pkg_name)
line = line.replace('#', pkg_name + '#')
line = line.replace("!", pkg_name + "!").replace(
pkg_name + pkg_name, pkg_name
)
line = line.replace("#", pkg_name + "#")
if not line:
continue
f.write('%s* %s' % (5*' ', line))
f.write("%s* %s" % (5 * " ", line))
def _write_commits_range(f, pkg_name, commits):
'''Writes all commits of a given package within a range, to the output file
"""Writes all commits of a given package within a range, to the output
file.
Args:
f: A :py:class:`File` ready to be written at
pkg_name: The name of the package we are writing tags of
commits: List of commits to be written
'''
"""
for commit in commits:
commit_title = commit.title
# skip commits that do not carry much useful information
if '[skip ci]' in commit_title or \
'Merge branch' in commit_title or \
'Increased stable' in commit_title:
if (
"[skip ci]" in commit_title
or "Merge branch" in commit_title
or "Increased stable" in commit_title
):
continue
commit_title = commit_title.strip()
commit_title = commit_title.replace('!', pkg_name + '!').replace(pkg_name + pkg_name, pkg_name)
commit_title = commit_title.replace('#', pkg_name + '#')
f.write('%s- %s\n' % (' ' * 5, commit_title))
commit_title = commit_title.replace("!", pkg_name + "!").replace(
pkg_name + pkg_name, pkg_name
)
commit_title = commit_title.replace("#", pkg_name + "#")
f.write("%s- %s\n" % (" " * 5, commit_title))
def _write_mergerequests_range(f, pkg_name, mrs):
'''Writes all merge-requests of a given package, with a range, to the
output file
"""Writes all merge-requests of a given package, with a range, to the
output file.
Args:
f: A :py:class:`File` ready to be written at
pkg_name: The name of the package we are writing tags of
mrs: The list of merge requests to write
'''
"""
for mr in mrs:
title = mr.title.strip().replace('\r','').replace('\n', ' ')
title = title.replace(' !', ' ' + pkg_name + '!')
title = title.replace(' #', ' ' + pkg_name + '#')
title = mr.title.strip().replace("\r", "").replace("\n", " ")
title = title.replace(" !", " " + pkg_name + "!")
title = title.replace(" #", " " + pkg_name + "#")
if mr.description is not None:
description = \
mr.description.strip().replace('\r','').replace('\n', ' ')
description = description.replace(' !', ' ' + pkg_name + '!')
description = description.replace(' #', ' ' + pkg_name + '#')
description = (
mr.description.strip().replace("\r", "").replace("\n", " ")
)
description = description.replace(" !", " " + pkg_name + "!")
description = description.replace(" #", " " + pkg_name + "#")
else:
description = 'No description for this MR'
space = ': ' if description else ''
log = ''' - {pkg}!{iid} {title}{space}{description}'''
f.write(log.format(pkg=pkg_name, iid=mr.iid, title=title, space=space, description=description))
f.write('\n')
description = "No description for this MR"
space = ": " if description else ""
log = """ - {pkg}!{iid} {title}{space}{description}"""
f.write(
log.format(
pkg=pkg_name,
iid=mr.iid,
title=title,
space=space,
description=description,
)
)
f.write("\n")
def write_tags_with_commits(f, gitpkg, since, mode):
'''Writes all tags and commits of a given package to the output file
"""Writes all tags and commits of a given package to the output file.
Args:
......@@ -214,14 +239,13 @@ def write_tags_with_commits(f, gitpkg, since, mode):
since: Starting date (as a datetime object)
mode: One of mrs (merge-requests), commits or tags indicating how to
list entries in the changelog for this package
'''
"""
# get tags since release and sort them
tags = gitpkg.tags.list()
# sort tags by date
tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
tags = [k for k in tags if parse_date(k.commit["committed_date"]) >= since]
tags = _sort_tags(tags, reverse=False)
# get commits since release date and sort them too
......@@ -231,8 +255,17 @@ def write_tags_with_commits(f, gitpkg, since, mode):
commits = _sort_commits(commits, reverse=False)
# get merge requests since the release data
mrs = list(reversed(gitpkg.mergerequests.list(state='merged', updated_after=since, order_by='updated_at', all=True)))
f.write('* %s\n' % (gitpkg.attributes['path_with_namespace'],))
mrs = list(
reversed(
gitpkg.mergerequests.list(
state="merged",
updated_after=since,
order_by="updated_at",
all=True,
)
)
)
f.write("* %s\n" % (gitpkg.attributes["path_with_namespace"],))
# go through tags and writes each with its message and corresponding
# commits
......@@ -240,65 +273,76 @@ def write_tags_with_commits(f, gitpkg, since, mode):
for tag in tags:
# write tag name and its text
_write_one_tag(f, gitpkg.attributes['path_with_namespace'], tag)
end_date = parse_date(tag.commit['committed_date'])
_write_one_tag(f, gitpkg.attributes["path_with_namespace"], tag)
end_date = parse_date(tag.commit["committed_date"])
if mode == 'commits':
if mode == "commits":
# write commits from the previous tag up to this one
commits4tag = [k for k in commits \
if (start_date < parse_date(k.committed_date) <= end_date)]
_write_commits_range(f, gitpkg.attributes['path_with_namespace'],
commits4tag)
elif mode == 'mrs':
commits4tag = [
k
for k in commits
if (start_date < parse_date(k.committed_date) <= end_date)
]
_write_commits_range(
f, gitpkg.attributes["path_with_namespace"], commits4tag
)
elif mode == "mrs":
# write merge requests from the previous tag up to this one
# the attribute 'merged_at' is not available in GitLab API as of 27
# June 2018
mrs4tag = [k for k in mrs \
if (start_date < parse_date(k.updated_at) <= end_date)]
_write_mergerequests_range(f,
gitpkg.attributes['path_with_namespace'], mrs4tag)
mrs4tag = [
k
for k in mrs
if (start_date < parse_date(k.updated_at) <= end_date)
]
_write_mergerequests_range(
f, gitpkg.attributes["path_with_namespace"], mrs4tag
)
start_date = end_date
if mode != 'tags':
if mode != "tags":
# write the tentative patch version bump for the future tag
f.write(' * patch\n')
f.write(" * patch\n")
if mode == 'mrs':
if mode == "mrs":
# write leftover merge requests
# the attribute 'merged_at' is not available in GitLab API as of 27
# June 2018
leftover_mrs = [k for k in mrs \
if parse_date(k.updated_at) > start_date]
_write_mergerequests_range(f,
gitpkg.attributes['path_with_namespace'], leftover_mrs)
leftover_mrs = [
k for k in mrs if parse_date(k.updated_at) > start_date
]
_write_mergerequests_range(
f, gitpkg.attributes["path_with_namespace"], leftover_mrs
)
else:
# write leftover commits that were not tagged yet
leftover_commits = [k for k in commits \
if parse_date(k.committed_date) > start_date]
_write_commits_range(f, gitpkg.attributes['path_with_namespace'],
leftover_commits)
leftover_commits = [
k for k in commits if parse_date(k.committed_date) > start_date
]
_write_commits_range(
f, gitpkg.attributes["path_with_namespace"], leftover_commits
)
def write_tags(f, gitpkg, since):
'''Writes all tags of a given package to the output file
"""Writes all tags of a given package to the output file.
Args:
f: A :py:class:`File` ready to be written at
gitpkg: A pointer to the gitlab package object
since: Starting date as a datetime object
'''
"""
tags = gitpkg.tags.list()
# sort tags by date
tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
tags = [k for k in tags if parse_date(k.commit["committed_date"]) >= since]
tags = _sort_tags(tags, reverse=False)
f.write('* %s\n')
f.write("* %s\n")
for tag in tags:
_write_one_tag(gitpkg.attributes['path_with_namespace'], tag)
_write_one_tag(gitpkg.attributes["path_with_namespace"], tag)
bob/devtools/ci.py
View file @ 1ff56b93
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Tools to help CI-based builds and artifact deployment'''
"""Tools to help CI-based builds and artifact deployment."""
import git
import distutils.version
from .log import get_logger
from .build import load_order_file
logger = get_logger(__name__)
def is_master(refname, tag, repodir):
'''Tells if we're on the master branch via ref_name or tag
"""Tells if we're on the master branch via ref_name or tag.
This function checks if the name of the branch being built is "master". If a
tag is set, then it checks if the tag is on the master branch. If so, then
also returns ``True``, otherwise, ``False``.
This function checks if the name of the branch being built is "master". If a
tag is set, then it checks if the tag is on the master branch. If so, then
also returns ``True``, otherwise, ``False``.
Args:
Args:
refname: The value of the environment variable ``CI_COMMIT_REF_NAME``
tag: The value of the environment variable ``CI_COMMIT_TAG`` - (may be
``None``)
refname: The value of the environment variable ``CI_COMMIT_REF_NAME``
tag: The value of the environment variable ``CI_COMMIT_TAG`` - (may be
``None``)
Returns: a boolean, indicating we're building the master branch **or** that
the tag being built was issued on the master branch.
'''
Returns: a boolean, indicating we're building the master branch **or** that
the tag being built was issued on the master branch.
"""
if tag is not None:
repo = git.Repo(repodir)
_tag = repo.tag('refs/tags/%s' % tag)
return _tag.commit in repo.iter_commits(rev='master')
if tag is not None:
repo = git.Repo(repodir)
_tag = repo.tag("refs/tags/%s" % tag)
return _tag.commit in repo.iter_commits(rev="master")
return refname == 'master'
return refname == "master"
def is_stable(package, refname, tag, repodir):
'''Determines if the package being published is stable
"""Determines if the package being published is stable.
This is done by checking if a tag was set for the package. If that is the
case, we still cross-check the tag is on the "master" branch. If everything
checks out, we return ``True``. Else, ``False``.
This is done by checking if a tag was set for the package. If that is the
case, we still cross-check the tag is on the "master" branch. If everything
checks out, we return ``True``. Else, ``False``.
Args:
Args:
package: Package name in the format "group/name"
refname: The current value of the environment ``CI_COMMIT_REF_NAME``
tag: The current value of the enviroment ``CI_COMMIT_TAG`` (may be
``None``)
repodir: The directory that contains the clone of the git repository
package: Package name in the format "group/name"
refname: The current value of the environment ``CI_COMMIT_REF_NAME``
tag: The current value of the enviroment ``CI_COMMIT_TAG`` (may be
``None``)
repodir: The directory that contains the clone of the git repository
Returns: a boolean, indicating if the current build is for a stable release
'''
Returns: a boolean, indicating if the current build is for a stable release
"""
if tag is not None:
logger.info('Project %s tag is "%s"', package, tag)
parsed_tag = distutils.version.LooseVersion(tag[1:]).version #remove 'v'
is_prerelease = any([isinstance(k, str) for k in parsed_tag])
if tag is not None:
logger.info('Project %s tag is "%s"', package, tag)
parsed_tag = distutils.version.LooseVersion(
tag[1:]
).version # remove 'v'
is_prerelease = any([isinstance(k, str) for k in parsed_tag])
if is_prerelease:
logger.warn('Pre-release detected - not publishing to stable channels')
return False
if is_prerelease:
logger.warn(
"Pre-release detected - not publishing to stable channels"
)
return False
if is_master(refname, tag, repodir):
return True
else:
logger.warn('Tag %s in non-master branch will be ignored', tag)
return False
if is_master(refname, tag, repodir):
return True
else:
logger.warn("Tag %s in non-master branch will be ignored", tag)
return False
logger.info('No tag information available at build')
logger.info('Considering this to be a pre-release build')
return False
logger.info("No tag information available at build")
logger.info("Considering this to be a pre-release build")
return False
def comment_cleanup(lines):
"""Cleans-up comments and empty lines from textual data read from files"""
def read_packages(filename):
"""Return a python list of tuples (repository, branch), given a file
containing one package (and branch) per line.
no_comments = [k.partition('#')[0].strip() for k in lines]
return [k for k in no_comments if k]
Comments are excluded
"""
lines = load_order_file(filename)
def read_packages(filename):
"""
Return a python list of tuples (repository, branch), given a file containing
one package (and branch) per line. Comments are excluded
packages = []
for line in lines:
if "," in line: # user specified a branch
path, branch = [k.strip() for k in line.split(",", 1)]
packages.append((path, branch))
else:
packages.append((line, "master"))
"""
# loads dirnames from order file (accepts # comments and empty lines)
with open(filename, 'rt') as f:
lines = comment_cleanup(f.readlines())
return packages
packages = []
for line in lines:
if ',' in line: #user specified a branch
path, branch = [k.strip() for k in line.split(',', 1)]
packages.append((path, branch))
else:
packages.append((line, 'master'))
return packages
def uniq(seq, idfun=None):
"""Very fast, order preserving uniq function."""