diff --git a/bob/devtools/changelog.py b/bob/devtools/changelog.py
index ddba8b29fb077415c633b0ab5a892d4d301861a8..355ba2b84a27674bc05b3289b6f26c0b903f05ce 100644
--- a/bob/devtools/changelog.py
+++ b/bob/devtools/changelog.py
@@ -43,7 +43,7 @@ def _sort_tags(tags, reverse):
def get_file_from_gitlab(gitpkg, path, ref="master"):
"""Retrieves a file from a Gitlab repository, returns a (StringIO) file."""
- return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
+ return io.StringIO(gitpkg.files.get(file_path=path, ref=ref).decode())
def get_last_tag(package):
@@ -229,18 +229,21 @@ def _write_mergerequests_range(f, pkg_name, mrs):
f.write("\n")
-def write_tags_with_commits(f, gitpkg, since, mode):
- """Writes all tags and commits of a given package to the output file.
+def get_changes_since(gitpkg, since):
+ """Gets the list of MRs, tags, and commits since the provided date
- Args:
+ Parameters
+ ----------
+ gitpkg : object
+ A gitlab pakcage object
+ since : object
+ A parsed date
- f: A :py:class:`File` ready to be written at
- gitpkg: A pointer to the gitlab package object
- since: Starting date (as a datetime object)
- mode: One of mrs (merge-requests), commits or tags indicating how to
- list entries in the changelog for this package
+ Returns
+ -------
+ tuple
+ mrs, tags, commits
"""
-
# get tags since release and sort them
tags = gitpkg.tags.list()
@@ -265,6 +268,21 @@ def write_tags_with_commits(f, gitpkg, since, mode):
)
)
)
+ return mrs, tags, commits
+
+def write_tags_with_commits(f, gitpkg, since, mode):
+ """Writes all tags and commits of a given package to the output file.
+
+ Args:
+
+ f: A :py:class:`File` ready to be written at
+ gitpkg: A pointer to the gitlab package object
+ since: Starting date (as a datetime object)
+ mode: One of mrs (merge-requests), commits or tags indicating how to
+ list entries in the changelog for this package
+ """
+ mrs, tags, commits = get_changes_since(gitpkg, since)
+
f.write("* %s\n" % (gitpkg.attributes["path_with_namespace"],))
# go through tags and writes each with its message and corresponding
diff --git a/bob/devtools/scripts/changelog.py b/bob/devtools/scripts/changelog.py
index 186b4e35eb0a2886b48d4c6d9b68ea82ed7124d3..ae1cd84ee1909f6b928581ca0786a64e50e6394d 100644
--- a/bob/devtools/scripts/changelog.py
+++ b/bob/devtools/scripts/changelog.py
@@ -1,16 +1,8 @@
#!/usr/bin/env python
-import os
-import sys
-import datetime
-
import click
from . import bdt
-from ..changelog import get_last_tag_date, write_tags_with_commits
-from ..changelog import parse_date
-from ..release import get_gitlab_instance
-
from ..log import verbosity_option, get_logger
logger = get_logger(__name__)
@@ -22,41 +14,39 @@ Examples:
1. Generates the changelog for a single package using merge requests:
- $ bdt gitlab changelog group/package.xyz changelog.md
+ $ bdt gitlab changelog -vvv group/package.xyz changelog.md
2. The same as above, but dumps the changelog to stdout instead of a file:
- $ bdt gitlab changelog group/package.xyz
+ $ bdt gitlab changelog -vvv group/package.xyz
3. Generates the changelog for a single package looking at commits
(not merge requests):
- $ bdt gitlab changelog --mode=commits group/package.xyz changelog.md
+ $ bdt gitlab changelog -vvv --mode=commits group/package.xyz changelog.md
4. Generates the changelog for a single package looking at merge requests starting from a given date of January 1, 2016:
\b
- $ bdt gitlab changelog --mode=mrs --since=2016-01-01 group/package.xyz changelog.md
+ $ bdt gitlab changelog -vvv --mode=mrs --since=2016-01-01 group/package.xyz changelog.md
- 5. Generates a complete list of changelogs for a list of packages (one per line:
+ 5. Generates a complete list of changelogs for a list of packages (one per line):
\b
$ bdt gitlab getpath bob/bob.nightlies order.txt
$ bdt gitlab lasttag bob/bob
# copy and paste date to next command
- $ bdt gitlab changelog --since="2018-07-17 10:23:40" order.txt changelog.md
+ $ bdt gitlab changelog -vvv --since="2018-07-17 10:23:40" order.txt changelog.md
"""
)
@click.argument("target")
@click.argument(
"changelog",
- type=click.Path(
- exists=False, dir_okay=False, file_okay=True, writable=True
- ),
+ type=click.Path(exists=False, dir_okay=False, file_okay=True, writable=True),
required=False,
)
@click.option(
@@ -102,10 +92,22 @@ def changelog(target, changelog, group, mode, since):
an existing file containing a list of packages that will be iterated on.
For each package, we will contact the Gitlab server and create a changelog
- using merge-requests (default), tags or commits since a given date. If a
- starting date is not passed, we'll use the date of the last tagged value or
- the date of the first commit, if no tags are available in the package.
+ using merge-requests (default), tags or commits since a given date. If a
+ starting date is not passed, we'll use the date of the last tagged value or
+ the date of the first commit, if no tags are available in the package.
"""
+ import os
+ import sys
+ import datetime
+
+ from ..changelog import (
+ get_last_tag_date,
+ write_tags_with_commits,
+ parse_date,
+ get_changes_since,
+ get_last_tag,
+ )
+ from ..release import get_gitlab_instance
gl = get_gitlab_instance()
@@ -119,15 +121,26 @@ def changelog(target, changelog, group, mode, since):
if k.strip() and not k.strip().startswith("#")
]
else:
- logger.info(
- "Assuming %s is a package name (file does not exist)...", target
- )
+ logger.info("Assuming %s is a package name (file does not exist)...", target)
packages = [target]
# if the user passed a date, convert it
if since:
since = parse_date(since)
+ # Since tagging packages requires bob.devtools to be tagged first. Add that to the
+ # list as well if bob.devtools has changed. Note that bob.devtools can release
+ # itself.
+ def bdt_has_changes():
+ gitpkg = gl.projects.get("bob/bob.devtools")
+ tag = get_last_tag(gitpkg)
+ last_tag_date = parse_date(tag.commit["committed_date"])
+ _, _, commits = get_changes_since(gitpkg, last_tag_date)
+ return len(commits)
+
+ if bdt_has_changes():
+ packages.insert(0, "bob/bob.devtools")
+
# iterates over the packages and dumps required information
for package in packages:
@@ -179,7 +192,5 @@ def changelog(target, changelog, group, mode, since):
changelog_file = open(changelog, "at")
# write_tags(f, use_package, last_release_date)
- write_tags_with_commits(
- changelog_file, use_package, last_release_date, mode
- )
+ write_tags_with_commits(changelog_file, use_package, last_release_date, mode)
changelog_file.flush()
diff --git a/bob/devtools/scripts/common_options.py b/bob/devtools/scripts/common_options.py
new file mode 100644
index 0000000000000000000000000000000000000000..df211a09622cfbedb3f50a38091125748c0e68f2
--- /dev/null
+++ b/bob/devtools/scripts/common_options.py
@@ -0,0 +1,18 @@
+import click
+
+
+def ref_option(**kwargs):
+ """An option for getting branch name."""
+
+ def custom_ref_option(func):
+
+ return click.option(
+ "-r",
+ "--ref",
+ default="master",
+ show_default=True,
+ help="Download path from the provided git reference (may be a branch, tag or commit hash)",
+ **kwargs
+ )(func)
+
+ return custom_ref_option
diff --git a/bob/devtools/scripts/getpath.py b/bob/devtools/scripts/getpath.py
index 04cba7715af3049bebe0451dacd2d00c5a79f821..a7aede79bc5f547a5ede77be8faf8a6067c6ef60 100644
--- a/bob/devtools/scripts/getpath.py
+++ b/bob/devtools/scripts/getpath.py
@@ -6,7 +6,7 @@ import click
from . import bdt
from ..release import get_gitlab_instance, download_path
-
+from .common_options import ref_option
from ..log import verbosity_option, get_logger
logger = get_logger(__name__)
@@ -34,13 +34,7 @@ Examples:
@click.argument("package")
@click.argument("path")
@click.argument("output", type=click.Path(exists=False), required=False)
-@click.option(
- "-r",
- "--ref",
- default="master",
- show_default=True,
- help="Download path from the provided git reference (may be a branch, tag or commit hash)",
-)
+@ref_option()
@verbosity_option()
@bdt.raise_on_error
def getpath(package, path, output, ref):
diff --git a/bob/devtools/scripts/gitlab.py b/bob/devtools/scripts/gitlab.py
index 90afda5183a82fd33a0c10c8866e8c12ebaeed05..7e66d36e2317788d7be9606f44c2b7beb6c8097c 100644
--- a/bob/devtools/scripts/gitlab.py
+++ b/bob/devtools/scripts/gitlab.py
@@ -1,7 +1,5 @@
#!/usr/bin/env python
-import os
-
import pkg_resources
import click
diff --git a/bob/devtools/scripts/release.py b/bob/devtools/scripts/release.py
index f666bc57f636ef07dc7d35eeb90fb95d44cbc8bc..11b932c34626dae0578db109a2398f90e503d2f0 100644
--- a/bob/devtools/scripts/release.py
+++ b/bob/devtools/scripts/release.py
@@ -22,28 +22,28 @@ Examples:
1. Releases a single package:
- $ bdt gitlab release --package=bob.package.xyz changelog.md
+ $ bdt gitlab release -vvv --package=bob/bob.package.xyz changelog.md
2. If there is a single package in the ``changelog.md`` file, the flag
``--package`` is not required:
- $ bdt gitlab release changelog.md
+ $ bdt gitlab release -vvv changelog.md
- 2. Releases the whole of bob using `changelog_since_last_release.md`:
+ 2. Releases the whole of bob using `changelog.md`:
- $ bdt gitlab release bob/devtools/data/changelog_since_last_release.md
+ $ bdt gitlab release -vvv changelog.md
3. In case of errors, resume the release of the whole of Bob:
- $ bdt gitlab release --resume bob/devtools/data/changelog_since_last_release.md
+ $ bdt gitlab release -vvv --resume --package=bob/bob.package.xyz changelog.md
4. The option `-dry-run` can be used to let the script print what it would do instead of actually doing it:
- $ bdt gitlab release --dry-run changelog_since_last_release.md
+ $ bdt gitlab release -vvv --dry-run changelog.md
"""
)
@click.argument("changelog", type=click.File("rt", lazy=False))
@@ -143,12 +143,6 @@ def release(changelog, group, package, resume, dry_run):
gl = get_gitlab_instance()
- # if we are releasing 'bob' metapackage, it's a simple thing, no GitLab
- # API
- if package == "bob":
- release_bob(changelog)
- return
-
# traverse all packages in the changelog, edit older tags with updated
# comments, tag them with a suggested version, then try to release, and
# wait until done to proceed to the next package
@@ -162,9 +156,7 @@ def release(changelog, group, package, resume, dry_run):
if package:
# get the index where the package first appears in the list
start_idx = [
- i
- for i, line in enumerate(changelogs)
- if line[1:].strip() == package
+ i for i, line in enumerate(changelogs) if line[1:].strip() == package
]
if not start_idx:
@@ -201,9 +193,7 @@ def release(changelog, group, package, resume, dry_run):
# release the package with the found tag and its comments
if use_package:
- pipeline_id = release_package(
- use_package, tag, tag_comments, dry_run
- )
+ pipeline_id = release_package(use_package, tag, tag_comments, dry_run)
# now, wait for the pipeline to finish, before we can release the
# next package
wait_for_pipeline_to_finish(use_package, pipeline_id, dry_run)
diff --git a/bob/devtools/scripts/update_bob.py b/bob/devtools/scripts/update_bob.py
new file mode 100644
index 0000000000000000000000000000000000000000..20f9a072e57bb69b89cd2530065c121d911cf01c
--- /dev/null
+++ b/bob/devtools/scripts/update_bob.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+
+import click
+
+from . import bdt
+from .common_options import ref_option
+
+from ..log import verbosity_option, get_logger
+
+logger = get_logger(__name__)
+
+
+@click.command(
+ epilog="""\b
+Examples:
+ bdt gitlab update-bob -vv
+ bdt gitlab update-bob -vv --stable
+"""
+)
+@click.option(
+ "--stable/--beta", help="To use the stable versions in the list and pin packages."
+)
+@verbosity_option()
+@bdt.raise_on_error
+def update_bob(stable):
+ """Updates the Bob meta package with new packages.
+ """
+ import tempfile
+ from ..ci import read_packages
+ from ..release import get_gitlab_instance, download_path, get_latest_tag_name
+
+ gl = get_gitlab_instance()
+
+ # download order.txt form bob.nightlies and get the list of packages
+ nightlies = gl.projects.get("bob/bob.nightlies")
+
+ with tempfile.NamedTemporaryFile() as f:
+ download_path(nightlies, "order.txt", f.name, ref="master")
+ packages = read_packages(f.name)
+
+ # find the list of public packages
+ public_packages, private_packages = [], []
+ for n, (package, branch) in enumerate(packages):
+
+ if package == "bob/bob":
+ continue
+
+ # determine package visibility
+ use_package = gl.projects.get(package)
+ is_public = use_package.attributes["visibility"] == "public"
+
+ if is_public:
+ public_packages.append(package.replace("bob/", ""))
+ else:
+ private_packages.append(package.replace("bob/", ""))
+
+ logger.debug("%s is %s", package, "public" if is_public else "not public")
+
+ logger.info("Found %d public packages", len(public_packages))
+ logger.info(
+ "The following packages were not public:\n%s", "\n".join(private_packages)
+ )
+
+ # if requires stable versions, add latest tag versions to the names
+ if stable:
+ logger.info("Getting latest tag names for the public packages")
+ tags = [
+ get_latest_tag_name(gl.projects.get(f"bob/{pkg}"))
+ for pkg in public_packages
+ ]
+ public_packages = [f"{pkg} =={tag}" for pkg, tag in zip(public_packages, tags)]
+
+
+ # modify conda/meta.yaml and requirements.txt in bob/bob
+ logger.info("Updating conda/meta.yaml")
+ start_tag = "# LIST OF BOB PACKAGES - START"
+ end_tag = "# LIST OF BOB PACKAGES - END"
+
+ with open("conda/meta.yaml") as f:
+ lines = f.read()
+ i1 = lines.find(start_tag) + len(start_tag)
+ i2 = lines.find(end_tag)
+
+ lines = (
+ lines[:i1] + "\n - ".join([""] + public_packages) + "\n " + lines[i2:]
+ )
+
+ with open("conda/meta.yaml", "w") as f:
+ f.write(lines)
+
+ logger.info("Updating requirements.txt")
+ with open("requirements.txt", "w") as f:
+ f.write("\n".join(public_packages) + "\n")
+
+ click.echo(
+ "You may need to add the ` # [linux]` tag in front of linux only "
+ "packages in conda/meta.yaml"
+ )
diff --git a/doc/release.rst b/doc/release.rst
index bcb5a77afdd14303febea17f0cadcc1543a15e65..f627fe639c1e593993fc363fd0b91b4d46e876c4 100644
--- a/doc/release.rst
+++ b/doc/release.rst
@@ -69,67 +69,77 @@ To manually update the changelog, follow these guidelines:
Releasing the Bob meta package
==============================
-.. todo:: These instructions may be outdated!!
-
Here are the instructions to release Bob meta package:
* Run:
.. code-block:: sh
- $ bdt gitlab getpath bob/bob.nightlies order.txt
- $ bdt gitlab visibility order.txt
+ $ cd bob
+ $ bdt gitlab update-bob -vvv --stable
-* Put the list of public packages in ../../bob/requirements.txt
-* Run ``bdt gitlab changelog`` first:
+* The script above cannot identify linux only packages. After running the script,
+ **you need to manually tag linux only packages** in both ``conda/meta.yaml`` and
+ ``requirements.txt``. For example, in ``conda/meta.yaml``::
- .. code-block:: sh
+ .. code-block:: yaml
- $ bdt gitlab changelog ../../bob/requirements.txt bob_changelog.md
+ - bob.ip.binseg ==1.1.0 # [linux]
-* Put the beta of version of the intended release version in
- ``../../bob/version.txt``
+ and, in ``requirements.txt``::
- * For example do ``$ echo 5.0.0b0 > version.txt`` for bob 5.0.0 release.
- * Commit only this change to master: ``$ git commit -m "prepare for bob 5 release" version.txt``
+ bob.ip.binseg ==1.1.0 ; sys_platform == 'linux'
-* Get the pinnings (``--bob-version`` needs to be changed):
+* Test the conda recipe of bob. You may want to cancel the
+ command below once it reaches the nosetests.:
.. code-block:: sh
- $ bdt gitlab release -p bob -c bob_changelog.md --bob-version 5.0.0 -- TOKEN
+ $ bdt build -vv --stable
+
+* Commit the changes and push:
-* Put the pinnings below in requirements.txt and meta.yaml (like ``bob.buildout
- == 2.1.6``) and meta.yaml (like ``bob.buildout 2.1.6``)
+ .. code-block:: sh
+
+ $ git commit -m "Pinning packages for the next release. [skip ci]" conda/meta.yaml requirements.txt
+ $ git push
- * Make sure you add `` # [linux]`` to Linux only packages.
-* Test the conda recipe:
+* Tag the package using the same changelog mechanism that you used to tag other
+ packages. Assuming the changelog has a ``* bob/bob`` entry:
.. code-block:: sh
- $ cd ../../bob
- $ conda render -m ../bob.admin/gitlab/conda_build_config.yaml -c https://www.idiap.ch/software/bob/conda conda
+ $ bdt gitlab release -vvv CHANGELOG --package bob/bob
-* Update the badges and version.txt to point to this version of Bob.
-* Commit, push and tag a new version manually:
+* When the script says ``Waiting for the pipeline *** of "bob/bob" to finish``, you may
+ cancel it and check the progress online.
+
+* To revert the pins while in beta run::
.. code-block:: sh
- $ git commit -am "Increased stable version to 4.0.0"
- $ git tag v4.0.0
- $ git push
- $ git push --tags
+ $ git pull --rebase
+ $ bdt gitlab update-bob -vvv --beta
-* Put ``bob_changelog.md`` inside bob's tag description.
-* Cancel the pipeline for master and make sure that tag pipeline passes before
- continuing.
-* Remove pinnings from bob's requirement.txt and meta.yaml and revert changes
- that went in ``README.rst`` back to master version.
-* Commit and push the following (not verbatim):
+* Like before, **tag the linux only packages manually**.
+
+* Commit and push the changes:
.. code-block:: sh
- $ echo 4.0.1b0 > version.txt
- $ git commit -am "Increased latest version to 4.0.1b0 [skip ci]"
+ $ git commit -m "Remove package pins while in beta. [skip ci]" conda/meta.yaml requirements.txt
$ git push
+
+You can see that if we could identify linux only packages automatically, the whole
+release process would have been only to run
+``bdt gitlab release -vvv CHANGELOG --package bob/bob``.
+Do you want to help fix that?
+
+
+Releasing the docs meta package
+===============================
+
+Don't forget to release ``bob/docs`` after the bob release has successfully finished.
+To do so, go to https://gitlab.idiap.ch/bob/docs/-/tags and click on ``New tag``.
+Use the same version number you used for bob.
diff --git a/setup.py b/setup.py
index f9ac0ca9b2d85691ec5c03520cae6c51d41e0db8..449ba6d330f33c97c335a99c42987d819bc7cce5 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,8 @@ setup(
'getpath = bob.devtools.scripts.getpath:getpath',
'process-pipelines = bob.devtools.scripts.pipelines:process_pipelines',
'get-pipelines- = bob.devtools.scripts.pipelines:get_pipelines',
- 'graph = bob.devtools.scripts.graph:graph'
+ 'graph = bob.devtools.scripts.graph:graph',
+ 'update-bob = bob.devtools.scripts.update_bob:update_bob',
],
'bdt.ci.cli': [