diff --git a/MANIFEST.in b/MANIFEST.in
index 6386c90ae42107d98dcec6d5af6579fb059f6288..09fc130a0450115612e5063b6a063347d2b9092a 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,3 @@
include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst
-recursive-include bob *.cpp *.h
-recursive-include bob/extension/examples *
-recursive-include bob/extension/data *
+recursive-include bob/devtools/data *.md
diff --git a/README.rst b/README.rst
index 06af8fbc664dfde646c4b746f00e82f8b7847415..27c36ba3ac48ce293c6c3dbc3d08b3d49c4e79d7 100644
--- a/README.rst
+++ b/README.rst
@@ -5,18 +5,24 @@
===============================================
This package is part of the signal-processing and machine learning toolbox
-Bob_. It provides some tools to help maintain Bob_.
+Bob_. It provides tools to help maintain Bob_.
+
Installation
------------
-This package needs to be installed in your base conda environment. To install
+This package needs to be installed in a conda environment. To install
this package, run::
- $ conda activate base
- # the dependency list below matches the ones in setup.py
- $ conda install pip click click-plugins conda-build
- $ pip install -e .
+ $ conda env create -f env.yml
+ $ conda activate bdt
+ (bdt) $ buildout
+ (bdt) $ ./bin/bdt --help
+ ...
+
+To build the documentation, just do::
+
+ (bdt) $ ./bin/sphinx-build doc sphinx
Contact
diff --git a/bob/__init__.py b/bob/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ab1e28b150f0549def9963e9e87de3fdd6b2579
--- /dev/null
+++ b/bob/__init__.py
@@ -0,0 +1,3 @@
+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
diff --git a/bob_tools/__init__.py b/bob/devtools/__init__.py
similarity index 100%
rename from bob_tools/__init__.py
rename to bob/devtools/__init__.py
diff --git a/bob/devtools/changelog.py b/bob/devtools/changelog.py
new file mode 100644
index 0000000000000000000000000000000000000000..812f41dffe46d9c6959bf98b268410813cf4fcb9
--- /dev/null
+++ b/bob/devtools/changelog.py
@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import io
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+
+import pytz
+import dateutil.parser
+
+from .release import ensure_correct_package
+
+
+def parse_date(d):
+ '''Parses any date supported by :py:func:`dateutil.parser.parse`'''
+
+ return dateutil.parser.parse(d, ignoretz=True).replace(
+ tzinfo=pytz.timezone("Europe/Zurich"))
+
+
+def _sort_commits(commits, reverse):
+ '''Sorts gitlab commit objects using their ``committed_date`` attribute'''
+
+ return sorted(commits,
+ key=lambda x: parse_date(x.committed_date),
+ reverse=reverse,
+ )
+
+
+def _sort_tags(tags, reverse):
+ '''Sorts gitlab tag objects using their ``committed_date`` attribute'''
+
+ return sorted(tags,
+ key=lambda x: parse_date(x.commit['committed_date']),
+ reverse=reverse,
+ )
+
+
+def get_file_from_gitlab(gitpkg, path, ref='master'):
+ '''Retrieves a file from a Gitlab repository, returns a (StringIO) file'''
+
+ return io.StringIO(gitpkg.files.get(file_path=path, ref=branch).decode())
+
+
+def get_last_tag(package):
+ '''Returns the last (gitlab object) tag for the given package
+
+ Args:
+
+ package: The gitlab project object from where to fetch the last release
+ date information
+
+
+ Returns: a tag object
+ '''
+
+ # according to the Gitlab API documentation, tags are sorted from the last
+ # updated to the first, by default - no need to do further sorting!
+ tag_list = package.tags.list()
+
+ if tag_list:
+ # there are tags, use these
+ return tag_list[0]
+
+
+def get_last_tag_date(package):
+ '''Returns the last release date for the given package
+
+ Falls back to the first commit date if the package has not yet been tagged
+
+
+ Args:
+
+ package: The gitlab project object from where to fetch the last release
+ date information
+
+
+ Returns: a datetime object that refers to the last date the package was
+ released. If the package was never released, then returns the
+ date just before the first commit.
+ '''
+
+ # according to the Gitlab API documentation, tags are sorted from the last
+ # updated to the first, by default - no need to do further sorting!
+ tag_list = package.tags.list()
+
+ if tag_list:
+ # there are tags, use these
+ last = tag_list[0]
+ logger.debug('Last tag for package %s (id=%d) is %s', package.name,
+ package.id, last.name)
+ return parse_date(last.commit['committed_date']) + \
+ datetime.timedelta(milliseconds=500)
+
+ else:
+ commit_list = package.commits.list(all=True)
+
+ if commit_list:
+ # there are commits, use these
+ first = _sort_commits(commit_list, reverse=False)[0]
+ logger.debug('First commit for package %s (id=%d) is from %s',
+ package.name, package.id, first.committed_date)
+ return parse_date(first.committed_date) - \
+ datetime.timedelta(milliseconds=500)
+
+ else:
+ # there are no commits nor tags - abort
+ raise RuntimeError('package %s (id=%d) does not have commits ' \
+ 'or tags so I cannot devise a good starting date' % \
+ (package.name, package.id))
+
+
+def _get_tag_changelog(tag):
+
+ try:
+ return tag.release['description']
+ except Exception:
+ return ''
+
+
+def _write_one_tag(f, pkg_name, tag):
+ '''Prints commit information for a single tag of a given package
+
+ Args:
+
+ f: A :py:class:`File` ready to be written at
+ pkg_name: The name of the package we are writing tags of
+ tag: The tag value
+
+ '''
+
+ git_date = parse_date(tag.commit['committed_date'])
+ f.write(' * %s (%s)\n' % (tag.name, git_date.strftime('%b %d, %Y %H:%M')))
+
+ for line in _get_tag_changelog(tag).replace('\r\n', '\n').split('\n'):
+
+ line = line.strip()
+ if line.startswith('* ') or line.startswith('- '):
+ line = line[2:]
+
+ line = line.replace('!', pkg_name + '!').replace(pkg_name + \
+ pkg_name, pkg_name)
+ line = line.replace('#', pkg_name + '#')
+ if not line:
+ continue
+ f.write('%s* %s' % (5*' ', line))
+
+
+def _write_commits_range(f, pkg_name, commits):
+ '''Writes all commits of a given package within a range, to the output file
+
+ Args:
+
+ f: A :py:class:`File` ready to be written at
+ pkg_name: The name of the package we are writing tags of
+ commits: List of commits to be written
+
+ '''
+
+
+ for commit in commits:
+ commit_title = commit.title
+
+ # skip commits that do not carry much useful information
+ if '[skip ci]' in commit_title or \
+ 'Merge branch' in commit_title or \
+ 'Increased stable' in commit_title:
+ continue
+
+ commit_title = commit_title.strip()
+ commit_title = commit_title.replace('!', pkg_name + '!').replace(pkg_name + pkg_name, pkg_name)
+ commit_title = commit_title.replace('#', pkg_name + '#')
+ f.write('%s- %s' % (' ' * 5, commit_title))
+
+
+def _write_mergerequests_range(f, pkg_name, mrs):
+ '''Writes all merge-requests of a given package, with a range, to the
+ output file
+
+ Args:
+
+ f: A :py:class:`File` ready to be written at
+ pkg_name: The name of the package we are writing tags of
+ mrs: The list of merge requests to write
+
+ '''
+
+ for mr in mrs:
+ title = mr.title.strip().replace('\r','').replace('\n', ' ')
+ title = title.replace(' !', ' ' + pkg_name + '!')
+ title = title.replace(' #', ' ' + pkg_name + '#')
+ description = mr.description.strip().replace('\r','').replace('\n', ' ')
+ description = description.replace(' !', ' ' + pkg_name + '!')
+ description = description.replace(' #', ' ' + pkg_name + '#')
+ space = ': ' if description else ''
+ log = ''' - {pkg}!{iid} {title}{space}{description}'''
+ f.write(log.format(pkg=pkg_name, iid=mr.iid, title=title, space=space, description=description))
+ f.write('\n')
+
+
+def write_tags_with_commits(f, gitpkg, since, mode):
+ '''Writes all tags and commits of a given package to the output file
+
+ Args:
+
+ f: A :py:class:`File` ready to be written at
+ gitpkg: A pointer to the gitlab package object
+ since: Starting date (as a datetime object)
+ mode: One of mrs (merge-requests), commits or tags indicating how to
+ list entries in the changelog for this package
+
+ '''
+
+ # get tags since release and sort them
+ tags = gitpkg.tags.list()
+
+ # sort tags by date
+ tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
+ tags = _sort_tags(tags, reverse=False)
+
+ # get commits since release date and sort them too
+ commits = gitpkg.commits.list(since=since, all=True)
+
+ # sort commits by date
+ commits = _sort_commits(commits, reverse=False)
+
+ # get merge requests since the release data
+ mrs = list(reversed(gitpkg.mergerequests.list(state='merged', updated_after=since, order_by='updated_at', all=True)))
+ f.write('* %s\n' % (gitpkg.name,))
+
+ # go through tags and writes each with its message and corresponding
+ # commits
+ start_date = since
+ for tag in tags:
+
+ # write tag name and its text
+ _write_one_tag(f, gitpkg.name, tag)
+ end_date = parse_date(tag.commit['committed_date'])
+
+ if mode == 'commits':
+ # write commits from the previous tag up to this one
+ commits4tag = [k for k in commits \
+ if (start_date < parse_date(k.committed_date) <= end_date)]
+ _write_commits_range(f, gitpkg.name, commits4tag)
+
+ elif mode == 'mrs':
+ # write merge requests from the previous tag up to this one
+ # the attribute 'merged_at' is not available in GitLab API as of 27
+ # June 2018
+ mrs4tag = [k for k in mrs \
+ if (start_date < parse_date(k.updated_at) <= end_date)]
+ _write_mergerequests_range(f, gitpkg.name, mrs4tag)
+
+ start_date = end_date
+
+ if mode != 'tags':
+
+ # write the tentative patch version bump for the future tag
+ f.write(' * patch\n')
+
+ if mode == 'mrs':
+ # write leftover merge requests
+ # the attribute 'merged_at' is not available in GitLab API as of 27
+ # June 2018
+ leftover_mrs = [k for k in mrs \
+ if parse_date(k.updated_at) > start_date]
+ _write_mergerequests_range(f, gitpkg.name, leftover_mrs)
+
+ else:
+ # write leftover commits that were not tagged yet
+ leftover_commits = [k for k in commits \
+ if parse_date(k.committed_date) > start_date]
+ _write_commits_range(f, gitpkg.name, leftover_commits)
+
+
+def write_tags(f, gitpkg, since):
+ '''Writes all tags of a given package to the output file
+
+ Args:
+
+ f: A :py:class:`File` ready to be written at
+ gitpkg: A pointer to the gitlab package object
+ since: Starting date as a datetime object
+
+ '''
+
+ tags = gitpkg.tags.list()
+ # sort tags by date
+ tags = [k for k in tags if parse_date(k.commit['committed_date']) >= since]
+ tags = _sort_tags(tags, reverse=False)
+ f.write('* %s\n')
+
+ for tag in tags:
+ _write_one_tag(gitpkg.name, tag)
diff --git a/bob_tools/utils/conda.py b/bob/devtools/conda.py
similarity index 100%
rename from bob_tools/utils/conda.py
rename to bob/devtools/conda.py
diff --git a/bob/devtools/data/changelog_since_last_release.md b/bob/devtools/data/changelog_since_last_release.md
new file mode 100644
index 0000000000000000000000000000000000000000..aa9133553ed590d612660a57f5afe8ee5af13a75
--- /dev/null
+++ b/bob/devtools/data/changelog_since_last_release.md
@@ -0,0 +1,587 @@
+* bob.buildout
+ * patch
+ - bob.buildout!30 Update guide.rst for automated environment creation
+* bob.extension
+ * v3.0.0 (Jun 27, 2018 13:53)
+ - Breaking and significant changes:
+ - Removed bob_new_version script bob.extension!76
+ - Implemented `bob` click command bob.extension!64 bob.extension!73
+ bob.extension!77 bob.extension!74 bob.extension!75
+ - Detailed changes:
+ - bob.extension!75 Fix a bug when commands where invoked multiple times
+ - bob.extension!72 Added the function bob.extension.download_and_unzip in
+ the core functionalities: Closes bob.extension#50
+ - bob.extension!77 Fix list option
+ - bob.extension!64 Implementation of the bob script using click: Fixes
+ bob.extension#44
+ - bob.extension!79 added support for pytorch doc
+ - bob.extension!80 Add scikit-learn intersphinx mapping
+ - bob.extension!81 Add prefix aliasing for bob commands
+ - bob.extension!83 Fixed issue with bz2 files: For some reason, it is not
+ possible to open some `bz2` files for reading ('r:bz2') using the
+ `tarfile` module. For instance, this is failing with the `dlib`
+ landmarks model. If I use the `bz2` module it works. This patch uses
+ the `bz2` module for `bz2` compressed files.
+ - bob.extension!82 Resolve "The `-DBOOST_VERSION` flag has unnecessary and
+ unwanted quotes": Closes bob.extension#58
+ - bob.extension!76 Resolve "Documentation should be improved": [doc] added
+ the reference to NumPy style docstrings, added note on new package
+ instructions, added corresponding links Closes bob.extension#55
+ * minor
+ - Improved the help option on all Bob click commands !84
+ - Improved the click bool option !85
+ - Added functionality to dump config file when calling a ConfigCommand !86
+ - Add an ignore variable to log_parameters function (click helper) !88
+ - bob.extension!87 Add a common_attribute argument to config.load: Fixes
+ bob.extension#64
+* bob.blitz
+ * patch
+ - bob.blitz!11 [sphinx] Fixed doctest: Close bob.blitz#12
+* bob.core
+ * patch
+ - bob.core!17 Adapted the documentation to the new behavior of version
+ exporting: When merging bob.extension!82, this MR will update the
+ according documentation.
+ - bob.core!16 Use log modules from bob.extension
+* bob.io.base
+ * patch
+ - bob.io.base!25 Resolve "HDF5_VERSION is computed but never used": Closes
+ bob.io.base#19
+* bob.math
+ * patch
+ - bob.math!18 Fixing SVD test: Just fixing the sign of the last
+ eigenvector. More info check it out bob.math#10
+* bob.measure
+ * major
+ - Breaking and significant changes:
+ - Removed the old plotting scripts (``compute_perf.py``, etc.).
+ - Added a new 2-column score format specific to bob.measure.
+ - Implemented generic metrics and plotting scripts. Do ``bob measure
+ --help`` to see the new scripts and refer to the documentation.
+ - Matplotlib 2.2 and above is required now.
+ - Some biometric-related-only functionality is moved to bob.bio.base.
+ - Detailed changes:
+ - bob.measure!52 generic plotting script for bob measure: From
+ bob.measure#37, provide generic plotting scripts: * bob measure
+ evaluate * bob measure hist * bob measure hter * ...
+ - bob.measure!55 Change option name criter to criterion
+ - bob.measure!56 Fix error in context name
+ - bob.measure!57 Extend bins number option
+ - bob.measure!58 Change variable name form criter to criterion
+ - bob.measure!59 Bug fix: incorrect input file reading
+ - bob.measure!60 Bugfix
+ - bob.measure!54 Refactors the score loading and scripts functionality:
+ This merge request: * Move the biometric related functionality of
+ bob.measure to bob.bio.base. * Add confidence interval calculations *
+ Provide a score format for bob.measure with load functionalities *
+ Provide a generic plotting script bob measure in bob.measure using
+ bob.measure input file format: * `bob measure metrics`: to compute
+ thresholds and evaluate performances * `bob measure roc`: to plot
+ ROC * `bob measure det`: to plot DET * `bob measure hist`:
+ to plot histograms * `bob measure epc` : to plot EPC * `bob
+ measure evaluate`: applies all the above commands at once * `bob
+ measure gen`: to generate fake scores for `bob.measure` Each command
+ accepts one or several (dev,eval) score(s) file(s) for each system.
+ - bob.measure!61 Title and histograms subplots: Account for this
+ bob.bio.base!146#note_28760.
+ * Titles can be remove using an empty string, i.e. `-t ' '` *
+ Histograms support subplot display, see options `--subplots` * Add
+ `--legends-ncol` option
+ - bob.measure!62 Improve legends in histograms
+ - bob.measure!48 recompute far values in roc_for_far: Fixes bob.measure#27
+ - bob.measure!44 Compute roc using roc_for_far internally: Fixes
+ bob.measure#26
+ - bob.measure!43 Resolve "FAR and FRR thresholds are computed even when
+ there is no data support": Closes bob.measure#27 Also changes
+ behavior of far_threshold and frr_threshold where the returned threshold
+ guarantees the at most the requested far/frr value.
+ - bob.measure!63 Enable semilogx option in roc curves: * Fixes
+ bob.measure#40 * Remove 0 points on x-axis in semilogx plots.
+ Matplotlib 2 was doing this automatically before but matplotlib 2.2
+ doesn't * Code clean-up
+ - bob.measure!64 Fix semilog plots for non numpy arrays: Fixes
+ bob.bio.base#117
+ - bob.measure!65 Modification of criterion_option: Correct display of
+ available criteria
+ - bob.measure!66 Add prefix aliasing: Prefix aliasing using AliasedGroup
+ for bob.extension
+ - bob.measure!68 Change --eval option default and Various fixes: Change
+ defaults, clean unused options for histograms. Fix bob.bio.base#112
+ - bob.measure!69 Explain that the thresholds might not satisfy the
+ requested criteria: * Add a helper eer function for convenience.
+ - bob.measure!50 Generic loading input file: Add a generic input file
+ loading function for bob measure and its corresponding test. From issue
+ bob.measure#39, comments bob.measure#39#note_26366.
+ - bob.measure!71 Fix issues with matplotlib.hist.: For some reason
+ `mpl.hist` hangs when `n_bins` is set to `auto`. This is related with
+ https://github.com/numpy/numpy#8203 This MR set the default value to
+ `doane`. Furthermore, allows you to pick one of the options here
+ https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram.html#numpy.histogram
+ This will fix one of the points here bob.pad.base!43
+ - bob.measure!72 Various improvements: Fix bob.measure#41 and
+ bob.measure#42
+ - bob.measure!70 Improvements to new metrics codes
+ - bob.measure!73 Histogram legends: Fix bob.measure#44
+ - bob.measure!74 Bins histograms: Option only depends on the number of
+ data plotter per histograms, not the number of system. For example, pad
+ hist requires 2 nbins and vuln hist requires 3 independently on the
+ number of systems to be plotted. Fix bob.measure#45.
+ - bob.measure!75 Fix issue with histo legends: Fix bob.measure#47 Also
+ change default for number of legend columns. Add comments and doc in
+ the code for Hist.
+ - bob.measure!76 Histo fix: Fix typo and bob.pad.base!43#note_31884
+ - bob.measure!77 dd lines for dev histograms: See
+ bob.pad.base!43#note_31903
+ - bob.measure!78 Metrics: All stuff related to bob.measure#46.
+ - bob.measure!79 Add a command for multi protocol (N-fold cross
+ validation) analysis
+ - bob.measure!80 Consider non 1 values negatives: This will make
+ bob.measure scripts work with FOFRA scores
+ - bob.measure!81 Compute HTER using FMR and FNMR: As discussed in the
+ meeting. Fixes bob.measure#48
+ - bob.measure!83 Enable grid on all histograms
+ - bob.measure!84 Improve the constrained layout option
+ - bob.measure!82 Various fixes: * Change measure metrics * Document and
+ change HTER * add decimal precision option for metric * Use acronyms
+ instead of full names in figures * Remove filenames form figures and
+ add log output instead
+ - bob.measure!86 Fix decimal number control for metrics: Fixes
+ bob.measure#52
+ - bob.measure!88 Fix tests: Should fix bob.nightlies#40
+ - bob.measure!67 Titles: Allow list of titles and remove `(development)`
+ `(evaluation)` when default titles are modified
+ - bob.measure!45 Condapackage
+ - bob.measure!85 Fix broken commands cref bob.extension!86
+ - bob.measure!53 Change the way the scores arguments are passed to the
+ compute() function: it now: Change the way the scores arguments are
+ passed to the compute() function: it now does not rely on dev,eval pairs
+ anymore and can take any number of different files (e.g. train)
+ - bob.measure!87 Update documentation and commands: FAR->FPR, FRR->FNR:
+ Fix bob.measure#54
+* bob.io.image
+ * patch
+ - bob.io.image!41 Resolve "versions of image libraries are evaluated by
+ hand and given as parameters on the compiler command line": Closes
+ bob.io.image#32
+* bob.db.base
+ * patch
+ - bob.db.base!42 Handle errors when loading db interfaces: Fixes
+ bob.db.base#24
+* bob.io.video
+ * v2.1.1 (Apr 17, 2018 11:06)
+ - Sligthly increased noise test parameters on mp4 files with mpeg2video codecs (closed #11)
+* bob.io.matlab
+ * patch
+* bob.io.audio
+ * patch
+ - bob.io.audio!7 Resolve "version relies on compiler command line
+ parameter": Closes bob.io.audio#6
+* bob.sp
+ * patch
+* bob.ap
+ * patch
+ - [sphinx] Fixed doc tests
+* bob.ip.base
+ * patch
+ - bob.ip.base!16 Add a block_generator function. Fixes bob.ip.base#11
+ - [sphinx] Fixed doc tests
+* bob.ip.color
+ * patch
+* bob.ip.draw
+ * patch
+* bob.ip.gabor
+ * patch
+* bob.learn.activation
+ * patch
+* bob.learn.libsvm
+ * patch
+ - bob.learn.libsvm!9 Resolve "LIBSVM_VERSION is passed as a string to the
+ compiler, but evaluated as uint64_t in the code": Closes
+ bob.learn.libsvm#10
+* bob.learn.linear
+ * patch
+ - [sphinx] Fixed doc tests
+* bob.learn.mlp
+ * patch
+ - [sphinx] Fixed doc tests
+* bob.learn.boosting
+ * patch
+* bob.db.iris
+ * patch
+* bob.learn.em
+ * patch
+ - [sphinx] Fixed doc tests
+* bob.db.wine
+ * patch
+* bob.db.mnist
+ * patch
+* bob.db.atnt
+ * patch
+* bob.ip.facedetect
+ * patch
+ - [sphinx] Fixed doc tests
+* bob.ip.optflow.hornschunck
+ * patch
+* bob.ip.optflow.liu
+ * patch
+* bob.ip.flandmark
+ * patch
+* gridtk
+ * patch
+ - gridtk!21 remove the submitted command line column when not long: Fixes
+ gridtk#27
+ - gridtk!20 Resolve "jman fails when jobs are deleted before they are
+ finished": Closes gridtk#26
+ - gridtk!22 Memory argument sets gpumem parameter for gpu queues: Fixes
+ gridtk#24
+ - gridtk!23 Accept a plus sign for specifying the job ranges
+* bob.ip.qualitymeasure
+ * patch
+* bob.ip.skincolorfilter
+ * patch
+* bob.ip.facelandmarks
+ * patch
+* bob.ip.dlib
+ * patch
+ - bob.ip.dlib!13 Removed bob_to_dlib_image_convertion functions and
+ replaced them to bob.io.image.to_matplotlib: CLoses bob.ip.dlib#6
+ - bob.ip.dlib!11 Fix in the download mechanism
+* bob.ip.mtcnn
+ * v0.0.1 (May 23, 2018 17:56)
+ * First release
+ - bob.ip.mtcnn!4 Added conda recipe: Closes bob.ip.mtcnn#2 We don't
+ have a caffe for Mac OSX in the defaults channel. This can be an issue
+ for the bob.pad.face builds.
+ - bob.ip.mtcnn!5 Deleted the fuctions bob_to_dlib_image_convertion and…:
+ Deleted the fuctions bob_to_dlib_image_convertion and
+ dlib_to_bob_image_convertion and replaced them by the
+ bob.io.color.to_matplotlib Closes bob.ip.mtcnn#3 [sphinx] Fixed
+ plots
+ - bob.ip.mtcnn!6 [conda] Fixed conda recipe for the nightlies build issue
+ bob.ip.mtcnn#5: Closes bob.ip.mtcnn#5
+ * v1.0.0 (May 23, 2018 20:01)
+ * First release
+ * patch
+* bob.db.arface
+ * patch
+* bob.db.asvspoof
+ * patch
+* bob.db.asvspoof2017
+ * patch
+ - bob.db.asvspoof2017!5 Update link to avspoof 2017
+* bob.db.atvskeystroke
+ * patch
+* bob.db.avspoof
+ * patch
+* bob.db.banca
+ * patch
+* bob.db.biosecure
+ * patch
+* bob.db.biosecurid.face
+ * patch
+* bob.db.casme2
+ * patch
+* bob.db.caspeal
+ * patch
+* bob.db.cohface
+ * patch
+* bob.db.frgc
+ * patch
+* bob.db.gbu
+ * patch
+* bob.db.hci_tagging
+ * patch
+* bob.db.kboc16
+ * patch
+* bob.db.lfw
+ * patch
+* bob.db.livdet2013
+ * patch
+* bob.db.mobio
+ * patch
+* bob.db.msu_mfsd_mod
+ * patch
+* bob.db.multipie
+ * patch
+* bob.db.nist_sre12
+ * patch
+* bob.db.putvein
+ * patch
+* bob.db.replay
+ * patch
+* bob.db.replaymobile
+ * patch
+ - bob.db.replaymobile!10 Ignore all exceptions when closing the session
+* bob.db.scface
+ * patch
+* bob.db.utfvp
+ * patch
+* bob.db.verafinger
+ * patch
+* bob.db.fv3d
+ * patch
+* bob.db.hkpu
+ * patch
+ - bob.db.hkpu!2 Antecipated issue with conda-build. Check bob.db.ijbc!1
+* bob.db.thufvdt
+ * patch
+* bob.db.mmcbnu6k
+ * patch
+ - bob.db.mmcbnu6k!2 Antecipated issue with conda-build. Check
+ bob.db.ijbc!1
+* bob.db.hmtvein
+ * patch
+ - bob.db.hmtvein!2 Antecipated issue with conda-build. Check bob.db.ijbc!1
+* bob.db.voicepa
+ * patch
+* bob.db.xm2vts
+ * patch
+* bob.db.youtube
+ * patch
+* bob.db.pericrosseye
+ * patch
+* bob.bio.base
+ * major
+ - Breaking and significant changes
+ - Removed the old ``evaluate.py`` script.
+ - Functionality to load biometric scores are now in ``bob.bio.base.score``
+ - Added new scripts for plotting and evaluations. Refer to docs.
+ - Added a new baselines concept. Refer to docs.
+ - Detailed changes
+ - bob.bio.base!147 Update installation instructions since conda's usage
+ has changed.
+ - bob.bio.base!148 Archive CSU: closes bob.bio.base#109
+ - bob.bio.base!146 Add 4-5-col files related functionalities and add
+ click commands: In this merge: * Add loading functionalities from
+ `bob.measure` * Add the following click commands (as substitutes for
+ old script evaluate.py) using 4- or 5 - scores input files: * `bob
+ bio metrics` * `bob bio roc` * `bob bio det` * `bob bio
+ epc` * `bob bio hist` * `bob bio evaluate` : calls all the
+ above commands at once * `bob bio cmc` * `bob bio dic` *
+ `bob bio gen` Plots follow ISO standards. The underlying
+ implementation of the mentioned commands uses `bob.measure` base
+ classes. Fixes bob.bio.base#108
+ - bob.bio.base!149 Set io-big flag for the demanding grid config: Closes
+ bob.bio.base#110 Anyone cares to review this one? It's harmless.
+ - bob.bio.base!143 Set of click commands for bio base: From
+ bob.bio.base#65 Provide commands in bio base: - bob bio metrics
+ - bob bio roc - bob bio evaluate (Very similar to evalute.py)
+ - bob.bio.base!152 Removed unused import imp and solving bob.bio.base#83:
+ Closes bob.bio.base#83
+ - bob.bio.base!153 Added the protocol argument issue bob.bio.base#111:
+ Closes bob.bio.base#111
+ - bob.bio.base!154 Fixes in ROC and DET labels
+ - bob.bio.base!157 Fixed bob bio dir x_labels and y_labels: The labels of
+ the DIR plot were incorrect.
+ - bob.bio.base!155 Write parameters in a temporary config file to enable
+ chain loading: Fixes bob.bio.base#116
+ - bob.bio.base!150 Exposing the method groups in our FileDatabase API
+ - bob.bio.base!158 Add prefix aliasing for Click commands
+ - bob.bio.base!160 Titltes: Allows a list of titles Fixes
+ bob.bio.base#121. Requires bob.measure!67
+ - bob.bio.base!159 Resolve "Documentation does not include a link to the
+ recordings of the IJCB tutorial": Closes bob.bio.base#122
+ - bob.bio.base!161 Change --eval option default and Various fixes: fixes
+ bob.bio.base#112. Add and clean histo options. See
+ bob.measure!67#note_30951 Requires bob.measure!68
+ - bob.bio.base!163 Reduce repition between commands: Depends on
+ bob.measure!70
+ - bob.bio.base!162 Removed traces of evaluate.py in the documentation
+ - bob.bio.base!164 Fix test according to changes in nbins option
+ - bob.bio.base!165 Set names for different bio metrics: Bio specific names
+ for metrics when using bob.measure Metrics
+ - bob.bio.base!166 Add a command for multi protocol (N-fold cross
+ validation) analysis: Similar to bob.measure!79
+ - bob.bio.base!167 Various fixes: Requires bob.measure!82 Similar to
+ bob.measure!82 for bio commands
+ - bob.bio.base!168 Documentation changes in bob bio annotate: Depends on
+ bob.extension!86
+ - bob.bio.base!156 Using the proper verify script depending on system:
+ Closes bob.bio.base#119
+ - bob.bio.base!151 Created the Baselines Concept
+ - bob.bio.base!169 Change assert to assert_click_runner_result
+* bob.bio.gmm
+ * patch
+ - bob.bio.gmm!19 Fix bob.measure->bob.bio.base related issues: Fixes
+ bob.bio.gmm#25
+ - bob.bio.gmm!20 argument allow_missing_files wrongly passed to qsub
+ - bob.bio.gmm!21 IVector - Fix LDA rank: With this MR we make sure that
+ the dimension of the LDA matrix is not higher than its rank.
+* bob.bio.face
+ * major
+ - Breaking changes:
+ - Dropped support for CSU baselines
+ - Detailed changes:
+ - bob.bio.face!47 Accept an annotator in FaceCrop: related to
+ bob.bio.face#26
+ - bob.bio.face!48 Dropped support to CSU baselines issue bob.bio.face#29
+ - bob.bio.face!51 Removing baselines: Related to this MR bob.bio.face!49
+ - bob.bio.face!50 Ijbc highlevel
+ - bob.bio.face!49 Refactoring baselines: Now we can all the facerec
+ baselines can be reached via ``bob bio baselines --help``. Refer to
+ docs.
+* bob.bio.spear
+ * patch
+ - bob.bio.spear!40 Handle mute audio correctly: Fixes bob.bio.spear#31
+* bob.bio.video
+ * patch
+ - bob.bio.video!35 Add video wrappers using chain loading: Fixes
+ bob.bio.video#12
+ - bob.bio.video!36 Load videos frame by frame in annotators: Related to
+ bob.bio.video#13
+* bob.bio.vein
+ * patch
+ - bob.bio.vein!42 Fix bob.measure->bob.bio.base related issues
+ - bob.bio.vein!43 Using the new bob.bio API
+ - bob.bio.vein!44 Fixing comparison: Closes bob.bio.vein#18
+* bob.db.voxforge
+ * patch
+ - bob.db.voxforge!20 Handling the --help command in download_and_untar
+ script: Fixes bob.db.voxforge#12
+ - bob.db.voxforge!18 Fixed the index.rst to match joint docs requirements
+* bob.rppg.base
+ * v1.1.0 (Jun 22, 2018 08:58)
+ - Initial conda release for rPPG algorithms
+ - Bug correction in the CHROM algorithm
+ - Python 2 and 3 compatibility
+ - Improved documentation (mostly Python API)
+ * v2.0.0 (Jun 27, 2018 15:52)
+ - getting rid of bob.db.* dependencies
+ - usage of configuration files
+ * patch
+ - bob.rppg.base!8 Resolve "Potential bug when computing average color from
+ mask": Closes bob.rppg.base#15
+ - bob.rppg.base!7 Fixed recipe to test all modules
+* bob.pad.base
+ * minor
+ - Significant changes:
+ - Added new plotting and evaluation scripts. Refer to docs.
+ - Detailed Changes:
+ - bob.pad.base!52 Change assert to assert_click_runner_result
+ - bob.pad.base!50 Add new classification algorithms: As mentioned here
+ bob.pad.base!33 here is the new branch
+ - bob.pad.base!51 Fix broken commands cref bob.extension!86
+ - bob.pad.base!48 Various fix: Requires bob.measure!82 Similar to
+ bob.measure!82 for PAD
+ - bob.pad.base!45 Allow PAD filelist database to be used in vulnerability
+ experiments
+ - bob.pad.base!47 Remove bob vuln metrics and evaluate commands: since
+ they were not well defined and we do not know what should be in there.
+ rename --hlines-at to --fnmr in bob vuln roc,det commands small
+ nit-pick fixes overall
+ - bob.pad.base!43 Finalization of plots: Fixes bob.pad.base#22. Requires
+ merge bob.measure!65. Requires bob.measure!67
+ - bob.pad.base!46 Add a command for multi protocol (N-fold cross
+ validation) analysis: Depends on bob.measure!79
+ - bob.pad.base!44 Remove the grid_search.py entrypoint: Fixes
+ bob.pad.base#23
+ - bob.pad.base!41 Set of click commands for pad: Provide commands: bob
+ pad metrics bob pad epc bob pad epsc bob pad gen
+ - bob.pad.base!42 Set of click commands for pad: Provide commands: bob
+ pad metrics bob pad epc bob pad epsc bob pad gen
+ - added MLP and LDA classifiers for PAD
+* bob.pad.face
+ * minor
+ - bob.pad.face!68 Improve load_utils.py: * Plus some minor fixes to the
+ frame-diff method
+ - bob.pad.face!67 HLDI for the CelebA database and quality estimation
+ script: This MR contains two contributions: 1. The High Level DB
+ interface for the CelebA database. 2. The quality assessment script +
+ config file allowing to estimate the quality of preprocessed CelebA
+ images.
+ - bob.pad.face!66 Added an option to exclude specific types of attacks
+ from train set in BATL DB: This allows to exclude specific PAIs from the
+ training set of the BATL DB. PAIs currently handled: makeup.
+ - bob.pad.face!65 Updated the HLDI of BATL DB, added FunnyEyes fix, and
+ protocol joining test and dev sets
+ - bob.pad.face!64 Add support for external annotations in replaymobile
+ - bob.pad.face!60 Change the API of yield_faces
+ - bob.pad.face!62 Add a script for analyzing database face sizes
+ - added preprocessors and extractors for pulse-based PAD.
+* bob.pad.voice
+ * v1.0.5 (Jun 27, 2018 16:00)
+ - Fix imports and add an alias for PadVoiceFile
+ - experimental audio eval: support for 2, 3 layers lstms
+ - Fix error related to bob.measure->bob.bio.base movings Fixes
+ bob.pad.voicebob.pad.voice#4
+ - bob.pad.voice!14 Fix error related to bob.measure->bob.bio.base movings:
+ Fixes bob.pad.voice#4
+ - bob.pad.voice!15 Fix imports and add an alias for PadVoiceFile
+ * patch
+ - Fix avspoof db interface (added annotations method) !16
+* bob.pad.vein
+ * patch
+ - Fix database tests
+* bob.fusion.base
+ * patch
+ - bob.fusion.base!7 Major refactoring: Fixes bob.fusion.base#3 Fixes
+ biometric/software#7 Fixes biometric/software#8
+ - bob.fusion.base!8 improve behaviour and tests
+ - bob.fusion.base!9 Fix the boundary script due to changes in
+ bob.extension: Fixes bob.fusion.base#7
+* bob.db.oulunpu
+ * patch
+ - bob.db.oulunpu!3 Training depends on protocol
+ - bob.db.oulunpu!4 Add protocol 1_2
+* bob.db.uvad
+ * patch
+ - bob.db.uvad!2 Convert annotations to properties
+* bob.db.swan
+ * patch
+ - bob.db.swan!2 DRY
+ - bob.db.swan!3 Recreate the PAD protocols and add a global face and voice
+ pad protocol
+* bob.db.cuhk_cufs
+ * v2.2.1 (May 23, 2018 16:55)
+ * Removed deprecated entrypoints
+ - bob.db.cuhk_cufs!8 Removing deprecated entry-points: Removing deprecated
+ entry-points fixing bob.db.cuhk_cufs!7
+ * patch
+* bob.db.cbsr_nir_vis_2
+ * v2.0.2 (May 23, 2018 17:35)
+ * Removed deprecated entrypoints
+ * Ported to the new CI
+ * patch
+* bob.db.nivl
+ * v0.0.1 (May 23, 2018 18:15)
+ * First release
+ * v1.0.0 (May 24, 2018 11:37)
+ * First release
+ * patch
+* bob.db.pola_thermal
+ * v0.0.1 (May 23, 2018 18:57)
+ * First release
+ * v1.0.0 (May 24, 2018 11:54)
+ * First release
+ * patch
+* bob.db.cuhk_cufsf
+ * v0.0.1 (May 23, 2018 19:10)
+ * First release
+ * v1.0.0 (May 24, 2018 12:09)
+ * First release
+ * patch
+* bob.db.ijba
+ * patch
+ - bob.db.ijba!13 Fix bob.measure->bob.bio.base related issues
+* bob.ip.tensorflow_extractor
+ * v0.0.2 (Apr 27, 2018 15:16)
+ - added DR-GAN extractor
+ * patch
+ - Replaced the download_model method to the new one implemented in
+ bob.extension !7
+ - Fixed conda-build issue !8
+* bob.ip.caffe_extractor
+ * v2.0.0 (Jun 30, 2018 12:22)
+ * Added Light CNN bob.ip.caffe_extractor!6 bob.ip.caffe_extractor!7
+ * Created conda package of it bob.ip.caffe_extractor!9
+ * Replaced the download_model method to the new one implemented in
+ bob.extension bob.ip.caffe_extractor!10
+ - bob.ip.caffe_extractor!10 Replaced the download_model method to the new
+ one implemented in bob.extension: Closes bob.ip.caffe_extractor#6 I
+ already updated the URL in this one
+ * patch
+* bob.bio.caffe_face
+ * patch
+* bob.db.maskattack
+ * patch
+ - first release
diff --git a/bob/devtools/release.py b/bob/devtools/release.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0b780f119a008986f693a8649e4042f5edf98fb
--- /dev/null
+++ b/bob/devtools/release.py
@@ -0,0 +1,464 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import time
+import gitlab
+
+import logging
+logger = logging.getLogger(__name__)
+
+from distutils.version import StrictVersion
+
+
+def get_gitlab_instance():
+ '''Returns an instance of the gitlab object for remote operations'''
+
+ # tries to figure if we can authenticate using a global configuration
+ cfgs = ['~/.python-gitlab.cfg', '/etc/python-gitlab.cfg']
+ cfgs = [os.path.expanduser(k) for k in cfgs]
+ if any([os.path.exists(k) for k in cfgs]):
+ gl = gitlab.Gitlab.from_config('idiap', cfgs)
+ else: #ask the user for a token
+ server = "https://gitlab.idiap.ch"
+ token = input("%s token: " % server)
+ gl = gitlab.Gitlab(server, private_token=token, api_version=4)
+
+ return gl
+
+
+def ensure_correct_package(candidates, group_name, pkg_name):
+
+ for pkg in candidates:
+ # make sure the name and the group name match exactly
+ if pkg.name == pkg_name and pkg.namespace['name'] == group_name:
+ return pkg
+
+ raise ValueError('Package "{0}" was not found inside group "{1}"'.format(pkg_name, group_name))
+
+
+def _update_readme(readme, version):
+ """
+ Inside text of the readme, replaces parts of the links to the provided
+ version. If version is not provided, replace to `stable` or `master`.
+
+ Args:
+
+ readme: Text of the README.rst file from a bob package
+ version: Format of the version string is '#.#.#'
+
+ Returns: New text of readme with all replaces done
+ """
+
+ # replace the badge in the readme's text with the given version
+ DOC_IMAGE = re.compile(r'\-(stable|(v\d+\.\d+\.\d+([abc]\d+)?))\-')
+ BRANCH_RE = re.compile(r'/(stable|master|(v\d+\.\d+\.\d+([abc]\d+)?))')
+
+ new_readme = []
+ for line in readme.splitlines():
+ if BRANCH_RE.search(line) is not None:
+ if "gitlab" in line: # gitlab links
+ replacement = "/v%s" % version if version is not None \
+ else "/master"
+ line = BRANCH_RE.sub(replacement, line)
+ if ("software/bob" in line) or \
+ ("software/beat" in line): # our doc server
+ if 'master' not in line: # don't replace 'latest' pointer
+ replacement = "/v%s" % version if version is not None \
+ else "/stable"
+ line = BRANCH_RE.sub(replacement, line)
+ if DOC_IMAGE.search(line) is not None:
+ replacement = '-v%s-' % version if version is not None \
+ else '-stable-'
+ line = DOC_IMAGE.sub(replacement, line)
+ new_readme.append(line)
+ return '\n'.join(new_readme) + '\n'
+
+
+def get_latest_tag_name(gitpkg):
+ """Find the name of the latest tag for a given package in the format '#.#.#'
+
+ Args:
+ gitpkg: gitlab package object
+
+ Returns: The name of the latest tag in format '#.#.#'. None if no tags for
+ the package were found.
+ """
+
+ # get 50 latest tags as a list
+ latest_tags = gitpkg.tags.list(all=True)
+ if not latest_tags:
+ return None
+ # create list of tags' names but ignore the first 'v' character in each name
+ # also filter out non version tags
+ tag_names = [tag.name[1:] for tag in latest_tags \
+ if StrictVersion.version_re.match(tag.name[1:])]
+ # sort them correctly according to each subversion number
+ tag_names.sort(key=StrictVersion)
+ # take the last one, as it is the latest tag in the sorted tags
+ latest_tag_name = tag_names[-1]
+ return latest_tag_name
+
+
+def get_parsed_tag(gitpkg, tag):
+ """
+ An older tag is formatted as 'v2.1.3 (Sep 22, 2017 10:37)', from which we
+ need only v2.1.3
+
+ The latest tag is either patch, minor, major, or none
+ """
+
+ m = re.search(r"(v\d+.\d+.\d+)", tag)
+ if m:
+ return m.group(0)
+ # tag = Version(tag)
+
+ # if we bump the version, we need to find the latest released version for
+ # this package
+ if 'patch' == tag or 'minor' == tag or 'major' == tag:
+
+ # find the correct latest tag of this package (without 'v' in front),
+ # None if there are no tags yet
+ latest_tag_name = get_latest_tag_name(gitpkg)
+
+ # if there were no tags yet, assume the very first version
+ if not latest_tag_name: return 'v0.0.1'
+
+ # check that it has expected format #.#.#
+ # latest_tag_name = Version(latest_tag_name)
+ m = re.match(r"(\d.\d.\d)", latest_tag_name)
+ if not m:
+ raise ValueError('The latest tag name {0} in package {1} has ' \
+ 'unknown format'.format('v' + latest_tag_name, gitpkg.name))
+
+ # increase the version accordingly
+ major, minor, patch = latest_tag_name.split('.')
+
+ if 'major' == tag:
+ # increment the first number in 'v#.#.#' but make minor and patch
+ # to be 0
+ return 'v' + str(int(major) + 1) + '.0.0'
+
+ if 'minor' == tag:
+ # increment the second number in 'v#.#.#' but make patch to be 0
+ return 'v' + major + '.' + str(int(minor) + 1) + '.0'
+
+ if 'patch' == tag:
+ # increment the last number in 'v#.#.#'
+ return 'v' + major + '.' + minor + '.' + str(int(patch) + 1)
+
+ if 'none' == tag:
+ # we do nothing in this case
+ return tag
+
+ raise ValueError('Cannot parse changelog tag {0} of the ' \
+ 'package {1}'.format(tag, gitpkg.name))
+
+
+def update_tag_comments(gitpkg, tag_name, tag_comments_list, dry_run=False):
+ """Write annotations inside the provided tag of a given package.
+
+ Args:
+
+ gitpkg: gitlab package object
+ tag_name: The name of the tag to update
+ tag_comments_list: New annotations for this tag in a form of list
+ dry_run: If True, nothing will be committed or pushed to GitLab
+
+ Returns: The gitlab object for the tag that was updated
+ """
+
+ # get tag and update its description
+ logger.info(tag_name)
+ tag = gitpkg.tags.get(tag_name)
+ tag_comments = '\n'.join(tag_comments_list)
+ logger.info('Found tag %s, updating its comments with:\n%s', tag.name,
+ tag_comments)
+ if not dry_run: tag.set_release_description(tag_comments)
+ return tag
+
+
+def commit_files(gitpkg, files_dict, message='Updated files', dry_run=False):
+ """Commit files of a given GitLab package.
+
+ Args:
+
+ gitpkg: gitlab package object
+ files_dict: Dictionary of file names and their contents (as text)
+ message: Commit message
+ dry_run: If True, nothing will be committed or pushed to GitLab
+
+ """
+
+ data = {
+ 'branch': 'master', # v4
+ 'commit_message': message,
+ 'actions': []
+ }
+
+ # add files to update
+ for filename in files_dict.keys():
+ update_action = dict(action='update', file_path=filename)
+ update_action['content'] = files_dict[filename]
+ data['actions'].append(update_action)
+
+ logger.info("Committing changes in files: %s", str(files_dict.keys()))
+ if not dry_run:
+ gitpkg.commits.create(data)
+
+
+def get_last_pipeline(gitpkg):
+ """Returns the last pipeline of the project
+
+ Args:
+
+ gitpkg: gitlab package object
+
+ Returns: The gtilab object of the pipeline
+ """
+
+ # wait for 10 seconds to ensure that if a pipeline was just submitted,
+ # we can retrieve it
+ time.sleep(10)
+
+ # get the last pipeline
+ return gitpkg.pipelines.list(per_page=1, page=1)[0]
+
+
+def just_build_package(gitpkg, dry_run=False):
+ """Creates the pipeline with the latest tag and starts it
+
+ Args:
+
+ gitpkg: gitlab package object
+ dry_run: If True, the pipeline will not be created on GitLab
+
+ Returns:
+
+ """
+
+ # get the latest tag
+ latest_tag_name = 'v' + get_latest_tag_name(gitpkg)
+
+ # create the pipeline with this tag and start it
+ logger.info("Creating and starting pipeline for tag %s", latest_tag_name)
+
+ if not dry_run:
+ new_pipeline = gitpkg.pipelines.create({'ref': latest_tag_name})
+ return new_pipeline.id
+
+ return None
+
+
+def wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run=False):
+ """Using sleep function, wait for the latest pipeline to finish building.
+
+ This function pauses the script until pipeline completes either
+ successfully or with error.
+
+ Args:
+
+ gitpkg: gitlab package object
+ pipeline_id: id of the pipeline for which we are waiting to finish
+ dry_run: If True, outputs log message and exit. There wil be no
+ waiting.
+
+ """
+
+ sleep_step = 30
+ max_sleep = 120 * 60 # two hours
+ # pipeline = get_last_pipeline(gitpkg, before_last=before_last)
+
+ logger.info('Waiting for the pipeline %s of package %s to finish. ' \
+ 'Do not interrupt.', pipeline_id, gitpkg.name)
+
+ if dry_run: return
+
+ # retrieve the pipeline we are waiting for
+ pipeline = gitpkg.pipelines.get(pipeline_id)
+
+ # probe and wait for the pipeline to finish
+ slept_so_far = 0
+
+ while pipeline.status == 'running' or pipeline.status == 'pending':
+
+ time.sleep(sleep_step)
+ slept_so_far += sleep_step
+ if slept_so_far > max_sleep:
+ raise ValueError('I cannot wait longer than {0} seconds for '
+ 'pipeline {1} to finish running!'.format(max_sleep, pipeline_id))
+ # probe gitlab to update the status of the pipeline
+ pipeline = gitpkg.pipelines.get(pipeline_id)
+
+ # finished running, now check if it succeeded
+ if pipeline.status != 'success':
+ raise ValueError('Pipeline {0} of project {1} exited with ' \
+ 'undesired status "{2}". Release is not possible.' \
+ .format(pipeline_id, gitpkg.name, pipeline.status))
+
+ logger.info('Pipeline %s of package %s SUCCEEDED. Continue processing.',
+ pipeline_id, gitpkg.name)
+
+
+def cancel_last_pipeline(gitpkg):
+ """ Cancel the last started pipeline of a package
+
+ Args:
+
+ gitpkg: gitlab package object
+
+ """
+
+ pipeline = get_last_pipeline(gitpkg)
+ logger.info('Cancelling the last pipeline %s of project %s', pipeline.id,
+ gitpkg.name)
+ pipeline.cancel()
+
+
+def release_package(gitpkg, tag_name, tag_comments_list, dry_run=False):
+ """Release package
+
+ The provided tag will be annotated with a given list of comments.
+ README.rst and version.txt files will also be updated according to the
+ release procedures.
+
+ Args:
+
+ gitpkg: gitlab package object
+ tag_name: The name of the release tag
+ tag_comments_list: New annotations for this tag in a form of list
+ dry_run: If True, nothing will be committed or pushed to GitLab
+
+ """
+
+ # if there is nothing to release, just rebuild the package
+ latest_tag = get_latest_tag_name(gitpkg)
+
+ if tag_name == 'none' or (latest_tag and ('v' + latest_tag) == tag_name):
+ logger.warn("Since the tag is 'none' or already exists, we just " \
+ "re-build the last pipeline")
+ return just_build_package(gitpkg, dry_run)
+
+ # 1. Replace branch tag in Readme to new tag, change version file to new
+ # version tag. Add and commit to gitlab
+ version_number = tag_name[1:] # remove 'v' in front
+ readme_file = gitpkg.files.get(file_path='README.rst', ref='master')
+ readme_content = readme_file.decode().decode()
+ readme_content = _update_readme(readme_content, version_number)
+ # commit and push changes
+ commit_files(gitpkg,
+ {
+ 'README.rst': readme_content,
+ 'version.txt': version_number
+ },
+ 'Increased stable version to %s' % version_number, dry_run)
+
+ if not dry_run:
+ # cancel running the pipeline triggered by the last commit
+ cancel_last_pipeline(gitpkg)
+
+ # 2. Tag package with new tag and push
+ logger.info("Creating tag %s", tag_name)
+ tag_comments = '\n'.join(tag_comments_list)
+ logger.info("Updating tag comments with:\n%s", tag_comments)
+ if not dry_run:
+ tag = gitpkg.tags.create({'tag_name': tag_name, 'ref': 'master'})
+ # update tag with comments
+ tag.set_release_description(tag_comments)
+
+ # get the pipeline that is actually running with no skips
+ running_pipeline = get_last_pipeline(gitpkg)
+
+ # 3. Replace branch tag in Readme to master, change version file to beta
+ # version tag. Git add, commit, and push.
+ readme_content = _update_readme(readme_content, None)
+ major, minor, patch = version_number.split('.')
+ version_number = '{}.{}.{}b0'.format(major, minor, int(patch)+1)
+ # commit and push changes
+ commit_files(gitpkg, {
+ 'README.rst': readme_content,
+ 'version.txt': version_number,
+ },
+ 'Increased latest version to %s [skip ci]' % version_number, dry_run)
+
+ return running_pipeline.id
+
+
+def parse_and_process_package_changelog(gl, bob_group, pkg_name,
+ package_changelog, dry_run):
+ """Process the changelog of a single package
+
+ Parse the log following specific format. Update annotations of the
+ provided older tags and release the package by following the last tag
+ description.
+
+ Args:
+
+ gl: Gitlab API object
+ bob_group: gitlab object for the group
+ pkg_name: name of the package
+ package_changelog: the changelog corresponding to the provided package
+ dry_run: If True, nothing will be committed or pushed to GitLab
+
+ Returns: gitlab handle for the package, name of the latest tag, and tag's
+ comments
+
+ """
+
+ cur_tag = None
+ cur_tag_comments = []
+
+ grpkg = ensure_correct_package(bob_group.projects.list(search=pkg_name),
+ bob_group.name, pkg_name)
+
+ # so, we need to retrieve the full info from GitLab using correct project id
+ gitpkg = gl.projects.get(id=grpkg.id)
+
+ # we assume that changelog is formatted as structured text
+ # first line is the name of the package
+ for line in package_changelog:
+ if ' *' == line[:3]: # a tag level
+ # write the comments collected for the previous tag
+ if cur_tag:
+ update_tag_comments(gitpkg, cur_tag, cur_tag_comments, dry_run)
+ cur_tag_comments = [] # reset comments
+
+ # parse the current tag name
+ cur_tag = get_parsed_tag(gitpkg, line[3:].strip())
+
+ else: # all other lines are assumed to be comments
+ cur_tag_comments.append(line.strip())
+
+ # return the last tag and comments for release
+ return gitpkg, cur_tag, cur_tag_comments
+
+
+def release_bob(changelog_file):
+ """Process the changelog and releases the ``bob`` metapackage"""
+
+ logger.info('Read the section "Releasing the Bob meta package" ' \
+ 'on the documentation')
+
+ # get the list of bob's dependencies.
+ # Get their latest tags (since bob's last release) and the tag's changelog
+ saw_a_new_package = True
+ latest_tag = None
+ latest_pkg = None
+ for line in changelog_file:
+ # if saw_a_new_package:
+ if line.startswith('*'):
+ pkg = line[2:].strip()
+ saw_a_new_package = True
+ logger.info('%s == %s', latest_pkg, latest_tag)
+ latest_pkg = pkg
+ latest_tag = None
+ continue
+ if line.startswith(' *'):
+ latest_tag = line.split()[1][1:]
+ saw_a_new_package = False
+ logger.info('%s == %s', latest_pkg, latest_tag)
+ readme = open('../../bob/README.rst').read()
+ readme = _update_readme(readme, bob_version)
+ open('../../bob/README.rst', 'wt').write(readme)
+ open('../../bob/version.txt', 'wt').write(bob_version)
diff --git a/bob_tools/scripts/__init__.py b/bob/devtools/scripts/__init__.py
similarity index 100%
rename from bob_tools/scripts/__init__.py
rename to bob/devtools/scripts/__init__.py
diff --git a/bob/devtools/scripts/bdt.py b/bob/devtools/scripts/bdt.py
new file mode 100644
index 0000000000000000000000000000000000000000..4af5720cea8c6ee66deaf62292efb77609f52e14
--- /dev/null
+++ b/bob/devtools/scripts/bdt.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Main entry point for bdt
+"""
+
+import pkg_resources
+
+import click
+from click_plugins import with_plugins
+
+import logging
+logger = logging.getLogger('bdt')
+
+
+def set_verbosity_level(logger, level):
+ """Sets the log level for the given logger.
+
+ Parameters
+ ----------
+ logger : :py:class:`logging.Logger` or str
+ The logger to generate logs for, or the name of the module to generate
+ logs for.
+ level : int
+ Possible log levels are: 0: Error; 1: Warning; 2: Info; 3: Debug.
+ Raises
+ ------
+ ValueError
+ If the level is not in range(0, 4).
+ """
+ if level not in range(0, 4):
+ raise ValueError(
+ "The verbosity level %d does not exist. Please reduce the number "
+ "of '--verbose' parameters in your command line" % level
+ )
+ # set up the verbosity level of the logging system
+ log_level = {
+ 0: logging.ERROR,
+ 1: logging.WARNING,
+ 2: logging.INFO,
+ 3: logging.DEBUG
+ }[level]
+
+ # set this log level to the logger with the specified name
+ if isinstance(logger, str):
+ logger = logging.getLogger(logger)
+ logger.setLevel(log_level)
+
+
+def verbosity_option(**kwargs):
+ """Adds a -v/--verbose option to a click command.
+
+ Parameters
+ ----------
+ **kwargs
+ All kwargs are passed to click.option.
+
+ Returns
+ -------
+ callable
+ A decorator to be used for adding this option.
+ """
+ def custom_verbosity_option(f):
+ def callback(ctx, param, value):
+ ctx.meta['verbosity'] = value
+ set_verbosity_level(logger, value)
+ logger.debug("`bdt' logging level set to %d", value)
+ return value
+ return click.option(
+ '-v', '--verbose', count=True,
+ expose_value=False, default=2,
+ help="Increase the verbosity level from 0 (only error messages) "
+ "to 1 (warnings), 2 (info messages), 3 (debug information) by "
+ "adding the --verbose option as often as desired "
+ "(e.g. '-vvv' for debug).",
+ callback=callback, **kwargs)(f)
+ return custom_verbosity_option
+
+
+class AliasedGroup(click.Group):
+ ''' Class that handles prefix aliasing for commands '''
+ def get_command(self, ctx, cmd_name):
+ rv = click.Group.get_command(self, ctx, cmd_name)
+ if rv is not None:
+ return rv
+ matches = [x for x in self.list_commands(ctx)
+ if x.startswith(cmd_name)]
+ if not matches:
+ return None
+ elif len(matches) == 1:
+ return click.Group.get_command(self, ctx, matches[0])
+ ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
+
+
+def raise_on_error(view_func):
+ """Raise a click exception if returned value is not zero.
+
+ Click exits successfully if anything is returned, in order to exit properly
+ when something went wrong an exception must be raised.
+ """
+
+ from functools import wraps
+
+ def _decorator(*args, **kwargs):
+ value = view_func(*args, **kwargs)
+ if value not in [None, 0]:
+ exception = click.ClickException("Error occured")
+ exception.exit_code = value
+ raise exception
+ return value
+ return wraps(view_func)(_decorator)
+
+
+@with_plugins(pkg_resources.iter_entry_points('bdt.cli'))
+@click.group(cls=AliasedGroup,
+ context_settings=dict(help_option_names=['-?', '-h', '--help']))
+@verbosity_option()
+def main():
+ """Bob Development Tools - see available commands below"""
diff --git a/bob/devtools/scripts/cb_output.py b/bob/devtools/scripts/cb_output.py
new file mode 100644
index 0000000000000000000000000000000000000000..35c98c0d8c4b1827d0efe53d5ea4d007b6cb1e39
--- /dev/null
+++ b/bob/devtools/scripts/cb_output.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import click
+from click.testing import CliRunner
+import conda_build.api as cb
+
+from . import bdt
+
+from ..conda import should_skip_build
+
+
+@click.command(context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+ epilog='''\b
+Examples:
+$ bdt cb-output conda_recipe_dir
+$ bdt cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
+'''
+)
+@click.argument('recipe_path')
+@click.option('-m', '--variant-config-files', help='see conda build --help')
+@click.option('--python', help='see conda build --help')
+@bdt.raise_on_error
+def cb_output(recipe_path, variant_config_files, python):
+ """Outputs name(s) of package(s) that would be generated by conda build.
+
+ This command accepts extra unknown arguments so you can give it the same
+ arguments that you would give to conda build.
+
+ As of now, it only parses -m/--variant_config_files and --python and other
+ arguments are ignored.
+ """
+ clirunner = CliRunner()
+ with clirunner.isolation():
+ # render
+ config = cb.get_or_merge_config(
+ None, variant_config_files=variant_config_files, python=python)
+ metadata_tuples = cb.render(recipe_path, config=config)
+
+ # check if build(s) should be skipped
+ if should_skip_build(metadata_tuples):
+ return 0
+
+ paths = cb.get_output_file_paths(metadata_tuples, config=config)
+ click.echo('\n'.join(sorted(paths)))
diff --git a/bob/devtools/scripts/changelog.py b/bob/devtools/scripts/changelog.py
new file mode 100644
index 0000000000000000000000000000000000000000..7765d96004e1c2c54d75abe5d53095269cbd6ec4
--- /dev/null
+++ b/bob/devtools/scripts/changelog.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import datetime
+
+import click
+
+from . import bdt
+from ..changelog import get_last_tag_date, write_tags_with_commits
+from ..changelog import parse_date
+from ..release import get_gitlab_instance
+
+
+@click.command(context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+ epilog='''
+Examples:
+
+ 1. Generates the changelog for a single package using merge requests:
+
+ $ bdt -vvv changelog group/package.xyz changelog.md
+
+
+ 2. The same as above, but dumps the changelog to stdout instead of a file
+
+ $ bdt -vvv changelog group/package.xyz -
+
+
+ 3. Generates the changelog for a single package looking at commits
+ (not merge requests):
+
+ $ bdt -vvv changelog --mode=commits group/package.xyz changelog.md
+
+
+ 4. Generates the changelog for a single package looking at merge requests starting from a given date of January 1, 2016:
+
+\b
+ $ bdt -vvv changelog --mode=mrs --since=2016-01-01 group/package.xyz changelog.md
+
+
+ 5. Generates a complete list of changelogs for a list of packages (one per line:
+
+\b
+ $ curl -o order.txt https://gitlab.idiap.ch/bob/bob.nightlies/raw/master/order.txt
+ $ bdt lasttag bob/bob
+ # copy and paste date to next command
+ $ bdt -vvv changelog --since="2018-07-17 10:23:40" order.txt changelog.md
+''')
+@click.argument('target')
+@click.argument('changelog', type=click.Path(exists=False, dir_okay=False,
+ file_okay=True, writable=True))
+@click.option('-g', '--group', default='bob', show_default=True,
+ help='Gitlab default group name where packages are located (if not ' \
+ 'specified using a "/" on the package name - e.g. ' \
+ '"bob/bob.extension")')
+@click.option('-m', '--mode', type=click.Choice(['mrs', 'tags', 'commits']),
+ default='mrs', show_default=True,
+ help='Changes the way we produce the changelog. By default, uses the ' \
+ 'text in every merge request (mode "mrs"). To use tag annotations, ' \
+ 'use mode "tags". If you use "commits" as mode, we use the text ' \
+ 'in commits to produce the changelog')
+@click.option('-s', '--since',
+ help='A starting date in any format accepted by dateutil.parser.parse() ' \
+ '(see https://dateutil.readthedocs.io/en/stable/parser.html) from ' \
+ 'which you want to generate the changelog. If not set, the package\'s' \
+ 'last release date will be used')
+@bdt.raise_on_error
+def changelog(target, changelog, group, mode, since):
+ """Generates changelog file for package(s) from the Gitlab server.
+
+ This script generates changelogs for either a single package or multiple
+ packages, depending on the value of TARGET. The changelog (in markdown
+ format) is written to the output file CHANGELOG.
+
+ There are two modes of operation: you may provide the package name in the
+ format ``<gitlab-group>/<package-name>`` (or simply ``<package-name>``, in
+ which case the value of ``--group`` will be used). Or, optionally, provide
+ an existing file containing a list of packages that will be iterated on.
+
+ For each package, we will contact the Gitlab server and create a changelog
+ using merge-requests (default), tags or commits since a given date. If a
+ starting date is not passed, we'll use the date of the last tagged value or
+ the date of the first commit, if no tags are available in the package.
+ """
+
+ gl = get_gitlab_instance()
+
+ # reads package list or considers name to be a package name
+ if os.path.exists(target) and os.path.isfile(target):
+ bdt.logger.info('Reading package names from file %s...', target)
+ with open(target, 'rb') as f:
+ packages = [k.strip() for k in f.readlines() if k and not \
+ k.strip().startswith('#')]
+ else:
+ bdt.logger.info('Assuming %s is a package name (file does not ' \
+ 'exist)...', target)
+ packages = [target]
+
+ # if the user passed a date, convert it
+ if since: since = parse_date(since)
+
+ # iterates over the packages and dumps required information
+ for package in packages:
+
+ if '/' not in package:
+ package = '/'.join(group, package)
+
+ # retrieves the gitlab package object
+ use_package = gl.projects.get(package)
+ bdt.logger.info('Found gitlab project %s (id=%d)',
+ use_package.attributes['path_with_namespace'], use_package.id)
+
+ last_release_date = since or get_last_tag_date(use_package)
+ bdt.logger.info('Retrieving data (mode=%s) since %s', mode,
+ last_release_date.strftime('%b %d, %Y %H:%M'))
+
+ # add 1s to avoid us retrieving previous release data
+ last_release_date += datetime.timedelta(seconds=1)
+
+ if mode == 'tags':
+ visibility = ('public',)
+ else:
+ visibility = ('public', 'private', 'internal')
+
+ if use_package.attributes['namespace'] == use_package.name:
+ # skip system meta-package
+ bdt.logger.warn('Skipping meta package %s...',
+ use_package.attributes['path_with_namespace'])
+ continue
+
+ if use_package.attributes['visibility'] not in visibility:
+ bdt.logger.warn('Skipping package %s (visibility not in ' \
+ '"%s")...', use_package.attributes['path_with_namespace'],
+ '|'.join(visibility))
+ continue
+
+ if changelog == '-':
+ changelog_file = sys.stdout
+ else:
+ changelog_file = open(changelog, 'at')
+
+ # write_tags(f, use_package, last_release_date)
+ write_tags_with_commits(changelog_file, use_package, last_release_date,
+ mode)
+ changelog_file.flush()
diff --git a/bob/devtools/scripts/lasttag.py b/bob/devtools/scripts/lasttag.py
new file mode 100644
index 0000000000000000000000000000000000000000..2826ad1868ae1498f5e04223c71e7c22de6317ba
--- /dev/null
+++ b/bob/devtools/scripts/lasttag.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+import os
+
+import click
+
+from . import bdt
+from ..changelog import get_last_tag, parse_date
+from ..release import get_gitlab_instance
+
+
+@click.command(context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+ epilog='''
+Examples:
+
+ 1. Get the last tag information of the bob/bob package
+
+ $ bdt lasttag bob/bob
+
+
+ 2. Get the last tag information of the beat/beat.core package
+
+ $ bdt lasttag beat/beat.core
+
+''')
+@click.argument('package')
+@bdt.raise_on_error
+def lasttag(package):
+ """Returns the last tag information on a given PACKAGE
+ """
+
+ if '/' not in package:
+ raise RuntimeError('PACKAGE should be specified as "group/name"')
+
+ gl = get_gitlab_instance()
+
+ # we lookup the gitlab group once
+ use_package = gl.projects.get(package)
+ bdt.logger.info('Found gitlab project %s (id=%d)',
+ use_package.attributes['path_with_namespace'], use_package.id)
+
+ tag = get_last_tag(use_package)
+ date = parse_date(tag.commit['committed_date'])
+ click.echo('Lastest tag for %s is %s (%s)' % \
+ (package, tag.name, date.strftime('%Y-%m-%d %H:%M:%S')))
diff --git a/bob/devtools/scripts/release.py b/bob/devtools/scripts/release.py
new file mode 100644
index 0000000000000000000000000000000000000000..63b8374f883e3c107b4a0235f105fd443c0f65d8
--- /dev/null
+++ b/bob/devtools/scripts/release.py
@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import os
+
+import click
+
+from . import bdt
+from ..release import release_bob, parse_and_process_package_changelog
+from ..release import release_package, wait_for_pipeline_to_finish
+from ..release import get_gitlab_instance
+
+@click.command(context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+ epilog='''
+Examples:
+
+ 1. Releases a single package:
+
+ $ bdt -vvv release --package=bob.package.xyz changelog.md
+
+
+ 2. If there is a single package in the ``changelog.md`` file, the flag
+ ``--package`` is not required:
+
+ $ bdt -vvv release changelog.md
+
+
+ 2. Releases the whole of bob using `changelog_since_last_release.md`:
+
+ $ bdt -vvv release bob/devtools/data/changelog_since_last_release.md
+
+
+ 3. In case of errors, resume the release of the whole of Bob:
+
+ $ bdt -vvv release --resume bob/devtools/data/changelog_since_last_release.md
+
+
+ 4. The option `-dry-run` can be used to let the script print what it would do instead of actually doing it:
+
+ $ bdt -vvv release --dry-run changelog_since_last_release.md
+'''
+)
+@click.argument('changelog', type=click.File('rb', lazy=False))
+@click.option('-g', '--group', default='bob', show_default=True,
+ help='Group name where all packages are located')
+@click.option('-p', '--package',
+ help='If the name of a package is provided, then this package will be ' \
+ 'found in the changelog file and the release will resume from it ' \
+ '(if option ``--resume`` is set) or only this package will be ' \
+ 'released. If there is only a single package in the changelog, ' \
+ 'then you do NOT need to set this flag')
+@click.option('-r', '--resume/--no-resume', default=False,
+ help='The overall release will resume from the provided package name')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@bdt.raise_on_error
+def release(changelog, group, package, resume, dry_run):
+ """\b
+ Tags packages on gitlab from an input CHANGELOG in markdown formatting
+
+ By using a CHANGELOG file as an input (that can be generated with the ``bdt
+ changelog`` command), this script goes through all packages in CHANGELOG
+ file (in order listed), tags them correctly as per the file, and pushes
+ this tag to gitlab them one by one. Tagged releases are treated specially
+ by the CI and are auto-deployed to our stable conda channels and PyPI.
+
+ This script uses the provided CHANGELOG file to release one or more
+ package. The CHANGELOG is expected to have the following structure:
+
+ \b
+ * package name
+ * tag1 name (date of the tag).
+ * tag description. Each line of the tag description starts with `*`
+ character.
+ - commits (from earliest to latest). Each line of the commit starts
+ with `-` character.
+ * tag2 name (date of the tag).
+ * tag description. Each line of the tag description starts with `*`
+ character.
+ - commits (from earliest to latest). Each line of the commit starts
+ with `-` character.
+ * patch
+ - leftover not-tagged commits (from earliest to latest)
+
+ This script can also be used to release a single package.
+
+ IMPORTANT: There are some considerations that needs to be taken into
+ account **before** you release a new version of a package:
+
+ \b
+ * In the changelog file:
+ - write the name of this package and write (at least) the next tag value.
+ For the next tag value, you can either indicate one of the special
+ values: ``patch``, ``minor`` or ``major``, and the package will be then
+ released with either patch, minor, or major version **bump**.
+ - Alternatively, you can specify the tag value directly (using
+ a ``vX.Y.Z`` format), but be careful that it is higher than the last
+ release tag of this package. Make sure that the version that you are
+ trying to release is not already released. You must follow semantic
+ versioning: http://semver.org.
+ - Then, under the desired new tag version of the package, please write
+ down the changes that are applied to the package between the last
+ released version and this version. This changes are written to
+ release tags of packages in the Gitlab interface. For an example
+ look at: https://gitlab.idiap.ch/bob/bob.extension/tags
+ * Make sure all the tests for the package are passing.
+ * Make sure the documentation is building with the following command:
+ ``sphinx-build -aEWn doc sphinx``
+ * Ensure all changes are committed to the git repository and pushed.
+ * Ensure the documentation badges in README.rst are pointing to:
+ https://www.idiap.ch/software/bob/docs/bob/...
+ * For database packages, ensure that the '.sql3' file or other metadata
+ files have been generated (if any).
+ * Ensure the nightlies build is green after the changes are submitted if
+ the package is a part of the nightlies.
+ * If your package depends on an unreleased version of another package,
+ you need to release that package first.
+ """
+
+ gl = get_gitlab_instance()
+
+ use_group = gl.groups.list(search='"%s"' % group)[0]
+
+ # if we are releasing 'bob' metapackage, it's a simple thing, no GitLab
+ # API
+ if package == 'bob':
+ release_bob(changelog)
+ return
+
+ # traverse all packages in the changelog, edit older tags with updated
+ # comments, tag them with a suggested version, then try to release, and
+ # wait until done to proceed to the next package
+ changelogs = changelog.readlines()
+
+ # find the starts of each package's description in the changelog
+ pkgs = [i for i, line in enumerate(changelogs) if line[0] == '*']
+ pkgs.append(len(changelogs)) #the end
+ start_idx = 0
+
+ if package:
+ # get the index where the package first appears in the list
+ start_idx = [i for i, line in enumerate(changelogs) \
+ if line[1:].strip() == package]
+
+ if not start_idx:
+ bdt.logger.error('Package %s was not found in the changelog',
+ package)
+ return
+
+ start_idx = pkgs.index(start_idx[0])
+
+ # if we are in a dry-run mode, let's let it be known
+ if dry_run:
+ bdt.logger.warn('!!!! DRY RUN MODE !!!!')
+ bdt.logger.warn('Nothing is being committed to Gitlab')
+
+ # go through the list of packages and release them starting from the
+ # start_idx
+ for i in range(start_idx, len(pkgs) - 1):
+ cur_package_name = changelogs[pkgs[i]][1:].strip()
+ bdt.logger.info('Processing package %s', changelogs[pkgs[i]])
+ gitpkg, tag, tag_comments = parse_and_process_package_changelog(gl,
+ use_group, cur_package_name,
+ changelogs[pkgs[i] + 1: pkgs[i + 1]], dry_run)
+
+ # release the package with the found tag and its comments
+ if gitpkg:
+ pipeline_id = release_package(gitpkg, tag, tag_comments, dry_run)
+ # now, wait for the pipeline to finish, before we can release the
+ # next package
+ wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run)
+
+ # if package name is provided and resume is not set, process only
+ # this package
+ if package == cur_package_name and not resume:
+ break
+
+ bdt.logger.info('Finished processing %s', changelog)
diff --git a/bob_tools/scripts/cb_output.py b/bob_tools/scripts/cb_output.py
deleted file mode 100644
index 6545ce9ae234f374e22373c050cec0e8cbb1bad4..0000000000000000000000000000000000000000
--- a/bob_tools/scripts/cb_output.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from ..utils.click import raise_on_error
-from ..utils.conda import should_skip_build
-from click.testing import CliRunner
-import click
-import conda_build.api as cb
-
-
-@click.command(context_settings=dict(
- ignore_unknown_options=True, allow_extra_args=True),
- epilog='''\b
-Examples:
-$ bob-tools cb-output conda_recipe_dir
-$ bob-tools cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
-'''
-)
-@click.argument('recipe_path')
-@click.option('-m', '--variant-config-files', help='see conda build --help')
-@click.option('--python', help='see conda build --help')
-@raise_on_error
-def cb_output(recipe_path, variant_config_files, python):
- """Outputs name(s) of package(s) that would be generated by conda build.
-
- This command accepts extra unknown arguments so you can give it the same
- arguments that you would give to conda build.
-
- As of now, it only parses -m/--variant_config_files and --python and other
- arguments are ignored.
- """
- clirunner = CliRunner()
- with clirunner.isolation():
- # render
- config = cb.get_or_merge_config(
- None, variant_config_files=variant_config_files, python=python)
- metadata_tuples = cb.render(recipe_path, config=config)
-
- # check if build(s) should be skipped
- if should_skip_build(metadata_tuples):
- return 0
-
- paths = cb.get_output_file_paths(metadata_tuples, config=config)
- click.echo('\n'.join(sorted(paths)))
diff --git a/bob_tools/scripts/main.py b/bob_tools/scripts/main.py
deleted file mode 100644
index a5d7a6ca967b6a369a9778ed9e0e548459aafb7f..0000000000000000000000000000000000000000
--- a/bob_tools/scripts/main.py
+++ /dev/null
@@ -1,13 +0,0 @@
-"""This is the main entry to bob_tools's scripts.
-"""
-import pkg_resources
-import click
-from click_plugins import with_plugins
-
-
-@with_plugins(pkg_resources.iter_entry_points('bob_tools.cli'))
-@click.group(context_settings=dict(help_option_names=['-?', '-h', '--help']))
-def main():
- """The main command line interface for bob tools. Look below for available
- commands."""
- pass
diff --git a/bob_tools/utils/__init__.py b/bob_tools/utils/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/bob_tools/utils/click.py b/bob_tools/utils/click.py
deleted file mode 100644
index e698e594dd5d8e7dde2df4f711285d3246146ad8..0000000000000000000000000000000000000000
--- a/bob_tools/utils/click.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from functools import wraps
-
-import click
-
-
-def raise_on_error(view_func):
- """Raise a click exception if returned value is not zero.
-
- Click exits successfully if anything is returned, in order to exit properly
- when something went wrong an exception must be raised.
- """
-
- def _decorator(*args, **kwargs):
- value = view_func(*args, **kwargs)
- if value not in [None, 0]:
- exception = click.ClickException("Error occurred")
- exception.exit_code = value
- raise exception
- return value
- return wraps(view_func)(_decorator)
diff --git a/buildout.cfg b/buildout.cfg
index 6e6c0804244c488edeb41d6ee24626bf7365e071..81fcf8385e402dea5302c310a4baed6b604864a1 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -4,8 +4,7 @@
[buildout]
parts = scripts
develop = .
-eggs = bob_tools
-
+eggs = bob.devtools
newest = false
[scripts]
diff --git a/doc/api.rst b/doc/api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6ca5b6c228a6862389db899bcf862964866d4deb
--- /dev/null
+++ b/doc/api.rst
@@ -0,0 +1,21 @@
+.. vim: set fileencoding=utf-8 :
+
+
+============
+ Python API
+============
+
+.. autosummary::
+ bob.devtools.conda
+ bob.devtools.release
+ bob.devtools.changelog
+
+
+Detailed Information
+--------------------
+
+.. automodule:: bob.devtools.conda
+
+.. automodule:: bob.devtools.release
+
+.. automodule:: bob.devtools.changelog
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..10c340bc7a526655c27d9c0fe529d204e8491a02
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+import os
+import sys
+import glob
+import pkg_resources
+
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+needs_sphinx = '1.3'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+ 'sphinx.ext.todo',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.ifconfig',
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.autosummary',
+ 'sphinx.ext.doctest',
+ 'sphinx.ext.graphviz',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon',
+ 'sphinx.ext.viewcode',
+ 'sphinx.ext.mathjax',
+ ]
+
+# Be picky about warnings
+nitpicky = False
+
+# Ignores stuff we can't easily resolve on other project's sphinx manuals
+nitpick_ignore = []
+
+# Allows the user to override warnings from a separate file
+if os.path.exists('nitpick-exceptions.txt'):
+ for line in open('nitpick-exceptions.txt'):
+ if line.strip() == "" or line.startswith("#"):
+ continue
+ dtype, target = line.split(None, 1)
+ target = target.strip()
+ try: # python 2.x
+ target = unicode(target)
+ except NameError:
+ pass
+ nitpick_ignore.append((dtype, target))
+
+# Always includes todos
+todo_include_todos = True
+
+# Generates auto-summary automatically
+autosummary_generate = True
+
+# Create numbers on figures with captions
+numfig = True
+
+# If we are on OSX, the 'dvipng' path maybe different
+dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
+if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'bob.devtools'
+import time
+copyright = u'%s, Idiap Research Institute' % time.strftime('%Y')
+
+# Grab the setup entry
+distribution = pkg_resources.require(project)[0]
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = distribution.version
+# The full version, including alpha/beta/rc tags.
+release = distribution.version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['links.rst']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# Some variables which are useful for generated material
+project_variable = project.replace('.', '_')
+short_description = u'Tools for development and CI integration of Bob packages'
+owner = [u'Idiap Research Institute']
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = project_variable
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+html_logo = 'img/logo.png'
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+html_favicon = 'img/favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = project_variable + u'_doc'
+
+
+# -- Post configuration --------------------------------------------------------
+
+# Included after all input documents
+rst_epilog = """
+.. |project| replace:: Bob
+.. |version| replace:: %s
+.. |current-year| date:: %%Y
+""" % (version,)
+
+# Default processing flags for sphinx
+autoclass_content = 'class'
+autodoc_member_order = 'bysource'
+autodoc_default_flags = [
+ 'members',
+ 'undoc-members',
+ 'show-inheritance',
+ ]
+
+# Adds simplejson, pyzmq links
+#intersphinx_mapping['http://simplejson.readthedocs.io/en/stable/'] = None
+#intersphinx_mapping['http://pyzmq.readthedocs.io/en/stable/'] = None
+
+# We want to remove all private (i.e. _. or __.__) members
+# that are not in the list of accepted functions
+accepted_private_functions = ['__array__']
+
+def member_function_test(app, what, name, obj, skip, options):
+ # test if we have a private function
+ if len(name) > 1 and name[0] == '_':
+ # test if this private function should be allowed
+ if name not in accepted_private_functions:
+ # omit privat functions that are not in the list of accepted private functions
+ return skip
+ else:
+ # test if the method is documented
+ if not hasattr(obj, '__doc__') or not obj.__doc__:
+ return skip
+ return False
+
+def setup(app):
+ app.connect('autodoc-skip-member', member_function_test)
diff --git a/doc/img/favicon.ico b/doc/img/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..4cc3264302627d40868261add69eb755856611b6
Binary files /dev/null and b/doc/img/favicon.ico differ
diff --git a/doc/img/logo.png b/doc/img/logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..b60858a7068bf45c1ed8e3da12fe244ccdcfe85d
Binary files /dev/null and b/doc/img/logo.png differ
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..9a5f8ba7c0db6c8fcfa87ec0e42cff0a97da0ebe
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,29 @@
+.. vim: set fileencoding=utf-8 :
+
+.. _bob.devtools:
+
+=======================
+ Bob Development Tools
+=======================
+
+This package provides tools to help maintain Bob_.
+
+
+Documentation
+-------------
+
+.. toctree::
+ :maxdepth: 2
+
+ release
+ api
+
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
+.. include:: links.rst
diff --git a/doc/links.rst b/doc/links.rst
new file mode 100644
index 0000000000000000000000000000000000000000..021350c1af4db81289616a50c1228b1f6a2f09b9
--- /dev/null
+++ b/doc/links.rst
@@ -0,0 +1,5 @@
+.. vim: set fileencoding=utf-8 :
+
+.. Place here references to all citations in lower case
+
+.. _bob: https://www.idiap.ch/software/bob
diff --git a/doc/release.rst b/doc/release.rst
new file mode 100644
index 0000000000000000000000000000000000000000..7e1c393668f7fbcd8e3fcb54aa4b0501bd06a498
--- /dev/null
+++ b/doc/release.rst
@@ -0,0 +1,141 @@
+.. vim: set fileencoding=utf-8 :
+
+.. _bob.devtools.release:
+
+
+Release Management
+------------------
+
+This package offers tools to release (tag) packages that follow Bob's
+development guidelines. It automatically updates the ``README.rst`` file of
+the package to setup the correct pointers for the release build and
+documentation badges. The tools are setup to ensure a changelog is provided
+with each release. The changelog can be autogenerated from merge-requests or
+commits in the target package.
+
+
+Setup
+=====
+
+These programs require access to your gitlab private token which you can pass
+at every iteration or setup at your ``~/.python-gitlab.cfg``. If you don't set
+it up, it will request for your API token on-the-fly, what can be cumbersome
+and repeatitive. Your ``~/.python-gitlab.cfg`` should roughly look like this
+(there must be an "idiap" section on it, at least):
+
+.. code-block:: ini
+
+ [global]
+ default = idiap
+ ssl_verify = true
+ timeout = 15
+
+ [idiap]
+ url = https://gitlab.idiap.ch
+ private_token = <obtain token at your settings page in gitlab>
+ api_version = 4
+
+We recommend you set ``chmod 600`` to this file to avoid prying us to read out
+your personal token. Once you have your token set up, communication should work
+transparently between these gitlab clients and the server.
+
+
+Usage
+=====
+
+Using these scripts is a 2-step process:
+
+1. Generate a changelog using ``bdt changelog``
+2. Tag a release using ``bdt release``
+
+Use the ``--help`` flag in each command to learn more about each command.
+
+
+Manually update changelog
+=========================
+
+.. todo:: These instructions may be outdated!!
+
+The changelog since the last release can be found in the file
+``bob/devtools/data/changelog_since_last_release.md``. The structure is
+documented as part of the help message of ``bdt release``. Read it.
+
+To manually update the changelog, follow these guidelines:
+
+ 1. For each tag, summarize the commits into several descriptive lines.
+ These summaries become tag descriptions and they should extend/update
+ the existing tag description. Therefore, each line of the summary should
+ also start with ``*`` character like the tag descriptions.
+ 2. The last tag name is called ``patch``. This indicates that a patch
+ version of this package will be automatically updated during the next
+ tag/release. You can change ``patch`` to ``minor`` or ``major``, and the
+ package will be then tagged/released with either minor or major version
+ bump.
+ 3. Once all commits were changed to corresponding tag descriptions (no more
+ lines start with ``-`` characters), this package is ready for release
+ and you can continue to another package in the changelog.
+
+
+Releasing the Bob meta package
+==============================
+
+.. todo:: These instructions may be outdated!!
+
+Here are the instructions to release Bob meta package:
+
+* Run ./check_private.sh bob.buildout bob.extension ...
+ with the list of packages from `bob/bob.nightlies' "order.txt"
+ <https://gitlab.idiap.ch/bob/bob.nightlies/blob/master/order.txt>`_
+* Put the list of public packages in ../../bob/requirements.txt
+* Run ``bdt changelog`` first:
+
+ .. code-block:: sh
+
+ $ bdt changelog -l ../../bob/requirements.txt -R -- TOKEN | tee bob_changelog.md
+
+* Put the beta of version of the intended release version in
+ ``../../bob/version.txt``
+
+ * For example do ``$ echo 5.0.0b0 > version.txt`` for bob 5.0.0 release.
+ * Commit only this change to master: ``$ git commit -m "prepare for bob 5 release" version.txt``
+
+* Get the pinnings (``--bob-version`` needs to be changed):
+
+ .. code-block:: sh
+
+ $ bdt release -p bob -c bob_changelog.md --bob-version 5.0.0 -- TOKEN
+
+* Put the pinnings below in requirements.txt and meta.yaml (like ``bob.buildout
+ == 2.1.6``) and meta.yaml (like ``bob.buildout 2.1.6``)
+
+ * Make sure you add `` # [linux]`` to Linux only packages.
+
+* Test the conda recipe:
+
+ .. code-block:: sh
+
+ $ cd ../../bob
+ $ conda render -m ../bob.admin/gitlab/conda_build_config.yaml -c https://www.idiap.ch/software/bob/conda conda
+
+* Update the badges and version.txt to point to this version of Bob.
+* Commit, push and tag a new version manually:
+
+ .. code-block:: sh
+
+ $ git commit -am "Increased stable version to 4.0.0"
+ $ git tag v4.0.0
+ $ git push
+ $ git push --tags
+
+* Put ``bob_changelog.md`` inside bob's tag description.
+* Cancel the pipeline for master and make sure that tag pipeline passes before
+ continuing.
+* Remove pinnings from bob's requirement.txt and meta.yaml and revert changes
+ that went in ``README.rst`` back to master version.
+* Commit and push the following (not verbatim):
+
+ .. code-block:: sh
+
+ $ echo 4.0.1b0 > version.txt
+ $ git commit -am "Increased latest version to 4.0.1b0 [skip ci]"
+ $ git push
diff --git a/env.yml b/env.yml
new file mode 100644
index 0000000000000000000000000000000000000000..6788cba40f47d192ffc9a0ab514a0f5898247b6d
--- /dev/null
+++ b/env.yml
@@ -0,0 +1,22 @@
+name: bdt
+channels:
+- https://www.idiap.ch/software/bob/conda
+- defaults
+dependencies:
+- python=3.6
+- bob.buildout
+- click
+- click-plugins
+- conda-build
+- ipdb
+- sphinx
+- sphinx_rtd_theme
+- pip
+- pytz
+- python-dateutil
+- chardet
+- idna
+- requests
+- urllib3
+- pip:
+ - python-gitlab
diff --git a/setup.py b/setup.py
index 477e8faa0396e06454bb7f04ee743098a03d0c45..a4160852f92081e45d75a84a1412f61ed2c3a4a3 100644
--- a/setup.py
+++ b/setup.py
@@ -1,17 +1,24 @@
#!/usr/bin/env python
-"""A package that contains tools to maintain Bob
-"""
from setuptools import setup, find_packages
# Define package version
version = open("version.txt").read().rstrip()
+requires = [
+ 'setuptools',
+ 'click',
+ 'click-plugins',
+ 'conda-build',
+ 'python-gitlab',
+ 'requests',
+ ]
+
setup(
- name="bob_tools",
+ name="bob.devtools",
version=version,
- description="Tools to maintain Bob packages",
- url='http://gitlab.idiap.ch/bob/bob_tools',
+ description="Tools for development and CI integration of Bob packages",
+ url='http://gitlab.idiap.ch/bob/bob.devtools',
license="BSD",
author='Bob Developers',
author_email='bob-devel@googlegroups.com',
@@ -22,14 +29,17 @@ setup(
zip_safe=False,
# when updating these dependencies, update the README too
- install_requires=['setuptools', 'click', 'click-plugins', 'conda_build'],
+ install_requires=requires,
entry_points={
'console_scripts': [
- 'bob-tools = bob_tools.scripts.main:main',
+ 'bdt = bob.devtools.scripts.bdt:main',
],
- 'bob_tools.cli': [
- 'cb-output = bob_tools.scripts.cb_output:cb_output',
+ 'bdt.cli': [
+ 'cb-output = bob.devtools.scripts.cb_output:cb_output',
+ 'release = bob.devtools.scripts.release:release',
+ 'changelog = bob.devtools.scripts.changelog:changelog',
+ 'lasttag = bob.devtools.scripts.lasttag:lasttag',
],
},
classifiers=[