diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f496a872523dc13fd5f234e4875c81eb6ca9013c..fcef105c4474f9b3edf9aa3da2a1ec351d7a9114 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,6 +81,10 @@ build_macosx_36: - source ${CONDA_ROOT}/etc/profile.d/conda.sh - conda activate bdt - bdt ci deploy -vv + after_script: + - source ${CONDA_ROOT}/etc/profile.d/conda.sh + - conda activate bdt + - bdt ci clean -vv dependencies: - build_linux_36 - build_macosx_36 diff --git a/bob/devtools/build.py b/bob/devtools/build.py index dfcb7fff5a1c9732f118e180225aa8b89bc2c87d..9a982fc696af63a1ea33ea2bc32cc4f5c88c5a49 100644 --- a/bob/devtools/build.py +++ b/bob/devtools/build.py @@ -7,6 +7,7 @@ import os import re import sys +import glob import json import shutil import platform @@ -370,26 +371,39 @@ def check_version(workdir, envtag): return version, is_prerelease -def git_clean_build(runner, arch): +def git_clean_build(runner, verbose): '''Runs git-clean to clean-up build products Args: runner: A pointer to the ``run_cmdline()`` function + verbose: A boolean flag indicating if the git command should report erased + files or not ''' - # runs git clean to clean everything that is not needed. This helps to keep - # the disk usage on CI machines to a minimum. + # glob wild card entries we'd like to keep exclude_from_cleanup = [ "miniconda.sh", #the installer, cached - "miniconda/pkgs/*.tar.bz2", #downloaded packages, cached "miniconda/pkgs/urls.txt", #download index, cached - "miniconda/conda-bld/%s/*.tar.bz2" % (arch,), #build artifact -- conda - "dist/*.zip", #build artifact -- pypi package "sphinx", #build artifact -- documentation ] - runner(['git', 'clean', '-qffdx'] + \ + + # cache + exclude_from_cleanup += glob.glob("miniconda/pkgs/*.tar.bz2") + + # artifacts + exclude_from_cleanup += glob.glob("miniconda/conda-bld/*/*.tar.bz2") + exclude_from_cleanup += glob.glob("dist/*.zip") + + logger.debug('Excluding the following paths from git-clean:\n - %s', + ' - '.join(exclude_from_cleanup)) + + # decide on verbosity + flags = '-ffdx' + if not verbose: flags += 'q' + + runner(['git', 'clean', flags] + \ ['--exclude=%s' % k for k in exclude_from_cleanup]) @@ -437,13 +451,15 @@ if __name__ == '__main__': bootstrap.setup_logger(logger, args.verbose) - bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, verbose=True) - bootstrap.set_environment('LANG', 'en_US.UTF-8', verbose=True) - bootstrap.set_environment('LC_ALL', os.environ['LANG'], verbose=True) + VERB = (verbose >= 2) + + bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, verbose=VERB) + bootstrap.set_environment('LANG', 'en_US.UTF-8', verbose=VERB) + bootstrap.set_environment('LC_ALL', os.environ['LANG'], verbose=VERB) # get information about the version of the package being built version, is_prerelease = check_version(args.work_dir, args.tag) - bootstrap.set_environment('BOB_PACKAGE_VERSION', version, verbose=True) + bootstrap.set_environment('BOB_PACKAGE_VERSION', version, verbose=VERB) # create the build configuration conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml') @@ -474,7 +490,7 @@ if __name__ == '__main__': build_number, _ = next_build_number(channels[0], args.name, version, args.python_version) bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number), - verbose=True) + verbose=VERB) # runs the build using the conda-build API arch = conda_arch() @@ -484,4 +500,4 @@ if __name__ == '__main__': conda_build.api.build(os.path.join(args.work_dir, 'conda'), config=conda_config) - git_clean_build(bootstrap.run_cmdline, arch) + git_clean_build(bootstrap.run_cmdline, verbose=VERB) diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py index 7a33f5a5c063e3d04de9a678aa953d0a9754ccf6..3c7e12f2487395f0b63afcc3a7dacecef8108d97 100644 --- a/bob/devtools/scripts/ci.py +++ b/bob/devtools/scripts/ci.py @@ -263,8 +263,6 @@ def build(ctx, dry_run): """ from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND - from ..build import conda_arch, git_clean_build - from ..bootstrap import run_cmdline from .build import build ctx.invoke(build, @@ -281,4 +279,26 @@ def build(ctx, dry_run): ci=True, ) - git_clean_build(run_cmdline, conda_arch()) + +@ci.command(epilog=''' +Examples: + + 1. Cleans the current build (and prints what it cleans) + + $ bdt ci clean -vv + +''') +@verbosity_option() +@bdt.raise_on_error +@click.pass_context +def clean(ctx): + """Cleans builds + + This command cleans builds in the CI infrastructure. It is **not** meant + to be used outside this context. + """ + + from ..build import git_clean_build + from ..bootstrap import run_cmdline + + git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2)) diff --git a/conda/meta.yaml b/conda/meta.yaml index afac334e459a70ee6323f6e382a11566a3bf84a6..29370595b82bb6435df83d49604462fc419c9b56 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -68,6 +68,8 @@ test: - bdt ci build --help - bdt ci deploy --help - bdt ci pypi --help + - bdt ci readme --help + - bdt ci clean --help - sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx about: diff --git a/setup.py b/setup.py index f687ff05862e857270484d33a52191c210fe0f2b..05884050e230d494f41f210cf254dcee72f59bd5 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ setup( 'bdt.ci.cli': [ 'build = bob.devtools.scripts.ci:build', + 'clean = bob.devtools.scripts.ci:clean', 'deploy = bob.devtools.scripts.ci:deploy', 'readme = bob.devtools.scripts.ci:readme', 'pypi = bob.devtools.scripts.ci:pypi',