diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20644394d3e0e2bee843554de12184bd36592a70..1298c6625204c4d43285975dfe0525276d7e0ea9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,17 +2,22 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/timothycrosley/isort - rev: 4.3.21-2 + rev: 5.8.0 hooks: - id: isort args: [-sl] - repo: https://github.com/psf/black - rev: stable + rev: 20.8b1 hooks: - id: black exclude: bob/devtools/templates/setup.py + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.0 + hooks: + - id: flake8 + exclude: bob/devtools/templates/setup\.py - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.0.0 + rev: v3.4.0 hooks: - id: check-ast exclude: bob/devtools/templates/setup.py @@ -22,10 +27,9 @@ repos: - id: debug-statements exclude: bob/devtools/templates/setup.py - id: check-added-large-files - - id: flake8 exclude: bob/devtools/templates/setup.py - id: check-yaml - exclude: ./*/meta.yaml + exclude: .*/meta.yaml - repo: local hooks: - id: sphinx-build diff --git a/bob/devtools/build.py b/bob/devtools/build.py index 852c4cf89ebfa69e32864391f9d1f3f065a67c7e..5b8f99582b8c578181f21495331e8b61130ac420 100644 --- a/bob/devtools/build.py +++ b/bob/devtools/build.py @@ -4,20 +4,20 @@ """Tools for self-building and other utilities.""" -import os -import re -import sys -import json -import glob +import contextlib import copy +import distutils.version +import glob +import json import logging +import os import platform +import re import subprocess -import contextlib -import distutils.version +import sys -import yaml import conda_build.api +import yaml logger = logging.getLogger(__name__) @@ -98,13 +98,11 @@ def next_build_number(channel_url, basename): (reversed) build-number. """ - from conda.exports import fetch_index from conda.core.index import calculate_channel_urls + from conda.exports import fetch_index # get the channel index - channel_urls = calculate_channel_urls( - [channel_url], prepend=False, use_local=False - ) + channel_urls = calculate_channel_urls([channel_url], prepend=False, use_local=False) logger.debug("Downloading channel index from %s", channel_urls) index = fetch_index(channel_urls=channel_urls) @@ -115,8 +113,7 @@ def next_build_number(channel_url, basename): name, version, build = basename[:-6].rsplit("-", 2) else: raise RuntimeError( - "Package name %s does not end in either " - ".tar.bz2 or .conda" % (basename,) + "Package name %s does not end in either " ".tar.bz2 or .conda" % (basename,) ) # remove the build number as we're looking for the next value @@ -204,9 +201,7 @@ def make_conda_config(config, python, append_file, condarc_options): # appropriate platform-specific subdir (e.g. win-64) if os.path.isdir(url): if not os.path.isabs(url): - url = os.path.normpath( - os.path.abspath(os.path.join(os.getcwd(), url)) - ) + url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url))) with root_logger_protection(): url = url_path(url) retval.channel_urls.append(url) @@ -264,8 +259,7 @@ def exists_on_channel(channel_url, basename): name, version, build = name[:-8].rsplit("-", 2) else: raise RuntimeError( - "Package name %s does not end in either " - ".tar.bz2 or .conda" % (name,) + "Package name %s does not end in either " ".tar.bz2 or .conda" % (name,) ) # remove the build number as we're looking for the next value @@ -313,11 +307,7 @@ def parse_dependencies(recipe_dir, config): + ["ipython"] # for ipdb # Also add anaconda compilers to make sure source installed packages are # compiled properly - + [ - "clangxx_osx-64" - if platform.system() == "Darwin" - else "gxx_linux-64" - ] + + ["clangxx_osx-64" if platform.system() == "Darwin" else "gxx_linux-64"] ) # by last, packages required for local dev @@ -452,8 +442,7 @@ def get_docserver_setup(public, stable, server, intranet, group): if (not public) and (not intranet): raise RuntimeError( "You cannot request for private channels and set" - " intranet=False (server=%s) - these are conflicting options" - % server + " intranet=False (server=%s) - these are conflicting options" % server ) entries = [] @@ -477,9 +466,7 @@ def get_docserver_setup(public, stable, server, intranet, group): server + prefix + "/docs/" + group + "/%(name)s/stable/", ] else: - entries += [ - server + prefix + "/docs/" + group + "/%(name)s/master/" - ] + entries += [server + prefix + "/docs/" + group + "/%(name)s/master/"] return "|".join(entries) @@ -516,8 +503,7 @@ def check_version(workdir, envtag): '"version.txt" indicates version is a ' 'pre-release (v%s) - but environment provided tag "%s", ' "which indicates this is a **stable** build. " - "Have you created the tag using ``bdt release``?" - % (version, envtag) + "Have you created the tag using ``bdt release``?" % (version, envtag) ) else: # it is a stable build if envtag is None: @@ -570,10 +556,7 @@ def git_clean_build(runner, verbose): if not verbose: flags += "q" - runner( - ["git", "clean", flags] - + ["--exclude=%s" % k for k in exclude_from_cleanup] - ) + runner(["git", "clean", flags] + ["--exclude=%s" % k for k in exclude_from_cleanup]) def base_build( @@ -630,24 +613,18 @@ def base_build( "\n - ".join(condarc_options["channels"]), ) logger.info("Merging conda configuration files...") - conda_config = make_conda_config( - conda_build_config, None, None, condarc_options - ) + conda_config = make_conda_config(conda_build_config, None, None, condarc_options) metadata = get_rendered_metadata(recipe_dir, conda_config) arch = conda_arch() # checks we should actually build this recipe if should_skip_build(metadata): - logger.warn( - 'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch - ) + logger.warn('Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch) return paths = get_output_path(metadata, conda_config) - urls = [ - exists_on_channel(upload_channel, os.path.basename(k)) for k in paths - ] + urls = [exists_on_channel(upload_channel, os.path.basename(k)) for k in paths] if all(urls): logger.info( @@ -676,9 +653,7 @@ if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser( - description="Builds bob.devtools on the CI" - ) + parser = argparse.ArgumentParser(description="Builds bob.devtools on the CI") parser.add_argument( "-g", "--group", @@ -697,8 +672,7 @@ if __name__ == "__main__": default=os.environ.get( "CONDA_ROOT", os.path.realpath(os.path.join(os.curdir, "miniconda")) ), - help="The location where we should install miniconda " - "[default: %(default)s]", + help="The location where we should install miniconda " "[default: %(default)s]", ) parser.add_argument( "-V", @@ -778,9 +752,7 @@ if __name__ == "__main__": bootstrap.set_environment("BOB_PACKAGE_VERSION", version) # create the build configuration - conda_build_config = os.path.join( - args.work_dir, "conda", "conda_build_config.yaml" - ) + conda_build_config = os.path.join(args.work_dir, "conda", "conda_build_config.yaml") recipe_append = os.path.join(args.work_dir, "data", "recipe_append.yaml") condarc = os.path.join(args.conda_root, "condarc") @@ -844,8 +816,7 @@ if __name__ == "__main__": "typically means this build is running on a shared builder and " "the file ~/.conda/environments.txt is polluted with other " "environment paths. To fix, empty that file and set its mode " - "to read-only for all." - % (path, os.path.join(args.conda_root, "conda-bld")) + "to read-only for all." % (path, os.path.join(args.conda_root, "conda-bld")) ) # retrieve the current build number(s) for this build diff --git a/bob/devtools/changelog.py b/bob/devtools/changelog.py index 9ec044eb049c06e936319b039b682c32e9aa4654..abaa3967fd4058fd9e1d1a9fe531157973f99f9d 100644 --- a/bob/devtools/changelog.py +++ b/bob/devtools/changelog.py @@ -32,7 +32,9 @@ def _sort_tags(tags, reverse): """Sorts gitlab tag objects using their ``committed_date`` attribute.""" return sorted( - tags, key=lambda x: parse_date(x.commit["committed_date"]), reverse=reverse, + tags, + key=lambda x: parse_date(x.commit["committed_date"]), + reverse=reverse, ) @@ -253,7 +255,10 @@ def get_changes_since(gitpkg, since): mrs = list( reversed( gitpkg.mergerequests.list( - state="merged", updated_after=since, order_by="updated_at", all=True, + state="merged", + updated_after=since, + order_by="updated_at", + all=True, ) ) ) diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py index 4a15fb443c63991b28821f099e5a1cb59d17e94e..30098d40eae818c7be064e60743b8c56395eb90b 100644 --- a/bob/devtools/ci.py +++ b/bob/devtools/ci.py @@ -230,9 +230,10 @@ def cleanup(dry_run, username, password, includes): """ - from .deploy import _setup_webdav_client - from .constants import WEBDAV_PATHS, SERVER + from .constants import SERVER + from .constants import WEBDAV_PATHS from .dav import remove_old_beta_packages + from .deploy import _setup_webdav_client for public in (True, False): diff --git a/bob/devtools/config.py b/bob/devtools/config.py index 84282d3d282b30588337e83b962ec63f3f346304..6f1992eeee11778d08b956c6438790782d1d6452 100644 --- a/bob/devtools/config.py +++ b/bob/devtools/config.py @@ -4,8 +4,8 @@ """Reads and treats configuration files""" -import os import configparser +import os def read_config(): diff --git a/bob/devtools/data/conda_build_config.yaml b/bob/devtools/data/conda_build_config.yaml index 38a4910e7494c42b8c48413768498091a4a07b5b..1cdf2185112051b9207a8679e9b03974290dca33 100644 --- a/bob/devtools/data/conda_build_config.yaml +++ b/bob/devtools/data/conda_build_config.yaml @@ -17,8 +17,6 @@ c_compiler_version: # [linux or osx] cxx_compiler_version: # [linux or osx] - 7.3.0 # [linux] - 10 # [osx] -fortran_compiler_version: # [linux] - - 7.3.0 # [linux] clang_variant: - clang diff --git a/bob/devtools/data/gitlab-ci/noarch.yaml b/bob/devtools/data/gitlab-ci/noarch.yaml index c804564f195a0153c3c57949bcec39c4a9669aff..e34164dcee07ab981864a22b08b784d3dd9481fd 100644 --- a/bob/devtools/data/gitlab-ci/noarch.yaml +++ b/bob/devtools/data/gitlab-ci/noarch.yaml @@ -36,6 +36,7 @@ stages: extends: .bootstrap stage: build script: + - bdt ci check -vv - bdt ci build -vv - bdt ci clean -vv artifacts: diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml index 164de137769cd43c73c73707fc9d0df4ac93b8fd..0f14aabe893586f65dfb4e7fbc6a1ac90b83eb0f 100644 --- a/bob/devtools/data/gitlab-ci/single-package.yaml +++ b/bob/devtools/data/gitlab-ci/single-package.yaml @@ -37,6 +37,7 @@ stages: extends: .bootstrap stage: build script: + - bdt ci check -vv - bdt ci build -vv - bdt ci clean -vv artifacts: diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py index 3e3d83bc3df8838fc04d8f6f52269c23897a69fe..f29f8fc89d60069cdce87fdcf31383111a802ff7 100644 --- a/bob/devtools/dav.py +++ b/bob/devtools/dav.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import configparser import hashlib -import os import pathlib import re @@ -34,9 +32,9 @@ def _get_config(): or "password" not in data["webdav"] ): raise KeyError( - f'If the configuration file {k} contains a "webdav" ' - f"section, it should contain 3 variables defined inside: " - f'"server", "username", "password".' + 'If the configuration file contains a "webdav" ' + "section, it should contain 3 variables defined inside: " + '"server", "username", "password".' ) else: # ask the user for the information, in case nothing available diff --git a/bob/devtools/deploy.py b/bob/devtools/deploy.py index 14fef895c5a86ab81dd1be02d92a48c2dd02a076..18122396344bd46493ddcb63c6c10cf6fc2e86cf 100644 --- a/bob/devtools/deploy.py +++ b/bob/devtools/deploy.py @@ -84,7 +84,16 @@ def deploy_conda_package( def deploy_documentation( - path, package, stable, latest, public, branch, tag, username, password, dry_run, + path, + package, + stable, + latest, + public, + branch, + tag, + username, + password, + dry_run, ): """Deploys sphinx documentation to the appropriate webdav locations. diff --git a/bob/devtools/graph.py b/bob/devtools/graph.py index d3aa2fde5bed8a4f7498a8e613a201a66d056351..0b61ce30f5203b64db5b537f2523cd99619eb3ce 100644 --- a/bob/devtools/graph.py +++ b/bob/devtools/graph.py @@ -259,7 +259,8 @@ def generate_graph(adjacence_matrix, deptypes, whitelist): for package, values in adjacence_matrix.items(): if not whitelist_compiled.match(values["name"]): logger.debug( - "Skipping main package %s (did not match whitelist)", values["name"], + "Skipping main package %s (did not match whitelist)", + values["name"], ) continue name = values["name"] + "\n" + values["version"] + "\n" + values["build_string"] diff --git a/bob/devtools/log.py b/bob/devtools/log.py index 8e55848371cb742b68a33e151648e99b1301bea1..b43a2a709fb0e8501d32f1f022e17e5c05f46480 100644 --- a/bob/devtools/log.py +++ b/bob/devtools/log.py @@ -3,9 +3,9 @@ """Logging utilities.""" +import logging import os import sys -import logging import click import termcolor diff --git a/bob/devtools/release.py b/bob/devtools/release.py index 0565a8a88b33be8548113e62997895a048a777a6..f606620f2caa81b7148fb8fdf0d613d3dbcdab76 100644 --- a/bob/devtools/release.py +++ b/bob/devtools/release.py @@ -35,10 +35,11 @@ def download_path(package, path, output=None, ref="master"): directory ref: the name of the git reference (branch, tag or commit hash) to use """ - from io import BytesIO import tarfile import tempfile + from io import BytesIO + output = output or os.path.realpath(os.curdir) logger.debug( @@ -188,7 +189,8 @@ def get_parsed_tag(gitpkg, tag): raise ValueError( "The latest tag name {0} in package {1} has " "unknown format".format( - "v" + latest_tag_name, gitpkg.attributes["path_with_namespace"], + "v" + latest_tag_name, + gitpkg.attributes["path_with_namespace"], ) ) @@ -444,7 +446,9 @@ def wait_for_pipeline_to_finish(gitpkg, pipeline_id, dry_run=False): raise ValueError( "Pipeline {0} of project {1} exited with " 'undesired status "{2}". Release is not possible.'.format( - pipeline_id, gitpkg.attributes["path_with_namespace"], pipeline.status, + pipeline_id, + gitpkg.attributes["path_with_namespace"], + pipeline.status, ) ) diff --git a/bob/devtools/scripts/alternative_nightlies.py b/bob/devtools/scripts/alternative_nightlies.py index 3726479b9a2f0c8b9ef098c54b498705dfa6b6fe..2e536f31f03a06ebb3c84f8bba079ed78ecb4f2e 100644 --- a/bob/devtools/scripts/alternative_nightlies.py +++ b/bob/devtools/scripts/alternative_nightlies.py @@ -24,7 +24,8 @@ Examples: nargs=1, ) @click.argument( - "variables", nargs=-1, + "variables", + nargs=-1, ) @verbosity_option() @bdt.raise_on_error diff --git a/bob/devtools/scripts/badges.py b/bob/devtools/scripts/badges.py index f5940eee5ca6060af70461c759fccde1b10094a6..fa0d04969a95f2dddf0e82e510bb79330c97ae87 100644 --- a/bob/devtools/scripts/badges.py +++ b/bob/devtools/scripts/badges.py @@ -160,7 +160,7 @@ def badges(package, update_readme, dry_run, server): # creates all stock badges, preserve positions info = dict(zip(("group", "name"), package.split("/", 1))) if not server: - server = f"https://www.idiap.ch/software/{group}" + server = f"https://www.idiap.ch/software/{info['group']}" info["idiap_server"] = server[:-1] if server.endswith("/") else server for position, badge in enumerate(PROJECT_BADGES): data = dict([(k, v.format(**info)) for (k, v) in badge.items()]) diff --git a/bob/devtools/scripts/bdt.py b/bob/devtools/scripts/bdt.py index 979d34442c1b82d3cef28bd322ad94869c5be27e..220cb93252c52fbdd43ff1994fa98487b4c54abe 100644 --- a/bob/devtools/scripts/bdt.py +++ b/bob/devtools/scripts/bdt.py @@ -60,13 +60,14 @@ if "LC_ALL" not in os.environ: @with_plugins(pkg_resources.iter_entry_points("bdt.cli")) @click.group( - cls=AliasedGroup, context_settings=dict(help_option_names=["-?", "-h", "--help"]), + cls=AliasedGroup, + context_settings=dict(help_option_names=["-?", "-h", "--help"]), ) def main(): """Bob Development Tools - see available commands below""" - from ..constants import CACERT from ..bootstrap import set_environment + from ..constants import CACERT # certificate setup: required for gitlab API interaction set_environment("SSL_CERT_FILE", CACERT, os.environ) diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py index d436c5931bb39fecb083c43a423ff34d28fe79fa..e7e557547bb424ac24943f236d03e82fa468c88b 100644 --- a/bob/devtools/scripts/build.py +++ b/bob/devtools/scripts/build.py @@ -18,8 +18,8 @@ from ..build import get_parsed_recipe from ..build import get_rendered_metadata from ..build import make_conda_config from ..build import next_build_number -from ..build import should_skip_build from ..build import root_logger_protection +from ..build import should_skip_build from ..constants import BASE_CONDARC from ..constants import CONDA_BUILD_CONFIG from ..constants import CONDA_RECIPE_APPEND @@ -214,9 +214,7 @@ def build( prefix = get_env_directory(os.environ["CONDA_EXE"], "base") condarc_options["croot"] = os.path.join(prefix, "conda-bld") - conda_config = make_conda_config( - config, python, append_file, condarc_options - ) + conda_config = make_conda_config(config, python, append_file, condarc_options) set_environment("MATPLOTLIBRC", MATPLOTLIB_RCDIR) @@ -253,9 +251,7 @@ def build( # checks if we should actually build this recipe if should_skip_build(metadata): - logger.info( - "Skipping UNSUPPORTED build of %s for %s", recipe_dir, arch - ) + logger.info("Skipping UNSUPPORTED build of %s for %s", recipe_dir, arch) continue rendered_recipe = get_parsed_recipe(metadata) @@ -266,9 +262,7 @@ def build( path = get_output_path(metadata, conda_config)[0] # gets the next build number - build_number, _ = next_build_number( - upload_channel, os.path.basename(path) - ) + build_number, _ = next_build_number(upload_channel, os.path.basename(path)) logger.info( "Building %s-%s-py%s (build: %d) for %s", @@ -284,9 +278,7 @@ def build( # get it right set_environment("BOB_BUILD_NUMBER", str(build_number)) with root_logger_protection(): - paths = conda_build.api.build( - d, config=conda_config, notest=no_test - ) + paths = conda_build.api.build(d, config=conda_config, notest=no_test) # if you get to this point, the package was successfully rebuilt # set environment to signal caller we may dispose of it os.environ["BDT_BUILD"] = ":".join(paths) diff --git a/bob/devtools/scripts/caupdate.py b/bob/devtools/scripts/caupdate.py index e274d5254fff429f39000a4778911bee73fe9685..e3ba10550b5c2c04672b4e695cbb45679104b93f 100644 --- a/bob/devtools/scripts/caupdate.py +++ b/bob/devtools/scripts/caupdate.py @@ -32,7 +32,10 @@ def caupdate(): """ import requests - from ..constants import CACERT, CACERT_URL, IDIAP_ROOT_CA + + from ..constants import CACERT + from ..constants import CACERT_URL + from ..constants import IDIAP_ROOT_CA logger.info("Retrieving %s...", CACERT_URL) r = requests.get(CACERT_URL, allow_redirects=True) diff --git a/bob/devtools/scripts/changelog.py b/bob/devtools/scripts/changelog.py index 538caa00a9a38fee17a3234afac9bc3b2f8b2921..bc560a08f07439d124d161e03135a2affd253510 100644 --- a/bob/devtools/scripts/changelog.py +++ b/bob/devtools/scripts/changelog.py @@ -97,17 +97,15 @@ def changelog(target, changelog, group, mode, since): starting date is not passed, we'll use the date of the last tagged value or the date of the first commit, if no tags are available in the package. """ + import datetime import os import sys - import datetime - from ..changelog import ( - get_last_tag_date, - write_tags_with_commits, - parse_date, - get_changes_since, - get_last_tag, - ) + from ..changelog import get_changes_since + from ..changelog import get_last_tag + from ..changelog import get_last_tag_date + from ..changelog import parse_date + from ..changelog import write_tags_with_commits from ..release import get_gitlab_instance gl = get_gitlab_instance() diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py index 57773b3d6f152f87470e5a2a53bb6098c5ac04c2..ca22a5b83d09446d4797671e5012bfb4ecc8e25a 100644 --- a/bob/devtools/scripts/ci.py +++ b/bob/devtools/scripts/ci.py @@ -3,6 +3,7 @@ import glob import os import shutil +import sys import click import pkg_resources @@ -32,7 +33,19 @@ logger = get_logger(__name__) @with_plugins(pkg_resources.iter_entry_points("bdt.ci.cli")) @click.group(cls=bdt.AliasedGroup) -def ci(): +@click.option( + "-l", + "--local", + is_flag=True, + default=False, + help="Setups CI like environment for running ci scripts locally.", +) +@click.option( + "--python", + default=".".join(str(v) for v in sys.version_info[:2]), + help="Change the Python version for local runs.", +) +def ci(local, python): """Commands for building packages and handling CI activities. Commands defined here are supposed to run on our CI, where a number @@ -40,7 +53,10 @@ def ci(): **NOT** attempt to run these commands in your own installation. Unexpected errors may occur. """ - pass + if local: + os.environ["CI_JOB_TOKEN"] = "0" + os.environ["CI_PROJECT_DIR"] = "." + os.environ["PYTHON_VERSION"] = python @ci.command( @@ -297,9 +313,10 @@ def pypi(package, dry_run): % os.environ["CI_PROJECT_PATH"] ) - from ..constants import CACERT from twine.settings import Settings + from ..constants import CACERT + settings = Settings( username=os.environ["PYPIUSER"], password=os.environ["PYPIPASS"], @@ -397,7 +414,8 @@ def base_build(order, group, dry_run): continue variants_file = select_conda_build_config( - paths=[recipe, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build configuration file: %s", variants_file) @@ -449,25 +467,31 @@ def test(ctx, dry_run): recipe_dir = os.path.join(os.path.realpath(os.curdir), "conda") condarc = select_user_condarc( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) if condarc is not None: logger.info("Condarc configuration file: %s", condarc) variants_file = select_conda_build_config( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build configuration file: %s", variants_file) append_file = select_conda_recipe_append( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build recipe-append file: %s", append_file) from .test import test base_path = os.path.join( - os.environ["CONDA_ROOT"], "conda-bld", "*", os.environ["CI_PROJECT_NAME"], + os.environ["CONDA_ROOT"], + "conda-bld", + "*", + os.environ["CI_PROJECT_NAME"], ) ctx.invoke( @@ -527,18 +551,21 @@ def build(ctx, dry_run, recipe_dir): # Use custom variants and append files if available on recipe-dir condarc = select_user_condarc( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) if condarc is not None: logger.info("Condarc configuration file: %s", condarc) variants_file = select_conda_build_config( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build configuration file: %s", variants_file) append_file = select_conda_recipe_append( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build recipe-append file: %s", append_file) @@ -582,8 +609,8 @@ def clean(ctx): meant to be used outside this context. """ - from ..build import git_clean_build from ..bootstrap import run_cmdline + from ..build import git_clean_build git_clean_build(run_cmdline, verbose=(ctx.meta["verbosity"] >= 3)) @@ -640,9 +667,11 @@ def nightlies(ctx, order, dry_run): token = os.environ["CI_JOB_TOKEN"] + from urllib.request import urlopen + import git + from .build import build - from urllib.request import urlopen # loaded all recipes, now cycle through them implementing what is described # in the documentation of this function @@ -676,21 +705,28 @@ def nightlies(ctx, order, dry_run): recipe_dir = os.path.join(clone_to, "conda") condarc = select_user_condarc( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) if condarc is not None: logger.info("Condarc configuration file: %s", condarc) variants_file = select_conda_build_config( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build configuration file: %s", variants_file) append_file = select_conda_recipe_append( - paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"), + paths=[recipe_dir, os.curdir], + branch=os.environ.get("CI_COMMIT_REF_NAME"), ) logger.info("Conda build recipe-append file: %s", append_file) + logger.info("Running checks") + ctx.invoke(check, root=clone_to) + + logger.info("Building") ctx.invoke( build, recipe_dir=[recipe_dir], @@ -951,3 +987,42 @@ def clean_betas(dry_run): password=os.environ["DOCPASS"], includes=includes, ) + + +@ci.command( + epilog=""" +Example: + + bdt ci check -vv +""" +) +@click.option( + "-d", + "--dir", + "root", + default=os.path.join(os.path.realpath(os.curdir)), + help="Path to the root folder of the package.", +) +@verbosity_option() +@bdt.raise_on_error +def check(root): + # checks if a pyproject.toml file exists + path = os.path.join(root, "pyproject.toml") + if not os.path.isfile(path): + raise RuntimeError( + "pyproject.toml file not found in the root folder of the package. " + "See https://gitlab.idiap.ch/bob/bob/-/wikis/ci-checks#pyprojecttoml" + ) + + # if there is a pre-commit file, run the tests + path = os.path.join(root, ".pre-commit-config.yaml") + if os.path.isfile(path): + from ..bootstrap import run_cmdline + + run_cmdline(["python", "-m", "pip", "install", "pre-commit"]) + try: + backup = os.environ.get("SKIP", "") + os.environ["SKIP"] = "sphinx-build,sphinx-doctest" + run_cmdline(["python", "-m", "pre_commit", "run", "--all-files"]) + finally: + os.environ["SKIP"] = backup diff --git a/bob/devtools/scripts/create.py b/bob/devtools/scripts/create.py index 0ae47da83249ad0b68b780026c8012731f7204d9..91de856e508720aad915d389d3f3ee91eae0dc2d 100644 --- a/bob/devtools/scripts/create.py +++ b/bob/devtools/scripts/create.py @@ -7,12 +7,12 @@ import sys import click import yaml -from ..config import read_config -from ..bootstrap import set_environment from ..bootstrap import run_cmdline +from ..bootstrap import set_environment from ..build import conda_create from ..build import make_conda_config from ..build import parse_dependencies +from ..config import read_config from ..constants import BASE_CONDARC from ..constants import CONDA_BUILD_CONFIG from ..constants import CONDA_RECIPE_APPEND @@ -95,8 +95,7 @@ Examples: "--python", default=("%d.%d" % sys.version_info[:2]), show_default=True, - help="Version of python to build the " - "environment for [default: %(default)s]", + help="Version of python to build the " "environment for [default: %(default)s]", ) @click.option( "-o", @@ -265,17 +264,13 @@ def create( "\n - ".join(condarc_options["channels"]), ) - conda_config = make_conda_config( - config, python, append_file, condarc_options - ) + conda_config = make_conda_config(config, python, append_file, condarc_options) deps = parse_dependencies(recipe_dir, conda_config) # when creating a local development environment, remove the always_yes # option del condarc_options["always_yes"] - conda_create( - conda, name, overwrite, condarc_options, deps, dry_run, use_local - ) + conda_create(conda, name, overwrite, condarc_options, deps, dry_run, use_local) # part 2: pip-install everything listed in pip-extras # mix-in stuff from ~/.bdtrc and command-line diff --git a/bob/devtools/scripts/graph.py b/bob/devtools/scripts/graph.py index 28d189c6a85836d747ebdd8a4b6fc349c6901b8d..9c18d1a86499dd05ce8817722d8bbd674d545147 100644 --- a/bob/devtools/scripts/graph.py +++ b/bob/devtools/scripts/graph.py @@ -54,7 +54,9 @@ Examples: help="Version of python to build the environment for", ) @click.option( - "-r", "--condarc", help="Use custom conda configuration file instead of our own", + "-r", + "--condarc", + help="Use custom conda configuration file instead of our own", ) @click.option( "-m", @@ -103,7 +105,11 @@ Examples: help="Use this flag to indicate the graph will be running on the CI", ) @click.option( - "-n", "--name", show_default=True, default="graph", help="set the graph name", + "-n", + "--name", + show_default=True, + default="graph", + help="set the graph name", ) @click.option( "-f", diff --git a/bob/devtools/scripts/jobs.py b/bob/devtools/scripts/jobs.py index 8c31b00e222d524d0f58ddd891f070691ca56198..34d679bce61e954636ef2566374282c38c6e44d0 100644 --- a/bob/devtools/scripts/jobs.py +++ b/bob/devtools/scripts/jobs.py @@ -65,7 +65,11 @@ def jobs(name, status): jobs = runner.jobs.list(all=True, status=status) echo_normal( "Runner %s (id=%d) -- %d running" - % (runner.attributes["description"], runner.attributes["id"], len(jobs),) + % ( + runner.attributes["description"], + runner.attributes["id"], + len(jobs), + ) ) for k in jobs: echo_info( diff --git a/bob/devtools/scripts/mirror.py b/bob/devtools/scripts/mirror.py index bd08b58293141d763aadec2998fb8306940ed2d5..f8ac0b00320e9d420f93d5efc4fdf3d399b28370 100644 --- a/bob/devtools/scripts/mirror.py +++ b/bob/devtools/scripts/mirror.py @@ -38,7 +38,8 @@ Examples: """ ) @click.argument( - "channel-url", required=True, + "channel-url", + required=True, ) @click.argument( "dest-dir", diff --git a/bob/devtools/scripts/rebuild.py b/bob/devtools/scripts/rebuild.py index 92457ca747430ffbf1f38d01f7215912f3681d4e..e653af0bb3d56e5ca7e4c71edeb25be2a51e105a 100644 --- a/bob/devtools/scripts/rebuild.py +++ b/bob/devtools/scripts/rebuild.py @@ -19,8 +19,8 @@ from ..build import get_parsed_recipe from ..build import get_rendered_metadata from ..build import make_conda_config from ..build import next_build_number -from ..build import should_skip_build from ..build import root_logger_protection +from ..build import should_skip_build from ..constants import BASE_CONDARC from ..constants import CONDA_BUILD_CONFIG from ..constants import CONDA_RECIPE_APPEND @@ -62,7 +62,9 @@ Examples: help="Version of python to build the environment for", ) @click.option( - "-r", "--condarc", help="Use custom conda configuration file instead of our own", + "-r", + "--condarc", + help="Use custom conda configuration file instead of our own", ) @click.option( "-m", @@ -176,7 +178,11 @@ def rebuild( # get potential channel upload and other auxiliary channels channels, upload_channel = get_channels( - public=(not private), stable=stable, server=server, intranet=ci, group=group, + public=(not private), + stable=stable, + server=server, + intranet=ci, + group=group, ) if condarc is not None: @@ -209,7 +215,11 @@ def rebuild( # and derived documentation building via Sphinx) set_environment("DOCSERVER", server) doc_urls = get_docserver_setup( - public=(not private), stable=stable, server=server, intranet=ci, group=group, + public=(not private), + stable=stable, + server=server, + intranet=ci, + group=group, ) set_environment("BOB_DOCUMENTATION_SERVER", doc_urls) @@ -242,7 +252,9 @@ def rebuild( path = get_output_path(metadata, conda_config)[0] # Get the latest build number - build_number, existing = next_build_number(upload_channel, os.path.basename(path)) + build_number, existing = next_build_number( + upload_channel, os.path.basename(path) + ) should_build = True diff --git a/bob/devtools/scripts/runners.py b/bob/devtools/scripts/runners.py index 5b64b4b2a7e19ff4a78684506958ad189f91cdfe..8eee83f811af54b37e715f677eca0506abdbaa11 100644 --- a/bob/devtools/scripts/runners.py +++ b/bob/devtools/scripts/runners.py @@ -1,13 +1,12 @@ #!/usr/bin/env python import os + import click -from click_plugins import with_plugins -import pkg_resources +from ..log import echo_normal from ..log import get_logger from ..log import verbosity_option -from ..log import echo_normal from ..release import get_gitlab_instance from . import bdt @@ -18,9 +17,7 @@ def _get_runner_from_description(gl, descr): # search for the runner to affect the_runner = [ - k - for k in gl.runners.list(all=True) - if k.attributes["description"] == descr + k for k in gl.runners.list(all=True) if k.attributes["description"] == descr ] if not the_runner: raise RuntimeError("Cannot find runner with description = %s", descr) @@ -81,9 +78,7 @@ def _get_projects_from_runner(gl, runner): packages = [] for k, proj in enumerate(the_runner.projects): packages.append(_get_project(gl, proj["id"])) - logger.debug( - "Got data from project %d/%d", k + 1, len(the_runner.projects) - ) + logger.debug("Got data from project %d/%d", k + 1, len(the_runner.projects)) return packages @@ -96,9 +91,7 @@ def _get_projects_from_file(gl, filename): logger.info("Loaded %d entries from file %s", len(lines), filename) for k, proj in enumerate(lines): packages.append(_get_project(gl, proj)) - logger.debug( - "Got data from project %d/%d", k + 1, len(lines) - ) + logger.debug("Got data from project %d/%d", k + 1, len(lines)) return packages diff --git a/bob/devtools/scripts/test.py b/bob/devtools/scripts/test.py index 076d6d9b0f43ad9d5cafd45db9ae28acc57b5a29..58e3b3a324b776d5a205fe888389f764cfe17a31 100644 --- a/bob/devtools/scripts/test.py +++ b/bob/devtools/scripts/test.py @@ -49,7 +49,9 @@ Examples: nargs=-1, ) @click.option( - "-r", "--condarc", help="Use custom conda configuration file instead of our own", + "-r", + "--condarc", + help="Use custom conda configuration file instead of our own", ) @click.option( "-m", @@ -187,7 +189,11 @@ def test( # and derived documentation building via Sphinx) set_environment("DOCSERVER", server) doc_urls = get_docserver_setup( - public=(not private), stable=stable, server=server, intranet=ci, group=group, + public=(not private), + stable=stable, + server=server, + intranet=ci, + group=group, ) set_environment("BOB_DOCUMENTATION_SERVER", doc_urls) diff --git a/bob/devtools/scripts/update_bob.py b/bob/devtools/scripts/update_bob.py index 889381dcb07ad8e4156f7dfa14e1489687a9792a..e1eb0fe29ed2f614ef18715af5073f2b80a9db3e 100644 --- a/bob/devtools/scripts/update_bob.py +++ b/bob/devtools/scripts/update_bob.py @@ -23,11 +23,13 @@ Examples: @verbosity_option() @bdt.raise_on_error def update_bob(stable): - """Updates the Bob meta package with new packages. - """ + """Updates the Bob meta package with new packages.""" import tempfile + from ..ci import read_packages - from ..release import get_gitlab_instance, download_path, get_latest_tag_name + from ..release import download_path + from ..release import get_gitlab_instance + from ..release import get_latest_tag_name gl = get_gitlab_instance() diff --git a/bob/devtools/webdav3/client.py b/bob/devtools/webdav3/client.py index 51fc57a4ee6abebd3471d87e9c2bb16cf242b100..8d08cb8394039f7e082234ac801bad5fa6bbe3d2 100644 --- a/bob/devtools/webdav3/client.py +++ b/bob/devtools/webdav3/client.py @@ -30,9 +30,11 @@ logger = get_logger(__name__) try: - from urllib.parse import unquote, urlsplit + from urllib.parse import unquote + from urllib.parse import urlsplit except ImportError: from urllib import unquote + from urlparse import urlsplit __version__ = "0.2" @@ -366,11 +368,15 @@ class Client(object): urn = Urn(remote_path) if self.is_dir(urn.path()): self.download_directory( - local_path=local_path, remote_path=remote_path, progress=progress, + local_path=local_path, + remote_path=remote_path, + progress=progress, ) else: self.download_file( - local_path=local_path, remote_path=remote_path, progress=progress, + local_path=local_path, + remote_path=remote_path, + progress=progress, ) def download_directory(self, remote_path, local_path, progress=None): @@ -397,7 +403,9 @@ class Client(object): ) _local_path = os.path.join(local_path, resource_name) self.download( - local_path=_local_path, remote_path=_remote_path, progress=progress, + local_path=_local_path, + remote_path=_remote_path, + progress=progress, ) @wrap_connection_error @@ -481,7 +489,9 @@ class Client(object): """ if os.path.isdir(local_path): self.upload_directory( - local_path=local_path, remote_path=remote_path, progress=progress, + local_path=local_path, + remote_path=remote_path, + progress=progress, ) else: self.upload_file(local_path=local_path, remote_path=remote_path) @@ -516,7 +526,9 @@ class Client(object): ) _local_path = os.path.join(local_path, resource_name) self.upload( - local_path=_local_path, remote_path=_remote_path, progress=progress, + local_path=_local_path, + remote_path=_remote_path, + progress=progress, ) @wrap_connection_error @@ -603,7 +615,9 @@ class Client(object): path=self.get_full_path(urn_to) ) self.execute_request( - action="copy", path=urn_from.quote(), headers_ext=[header_destination], + action="copy", + path=urn_from.quote(), + headers_ext=[header_destination], ) @wrap_connection_error @@ -886,7 +900,9 @@ class Resource(object): def read_async(self, local_path, callback=None): return self.client.upload_async( - local_path=local_path, remote_path=self.urn.path(), callback=callback, + local_path=local_path, + remote_path=self.urn.path(), + callback=callback, ) def write_to(self, buff): @@ -899,7 +915,9 @@ class Resource(object): def write_async(self, local_path, callback=None): return self.client.download_async( - local_path=local_path, remote_path=self.urn.path(), callback=callback, + local_path=local_path, + remote_path=self.urn.path(), + callback=callback, ) def publish(self): diff --git a/bob/devtools/webdav3/urn.py b/bob/devtools/webdav3/urn.py index d822bdb08c266ab1a17807abe9f06c6d689ecfa9..342f8afc0fcac1f72dd54efb135439c0af0ddc87 100644 --- a/bob/devtools/webdav3/urn.py +++ b/bob/devtools/webdav3/urn.py @@ -1,5 +1,7 @@ try: - from urllib.parse import unquote, quote, urlsplit + from urllib.parse import quote + from urllib.parse import unquote + from urllib.parse import urlsplit except ImportError: from urllib import unquote, quote from urlparse import urlsplit diff --git a/conda/meta.yaml b/conda/meta.yaml index 821b281f05afce4ca7e7bf34d665bb14924943e2..8d3b99cdd23e41934cc876e187df38e211df072c 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -51,6 +51,7 @@ requirements: - psutil - tabulate - python-graphviz + - pip test: requires: @@ -98,6 +99,8 @@ test: - bdt ci nightlies --help - bdt ci docs --help - bdt ci clean-betas --help + - bdt ci check --help + - bdt ci check -vv --dir {{ environ.get('RECIPE_DIR') + '/..' }} - bdt dav --help - bdt dav list --help - bdt dav makedirs --help diff --git a/deps/repodata-patches/show_diff.py b/deps/repodata-patches/show_diff.py index d85ff3796b1b933ff088862ad4d3f1c1579db2cd..4141682a261ba6dc836ea95d3160e8a5e81f123e 100755 --- a/deps/repodata-patches/show_diff.py +++ b/deps/repodata-patches/show_diff.py @@ -7,7 +7,6 @@ import os import urllib from conda_build.index import _apply_instructions - from gen_patch_json import BASE_URL from gen_patch_json import SUBDIRS from gen_patch_json import gen_new_index_and_patch_instructions diff --git a/setup.py b/setup.py index 9bf5e1f69d0dd3275695f131b8cf76f054957077..88dbd8b674a3e8fb5ccb84fb1620920934771184 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ setup( "bdt.ci.cli": [ "base-build = bob.devtools.scripts.ci:base_build", "build = bob.devtools.scripts.ci:build", + "check = bob.devtools.scripts.ci:check", "test = bob.devtools.scripts.ci:test", "clean = bob.devtools.scripts.ci:clean", "base-deploy = bob.devtools.scripts.ci:base_deploy", diff --git a/version.txt b/version.txt index 868d7b27226d928488858c586826bc859393fe0b..ac54c0b9afedd1d236b2da0e956ad4fe33770a55 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.2.1b0 \ No newline at end of file +3.2.1b0