From ceb946a0b345386894895d2dd774772edb1c4267 Mon Sep 17 00:00:00 2001
From: Amir MOHAMMADI <amir.mohammadi@idiap.ch>
Date: Thu, 25 Mar 2021 14:24:26 +0100
Subject: [PATCH] Add a pyproject.toml check in the CI
---
.pre-commit-config.yaml | 13 ++-
bob/devtools/data/gitlab-ci/noarch.yaml | 1 +
.../data/gitlab-ci/single-package.yaml | 1 +
bob/devtools/scripts/ci.py | 94 +++++++++++++++----
conda/meta.yaml | 1 +
setup.py | 1 +
6 files changed, 90 insertions(+), 21 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 20644394..e37d785c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,17 +2,21 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/timothycrosley/isort
- rev: 4.3.21-2
+ rev: 5.8.0
hooks:
- id: isort
args: [-sl]
- repo: https://github.com/psf/black
- rev: stable
+ rev: 20.8b1
hooks:
- id: black
exclude: bob/devtools/templates/setup.py
+ - repo: https://gitlab.com/pycqa/flake8
+ rev: 3.9.0
+ hooks:
+ - id: flake8
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v2.0.0
+ rev: v3.4.0
hooks:
- id: check-ast
exclude: bob/devtools/templates/setup.py
@@ -22,10 +26,9 @@ repos:
- id: debug-statements
exclude: bob/devtools/templates/setup.py
- id: check-added-large-files
- - id: flake8
exclude: bob/devtools/templates/setup.py
- id: check-yaml
- exclude: ./*/meta.yaml
+ exclude: .*/meta.yaml
- repo: local
hooks:
- id: sphinx-build
diff --git a/bob/devtools/data/gitlab-ci/noarch.yaml b/bob/devtools/data/gitlab-ci/noarch.yaml
index c804564f..e34164dc 100644
--- a/bob/devtools/data/gitlab-ci/noarch.yaml
+++ b/bob/devtools/data/gitlab-ci/noarch.yaml
@@ -36,6 +36,7 @@ stages:
extends: .bootstrap
stage: build
script:
+ - bdt ci check -vv
- bdt ci build -vv
- bdt ci clean -vv
artifacts:
diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml
index 164de137..0f14aabe 100644
--- a/bob/devtools/data/gitlab-ci/single-package.yaml
+++ b/bob/devtools/data/gitlab-ci/single-package.yaml
@@ -37,6 +37,7 @@ stages:
extends: .bootstrap
stage: build
script:
+ - bdt ci check -vv
- bdt ci build -vv
- bdt ci clean -vv
artifacts:
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
index 57773b3d..6eaa9a0b 100644
--- a/bob/devtools/scripts/ci.py
+++ b/bob/devtools/scripts/ci.py
@@ -3,6 +3,7 @@
import glob
import os
import shutil
+import sys
import click
import pkg_resources
@@ -32,7 +33,19 @@ logger = get_logger(__name__)
@with_plugins(pkg_resources.iter_entry_points("bdt.ci.cli"))
@click.group(cls=bdt.AliasedGroup)
-def ci():
+@click.option(
+ "-l",
+ "--local",
+ is_flag=True,
+ default=False,
+ help="Setups CI like environment for running ci scripts locally.",
+)
+@click.option(
+ "--python",
+ default=".".join(str(v) for v in sys.version_info[:2]),
+ help="Change the Python version for local runs.",
+)
+def ci(local, python):
"""Commands for building packages and handling CI activities.
Commands defined here are supposed to run on our CI, where a number
@@ -40,7 +53,10 @@ def ci():
**NOT** attempt to run these commands in your own installation.
Unexpected errors may occur.
"""
- pass
+ if local:
+ os.environ["CI_JOB_TOKEN"] = "0"
+ os.environ["CI_PROJECT_DIR"] = "."
+ os.environ["PYTHON_VERSION"] = python
@ci.command(
@@ -297,9 +313,10 @@ def pypi(package, dry_run):
% os.environ["CI_PROJECT_PATH"]
)
- from ..constants import CACERT
from twine.settings import Settings
+ from ..constants import CACERT
+
settings = Settings(
username=os.environ["PYPIUSER"],
password=os.environ["PYPIPASS"],
@@ -397,7 +414,8 @@ def base_build(order, group, dry_run):
continue
variants_file = select_conda_build_config(
- paths=[recipe, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build configuration file: %s", variants_file)
@@ -449,25 +467,31 @@ def test(ctx, dry_run):
recipe_dir = os.path.join(os.path.realpath(os.curdir), "conda")
condarc = select_user_condarc(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
if condarc is not None:
logger.info("Condarc configuration file: %s", condarc)
variants_file = select_conda_build_config(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build configuration file: %s", variants_file)
append_file = select_conda_recipe_append(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build recipe-append file: %s", append_file)
from .test import test
base_path = os.path.join(
- os.environ["CONDA_ROOT"], "conda-bld", "*", os.environ["CI_PROJECT_NAME"],
+ os.environ["CONDA_ROOT"],
+ "conda-bld",
+ "*",
+ os.environ["CI_PROJECT_NAME"],
)
ctx.invoke(
@@ -527,18 +551,21 @@ def build(ctx, dry_run, recipe_dir):
# Use custom variants and append files if available on recipe-dir
condarc = select_user_condarc(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
if condarc is not None:
logger.info("Condarc configuration file: %s", condarc)
variants_file = select_conda_build_config(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build configuration file: %s", variants_file)
append_file = select_conda_recipe_append(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build recipe-append file: %s", append_file)
@@ -582,8 +609,8 @@ def clean(ctx):
meant to be used outside this context.
"""
- from ..build import git_clean_build
from ..bootstrap import run_cmdline
+ from ..build import git_clean_build
git_clean_build(run_cmdline, verbose=(ctx.meta["verbosity"] >= 3))
@@ -640,9 +667,11 @@ def nightlies(ctx, order, dry_run):
token = os.environ["CI_JOB_TOKEN"]
+ from urllib.request import urlopen
+
import git
+
from .build import build
- from urllib.request import urlopen
# loaded all recipes, now cycle through them implementing what is described
# in the documentation of this function
@@ -676,21 +705,28 @@ def nightlies(ctx, order, dry_run):
recipe_dir = os.path.join(clone_to, "conda")
condarc = select_user_condarc(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
if condarc is not None:
logger.info("Condarc configuration file: %s", condarc)
variants_file = select_conda_build_config(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build configuration file: %s", variants_file)
append_file = select_conda_recipe_append(
- paths=[recipe_dir, os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME"),
+ paths=[recipe_dir, os.curdir],
+ branch=os.environ.get("CI_COMMIT_REF_NAME"),
)
logger.info("Conda build recipe-append file: %s", append_file)
+ logger.info("Running checks")
+ ctx.invoke(check, root=clone_to)
+
+ logger.info("Building")
ctx.invoke(
build,
recipe_dir=[recipe_dir],
@@ -951,3 +987,29 @@ def clean_betas(dry_run):
password=os.environ["DOCPASS"],
includes=includes,
)
+
+
+@ci.command(
+ epilog="""
+Example:
+
+ bdt ci check -vv
+"""
+)
+@click.option(
+ "-d",
+ "--dir",
+ "root",
+ default=os.path.join(os.path.realpath(os.curdir)),
+ help="Path to the root folder of the package.",
+)
+@verbosity_option()
+@bdt.raise_on_error
+def check(root):
+ # checks if a pyproject.toml file exists
+ path = os.path.join(root, "pyproject.toml")
+ if not os.path.isfile(path):
+ raise RuntimeError(
+ "pyproject.toml file not found in the root folder of the package. "
+ "See https://gitlab.idiap.ch/bob/bob/-/wikis/ci-checks#pyprojecttoml"
+ )
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 821b281f..2ac890ed 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -98,6 +98,7 @@ test:
- bdt ci nightlies --help
- bdt ci docs --help
- bdt ci clean-betas --help
+ - bdt ci check -vv
- bdt dav --help
- bdt dav list --help
- bdt dav makedirs --help
diff --git a/setup.py b/setup.py
index 9bf5e1f6..88dbd8b6 100644
--- a/setup.py
+++ b/setup.py
@@ -73,6 +73,7 @@ setup(
"bdt.ci.cli": [
"base-build = bob.devtools.scripts.ci:base_build",
"build = bob.devtools.scripts.ci:build",
+ "check = bob.devtools.scripts.ci:check",
"test = bob.devtools.scripts.ci:test",
"clean = bob.devtools.scripts.ci:clean",
"base-deploy = bob.devtools.scripts.ci:base_deploy",
--
GitLab