diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7bb4121fd20252cd89eb1672894bfc8da69844fa..4badf4ceeba996c80b83b3e0cd893461033c483b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,17 +2,17 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/timothycrosley/isort
- rev: 5.8.0
+ rev: 5.10.1
hooks:
- id: isort
args: [--settings-path, "pyproject.toml"]
- repo: https://github.com/psf/black
- rev: 21.7b0
+ rev: 22.3.0
hooks:
- id: black
exclude: bob/devtools/templates/setup.py
- repo: https://gitlab.com/pycqa/flake8
- rev: 3.9.2
+ rev: 4.0.1
hooks:
- id: flake8
exclude: |
@@ -21,7 +21,7 @@ repos:
deps/bob-devel/run_test.py
)$
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.0.1
+ rev: v4.1.0
hooks:
- id: check-ast
exclude: bob/devtools/templates/setup.py
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
index 0f91e4c99107cae490ef7e43fc1351be493e8581..f95b415d9499d9b2a523b4b89156a3bed01d15c2 100644
--- a/bob/devtools/bootstrap.py
+++ b/bob/devtools/bootstrap.py
@@ -5,6 +5,7 @@
"""Bootstraps a new miniconda installation and prepares it for development."""
import glob
+import hashlib
import logging
import os
import platform
@@ -218,7 +219,7 @@ def ensure_miniconda_sh():
if platform.system().lower() == "darwin": # apple silicon
system = "MacOSX"
if platform.machine().lower() == "arm64":
- sha256 = "3cd1f11743f936ba522709eb7a173930c299ac681671a909b664222329a56290"
+ sha256 = "c753e99380e3f777d690e7131fc79c6f9cb8fb79af23fb53c7b8a0ade3361fec"
machine = "arm64"
else: # intel
sha256 = "955a6255871d9b53975e1c1581910844bcf33cbca613c7dba2842f6269917da6"
@@ -226,7 +227,7 @@ def ensure_miniconda_sh():
else:
system = "Linux"
if platform.machine().lower() == "aarch64": # raspberry pi
- sha256 = "d597961defe8c7889f3e924d0dc7624fab2c8845abccdd8ffa8da8018ff3dc6e"
+ sha256 = "b6d3c0af4ba6202dc9994e70933d2de47ef8c4e6891afce768889a7d44e1db28"
machine = "aarch64"
else: # intel
sha256 = "c63907ba0971d2ca9a8775bd7ea48b635b2bdce4838b2f2d3a8e751876849595"
@@ -235,7 +236,6 @@ def ensure_miniconda_sh():
if os.path.exists("miniconda.sh"):
logger.info("(check) miniconda.sh sha256 (== %s?)", sha256)
- import hashlib
actual_sha256 = hashlib.sha256(
open("miniconda.sh", "rb").read()
@@ -245,7 +245,7 @@ def ensure_miniconda_sh():
return
else:
logger.info(
- "Erasing cached miniconda3 installer (%s does NOT " "match)",
+ "Erasing cached miniconda3 installer (%s does NOT match)",
actual_sha256,
)
os.unlink("miniconda.sh")
@@ -259,6 +259,23 @@ def ensure_miniconda_sh():
with open(dst, "wb") as f:
f.write(response.read())
+ # checks that the checksum is correct on this file
+ actual_sha256 = hashlib.sha256(
+ open("miniconda.sh", "rb").read()
+ ).hexdigest()
+ if actual_sha256 != sha256:
+ os.unlink("miniconda.sh")
+ raise RuntimeError(
+ "Just downloaded miniconda3 installer sha256 checksum (%s) does "
+ "NOT match expected value (%s). Removing downloaded installer. "
+ "A wrong checksum may end up making the CI download too many copies "
+ "and be banned! You must fix this ASAP."
+ % (
+ actual_sha256,
+ sha256,
+ )
+ )
+
def install_miniconda(prefix, name):
"""Creates a new miniconda installation.
diff --git a/bob/devtools/data/gitlab-ci/single-package.yaml b/bob/devtools/data/gitlab-ci/single-package.yaml
index 6375dda2d9d82d2cecd91aace793af4a75d853cb..10aa2b547bc39355c5ed7039c1179d1556b931d8 100644
--- a/bob/devtools/data/gitlab-ci/single-package.yaml
+++ b/bob/devtools/data/gitlab-ci/single-package.yaml
@@ -105,7 +105,6 @@ build_macos_arm_39:
extends: .build_macos_arm_template
variables:
PYTHON_VERSION: "3.9"
- allow_failure: true
cache:
key: "build-py39"
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
index 67d16f3044f68237a586a2f55d347e2780d862fa..a7f005056f87ed10b6f408fcd231a04ca210adfe 100644
--- a/bob/devtools/scripts/dav.py
+++ b/bob/devtools/scripts/dav.py
@@ -2,6 +2,8 @@
# -*- coding: utf-8 -*-
import os
+import re
+import tempfile
import click
import pkg_resources
@@ -275,11 +277,17 @@ def upload(private, execute, checksum, local, remote):
return 1
for k in local:
- path_with_hash = k
- if checksum:
- path_with_hash = augment_path_with_hash(k)
- actual_remote = remote + os.path.basename(path_with_hash)
- remote_path = cl.get_url(actual_remote)
+
+ if not os.path.isdir(k):
+ path_with_hash = k
+ if checksum:
+ path_with_hash = augment_path_with_hash(k)
+ actual_remote = remote + os.path.basename(path_with_hash)
+ remote_path = cl.get_url(actual_remote)
+ else:
+ actual_remote = "/".join((remote, os.path.basename(k)))
+ actual_remote = re.sub("/+", "/", actual_remote)
+ remote_path = cl.get_url(actual_remote)
if cl.check(actual_remote):
echo_warning("resource %s already exists" % (remote_path,))
@@ -287,9 +295,37 @@ def upload(private, execute, checksum, local, remote):
continue
if os.path.isdir(k):
- echo_info("cp -r %s %s" % (k, remote_path))
- if execute:
- cl.upload_directory(local_path=k, remote_path=actual_remote)
+ if checksum:
+ # checksumming requires we create a new temporary directory
+ # structure in which the filenames are already hashed
+ # correctly, as there are no means to pass a set of remote
+ # paths at client call.
+ with tempfile.TemporaryDirectory() as d:
+ for root, __, files in os.walk(k):
+ for f in files:
+ rel_dir = os.path.relpath(root, k)
+ os.makedirs(os.path.join(d, rel_dir), exist_ok=True)
+ src = os.path.join(k, rel_dir, f)
+ path_with_hash = augment_path_with_hash(src)
+ os.symlink(
+ os.path.join(os.path.realpath(k), rel_dir, f),
+ os.path.join(
+ d,
+ rel_dir,
+ os.path.basename(path_with_hash),
+ ),
+ )
+ echo_info("cp -r %s %s" % (d, remote_path))
+ if execute:
+ cl.upload_directory(
+ local_path=d, remote_path=actual_remote
+ )
+ else:
+ # it is a simple upload, you can use the actual local directory
+ # as a pointer
+ echo_info("cp -r %s %s" % (k, remote_path))
+ if execute:
+ cl.upload_directory(local_path=k, remote_path=actual_remote)
else:
echo_info("cp %s %s" % (k, remote_path))
if execute:
diff --git a/doc/conf.py b/doc/conf.py
index e2d7899a429ca3d866194add0379c009e0affe68..c9fe9127598d1b5bd97ebd8d01a782a85760caf9 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -74,8 +74,8 @@ source_suffix = ".rst"
master_doc = "index"
# General information about the project.
-project = u"bob.devtools"
-copyright = u"%s, Idiap Research Institute" % time.strftime("%Y")
+project = "bob.devtools"
+copyright = "%s, Idiap Research Institute" % time.strftime("%Y")
# Grab the setup entry
distribution = pkg_resources.require(project)[0]
@@ -125,8 +125,8 @@ pygments_style = "sphinx"
# Some variables which are useful for generated material
project_variable = project.replace(".", "_")
-short_description = u"Tools for development and CI integration of Bob packages"
-owner = [u"Idiap Research Institute"]
+short_description = "Tools for development and CI integration of Bob packages"
+owner = ["Idiap Research Institute"]
# -- Options for HTML output ---------------------------------------------------
@@ -204,7 +204,7 @@ html_favicon = "img/favicon.ico"
# html_file_suffix = None
# Output file base name for HTML help builder.
-htmlhelp_basename = project_variable + u"_doc"
+htmlhelp_basename = project_variable + "_doc"
# -- Post configuration --------------------------------------------------------