Merge branch 'issue-94' into 'master'

Fixes recursive directory upload with checksumming

Closes #94

See merge request !281

.pre-commit-config.yaml

@@ -2,17 +2,17 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.8.0
+    rev: 5.10.1
     hooks:
       - id: isort
         args: [--settings-path, "pyproject.toml"]
   - repo: https://github.com/psf/black
-    rev: 21.7b0
+    rev: 22.3.0
     hooks:
       - id: black
         exclude: bob/devtools/templates/setup.py
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+    rev: 4.0.1
     hooks:
       - id: flake8
         exclude: |
@@ -21,7 +21,7 @@ repos:
                   deps/bob-devel/run_test.py
               )$
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks:
       - id: check-ast
         exclude: bob/devtools/templates/setup.py

bob/devtools/bootstrap.py

@@ -5,6 +5,7 @@
 """Bootstraps a new miniconda installation and prepares it for development."""
 
 import glob
+import hashlib
 import logging
 import os
 import platform
@@ -218,7 +219,7 @@ def ensure_miniconda_sh():
     if platform.system().lower() == "darwin":  # apple silicon
         system = "MacOSX"
         if platform.machine().lower() == "arm64":
-            sha256 = "3cd1f11743f936ba522709eb7a173930c299ac681671a909b664222329a56290"
+            sha256 = "c753e99380e3f777d690e7131fc79c6f9cb8fb79af23fb53c7b8a0ade3361fec"
             machine = "arm64"
         else:  # intel
             sha256 = "955a6255871d9b53975e1c1581910844bcf33cbca613c7dba2842f6269917da6"
@@ -226,7 +227,7 @@ def ensure_miniconda_sh():
     else:
         system = "Linux"
         if platform.machine().lower() == "aarch64":  # raspberry pi
-            sha256 = "d597961defe8c7889f3e924d0dc7624fab2c8845abccdd8ffa8da8018ff3dc6e"
+            sha256 = "b6d3c0af4ba6202dc9994e70933d2de47ef8c4e6891afce768889a7d44e1db28"
             machine = "aarch64"
         else:  # intel
             sha256 = "c63907ba0971d2ca9a8775bd7ea48b635b2bdce4838b2f2d3a8e751876849595"
@@ -235,7 +236,6 @@ def ensure_miniconda_sh():
 
     if os.path.exists("miniconda.sh"):
         logger.info("(check) miniconda.sh sha256 (== %s?)", sha256)
-        import hashlib
 
         actual_sha256 = hashlib.sha256(
             open("miniconda.sh", "rb").read()
@@ -245,7 +245,7 @@ def ensure_miniconda_sh():
             return
         else:
             logger.info(
-                "Erasing cached miniconda3 installer (%s does NOT " "match)",
+                "Erasing cached miniconda3 installer (%s does NOT match)",
                 actual_sha256,
             )
             os.unlink("miniconda.sh")
@@ -259,6 +259,23 @@ def ensure_miniconda_sh():
     with open(dst, "wb") as f:
         f.write(response.read())
 
+    # checks that the checksum is correct on this file
+    actual_sha256 = hashlib.sha256(
+        open("miniconda.sh", "rb").read()
+    ).hexdigest()
+    if actual_sha256 != sha256:
+        os.unlink("miniconda.sh")
+        raise RuntimeError(
+            "Just downloaded miniconda3 installer sha256 checksum (%s) does "
+            "NOT match expected value (%s). Removing downloaded installer. "
+            "A wrong checksum may end up making the CI download too many copies "
+            "and be banned! You must fix this ASAP."
+            % (
+                actual_sha256,
+                sha256,
+            )
+        )
+
 
 def install_miniconda(prefix, name):
     """Creates a new miniconda installation.

bob/devtools/data/gitlab-ci/single-package.yaml

@@ -105,7 +105,6 @@ build_macos_arm_39:
   extends: .build_macos_arm_template
   variables:
     PYTHON_VERSION: "3.9"
-  allow_failure: true
   cache:
     key: "build-py39"
 

bob/devtools/scripts/dav.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 
 import os
+import re
+import tempfile
 
 import click
 import pkg_resources
@@ -275,11 +277,17 @@ def upload(private, execute, checksum, local, remote):
         return 1
 
     for k in local:
-        path_with_hash = k
-        if checksum:
-            path_with_hash = augment_path_with_hash(k)
-        actual_remote = remote + os.path.basename(path_with_hash)
-        remote_path = cl.get_url(actual_remote)
+
+        if not os.path.isdir(k):
+            path_with_hash = k
+            if checksum:
+                path_with_hash = augment_path_with_hash(k)
+            actual_remote = remote + os.path.basename(path_with_hash)
+            remote_path = cl.get_url(actual_remote)
+        else:
+            actual_remote = "/".join((remote, os.path.basename(k)))
+            actual_remote = re.sub("/+", "/", actual_remote)
+            remote_path = cl.get_url(actual_remote)
 
         if cl.check(actual_remote):
             echo_warning("resource %s already exists" % (remote_path,))
@@ -287,9 +295,37 @@ def upload(private, execute, checksum, local, remote):
             continue
 
         if os.path.isdir(k):
-            echo_info("cp -r %s %s" % (k, remote_path))
-            if execute:
-                cl.upload_directory(local_path=k, remote_path=actual_remote)
+            if checksum:
+                # checksumming requires we create a new temporary directory
+                # structure in which the filenames are already hashed
+                # correctly, as there are no means to pass a set of remote
+                # paths at client call.
+                with tempfile.TemporaryDirectory() as d:
+                    for root, __, files in os.walk(k):
+                        for f in files:
+                            rel_dir = os.path.relpath(root, k)
+                            os.makedirs(os.path.join(d, rel_dir), exist_ok=True)
+                            src = os.path.join(k, rel_dir, f)
+                            path_with_hash = augment_path_with_hash(src)
+                            os.symlink(
+                                os.path.join(os.path.realpath(k), rel_dir, f),
+                                os.path.join(
+                                    d,
+                                    rel_dir,
+                                    os.path.basename(path_with_hash),
+                                ),
+                            )
+                    echo_info("cp -r %s %s" % (d, remote_path))
+                    if execute:
+                        cl.upload_directory(
+                            local_path=d, remote_path=actual_remote
+                        )
+            else:
+                # it is a simple upload, you can use the actual local directory
+                # as a pointer
+                echo_info("cp -r %s %s" % (k, remote_path))
+                if execute:
+                    cl.upload_directory(local_path=k, remote_path=actual_remote)
         else:
             echo_info("cp %s %s" % (k, remote_path))
             if execute:

doc/conf.py

@@ -74,8 +74,8 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # General information about the project.
-project = u"bob.devtools"
-copyright = u"%s, Idiap Research Institute" % time.strftime("%Y")
+project = "bob.devtools"
+copyright = "%s, Idiap Research Institute" % time.strftime("%Y")
 
 # Grab the setup entry
 distribution = pkg_resources.require(project)[0]
@@ -125,8 +125,8 @@ pygments_style = "sphinx"
 
 # Some variables which are useful for generated material
 project_variable = project.replace(".", "_")
-short_description = u"Tools for development and CI integration of Bob packages"
-owner = [u"Idiap Research Institute"]
+short_description = "Tools for development and CI integration of Bob packages"
+owner = ["Idiap Research Institute"]
 
 
 # -- Options for HTML output ---------------------------------------------------
@@ -204,7 +204,7 @@ html_favicon = "img/favicon.ico"
 # html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = project_variable + u"_doc"
+htmlhelp_basename = project_variable + "_doc"
 
 
 # -- Post configuration --------------------------------------------------------