Merge branch 'dav-hash-upload' into 'master'

bdt dav upload automatically adds checksum to filename on remote

Closes #64

See merge request !210

bob/devtools/dav.py

@@ -2,15 +2,17 @@
 # -*- coding: utf-8 -*-
 
 import configparser
+import hashlib
 import os
+import pathlib
 import re
 
 from distutils.version import StrictVersion
 
 import dateutil.parser
 
-from .deploy import _setup_webdav_client
 from .config import read_config
+from .deploy import _setup_webdav_client
 from .log import echo_normal
 from .log import echo_warning
 from .log import get_logger
@@ -26,20 +28,23 @@ def _get_config():
 
     # this does some sort of validation for the "webdav" data...
     if "webdav" in data:
-        if ("server" not in data["webdav"]
-                or "username" not in data["webdav"]
-                or "password" not in data["webdav"]
-                ):
+        if (
+            "server" not in data["webdav"]
+            or "username" not in data["webdav"]
+            or "password" not in data["webdav"]
+        ):
             raise KeyError(
-                f"If the configuration file {k} contains a \"webdav\" " \
-                f"section, it should contain 3 variables defined inside: " \
+                f'If the configuration file {k} contains a "webdav" '
+                f"section, it should contain 3 variables defined inside: "
                 f'"server", "username", "password".'
             )
     else:
         # ask the user for the information, in case nothing available
-        logger.warn("Requesting server information for webDAV operation. " \
-                "(To create a configuration file, and avoid these, follow " \
-                "the Setup subsection at our Installation manual.)")
+        logger.warn(
+            "Requesting server information for webDAV operation. "
+            "(To create a configuration file, and avoid these, follow "
+            "the Setup subsection at our Installation manual.)"
+        )
         webdav_data = dict()
         webdav_data["server"] = input("The base address of the server: ")
         webdav_data["username"] = input("Username: ")
@@ -49,6 +54,33 @@ def _get_config():
     return data["webdav"]
 
 
+def compute_sha256(path):
+    sha256_hash = hashlib.sha256()
+    with open(path, "rb") as f:
+        for byte_block in iter(lambda: f.read(65535), b""):
+            sha256_hash.update(byte_block)
+    file_hash = sha256_hash.hexdigest()
+    return file_hash
+
+
+def augment_path_with_hash(path):
+    """Adds the first 8 digits of sha256sum of a file to its name.
+
+    Example::
+
+        augment_path_with_hash('/datasets/pad-face-replay-attack.tar.gz')
+        '/datasets/pad-face-replay-attack-a8e31cc3.tar.gz'
+    """
+    path = pathlib.Path(path)
+    if not path.is_file():
+        raise ValueError(f"Can only augment path to files with a hash. Got: {path}")
+    file_hash = compute_sha256(path)[:8]
+    suffix = "".join(path.suffixes)
+    base_name = str(path.name)[: -len(suffix) or None]
+    new_path = path.parent / f"{base_name}-{file_hash}{suffix}"
+    return str(new_path)
+
+
 def setup_webdav_client(private):
     """Returns a ready-to-use WebDAV client"""
 

bob/devtools/scripts/dav.py

@@ -8,6 +8,7 @@ import pkg_resources
 
 from click_plugins import with_plugins
 
+from ..dav import augment_path_with_hash
 from ..dav import remove_old_beta_packages
 from ..dav import setup_webdav_client
 from ..log import echo_info
@@ -66,13 +67,14 @@ Examples:
     help="If set, print details about each listed file",
 )
 @click.argument(
-    "path", default="/", required=False,
+    "path",
+    default="/",
+    required=False,
 )
 @verbosity_option()
 @bdt.raise_on_error
 def list(private, long_format, path):
-    """List the contents of a given WebDAV directory.
-    """
+    """List the contents of a given WebDAV directory."""
 
     if not path.startswith("/"):
         path = "/" + path
@@ -105,7 +107,8 @@ Examples:
     help="If set, use the 'private' area instead of the public one",
 )
 @click.argument(
-    "path", required=True,
+    "path",
+    required=True,
 )
 @verbosity_option()
 @bdt.raise_on_error
@@ -165,7 +168,8 @@ Examples:
     help="If this flag is set, then execute the removal",
 )
 @click.argument(
-    "path", required=True,
+    "path",
+    required=True,
 )
 @verbosity_option()
 @bdt.raise_on_error
@@ -199,12 +203,12 @@ Examples:
 
   1. Uploads a single file to a specific location:
 
-     $ bdt dav -vv copy local/file remote
+     $ bdt dav upload -vv --checksum local/file remote
 
 
   2. Uploads various resources at once:
 
-     $ bdt dav -vv copy local/file1 local/dir local/file2 remote
+     $ bdt dav upload -vv --checksum local/file1 local/dir local/file2 remote
 
 """
 )
@@ -220,6 +224,12 @@ Examples:
     default=False,
     help="If this flag is set, then execute the removal",
 )
+@click.option(
+    "-c",
+    "--checksum/--no-checksum",
+    default=False,
+    help="If set, will augment the filename(s) on the server with 8 first characters of their sha256 checksum.",
+)
 @click.argument(
     "local",
     required=True,
@@ -227,11 +237,12 @@ Examples:
     nargs=-1,
 )
 @click.argument(
-    "remote", required=True,
+    "remote",
+    required=True,
 )
 @verbosity_option()
 @bdt.raise_on_error
-def upload(private, execute, local, remote):
+def upload(private, execute, checksum, local, remote):
     """Uploads a local resource (file or directory) to a remote destination
 
     If the local resource is a directory, it is uploaded recursively.  If the
@@ -258,7 +269,10 @@ def upload(private, execute, local, remote):
         return 1
 
     for k in local:
-        actual_remote = remote + os.path.basename(k)
+        path_with_hash = k
+        if checksum:
+            path_with_hash = augment_path_with_hash(k)
+        actual_remote = remote + os.path.basename(path_with_hash)
         remote_path = cl.get_url(actual_remote)
 
         if cl.check(actual_remote):
@@ -308,7 +322,8 @@ Examples:
     help="If this flag is set, then execute the removal",
 )
 @click.argument(
-    "path", required=True,
+    "path",
+    required=True,
 )
 @verbosity_option()
 @bdt.raise_on_error