Skip to content
Snippets Groups Projects
Commit 40f08faf authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

bdt dav upload automatically adds checksum to filename on remote

Adds an option (--checksum) to let bdt automatically augment the
filename on the remote with the first 8 digits of its sha256sum.
Fixes #64
parent 658fe3b5
Branches
Tags
1 merge request!210bdt dav upload automatically adds checksum to filename on remote
Pipeline #48736 passed
......@@ -2,15 +2,17 @@
# -*- coding: utf-8 -*-
import configparser
import hashlib
import os
import pathlib
import re
from distutils.version import StrictVersion
import dateutil.parser
from .deploy import _setup_webdav_client
from .config import read_config
from .deploy import _setup_webdav_client
from .log import echo_normal
from .log import echo_warning
from .log import get_logger
......@@ -26,20 +28,23 @@ def _get_config():
# this does some sort of validation for the "webdav" data...
if "webdav" in data:
if ("server" not in data["webdav"]
or "username" not in data["webdav"]
or "password" not in data["webdav"]
):
if (
"server" not in data["webdav"]
or "username" not in data["webdav"]
or "password" not in data["webdav"]
):
raise KeyError(
f"If the configuration file {k} contains a \"webdav\" " \
f"section, it should contain 3 variables defined inside: " \
f'If the configuration file {k} contains a "webdav" '
f"section, it should contain 3 variables defined inside: "
f'"server", "username", "password".'
)
else:
# ask the user for the information, in case nothing available
logger.warn("Requesting server information for webDAV operation. " \
"(To create a configuration file, and avoid these, follow " \
"the Setup subsection at our Installation manual.)")
logger.warn(
"Requesting server information for webDAV operation. "
"(To create a configuration file, and avoid these, follow "
"the Setup subsection at our Installation manual.)"
)
webdav_data = dict()
webdav_data["server"] = input("The base address of the server: ")
webdav_data["username"] = input("Username: ")
......@@ -49,6 +54,33 @@ def _get_config():
return data["webdav"]
def compute_sha256(path):
sha256_hash = hashlib.sha256()
with open(path, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
file_hash = sha256_hash.hexdigest()
return file_hash
def augment_path_with_hash(path):
"""Adds the first 8 digits of sha256sum of a file to its name.
Example::
augment_path_with_hash('/datasets/pad-face-replay-attack.tar.gz')
'/datasets/pad-face-replay-attack-a8e31cc3.tar.gz'
"""
path = pathlib.Path(path)
if not path.is_file():
raise ValueError(f"Can only augment path to files with a hash. Got: {path}")
file_hash = compute_sha256(path)[:8]
suffix = "".join(path.suffixes)
base_name = str(path.name)[: -len(suffix) or None]
new_path = path.parent / f"{base_name}-{file_hash}{suffix}"
return str(new_path)
def setup_webdav_client(private):
"""Returns a ready-to-use WebDAV client"""
......
......@@ -8,6 +8,7 @@ import pkg_resources
from click_plugins import with_plugins
from ..dav import augment_path_with_hash
from ..dav import remove_old_beta_packages
from ..dav import setup_webdav_client
from ..log import echo_info
......@@ -66,13 +67,14 @@ Examples:
help="If set, print details about each listed file",
)
@click.argument(
"path", default="/", required=False,
"path",
default="/",
required=False,
)
@verbosity_option()
@bdt.raise_on_error
def list(private, long_format, path):
"""List the contents of a given WebDAV directory.
"""
"""List the contents of a given WebDAV directory."""
if not path.startswith("/"):
path = "/" + path
......@@ -105,7 +107,8 @@ Examples:
help="If set, use the 'private' area instead of the public one",
)
@click.argument(
"path", required=True,
"path",
required=True,
)
@verbosity_option()
@bdt.raise_on_error
......@@ -165,7 +168,8 @@ Examples:
help="If this flag is set, then execute the removal",
)
@click.argument(
"path", required=True,
"path",
required=True,
)
@verbosity_option()
@bdt.raise_on_error
......@@ -199,12 +203,12 @@ Examples:
1. Uploads a single file to a specific location:
$ bdt dav -vv copy local/file remote
$ bdt dav upload -vv --checksum local/file remote
2. Uploads various resources at once:
$ bdt dav -vv copy local/file1 local/dir local/file2 remote
$ bdt dav upload -vv --checksum local/file1 local/dir local/file2 remote
"""
)
......@@ -220,6 +224,12 @@ Examples:
default=False,
help="If this flag is set, then execute the removal",
)
@click.option(
"-c",
"--checksum/--no-checksum",
default=False,
help="If set, will augment the filename(s) on the server with 8 first characters of their sha256 checksum.",
)
@click.argument(
"local",
required=True,
......@@ -227,11 +237,12 @@ Examples:
nargs=-1,
)
@click.argument(
"remote", required=True,
"remote",
required=True,
)
@verbosity_option()
@bdt.raise_on_error
def upload(private, execute, local, remote):
def upload(private, execute, checksum, local, remote):
"""Uploads a local resource (file or directory) to a remote destination
If the local resource is a directory, it is uploaded recursively. If the
......@@ -258,7 +269,10 @@ def upload(private, execute, local, remote):
return 1
for k in local:
actual_remote = remote + os.path.basename(k)
path_with_hash = k
if checksum:
path_with_hash = augment_path_with_hash(k)
actual_remote = remote + os.path.basename(path_with_hash)
remote_path = cl.get_url(actual_remote)
if cl.check(actual_remote):
......@@ -308,7 +322,8 @@ Examples:
help="If this flag is set, then execute the removal",
)
@click.argument(
"path", required=True,
"path",
required=True,
)
@verbosity_option()
@bdt.raise_on_error
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment