From 000d87e7318395079a6c92bf1aa7b8f91b2171e2 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Tue, 13 Aug 2019 14:36:06 +0200
Subject: [PATCH] [dav] Implement easy WebDAV access
---
bob/devtools/dav.py | 57 +++++++
bob/devtools/scripts/dav.py | 301 ++++++++++++++++++++++++++++++++++++
setup.py | 9 ++
3 files changed, 367 insertions(+)
create mode 100644 bob/devtools/dav.py
create mode 100644 bob/devtools/scripts/dav.py
diff --git a/bob/devtools/dav.py b/bob/devtools/dav.py
new file mode 100644
index 00000000..cd5c3d44
--- /dev/null
+++ b/bob/devtools/dav.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import configparser
+
+from .log import get_logger
+from .deploy import _setup_webdav_client
+
+logger = get_logger(__name__)
+
+
+def _get_config():
+ """Returns a dictionary with server parameters, or ask them to the user"""
+
+ # tries to figure if we can authenticate using a global configuration
+ cfgs = ["~/.bdt-webdav.cfg"]
+ cfgs = [os.path.expanduser(k) for k in cfgs]
+ for k in cfgs:
+ if os.path.exists(k):
+ data = configparser.ConfigParser()
+ data.read(k)
+ if 'global' not in data or \
+ 'server' not in data['global'] or \
+ 'username' not in data['global'] or \
+ 'password' not in data['global']:
+ assert KeyError, 'The file %s should contain a single ' \
+ '"global" section with 3 variables defined inside: ' \
+ '"server", "username", "password".' % (k,)
+ return data['global']
+
+ # ask the user for the information, cache credentials for future use
+ retval = dict()
+ retval['server'] = input("The base address of the server: ")
+ retval['username'] = input("Username: ")
+ retval['password'] = input("Password: ")
+
+ # record file for the user
+ data = configparser.ConfigParser()
+ data['global'] = retval
+ with open(cfgs[0], 'w') as f:
+ logger.warn('Recorded "%s" configuration file for next queries')
+ data.write(f)
+ os.chmod(cfgs[0], 0o600)
+ logger.warn('Changed mode of "%s" to be read-only to you')
+
+ return retval
+
+
+def setup_webdav_client(private):
+ """Returns a ready-to-use WebDAV client"""
+
+ config = _get_config()
+ root = '/private-upload' if private else '/public-upload'
+ c = _setup_webdav_client(config['server'], root, config['username'],
+ config['password'])
+ return c
diff --git a/bob/devtools/scripts/dav.py b/bob/devtools/scripts/dav.py
new file mode 100644
index 00000000..cb66d5b7
--- /dev/null
+++ b/bob/devtools/scripts/dav.py
@@ -0,0 +1,301 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+import click
+import pkg_resources
+from click_plugins import with_plugins
+
+from . import bdt
+
+from ..dav import setup_webdav_client
+from ..log import verbosity_option, get_logger, echo_normal, echo_info, \
+ echo_warning
+
+logger = get_logger(__name__)
+
+
+@with_plugins(pkg_resources.iter_entry_points("bdt.dav.cli"))
+@click.group(cls=bdt.AliasedGroup)
+def dav():
+ """Commands for reading/listing/renaming/copying content to a WebDAV server
+
+ Commands defined here may require a username and a password to operate
+ properly.
+ """
+ pass
+
+
+@dav.command(
+ epilog="""
+Examples:
+
+ 1. List contents of 'public':
+
+ $ bdt dav -vv list
+
+
+ 2. List contents of 'public/databases/latest':
+
+ $ bdt dav -vv list databases/latest
+
+
+ 3. List contents of 'private/docs':
+
+ $ bdt dav -vv list -p docs
+
+"""
+)
+@click.option(
+ "-p",
+ "--private/--no-private",
+ default=False,
+ help="If set, use the 'private' area instead of the public one",
+)
+@click.option(
+ "-l",
+ "--long-format/--no-long-format",
+ default=False,
+ help="If set, print details about each listed file",
+)
+@click.argument(
+ "path",
+ default="/",
+ required=False,
+)
+@verbosity_option()
+@bdt.raise_on_error
+def list(private, long_format, path):
+ """List the contents of a given WebDAV directory.
+ """
+
+ if not path.startswith('/'): path = '/' + path
+ cl = setup_webdav_client(private)
+ contents = cl.list(path)
+ remote_path = cl.get_url(path)
+ echo_info('ls %s' % (remote_path,))
+ for k in contents:
+ if long_format:
+ info = cl.info('/'.join((path, k)))
+ echo_normal('%-20s %-10s %s' % (info['created'], info['size'], k))
+ else:
+ echo_normal(k)
+
+
+@dav.command(
+ epilog="""
+Examples:
+
+ 1. Creates directory 'foo/bar' on the remote server:
+
+ $ bdt dav -vv mkdir foo/bar
+
+"""
+)
+@click.option(
+ "-p",
+ "--private/--no-private",
+ default=False,
+ help="If set, use the 'private' area instead of the public one",
+)
+@click.argument(
+ "path",
+ required=True,
+)
+@verbosity_option()
+@bdt.raise_on_error
+def makedirs(private, path):
+ """Creates a given directory, recursively (if necessary)
+
+ Gracefully exists if the directory is already there.
+ """
+
+ if not path.startswith('/'): path = '/' + path
+ cl = setup_webdav_client(private)
+ remote_path = cl.get_url(path)
+
+ if cl.check(path):
+ echo_warning('directory %s already exists' % (remote_path,))
+
+ rpath = ''
+ for k in path.split('/'):
+ rpath = '/'.join((rpath, k)) if rpath else k
+ if not cl.check(rpath):
+ echo_info('mkdir %s' % (rpath,))
+ cl.mkdir(rpath)
+
+
+@dav.command(
+ epilog="""
+Examples:
+
+ 1. Removes (recursively), everything under the 'remote/path/foo/bar' path:
+
+ $ bdt dav -vv rmtree remote/path/foo/bar
+
+ Notice this does not do anything for security. It just displays what it
+ would do. To actually run the rmtree comment pass the --execute flag (or
+ -x)
+
+
+ 2. Realy removes (recursively), everything under the 'remote/path/foo/bar'
+ path:
+
+ $ bdt dav -vv rmtree --execute remote/path/foo/bar
+
+
+"""
+)
+@click.option(
+ "-p",
+ "--private/--no-private",
+ default=False,
+ help="If set, use the 'private' area instead of the public one",
+)
+@click.option(
+ "-x",
+ "--execute/--no-execute",
+ default=False,
+ help="If this flag is set, then execute the removal",
+)
+@click.argument(
+ "path",
+ required=True,
+)
+@verbosity_option()
+@bdt.raise_on_error
+def rmtree(private, execute, path):
+ """Removes a whole directory tree from the WebDAV server
+
+ ATTENTION: There is no undo! Use --execute to execute.
+ """
+
+ if not execute:
+ echo_warning("!!!! DRY RUN MODE !!!!")
+ echo_warning("Nothing is being executed on server. Use -x to execute.")
+
+ if not path.startswith('/'): path = '/' + path
+ cl = setup_webdav_client(private)
+ remote_path = cl.get_url(path)
+
+ if not cl.check(path):
+ echo_warning('resource %s does not exist' % (remote_path,))
+ return
+
+ echo_info('rm -rf %s' % (remote_path,))
+ if execute:
+ cl.clean(path)
+
+
+@dav.command(
+ epilog="""
+Examples:
+
+ 1. Uploads a single file to a specific location:
+
+ $ bdt dav -vv copy local/file remote
+
+
+ 2. Uploads various resources at once:
+
+ $ bdt dav -vv copy local/file1 local/dir local/file2 remote
+
+"""
+)
+@click.option(
+ "-p",
+ "--private/--no-private",
+ default=False,
+ help="If set, use the 'private' area instead of the public one",
+)
+@click.option(
+ "-x",
+ "--execute/--no-execute",
+ default=False,
+ help="If this flag is set, then execute the removal",
+)
+@click.argument(
+ "local",
+ required=True,
+ type=click.Path(file_okay=True, dir_okay=True, exists=True),
+ nargs=-1,
+)
+@click.argument(
+ "remote",
+ required=True,
+)
+@verbosity_option()
+@bdt.raise_on_error
+def upload(private, execute, local, remote):
+ """Uploads a local resource (file or directory) to a remote destination
+
+ If the local resource is a directory, it is uploaded recursively. If the
+ remote resource with the same name already exists, an error is raised (use
+ rmtree to remove it first).
+
+ If the remote location does not exist, it is an error as well. As a
+ consequence, you cannot change the name of the resource being uploaded with
+ this command.
+
+ ATTENTION: There is no undo! Use --execute to execute.
+ """
+
+ if not execute:
+ echo_warning("!!!! DRY RUN MODE !!!!")
+ echo_warning("Nothing is being executed on server. Use -x to execute.")
+
+ if not remote.startswith('/'): remote = '/' + remote
+ cl = setup_webdav_client(private)
+
+ if not cl.check(remote):
+ echo_warning('base remote directory for upload %s does not exist' %
+ (remote,))
+ return 1
+
+ for k in local:
+ actual_remote = remote + os.path.basename(k)
+ remote_path = cl.get_url(actual_remote)
+
+ if cl.check(actual_remote):
+ echo_warning('resource %s already exists' % (remote_path,))
+ echo_warning('remove it first before uploading a new copy')
+ continue
+
+ if os.path.isdir(k):
+ echo_info('cp -r %s %s' % (k, remote_path))
+ if execute:
+ cl.upload_directory(local_path=k, remote_path=actual_remote)
+ else:
+ echo_info('cp %s %s' % (k, remote_path))
+ if execute:
+ cl.upload_file(local_path=k, remote_path=actual_remote)
+
+
+@dav.command(
+ epilog="""
+Examples:
+
+ 1. Lists the amount of free disk space on the WebDAV server:
+
+ $ bdt dav -vv free
+
+"""
+)
+@click.option(
+ "-p",
+ "--private/--no-private",
+ default=False,
+ help="If set, use the 'private' area instead of the public one",
+)
+@verbosity_option()
+@bdt.raise_on_error
+def free(private):
+ """Lists the amount of free space on the webserver disk
+ """
+
+ cl = setup_webdav_client(private)
+ echo_info('free')
+ data = cl.free()
+ echo_normal(data)
diff --git a/setup.py b/setup.py
index 995048cb..4d825c58 100644
--- a/setup.py
+++ b/setup.py
@@ -52,6 +52,7 @@ setup(
'test = bob.devtools.scripts.test:test',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
'ci = bob.devtools.scripts.ci:ci',
+ 'dav = bob.devtools.scripts.dav:dav',
'local = bob.devtools.scripts.local:local',
'gitlab = bob.devtools.scripts.gitlab:gitlab',
],
@@ -86,6 +87,14 @@ setup(
'base-build = bob.devtools.scripts.local:base_build',
],
+ 'bdt.dav.cli': [
+ 'list = bob.devtools.scripts.dav:list',
+ 'makedirs = bob.devtools.scripts.dav:makedirs',
+ 'rmtree = bob.devtools.scripts.dav:rmtree',
+ 'upload = bob.devtools.scripts.dav:upload',
+ #'free = bob.devtools.scripts.dav:free',
+ ],
+
},
classifiers=[
'Framework :: Bob',
--
GitLab