diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c8933e4a83c854dcee537489d55fbeae3f9a95d..2d8dbdab73acfe1f12dee2974dbd69899ee68119 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -77,7 +77,7 @@ build_macosx_36:
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
- - bdt --help
+ - bdt ci deploy -vv --dry-run
dependencies:
- build_linux_36
- build_macosx_36
diff --git a/LICENSE b/LICENSE
index 0c3160331a4198b350981b4eb39a1b2f851813f6..782e5e5e3ae29d87e82a55fbea22a0fada00d47f 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/
+Copyright (c) 2018 Idiap Research Institute, http://www.idiap.ch/
Written by Andre Anjos <andre.anjos@idiap.ch>
Redistribution and use in source and binary forms, with or without
@@ -25,3 +25,36 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------------------------------------------------------------------------------
+
+Code from the "webdav3" directory was copied from the Github repository
+https://github.com/ezhov-evgeny/webdav-client-python-3, but later modified and
+repackaged as part of this package.
+
+The authors asked to reproduce the following license text.
+
+
+COPYRIGHT AND PERMISSION NOTICE
+-------------------------------
+
+Copyright (c) 2016, The WDC Project, and many contributors, see the THANKS
+file.
+
+All rights reserved.
+
+Permission to use, copy, modify, and distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
+NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall not be
+used in advertising or otherwise to promote the sale, use or other dealings in
+this Software without prior written authorization of the copyright holder.
diff --git a/bob/devtools/ci.py b/bob/devtools/ci.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f79497682963132fef9ab0a25d1dbd30c86c8f4
--- /dev/null
+++ b/bob/devtools/ci.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''Tools to help CI-based builds and artifact deployment'''
+
+
+import os
+
+import logging
+logger = logging.getLogger(__name__)
+
+import git
+import packaging.version
+
+
+def is_master(refname, tag):
+ '''Tells if we're on the master branch via ref_name or tag
+
+ This function checks if the name of the branch being built is "master". If a
+ tag is set, then it checks if the tag is on the master branch. If so, then
+ also returns ``True``, otherwise, ``False``.
+
+ Args:
+
+ refname: The value of the environment variable ``CI_COMMIT_REF_NAME``
+ tag: The value of the environment variable ``CI_COMMIT_TAG`` - (may be
+ ``None``)
+
+ Returns: a boolean, indicating we're building the master branch **or** that
+ the tag being built was issued on the master branch.
+ '''
+
+ if tag is not None:
+ repo = git.Repo(os.environ['CI_PROJECT_DIR'])
+ _tag = repo.tag('refs/tags/%s' % tag)
+ return _tag.commit in repo.iter_commits(rev='master')
+
+ return refname == 'master'
+
+
+def is_visible_outside(package, visibility):
+ '''Determines if the project is visible outside Idiap'''
+
+ logger.info('Project %s visibility is "%s"', package, visibility)
+
+ if visibility == 'internal':
+ visibility = 'private' #the same thing for this command
+ logger.warn('Project %s visibility switched to "%s". ' \
+ 'For this command, it all boils down to the same...', package,
+ visibility)
+
+ return visibility == 'public'
+
+
+def is_stable(package, refname, tag):
+ '''Determines if the package being published is stable
+
+ This is done by checking if a tag was set for the package. If that is the
+ case, we still cross-check the tag is on the "master" branch. If everything
+ checks out, we return ``True``. Else, ``False``.
+
+ Args:
+
+ package: Package name in the format "group/name"
+ refname: The current value of the environment ``CI_COMMIT_REF_NAME``
+ tag: The current value of the enviroment ``CI_COMMIT_TAG`` (may be
+ ``None``)
+
+ Returns: a boolean, indicating if the current build is for a stable release
+ '''
+
+ if tag is not None:
+ logger.info('Project %s tag is "%s"', package, tag)
+ parsed_tag = packaging.version.Version(tag)
+
+ if parsed_tag.is_prerelease:
+ logger.warn('Pre-release detected - not publishing to stable channels')
+ return False
+
+ if is_master(os.environ['CI_COMMIT_REF_NAME'], tag):
+ return True
+ else:
+ logger.warn('Tag %s in non-master branch will be ignored', tag)
+ return False
+
+ logger.info('No tag information available at build')
+ logger.info('Considering this to be a pre-release build')
+ return False
diff --git a/bob/devtools/constants.py b/bob/devtools/constants.py
index a5179b1d9e9fe4ef5be099fa11dd0c166256a2ce..b186d73ec0838f329dd66382a0fba30d26a519c9 100644
--- a/bob/devtools/constants.py
+++ b/bob/devtools/constants.py
@@ -29,6 +29,48 @@ SERVER = 'http://www.idiap.ch'
'''This is the default server use use to store data and build artifacts'''
+CONDA_CHANNELS = {
+ True: { #stable?
+ False: '/private/conda', #visible outside?
+ True: '/public/conda',
+ },
+ False: {
+ False: '/private/conda/label/beta', #visible outside?
+ True: '/public/conda/label/beta',
+ },
+ }
+'''Default locations of our stable, beta, public and private conda channels'''
+
+
+WEBDAV_PATHS = {
+ True: { #stable?
+ False: { #visible?
+ 'root': '/private-upload',
+ 'conda': '/conda',
+ 'docs': '/docs',
+ },
+ True: { #visible?
+ 'root': '/public-upload',
+ 'conda': '/conda',
+ 'docs': '/docs',
+ },
+ },
+ False: { #stable?
+ False: { #visible?
+ 'root': '/private-upload',
+ 'conda': '/conda/label/beta',
+ 'docs': '/docs',
+ },
+ True: { #visible?
+ 'root': '/public-upload',
+ 'conda': '/conda/label/beta',
+ 'docs': '/docs',
+ },
+ },
+ }
+'''Default locations of our webdav upload paths'''
+
+
IDIAP_ROOT_CA = b'''
Idiap Root CA 2016 - for internal use
=====================================
diff --git a/bob/devtools/release.py b/bob/devtools/release.py
index 2dad02a879f887445448312db33cb5b941d13bca..8e0bbc793fb3a491b3700473dd029b0ae1c339e5 100644
--- a/bob/devtools/release.py
+++ b/bob/devtools/release.py
@@ -61,9 +61,13 @@ def get_gitlab_instance():
cfgs = [os.path.expanduser(k) for k in cfgs]
if any([os.path.exists(k) for k in cfgs]):
gl = gitlab.Gitlab.from_config('idiap', cfgs)
- else: #ask the user for a token
+ else: #ask the user for a token or use one from the current runner
server = "https://gitlab.idiap.ch"
- token = input("%s token: " % server)
+ token = os.environ.get('CI_JOB_TOKEN')
+ if token is None:
+ logger.debug('Did not find any of %s nor CI_JOB_TOKEN is defined. ' \
+ 'Asking for user token on the command line...', '|'.join(cfgs))
+ token = input("Your %s (private) token: " % server)
gl = gitlab.Gitlab(server, private_token=token, api_version=4)
return gl
diff --git a/bob/devtools/scripts/ci.py b/bob/devtools/scripts/ci.py
new file mode 100644
index 0000000000000000000000000000000000000000..5078e25954e9fb56f0ae12e69e139c598b699958
--- /dev/null
+++ b/bob/devtools/scripts/ci.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+
+import os
+import re
+import glob
+import logging
+logger = logging.getLogger(__name__)
+
+import click
+import pkg_resources
+from click_plugins import with_plugins
+
+from . import bdt
+from ..log import verbosity_option
+from ..ci import is_stable, is_visible_outside
+from ..constants import SERVER, WEBDAV_PATHS
+from .. import webdav3
+
+
+@with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
+@click.group(cls=bdt.AliasedGroup)
+def ci():
+ """Commands for building packages and handling CI activities
+
+ Commands defined here are supposed to run on our CI, where a number of
+ variables that define their behavior is correctly defined. Do **NOT**
+ attempt to run these commands in your own installation. Unexpected errors
+ may occur.
+ """
+ pass
+
+
+@ci.command(epilog='''
+Examples:
+
+ 1. Deploys current build artifacts to the appropriate channels:
+
+ $ bdt ci deploy -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+ help='Only goes through the actions, but does not execute them ' \
+ '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+ 'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def deploy(dry_run):
+ """Deploys build artifacts (conda packages and sphinx documentation)
+
+ Deployment happens at the "right" locations - conda packages which do not
+ represent stable releases are deployed to our conda "beta" channel, while
+ stable packages to our root channel. Sphinx documentation from unstable
+ builds (typically the master branch) is deployed to the documentation
+ server in a subdirectory named after the current branch name, while stable
+ documentation is deployed to a special subdirectory named "stable" and to
+ the respective tag name.
+ """
+
+ if dry_run:
+ logger.warn('!!!! DRY RUN MODE !!!!')
+ logger.warn('Nothing is being deployed to server')
+
+ package = os.environ['CI_PROJECT_PATH']
+
+ # determine project visibility
+ visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
+
+ # determine if building master branch or tag - and if tag is on master
+ tag = os.environ.get('CI_COMMIT_TAG')
+ stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag)
+
+ server_info = WEBDAV_PATHS[stable][visible]
+
+ logger.info('Deploying conda packages to %s/%s%s...', SERVER,
+ server_info['root'], server_info['conda'])
+
+ # setup webdav connection
+ webdav_options = {
+ 'webdav_hostname': SERVER,
+ 'webdav_root': server_info['root'],
+ 'webdav_login': os.environ['DOCUSER'],
+ 'webdav_password': os.environ['DOCPASS'],
+ }
+ davclient = Client(options)
+ assert davclient.valid()
+
+ group, name = package.split('/')
+
+ # uploads conda package artificats
+ for arch in ('linux-64', 'osx-64', 'noarch'):
+ # finds conda packages and uploads what we can find
+ package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
+ name + '*.tar.bz2')
+ deploy_packages = glob.glob(package_path)
+ if len(deploy_packages):
+ logger.info('Deploying %d conda package(s) for %s',
+ len(deploy_packages), arch)
+ for k in deploy_packages:
+ remote_path = '%s%s/%s' % (server_info['root'], server_info['conda'],
+ os.path.basename(k))
+ if davclient.check(remote_path):
+ raise RuntimeError('The file %s/%s already exists on the server ' \
+ '- this can be due to more than one build with deployment ' \
+ 'running at the same time. Re-running the broken builds ' \
+ 'normally fixes it' % (SERVER, remote_path))
+ if not dry_run:
+ davclient.upload(local_path=k, remote_path=remote_path)
+
+ # uploads documentation artifacts
+ local_docs = os.path.join(os.environ['CI_PROJECT_DIR'], 'sphinx')
+ if not os.path.exists(local_docs):
+ raise RuntimeError('Documentation is not available at %s - ' \
+ 'ensure documentation is being produced for your project!' % \
+ local_docs)
+
+ remote_path_prefix = '%s%s/%s' % (server_info['root'], server_info['docs'],
+ package)
+
+ # finds out the correct mixture of sub-directories we should deploy to.
+ # 1. if ref-name is a tag, don't forget to publish to 'master' as well -
+ # all tags are checked to come from that branch
+ # 2. if ref-name is a branch name, deploy to it
+ # 3. in case a tag is being published, make sure to deploy to the special
+ # "stable" subdir as well
+ deploy_docs_to = set([os.environ['CI_COMMIT_REF_NAME']])
+ if stable:
+ deploy_docs_to.add('master')
+ if os.environ.get('CI_COMMIT_TAG') is not None:
+ deploy_docs_to.add(os.environ['CI_COMMIT_TAG'])
+ deploy_docs_to.add('stable')
+
+ for k in deploy_docs_to:
+ remote_path = '%s/%s' % (remote_path_prefix, k)
+ logger.info('Deploying package documentation to %s/%s...', SERVER,
+ remote_path)
+ if not dry_run:
+ client.upload_directory(local_path=local_docs, remote_path=remote_path)
diff --git a/bob/devtools/webdav3/__init__.py b/bob/devtools/webdav3/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/bob/devtools/webdav3/client.py b/bob/devtools/webdav3/client.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1850d705ac91c92c4ecffb01a0ed4685231bd3c
--- /dev/null
+++ b/bob/devtools/webdav3/client.py
@@ -0,0 +1,975 @@
+# -*- coding: utf-8
+
+import functools
+import logging
+import os
+import shutil
+import threading
+from io import BytesIO
+from re import sub
+
+import lxml.etree as etree
+import requests
+
+from .connection import *
+from .exceptions import *
+from .urn import Urn
+
+try:
+ from urllib.parse import unquote, urlsplit
+except ImportError:
+ from urllib import unquote
+ from urlparse import urlsplit
+
+__version__ = "0.2"
+log = logging.getLogger(__name__)
+
+
+def listdir(directory):
+ """Returns list of nested files and directories for local directory by path
+
+ :param directory: absolute or relative path to local directory
+ :return: list nested of file or directory names
+ """
+ file_names = list()
+ for filename in os.listdir(directory):
+ file_path = os.path.join(directory, filename)
+ if os.path.isdir(file_path):
+ filename = "{filename}{separate}".format(filename=filename, separate=os.path.sep)
+ file_names.append(filename)
+ return file_names
+
+
+def get_options(option_type, from_options):
+ """Extract options for specified option type from all options
+
+ :param option_type: the object of specified type of options
+ :param from_options: all options dictionary
+ :return: the dictionary of options for specified type, each option can be filled by value from all options
+ dictionary or blank in case the option for specified type is not exist in all options dictionary
+ """
+ _options = dict()
+
+ for key in option_type.keys:
+ key_with_prefix = "{prefix}{key}".format(prefix=option_type.prefix, key=key)
+ if key not in from_options and key_with_prefix not in from_options:
+ _options[key] = ""
+ elif key in from_options:
+ _options[key] = from_options.get(key)
+ else:
+ _options[key] = from_options.get(key_with_prefix)
+
+ return _options
+
+
+def wrap_connection_error(fn):
+ @functools.wraps(fn)
+ def _wrapper(self, *args, **kw):
+ log.debug("Requesting %s(%s, %s)", fn, args, kw)
+ try:
+ res = fn(self, *args, **kw)
+ except requests.ConnectionError:
+ raise NoConnection(self.webdav.hostname)
+ except requests.RequestException as re:
+ raise ConnectionException(re)
+ else:
+ return res
+ return _wrapper
+
+
+class Client(object):
+ """The client for WebDAV servers provides an ability to control files on remote WebDAV server.
+ """
+ # path to root directory of WebDAV
+ root = '/'
+
+ # Max size of file for uploading
+ large_size = 2 * 1024 * 1024 * 1024
+
+ # request timeout in seconds
+ timeout = 30
+
+ # HTTP headers for different actions
+ http_header = {
+ 'list': ["Accept: */*", "Depth: 1"],
+ 'free': ["Accept: */*", "Depth: 0", "Content-Type: text/xml"],
+ 'copy': ["Accept: */*"],
+ 'move': ["Accept: */*"],
+ 'mkdir': ["Accept: */*", "Connection: Keep-Alive"],
+ 'clean': ["Accept: */*", "Connection: Keep-Alive"],
+ 'check': ["Accept: */*"],
+ 'info': ["Accept: */*", "Depth: 1"],
+ 'get_property': ["Accept: */*", "Depth: 1", "Content-Type: application/x-www-form-urlencoded"],
+ 'set_property': ["Accept: */*", "Depth: 1", "Content-Type: application/x-www-form-urlencoded"]
+ }
+
+ def get_headers(self, action, headers_ext=None):
+ """Returns HTTP headers of specified WebDAV actions.
+
+ :param action: the identifier of action.
+ :param headers_ext: (optional) the addition headers list witch sgould be added to basic HTTP headers for
+ the specified action.
+ :return: the dictionary of headers for specified action.
+ """
+ if action in Client.http_header:
+ try:
+ headers = Client.http_header[action].copy()
+ except AttributeError:
+ headers = Client.http_header[action][:]
+ else:
+ headers = list()
+
+ if headers_ext:
+ headers.extend(headers_ext)
+
+ if self.webdav.token:
+ webdav_token = "Authorization: OAuth {token}".format(token=self.webdav.token)
+ headers.append(webdav_token)
+ return dict([map(lambda s: s.strip(), i.split(':')) for i in headers])
+
+ def get_url(self, path):
+ """Generates url by uri path.
+
+ :param path: uri path.
+ :return: the url string.
+ """
+ url = {'hostname': self.webdav.hostname, 'root': self.webdav.root, 'path': path}
+ return "{hostname}{root}{path}".format(**url)
+
+ def get_full_path(self, urn):
+ """Generates full path to remote resource exclude hostname.
+
+ :param urn: the URN to resource.
+ :return: full path to resource with root path.
+ """
+ return "{root}{path}".format(root=self.webdav.root, path=urn.path())
+
+ def execute_request(self, action, path, data=None, headers_ext=None):
+ """Generate request to WebDAV server for specified action and path and execute it.
+
+ :param action: the action for WebDAV server which should be executed.
+ :param path: the path to resource for action
+ :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes,
+ or file-like object to send in the body of the :class:`Request`.
+ :param headers_ext: (optional) the addition headers list witch should be added to basic HTTP headers for
+ the specified action.
+ :return: HTTP response of request.
+ """
+ response = requests.request(
+ method=Client.requests[action],
+ url=self.get_url(path),
+ auth=(self.webdav.login, self.webdav.password),
+ headers=self.get_headers(action, headers_ext),
+ timeout=self.timeout,
+ data=data
+ )
+ if response.status_code == 507:
+ raise NotEnoughSpace()
+ if response.status_code >= 400:
+ raise ResponseErrorCode(url=self.get_url(path), code=response.status_code, message=response.content)
+ return response
+
+ # mapping of actions to WebDAV methods
+ requests = {
+ 'download': "GET",
+ 'upload': "PUT",
+ 'copy': "COPY",
+ 'move': "MOVE",
+ 'mkdir': "MKCOL",
+ 'clean': "DELETE",
+ 'check': "HEAD",
+ 'list': "PROPFIND",
+ 'free': "PROPFIND",
+ 'info': "PROPFIND",
+ 'publish': "PROPPATCH",
+ 'unpublish': "PROPPATCH",
+ 'published': "PROPPATCH",
+ 'get_property': "PROPFIND",
+ 'set_property': "PROPPATCH"
+ }
+
+ meta_xmlns = {
+ 'https://webdav.yandex.ru': "urn:yandex:disk:meta",
+ }
+
+ def __init__(self, options):
+ """Constructor of WebDAV client
+
+ :param options: the dictionary of connection options to WebDAV can include proxy server options.
+ WebDev settings:
+ `webdav_hostname`: url for WebDAV server should contain protocol and ip address or domain name.
+ Example: `https://webdav.server.com`.
+ `webdav_login`: (optional) login name for WebDAV server can be empty in case using of token auth.
+ `webdav_password`: (optional) password for WebDAV server can be empty in case using of token auth.
+ `webdav_token': (optional) token for WebDAV server can be empty in case using of login/password auth.
+ `webdav_root`: (optional) root directory of WebDAV server. Defaults is `/`.
+ `webdav_cert_path`: (optional) path to certificate.
+ `webdav_key_path`: (optional) path to private key.
+ `webdav_recv_speed`: (optional) rate limit data download speed in Bytes per second.
+ Defaults to unlimited speed.
+ `webdav_send_speed`: (optional) rate limit data upload speed in Bytes per second.
+ Defaults to unlimited speed.
+ `webdav_verbose`: (optional) set verbose mode on.off. By default verbose mode is off.
+ Proxy settings (optional):
+ `proxy_hostname`: url to proxy server should contain protocol and ip address or domain name and if needed
+ port. Example: `https://proxy.server.com:8383`.
+ `proxy_login`: login name for proxy server.
+ `proxy_password`: password for proxy server.
+ """
+ webdav_options = get_options(option_type=WebDAVSettings, from_options=options)
+ proxy_options = get_options(option_type=ProxySettings, from_options=options)
+
+ self.webdav = WebDAVSettings(webdav_options)
+ self.proxy = ProxySettings(proxy_options)
+ self.default_options = {}
+
+ def valid(self):
+ """Validates of WebDAV and proxy settings.
+
+ :return: True in case settings are valid and False otherwise.
+ """
+ return True if self.webdav.valid() and self.proxy.valid() else False
+
+ @wrap_connection_error
+ def list(self, remote_path=root):
+ """Returns list of nested files and directories for remote WebDAV directory by path.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
+
+ :param remote_path: path to remote directory.
+ :return: list of nested file or directory names.
+ """
+ directory_urn = Urn(remote_path, directory=True)
+ if directory_urn.path() != Client.root:
+ if not self.check(directory_urn.path()):
+ raise RemoteResourceNotFound(directory_urn.path())
+
+ response = self.execute_request(action='list', path=directory_urn.quote())
+ urns = WebDavXmlUtils.parse_get_list_response(response.content)
+
+ path = Urn.normalize_path(self.get_full_path(directory_urn))
+ return [urn.filename() for urn in urns if Urn.compare_path(path, urn.path()) is False]
+
+ @wrap_connection_error
+ def free(self):
+ """Returns an amount of free space on remote WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
+
+ :return: an amount of free space in bytes.
+ """
+ data = WebDavXmlUtils.create_free_space_request_content()
+ response = self.execute_request(action='free', path='', data=data)
+ return WebDavXmlUtils.parse_free_space_response(response.content, self.webdav.hostname)
+
+ @wrap_connection_error
+ def check(self, remote_path=root):
+ """Checks an existence of remote resource on WebDAV server by remote path.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
+
+ :param remote_path: (optional) path to resource on WebDAV server. Defaults is root directory of WebDAV.
+ :return: True if resource is exist or False otherwise
+ """
+ urn = Urn(remote_path)
+ try:
+ response = self.execute_request(action='check', path=urn.quote())
+ except ResponseErrorCode:
+ return False
+
+ if int(response.status_code) == 200:
+ return True
+ return False
+
+ @wrap_connection_error
+ def mkdir(self, remote_path):
+ """Makes new directory on WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MKCOL
+
+ :param remote_path: path to directory
+ :return: True if request executed with code 200 or 201 and False otherwise.
+
+ """
+ directory_urn = Urn(remote_path, directory=True)
+ if not self.check(directory_urn.parent()):
+ raise RemoteParentNotFound(directory_urn.path())
+
+ response = self.execute_request(action='mkdir', path=directory_urn.quote())
+ return response.status_code in (200, 201)
+
+ @wrap_connection_error
+ def download_from(self, buff, remote_path):
+ """Downloads file from WebDAV and writes it in buffer.
+
+ :param buff: buffer object for writing of downloaded file content.
+ :param remote_path: path to file on WebDAV server.
+ """
+ urn = Urn(remote_path)
+ if self.is_dir(urn.path()):
+ raise OptionNotValid(name="remote_path", value=remote_path)
+
+ if not self.check(urn.path()):
+ raise RemoteResourceNotFound(urn.path())
+
+ response = self.execute_request(action='download', path=urn.quote())
+ buff.write(response.content)
+
+ def download(self, remote_path, local_path, progress=None):
+ """Downloads remote resource from WebDAV and save it in local path.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
+
+ :param remote_path: the path to remote resource for downloading can be file and directory.
+ :param local_path: the path to save resource locally.
+ :param progress: progress function. Not supported now.
+ """
+ urn = Urn(remote_path)
+ if self.is_dir(urn.path()):
+ self.download_directory(local_path=local_path, remote_path=remote_path, progress=progress)
+ else:
+ self.download_file(local_path=local_path, remote_path=remote_path, progress=progress)
+
+ def download_directory(self, remote_path, local_path, progress=None):
+ """Downloads directory and downloads all nested files and directories from remote WebDAV to local.
+ If there is something on local path it deletes directories and files then creates new.
+
+ :param remote_path: the path to directory for downloading form WebDAV server.
+ :param local_path: the path to local directory for saving downloaded files and directories.
+ :param progress: Progress function. Not supported now.
+ """
+ urn = Urn(remote_path, directory=True)
+ if not self.is_dir(urn.path()):
+ raise OptionNotValid(name="remote_path", value=remote_path)
+
+ if os.path.exists(local_path):
+ shutil.rmtree(local_path)
+
+ os.makedirs(local_path)
+
+ for resource_name in self.list(urn.path()):
+ _remote_path = "{parent}{name}".format(parent=urn.path(), name=resource_name)
+ _local_path = os.path.join(local_path, resource_name)
+ self.download(local_path=_local_path, remote_path=_remote_path, progress=progress)
+
+ @wrap_connection_error
+ def download_file(self, remote_path, local_path, progress=None):
+ """Downloads file from WebDAV server and save it locally.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#rfc.section.9.4
+
+ :param remote_path: the path to remote file for downloading.
+ :param local_path: the path to save file locally.
+ :param progress: progress function. Not supported now.
+ """
+ urn = Urn(remote_path)
+ if self.is_dir(urn.path()):
+ raise OptionNotValid(name="remote_path", value=remote_path)
+
+ if os.path.isdir(local_path):
+ raise OptionNotValid(name="local_path", value=local_path)
+
+ if not self.check(urn.path()):
+ raise RemoteResourceNotFound(urn.path())
+
+ with open(local_path, 'wb') as local_file:
+ response = self.execute_request('download', urn.quote())
+ for block in response.iter_content(1024):
+ local_file.write(block)
+
+ def download_sync(self, remote_path, local_path, callback=None):
+ """Downloads remote resources from WebDAV server synchronously.
+
+ :param remote_path: the path to remote resource on WebDAV server. Can be file and directory.
+ :param local_path: the path to save resource locally.
+ :param callback: the callback which will be invoked when downloading is complete.
+ """
+ self.download(local_path=local_path, remote_path=remote_path)
+ if callback:
+ callback()
+
+ def download_async(self, remote_path, local_path, callback=None):
+ """Downloads remote resources from WebDAV server asynchronously
+
+ :param remote_path: the path to remote resource on WebDAV server. Can be file and directory.
+ :param local_path: the path to save resource locally.
+ :param callback: the callback which will be invoked when downloading is complete.
+ """
+ target = (lambda: self.download_sync(local_path=local_path, remote_path=remote_path, callback=callback))
+ threading.Thread(target=target).start()
+
+ @wrap_connection_error
+ def upload_to(self, buff, remote_path):
+ """Uploads file from buffer to remote path on WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT
+
+ :param buff: the buffer with content for file.
+ :param remote_path: the path to save file remotely on WebDAV server.
+ """
+ urn = Urn(remote_path)
+ if urn.is_dir():
+ raise OptionNotValid(name="remote_path", value=remote_path)
+
+ if not self.check(urn.parent()):
+ raise RemoteParentNotFound(urn.path())
+
+ self.execute_request(action='upload', path=urn.quote(), data=buff)
+
+ def upload(self, remote_path, local_path, progress=None):
+ """Uploads resource to remote path on WebDAV server.
+ In case resource is directory it will upload all nested files and directories.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT
+
+ :param remote_path: the path for uploading resources on WebDAV server. Can be file and directory.
+ :param local_path: the path to local resource for uploading.
+ :param progress: Progress function. Not supported now.
+ """
+ if os.path.isdir(local_path):
+ self.upload_directory(local_path=local_path, remote_path=remote_path, progress=progress)
+ else:
+ self.upload_file(local_path=local_path, remote_path=remote_path)
+
+ def upload_directory(self, remote_path, local_path, progress=None):
+ """Uploads directory to remote path on WebDAV server.
+ In case directory is exist on remote server it will delete it and then upload directory with nested files and
+ directories.
+
+ :param remote_path: the path to directory for uploading on WebDAV server.
+ :param local_path: the path to local directory for uploading.
+ :param progress: Progress function. Not supported now.
+ """
+ urn = Urn(remote_path, directory=True)
+ if not urn.is_dir():
+ raise OptionNotValid(name="remote_path", value=remote_path)
+
+ if not os.path.isdir(local_path):
+ raise OptionNotValid(name="local_path", value=local_path)
+
+ if not os.path.exists(local_path):
+ raise LocalResourceNotFound(local_path)
+
+ if self.check(urn.path()):
+ self.clean(urn.path())
+
+ self.mkdir(remote_path)
+
+ for resource_name in listdir(local_path):
+ _remote_path = "{parent}{name}".format(parent=urn.path(), name=resource_name)
+ _local_path = os.path.join(local_path, resource_name)
+ self.upload(local_path=_local_path, remote_path=_remote_path, progress=progress)
+
+ @wrap_connection_error
+ def upload_file(self, remote_path, local_path, progress=None):
+ """Uploads file to remote path on WebDAV server. File should be 2Gb or less.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PUT
+
+ :param remote_path: the path to uploading file on WebDAV server.
+ :param local_path: the path to local file for uploading.
+ :param progress: Progress function. Not supported now.
+ """
+ if not os.path.exists(local_path):
+ raise LocalResourceNotFound(local_path)
+
+ urn = Urn(remote_path)
+ if urn.is_dir():
+ raise OptionNotValid(name="remote_path", value=remote_path)
+
+ if os.path.isdir(local_path):
+ raise OptionNotValid(name="local_path", value=local_path)
+
+ if not self.check(urn.parent()):
+ raise RemoteParentNotFound(urn.path())
+
+ with open(local_path, "rb") as local_file:
+ file_size = os.path.getsize(local_path)
+ if file_size > self.large_size:
+ raise ResourceTooBig(path=local_path, size=file_size, max_size=self.large_size)
+
+ self.execute_request(action='upload', path=urn.quote(), data=local_file)
+
+ def upload_sync(self, remote_path, local_path, callback=None):
+ """Uploads resource to remote path on WebDAV server synchronously.
+ In case resource is directory it will upload all nested files and directories.
+
+ :param remote_path: the path for uploading resources on WebDAV server. Can be file and directory.
+ :param local_path: the path to local resource for uploading.
+ :param callback: the callback which will be invoked when downloading is complete.
+ """
+ self.upload(local_path=local_path, remote_path=remote_path)
+
+ if callback:
+ callback()
+
+ def upload_async(self, remote_path, local_path, callback=None):
+ """Uploads resource to remote path on WebDAV server asynchronously.
+ In case resource is directory it will upload all nested files and directories.
+
+ :param remote_path: the path for uploading resources on WebDAV server. Can be file and directory.
+ :param local_path: the path to local resource for uploading.
+ :param callback: the callback which will be invoked when downloading is complete.
+ """
+ target = (lambda: self.upload_sync(local_path=local_path, remote_path=remote_path, callback=callback))
+ threading.Thread(target=target).start()
+
+ @wrap_connection_error
+ def copy(self, remote_path_from, remote_path_to):
+ """Copies resource from one place to another on WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_COPY
+
+ :param remote_path_from: the path to resource which will be copied,
+ :param remote_path_to: the path where resource will be copied.
+ """
+ urn_from = Urn(remote_path_from)
+ if not self.check(urn_from.path()):
+ raise RemoteResourceNotFound(urn_from.path())
+
+ urn_to = Urn(remote_path_to)
+ if not self.check(urn_to.parent()):
+ raise RemoteParentNotFound(urn_to.path())
+
+ header_destination = "Destination: {path}".format(path=self.get_full_path(urn_to))
+ self.execute_request(action='copy', path=urn_from.quote(), headers_ext=[header_destination])
+
+ @wrap_connection_error
+ def move(self, remote_path_from, remote_path_to, overwrite=False):
+ """Moves resource from one place to another on WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_MOVE
+
+ :param remote_path_from: the path to resource which will be moved,
+ :param remote_path_to: the path where resource will be moved.
+ :param overwrite: (optional) the flag, overwrite file if it exists. Defaults is False
+ """
+ urn_from = Urn(remote_path_from)
+ if not self.check(urn_from.path()):
+ raise RemoteResourceNotFound(urn_from.path())
+
+ urn_to = Urn(remote_path_to)
+ if not self.check(urn_to.parent()):
+ raise RemoteParentNotFound(urn_to.path())
+
+ header_destination = "Destination: {path}".format(path=self.get_full_path(urn_to))
+ header_overwrite = "Overwrite: {flag}".format(flag="T" if overwrite else "F")
+ self.execute_request(action='move', path=urn_from.quote(), headers_ext=[header_destination, header_overwrite])
+
+ @wrap_connection_error
+ def clean(self, remote_path):
+ """Cleans (Deletes) a remote resource on WebDAV server. The name of method is not changed for back compatibility
+ with original library.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_DELETE
+
+ :param remote_path: the remote resource whisch will be deleted.
+ """
+ urn = Urn(remote_path)
+ self.execute_request(action='clean', path=urn.quote())
+
+ @wrap_connection_error
+ def info(self, remote_path):
+ """Gets information about resource on WebDAV.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
+
+ :param remote_path: the path to remote resource.
+ :return: a dictionary of information attributes and them values with following keys:
+ `created`: date of resource creation,
+ `name`: name of resource,
+ `size`: size of resource,
+ `modified`: date of resource modification.
+ """
+ urn = Urn(remote_path)
+ if not self.check(urn.path()) and not self.check(Urn(remote_path, directory=True).path()):
+ raise RemoteResourceNotFound(remote_path)
+
+ response = self.execute_request(action='info', path=urn.quote())
+ path = self.get_full_path(urn)
+ return WebDavXmlUtils.parse_info_response(content=response.content, path=path, hostname=self.webdav.hostname)
+
+ @wrap_connection_error
+ def is_dir(self, remote_path):
+ """Checks is the remote resource directory.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
+
+ :param remote_path: the path to remote resource.
+ :return: True in case the remote resource is directory and False otherwise.
+ """
+ urn = Urn(remote_path)
+ parent_urn = Urn(urn.parent())
+ if not self.check(urn.path()) and not self.check(Urn(remote_path, directory=True).path()):
+ raise RemoteResourceNotFound(remote_path)
+
+ response = self.execute_request(action='info', path=parent_urn.quote())
+ path = self.get_full_path(urn)
+ return WebDavXmlUtils.parse_is_dir_response(content=response.content, path=path, hostname=self.webdav.hostname)
+
+ @wrap_connection_error
+ def get_property(self, remote_path, option):
+ """Gets metadata property of remote resource on WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND
+
+ :param remote_path: the path to remote resource.
+ :param option: the property attribute as dictionary with following keys:
+ `namespace`: (optional) the namespace for XML property which will be set,
+ `name`: the name of property which will be set.
+ :return: the value of property or None if property is not found.
+ """
+ urn = Urn(remote_path)
+ if not self.check(urn.path()):
+ raise RemoteResourceNotFound(urn.path())
+
+ data = WebDavXmlUtils.create_get_property_request_content(option)
+ response = self.execute_request(action='get_property', path=urn.quote(), data=data)
+ return WebDavXmlUtils.parse_get_property_response(response.content, option['name'])
+
+ @wrap_connection_error
+ def set_property(self, remote_path, option):
+ """Sets metadata property of remote resource on WebDAV server.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPPATCH
+
+ :param remote_path: the path to remote resource.
+ :param option: the property attribute as dictionary with following keys:
+ `namespace`: (optional) the namespace for XML property which will be set,
+ `name`: the name of property which will be set,
+ `value`: (optional) the value of property which will be set. Defaults is empty string.
+ """
+ self.set_property_batch(remote_path=remote_path, option=[option])
+
+ @wrap_connection_error
+ def set_property_batch(self, remote_path, option):
+ """Sets batch metadata properties of remote resource on WebDAV server in batch.
+ More information you can find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPPATCH
+
+ :param remote_path: the path to remote resource.
+ :param option: the property attributes as list of dictionaries with following keys:
+ `namespace`: (optional) the namespace for XML property which will be set,
+ `name`: the name of property which will be set,
+ `value`: (optional) the value of property which will be set. Defaults is empty string.
+ """
+ urn = Urn(remote_path)
+ if not self.check(urn.path()):
+ raise RemoteResourceNotFound(urn.path())
+
+ data = WebDavXmlUtils.create_set_property_batch_request_content(option)
+ self.execute_request(action='set_property', path=urn.quote(), data=data)
+
+ def resource(self, remote_path):
+ urn = Urn(remote_path)
+ return Resource(self, urn)
+
+ def push(self, remote_directory, local_directory):
+
+ def prune(src, exp):
+ return [sub(exp, "", item) for item in src]
+
+ urn = Urn(remote_directory, directory=True)
+
+ if not self.is_dir(urn.path()):
+ raise OptionNotValid(name="remote_path", value=remote_directory)
+
+ if not os.path.isdir(local_directory):
+ raise OptionNotValid(name="local_path", value=local_directory)
+
+ if not os.path.exists(local_directory):
+ raise LocalResourceNotFound(local_directory)
+
+ paths = self.list(urn.path())
+ expression = "{begin}{end}".format(begin="^", end=urn.path())
+ remote_resource_names = prune(paths, expression)
+
+ for local_resource_name in listdir(local_directory):
+
+ local_path = os.path.join(local_directory, local_resource_name)
+ remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(),
+ resource_name=local_resource_name)
+
+ if os.path.isdir(local_path):
+ if not self.check(remote_path=remote_path):
+ self.mkdir(remote_path=remote_path)
+ self.push(remote_directory=remote_path, local_directory=local_path)
+ else:
+ if local_resource_name in remote_resource_names:
+ continue
+ self.upload_file(remote_path=remote_path, local_path=local_path)
+
+ def pull(self, remote_directory, local_directory):
+
+ def prune(src, exp):
+ return [sub(exp, "", item) for item in src]
+
+ urn = Urn(remote_directory, directory=True)
+
+ if not self.is_dir(urn.path()):
+ raise OptionNotValid(name="remote_path", value=remote_directory)
+
+ if not os.path.exists(local_directory):
+ raise LocalResourceNotFound(local_directory)
+
+ local_resource_names = listdir(local_directory)
+
+ paths = self.list(urn.path())
+ expression = "{begin}{end}".format(begin="^", end=remote_directory)
+ remote_resource_names = prune(paths, expression)
+
+ for remote_resource_name in remote_resource_names:
+
+ local_path = os.path.join(local_directory, remote_resource_name)
+ remote_path = "{remote_directory}{resource_name}".format(remote_directory=urn.path(),
+ resource_name=remote_resource_name)
+
+ remote_urn = Urn(remote_path)
+
+ if self.is_dir(remote_urn.path()):
+ if not os.path.exists(local_path):
+ os.mkdir(local_path)
+ self.pull(remote_directory=remote_path, local_directory=local_path)
+ else:
+ if remote_resource_name in local_resource_names:
+ continue
+ self.download_file(remote_path=remote_path, local_path=local_path)
+
+ def sync(self, remote_directory, local_directory):
+
+ self.pull(remote_directory=remote_directory, local_directory=local_directory)
+ self.push(remote_directory=remote_directory, local_directory=local_directory)
+
+
+class Resource(object):
+ def __init__(self, client, urn):
+ self.client = client
+ self.urn = urn
+
+ def __str__(self):
+ return "resource {path}".format(path=self.urn.path())
+
+ def is_dir(self):
+ return self.client.is_dir(self.urn.path())
+
+ def rename(self, new_name):
+ old_path = self.urn.path()
+ parent_path = self.urn.parent()
+ new_name = Urn(new_name).filename()
+ new_path = "{directory}{filename}".format(directory=parent_path, filename=new_name)
+
+ self.client.move(remote_path_from=old_path, remote_path_to=new_path)
+ self.urn = Urn(new_path)
+
+ def move(self, remote_path):
+ new_urn = Urn(remote_path)
+ self.client.move(remote_path_from=self.urn.path(), remote_path_to=new_urn.path())
+ self.urn = new_urn
+
+ def copy(self, remote_path):
+ urn = Urn(remote_path)
+ self.client.copy(remote_path_from=self.urn.path(), remote_path_to=remote_path)
+ return Resource(self.client, urn)
+
+ def info(self, params=None):
+ info = self.client.info(self.urn.path())
+ if not params:
+ return info
+
+ return {key: value for (key, value) in info.items() if key in params}
+
+ def clean(self):
+ return self.client.clean(self.urn.path())
+
+ def check(self):
+ return self.client.check(self.urn.path())
+
+ def read_from(self, buff):
+ self.client.upload_to(buff=buff, remote_path=self.urn.path())
+
+ def read(self, local_path):
+ return self.client.upload_sync(local_path=local_path, remote_path=self.urn.path())
+
+ def read_async(self, local_path, callback=None):
+ return self.client.upload_async(local_path=local_path, remote_path=self.urn.path(), callback=callback)
+
+ def write_to(self, buff):
+ return self.client.download_from(buff=buff, remote_path=self.urn.path())
+
+ def write(self, local_path):
+ return self.client.download_sync(local_path=local_path, remote_path=self.urn.path())
+
+ def write_async(self, local_path, callback=None):
+ return self.client.download_async(local_path=local_path, remote_path=self.urn.path(), callback=callback)
+
+ def publish(self):
+ return self.client.publish(self.urn.path())
+
+ def unpublish(self):
+ return self.client.unpublish(self.urn.path())
+
+ @property
+ def property(self, option):
+ return self.client.get_property(remote_path=self.urn.path(), option=option)
+
+ @property.setter
+ def property(self, option, value):
+ option['value'] = value.__str__()
+ self.client.set_property(remote_path=self.urn.path(), option=option)
+
+
+class WebDavXmlUtils:
+ def __init__(self):
+ pass
+
+ @staticmethod
+ def parse_get_list_response(content):
+ """Parses of response content XML from WebDAV server and extract file and directory names.
+
+ :param content: the XML content of HTTP response from WebDAV server for getting list of files by remote path.
+ :return: list of extracted file or directory names.
+ """
+ try:
+ tree = etree.fromstring(content)
+ hrees = [Urn.separate + unquote(urlsplit(hree.text).path) for hree in tree.findall(".//{DAV:}href")]
+ return [Urn(hree) for hree in hrees]
+ except etree.XMLSyntaxError:
+ return list()
+
+ @staticmethod
+ def create_free_space_request_content():
+ """Creates an XML for requesting of free space on remote WebDAV server.
+
+ :return: the XML string of request content.
+ """
+ root = etree.Element("propfind", xmlns="DAV:")
+ prop = etree.SubElement(root, "prop")
+ etree.SubElement(prop, "quota-available-bytes")
+ etree.SubElement(prop, "quota-used-bytes")
+ tree = etree.ElementTree(root)
+ return WebDavXmlUtils.etree_to_string(tree)
+
+ @staticmethod
+ def parse_free_space_response(content, hostname):
+ """Parses of response content XML from WebDAV server and extract an amount of free space.
+
+ :param content: the XML content of HTTP response from WebDAV server for getting free space.
+ :param hostname: the server hostname.
+ :return: an amount of free space in bytes.
+ """
+ try:
+ tree = etree.fromstring(content)
+ node = tree.find('.//{DAV:}quota-available-bytes')
+ if node is not None:
+ return int(node.text)
+ else:
+ raise MethodNotSupported(name='free', server=hostname)
+ except TypeError:
+ raise MethodNotSupported(name='free', server=hostname)
+ except etree.XMLSyntaxError:
+ return str()
+
+ @staticmethod
+ def parse_info_response(content, path, hostname):
+ """Parses of response content XML from WebDAV server and extract an information about resource.
+
+ :param content: the XML content of HTTP response from WebDAV server.
+ :param path: the path to resource.
+ :param hostname: the server hostname.
+ :return: a dictionary of information attributes and them values with following keys:
+ `created`: date of resource creation,
+ `name`: name of resource,
+ `size`: size of resource,
+ `modified`: date of resource modification.
+ """
+ response = WebDavXmlUtils.extract_response_for_path(content=content, path=path, hostname=hostname)
+ find_attributes = {
+ 'created': ".//{DAV:}creationdate",
+ 'name': ".//{DAV:}displayname",
+ 'size': ".//{DAV:}getcontentlength",
+ 'modified': ".//{DAV:}getlastmodified"
+ }
+ info = dict()
+ for (name, value) in find_attributes.items():
+ info[name] = response.findtext(value)
+ return info
+
+ @staticmethod
+ def parse_is_dir_response(content, path, hostname):
+ """Parses of response content XML from WebDAV server and extract an information about resource.
+
+ :param content: the XML content of HTTP response from WebDAV server.
+ :param path: the path to resource.
+ :param hostname: the server hostname.
+ :return: True in case the remote resource is directory and False otherwise.
+ """
+ response = WebDavXmlUtils.extract_response_for_path(content=content, path=path, hostname=hostname)
+ resource_type = response.find(".//{DAV:}resourcetype")
+ if resource_type is None:
+ raise MethodNotSupported(name="is_dir", server=hostname)
+ dir_type = resource_type.find("{DAV:}collection")
+
+ return True if dir_type is not None else False
+
+ @staticmethod
+ def create_get_property_request_content(option):
+ """Creates an XML for requesting of getting a property value of remote WebDAV resource.
+
+ :param option: the property attributes as dictionary with following keys:
+ `namespace`: (optional) the namespace for XML property which will be get,
+ `name`: the name of property which will be get.
+ :return: the XML string of request content.
+ """
+ root = etree.Element("propfind", xmlns="DAV:")
+ prop = etree.SubElement(root, "prop")
+ etree.SubElement(prop, option.get('name', ""), xmlns=option.get('namespace', ""))
+ tree = etree.ElementTree(root)
+ return WebDavXmlUtils.etree_to_string(tree)
+
+ @staticmethod
+ def parse_get_property_response(content, name):
+ """Parses of response content XML from WebDAV server for getting metadata property value for some resource.
+
+ :param content: the XML content of response as string.
+ :param name: the name of property for finding a value in response
+ :return: the value of property if it has been found or None otherwise.
+ """
+ tree = etree.fromstring(content)
+ return tree.xpath('//*[local-name() = $name]', name=name)[0].text
+
+ @staticmethod
+ def create_set_property_batch_request_content(options):
+ """Creates an XML for requesting of setting a property values for remote WebDAV resource in batch.
+
+ :param options: the property attributes as list of dictionaries with following keys:
+ `namespace`: (optional) the namespace for XML property which will be set,
+ `name`: the name of property which will be set,
+ `value`: (optional) the value of property which will be set. Defaults is empty string.
+ :return: the XML string of request content.
+ """
+ root_node = etree.Element('propertyupdate', xmlns='DAV:')
+ set_node = etree.SubElement(root_node, 'set')
+ prop_node = etree.SubElement(set_node, 'prop')
+ for option in options:
+ opt_node = etree.SubElement(prop_node, option['name'], xmlns=option.get('namespace', ''))
+ opt_node.text = option.get('value', '')
+ tree = etree.ElementTree(root_node)
+ return WebDavXmlUtils.etree_to_string(tree)
+
+ @staticmethod
+ def etree_to_string(tree):
+ """Creates string from lxml.etree.ElementTree with XML declaration and UTF-8 encoding.
+
+ :param tree: the instance of ElementTree
+ :return: the string of XML.
+ """
+ buff = BytesIO()
+ tree.write(buff, xml_declaration=True, encoding='UTF-8')
+ return buff.getvalue()
+
+ @staticmethod
+ def extract_response_for_path(content, path, hostname):
+ """Extracts single response for specified remote resource.
+
+ :param content: raw content of response as string.
+ :param path: the path to needed remote resource.
+ :param hostname: the server hostname.
+ :return: XML object of response for the remote resource defined by path.
+ """
+ try:
+ tree = etree.fromstring(content)
+ responses = tree.findall("{DAV:}response")
+
+ n_path = Urn.normalize_path(path)
+
+ for resp in responses:
+ href = resp.findtext("{DAV:}href")
+
+ if Urn.compare_path(n_path, href) is True:
+ return resp
+ raise RemoteResourceNotFound(path)
+ except etree.XMLSyntaxError:
+ raise MethodNotSupported(name="is_dir", server=hostname)
diff --git a/bob/devtools/webdav3/connection.py b/bob/devtools/webdav3/connection.py
new file mode 100644
index 0000000000000000000000000000000000000000..392e5d9d35b5f075d334f723daafe17428f71697
--- /dev/null
+++ b/bob/devtools/webdav3/connection.py
@@ -0,0 +1,96 @@
+from os.path import exists
+
+from .exceptions import *
+from .urn import Urn
+
+
+class ConnectionSettings:
+ def is_valid(self):
+ pass
+
+ def valid(self):
+
+ try:
+ self.is_valid()
+ except OptionNotValid:
+ return False
+ else:
+ return True
+
+
+class WebDAVSettings(ConnectionSettings):
+ ns = "webdav:"
+ prefix = "webdav_"
+ keys = {'hostname', 'login', 'password', 'token', 'root', 'cert_path', 'key_path', 'recv_speed', 'send_speed',
+ 'verbose'}
+
+ hostname = None
+ login = None
+ password = None
+ token = None
+ root = None
+ cert_path = None
+ key_path = None
+ recv_speed = None
+ send_speed = None
+ verbose = None
+
+ def __init__(self, options):
+
+ self.options = dict()
+
+ for key in self.keys:
+ value = options.get(key, '')
+ self.options[key] = value
+ self.__dict__[key] = value
+
+ self.root = Urn(self.root).quote() if self.root else ''
+ self.root = self.root.rstrip(Urn.separate)
+
+ def is_valid(self):
+
+ if not self.hostname:
+ raise OptionNotValid(name="hostname", value=self.hostname, ns=self.ns)
+
+ if self.cert_path and not exists(self.cert_path):
+ raise OptionNotValid(name="cert_path", value=self.cert_path, ns=self.ns)
+
+ if self.key_path and not exists(self.key_path):
+ raise OptionNotValid(name="key_path", value=self.key_path, ns=self.ns)
+
+ if self.key_path and not self.cert_path:
+ raise OptionNotValid(name="cert_path", value=self.cert_path, ns=self.ns)
+
+ if self.password and not self.login:
+ raise OptionNotValid(name="login", value=self.login, ns=self.ns)
+
+ if not self.token and not self.login:
+ raise OptionNotValid(name="login", value=self.login, ns=self.ns)
+
+
+class ProxySettings(ConnectionSettings):
+ ns = "proxy:"
+ prefix = "proxy_"
+ keys = {'hostname', 'login', 'password'}
+
+ hostname = None
+ login = None
+ password = None
+
+ def __init__(self, options):
+
+ self.options = dict()
+
+ for key in self.keys:
+ value = options.get(key, '')
+ self.options[key] = value
+ self.__dict__[key] = value
+
+ def is_valid(self):
+
+ if self.password and not self.login:
+ raise OptionNotValid(name="login", value=self.login, ns=self.ns)
+
+ if self.login or self.password:
+ if not self.hostname:
+ raise OptionNotValid(name="hostname", value=self.hostname, ns=self.ns)
diff --git a/bob/devtools/webdav3/exceptions.py b/bob/devtools/webdav3/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..b30abbc62339fc292c393f27124bc4f00bbd2c83
--- /dev/null
+++ b/bob/devtools/webdav3/exceptions.py
@@ -0,0 +1,115 @@
+class WebDavException(Exception):
+ pass
+
+
+class NotValid(WebDavException):
+ pass
+
+
+class OptionNotValid(NotValid):
+ def __init__(self, name, value, ns=""):
+ self.name = name
+ self.value = value
+ self.ns = ns
+
+ def __str__(self):
+ return "Option ({ns}{name}={value}) have invalid name or value".format(ns=self.ns, name=self.name,
+ value=self.value)
+
+
+class CertificateNotValid(NotValid):
+ pass
+
+
+class NotFound(WebDavException):
+ pass
+
+
+class LocalResourceNotFound(NotFound):
+ def __init__(self, path):
+ self.path = path
+
+ def __str__(self):
+ return "Local file: {path} not found".format(path=self.path)
+
+
+class RemoteResourceNotFound(NotFound):
+ def __init__(self, path):
+ self.path = path
+
+ def __str__(self):
+ return "Remote resource: {path} not found".format(path=self.path)
+
+
+class RemoteParentNotFound(NotFound):
+ def __init__(self, path):
+ self.path = path
+
+ def __str__(self):
+ return "Remote parent for: {path} not found".format(path=self.path)
+
+
+class ResourceTooBig(WebDavException):
+ def __init__(self, path, size, max_size):
+ self.path = path
+ self.size = size
+ self.max_size = max_size
+
+ def __str__(self):
+ return "Resource {path} is too big, it should be less then {max_size} but actually: {size}".format(
+ path=self.path,
+ max_size=self.max_size,
+ size=self.size)
+
+
+class MethodNotSupported(WebDavException):
+ def __init__(self, name, server):
+ self.name = name
+ self.server = server
+
+ def __str__(self):
+ return "Method {name} not supported for {server}".format(name=self.name, server=self.server)
+
+
+class ConnectionException(WebDavException):
+ def __init__(self, exception):
+ self.exception = exception
+
+ def __str__(self):
+ return self.exception.__str__()
+
+
+class NoConnection(WebDavException):
+ def __init__(self, hostname):
+ self.hostname = hostname
+
+ def __str__(self):
+ return "Not connection with {hostname}".format(hostname=self.hostname)
+
+
+# This exception left only for supporting original library interface.
+class NotConnection(WebDavException):
+ def __init__(self, hostname):
+ self.hostname = hostname
+
+ def __str__(self):
+ return "No connection with {hostname}".format(hostname=self.hostname)
+
+
+class ResponseErrorCode(WebDavException):
+ def __init__(self, url, code, message):
+ self.url = url
+ self.code = code
+ self.message = message
+
+ def __str__(self):
+ return "Request to {url} failed with code {code} and message: {message}".format(url=self.url, code=self.code,
+ message=self.message)
+
+
+class NotEnoughSpace(WebDavException):
+ def __init__(self):
+ pass
+
+ def __str__(self):
+ return "Not enough space on the server"
diff --git a/bob/devtools/webdav3/urn.py b/bob/devtools/webdav3/urn.py
new file mode 100644
index 0000000000000000000000000000000000000000..6279de26c5c26de20f0c51252d6d70d755a3419b
--- /dev/null
+++ b/bob/devtools/webdav3/urn.py
@@ -0,0 +1,67 @@
+try:
+ from urllib.parse import unquote, quote, urlsplit
+except ImportError:
+ from urllib import unquote, quote
+ from urlparse import urlsplit
+
+from re import sub
+
+
+class Urn(object):
+
+ separate = "/"
+
+ def __init__(self, path, directory=False):
+
+ self._path = quote(path)
+ expressions = "/\.+/", "/+"
+ for expression in expressions:
+ self._path = sub(expression, Urn.separate, self._path)
+
+ if not self._path.startswith(Urn.separate):
+ self._path = "{begin}{end}".format(begin=Urn.separate, end=self._path)
+
+ if directory and not self._path.endswith(Urn.separate):
+ self._path = "{begin}{end}".format(begin=self._path, end=Urn.separate)
+
+ def __str__(self):
+ return self.path()
+
+ def path(self):
+ return unquote(self._path)
+
+ def quote(self):
+ return self._path
+
+ def filename(self):
+
+ path_split = self._path.split(Urn.separate)
+ name = path_split[-2] + Urn.separate if path_split[-1] == '' else path_split[-1]
+ return unquote(name)
+
+ def parent(self):
+
+ path_split = self._path.split(Urn.separate)
+ nesting_level = self.nesting_level()
+ parent_path_split = path_split[:nesting_level]
+ parent = self.separate.join(parent_path_split) if nesting_level != 1 else Urn.separate
+ if not parent.endswith(Urn.separate):
+ return unquote(parent + Urn.separate)
+ else:
+ return unquote(parent)
+
+ def nesting_level(self):
+ return self._path.count(Urn.separate, 0, -1)
+
+ def is_dir(self):
+ return self._path[-1] == Urn.separate
+
+ @staticmethod
+ def normalize_path(path):
+ result = sub('/{2,}', '/', path)
+ return result if len(result) < 1 or result[-1] != Urn.separate else result[:-1]
+
+ @staticmethod
+ def compare_path(path_a, href):
+ unqouted_path = Urn.separate + unquote(urlsplit(href).path)
+ return Urn.normalize_path(path_a) == Urn.normalize_path(unqouted_path)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 14abefcda941ac89a98df0a120fa997212b306fa..542a1b417c833f7420ddf4ff3b6eea944706d985 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -44,6 +44,8 @@ requirements:
- sphinx
- pyyaml
- twine
+ - packaging
+ - lxml
test:
requires:
diff --git a/setup.py b/setup.py
index 012596bf142b60ffe7e27c568a4d309c09a06fae..8863c4ac60cc39a075967c8215fe35863e60e576 100644
--- a/setup.py
+++ b/setup.py
@@ -17,6 +17,8 @@ requires = [
'sphinx',
'pyyaml',
'twine',
+ 'packaging',
+ 'lxml',
]
setup(
@@ -41,16 +43,21 @@ setup(
'bdt = bob.devtools.scripts.bdt:main',
],
'bdt.cli': [
- 'release = bob.devtools.scripts.release:release',
- 'changelog = bob.devtools.scripts.changelog:changelog',
- 'lasttag = bob.devtools.scripts.lasttag:lasttag',
- 'visibility = bob.devtools.scripts.visibility:visibility',
- 'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
- 'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
- 'build = bob.devtools.scripts.build:build',
- 'getpath = bob.devtools.scripts.getpath:getpath',
- 'caupdate = bob.devtools.scripts.caupdate:caupdate',
- ],
+ 'release = bob.devtools.scripts.release:release',
+ 'changelog = bob.devtools.scripts.changelog:changelog',
+ 'lasttag = bob.devtools.scripts.lasttag:lasttag',
+ 'visibility = bob.devtools.scripts.visibility:visibility',
+ 'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
+ 'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
+ 'build = bob.devtools.scripts.build:build',
+ 'getpath = bob.devtools.scripts.getpath:getpath',
+ 'caupdate = bob.devtools.scripts.caupdate:caupdate',
+ 'ci = bob.devtools.scripts.ci:ci',
+ ],
+
+ 'bdt.ci.cli': [
+ 'deploy = bob.devtools.scripts.ci:deploy',
+ ],
},
classifiers=[
'Framework :: Bob',