diff --git a/LICENSE b/LICENSE
index 782e5e5e3ae29d87e82a55fbea22a0fada00d47f..d608c01c3332c908d3f1bae6e746a7d125ffe04c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -25,36 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-------------------------------------------------------------------------------
-
-Code from the "webdav3" directory was copied from the Github repository
-https://github.com/ezhov-evgeny/webdav-client-python-3, but later modified and
-repackaged as part of this package.
-
-The authors asked to reproduce the following license text.
-
-
-COPYRIGHT AND PERMISSION NOTICE
--------------------------------
-
-Copyright (c) 2016, The WDC Project, and many contributors, see the THANKS
-file.
-
-All rights reserved.
-
-Permission to use, copy, modify, and distribute this software for any purpose
-with or without fee is hereby granted, provided that the above copyright notice
-and this permission notice appear in all copies.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
-NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
-
-Except as contained in this notice, the name of a copyright holder shall not be
-used in advertising or otherwise to promote the sale, use or other dealings in
-this Software without prior written authorization of the copyright holder.
diff --git a/bob/devtools/constants.py b/bob/devtools/constants.py
index dca906406c9fe6cd14b0df90f2d2f1926b2dc5b7..9ae15d91d9c005400f1e3b1fea09620dfe511b76 100644
--- a/bob/devtools/constants.py
+++ b/bob/devtools/constants.py
@@ -7,7 +7,7 @@ import os
import pkg_resources
-from . import bootstrap
+from . import bootstrap, deploy
from .log import get_logger
logger = get_logger(__name__)
@@ -30,38 +30,12 @@ CONDA_RECIPE_APPEND = pkg_resources.resource_filename(
SERVER = bootstrap._SERVER
+assert SERVER == deploy._SERVER, "Also change deploy._SERVER to match this!"
"""This is the default server use use to store data and build artifacts"""
-
-WEBDAV_PATHS = {
- True: { # stable?
- False: { # visible?
- "root": "/private-upload",
- "conda": "/conda",
- "docs": "/docs",
- },
- True: { # visible?
- "root": "/public-upload",
- "conda": "/conda",
- "docs": "/docs",
- },
- },
- False: { # stable?
- False: { # visible?
- "root": "/private-upload",
- "conda": "/conda/label/beta",
- "docs": "/docs",
- },
- True: { # visible?
- "root": "/public-upload",
- "conda": "/conda/label/beta",
- "docs": "/docs",
- },
- },
-}
+WEBDAV_PATHS = deploy._WEBDAV_PATHS
"""Default locations of our webdav upload paths"""
-
IDIAP_ROOT_CA = b"""
Idiap Root CA 2016 - for internal use
=====================================
diff --git a/bob/devtools/data/gitlab-ci/python-package.yaml b/bob/devtools/data/gitlab-ci/python-package.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f13e63bfb19bf0f7063e079d6820b557461bd839
--- /dev/null
+++ b/bob/devtools/data/gitlab-ci/python-package.yaml
@@ -0,0 +1,135 @@
+# This YAML file contains descriptions for the CI of python-only packages
+# - do **not** modify it unless you know what you're doing (and up to!)
+
+# Definition of global variables (all stages)
+variables:
+ PYTHONUNBUFFERED: "1"
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+ PRE_COMMIT_HOME: "${CI_PROJECT_DIR}/.cache/pre-commit"
+ DEPLOY: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/deploy.py"
+
+
+# Definition of our build pipeline order
+stages:
+ - build
+ - test
+ - deploy
+ - pypi
+
+
+# Build targets
+build:
+ image: python:latest
+ tags:
+ - docker
+ stage: build
+ before_script:
+ - pip install twine pre-commit sphinx sphinx-rtd-theme
+ script:
+ - "[ -r .pre-commit-config.yaml ] && pre-commit run --all-files --show-diff-on-failure --verbose"
+ - python setup.py sdist --formats=zip
+ - twine check dist/*.zip
+ - pip install -e .
+ - "[ -e doc ] && sphinx-build doc html"
+ artifacts:
+ paths:
+ - dist/*.zip
+ - html
+ expire_in: 1 week
+ cache:
+ key: "build-py"
+ paths:
+ - ${PRE_COMMIT_HOME}
+ - ${PIP_CACHE_DIR}
+
+
+# Test targets
+.test_template:
+ tags:
+ - docker
+ stage: test
+ dependencies:
+ - build
+ before_script:
+ - pip install tox
+ cache:
+ key: "test-py"
+ paths:
+ - ${PIP_CACHE_DIR}
+
+
+test_py38:
+ extends: .test_template
+ image: python:3.8
+ script:
+ - tox -e py38
+
+test_py39:
+ extends: .test_template
+ image: python:3.9
+ script:
+ - tox -e py39
+
+test_py310:
+ extends: .test_template
+ image: python:3.10
+ script:
+ - tox -e py310
+
+
+.deploy_template:
+ image: python:latest
+ tags:
+ - docker
+ stage: deploy
+ dependencies:
+ - test_py38
+ - test_py39
+ - test_py310
+ - build
+ before_script:
+ - pip install webdavclient3
+ - curl --silent "${DEPLOY}" --output "deploydocs.py"
+ script:
+ - python ./deploydocs.py -v html
+
+deploy_beta:
+ extends: .deploy_template
+ environment: beta
+ only:
+ - master
+
+deploy_stable:
+ extends: .deploy_template
+ environment: stable
+ only:
+ - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
+ except:
+ - branches
+
+
+pypi:
+ image: python:latest
+ tags:
+ - docker
+ stage: pypi
+ environment: pypi
+ only:
+ refs:
+ - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
+ variables:
+ - $CI_PROJECT_VISIBILITY == "public"
+ except:
+ - branches
+ dependencies:
+ - test_py38
+ - test_py39
+ - test_py310
+ - build
+ before_script:
+ - pip install twine
+ script:
+ - twine --skip-existing --username=${PYPIUSER} --password=${PYPIPASS} dist/*.zip
+ cache:
+ paths:
+ - ${PIP_CACHE_DIR}
diff --git a/bob/devtools/deploy.py b/bob/devtools/deploy.py
index 634fbb59be33974c50a6b7610ce39c8deffd6d90..3436308718cd7d1919e310382f2445fbd10651c9 100644
--- a/bob/devtools/deploy.py
+++ b/bob/devtools/deploy.py
@@ -4,12 +4,41 @@
"""Deployment utilities for conda packages and documentation via webDAV."""
+import logging
import os
-from .constants import SERVER, WEBDAV_PATHS
-from .log import get_logger
+logger = logging.getLogger(__name__)
-logger = get_logger(__name__)
+# This must be a copy of what is in bootstrap.py.
+# Notice this script is also called independently of bob.devtools!
+_SERVER = "http://www.idiap.ch"
+
+_WEBDAV_PATHS = {
+ True: { # stable?
+ False: { # visible?
+ "root": "/private-upload",
+ "conda": "/conda",
+ "docs": "/docs",
+ },
+ True: { # visible?
+ "root": "/public-upload",
+ "conda": "/conda",
+ "docs": "/docs",
+ },
+ },
+ False: { # stable?
+ False: { # visible?
+ "root": "/private-upload",
+ "conda": "/conda/label/beta",
+ "docs": "/docs",
+ },
+ True: { # visible?
+ "root": "/public-upload",
+ "conda": "/conda/label/beta",
+ "docs": "/docs",
+ },
+ },
+}
def _setup_webdav_client(server, root, username, password):
@@ -23,7 +52,7 @@ def _setup_webdav_client(server, root, username, password):
webdav_password=password,
)
- from .webdav3 import client as webdav
+ from webdav3 import client as webdav
retval = webdav.Client(webdav_options)
assert retval.valid()
@@ -56,9 +85,9 @@ def deploy_conda_package(
messages.
"""
- server_info = WEBDAV_PATHS[stable][public]
+ server_info = _WEBDAV_PATHS[stable][public]
davclient = _setup_webdav_client(
- SERVER, server_info["root"], username, password
+ _SERVER, server_info["root"], username, password
)
basename = os.path.basename(package)
@@ -71,18 +100,18 @@ def deploy_conda_package(
"The file %s/%s already exists on the server "
"- this can be due to more than one build with deployment "
"running at the same time. Re-running the broken builds "
- "normally fixes it" % (SERVER, remote_path)
+ "normally fixes it" % (_SERVER, remote_path)
)
else:
logger.info(
- "[dav] rm -f %s%s%s", SERVER, server_info["root"], remote_path
+ "[dav] rm -f %s%s%s", _SERVER, server_info["root"], remote_path
)
if not dry_run:
davclient.clean(remote_path)
logger.info(
- "[dav] %s -> %s%s%s", package, SERVER, server_info["root"], remote_path
+ "[dav] %s -> %s%s%s", package, _SERVER, server_info["root"], remote_path
)
if not dry_run:
davclient.upload(local_path=package, remote_path=remote_path)
@@ -132,9 +161,9 @@ def deploy_documentation(
"ensure documentation is being produced for your project!" % path
)
- server_info = WEBDAV_PATHS[stable][public]
+ server_info = _WEBDAV_PATHS[stable][public]
davclient = _setup_webdav_client(
- SERVER, server_info["root"], username, password
+ _SERVER, server_info["root"], username, password
)
remote_path_prefix = "%s/%s" % (server_info["docs"], package)
@@ -161,7 +190,85 @@ def deploy_documentation(
davclient.mkdir(remote_path_prefix)
remote_path = "%s/%s" % (remote_path_prefix, k)
logger.info(
- "[dav] %s -> %s%s%s", path, SERVER, server_info["root"], remote_path
+ "[dav] %s -> %s%s%s",
+ path,
+ _SERVER,
+ server_info["root"],
+ remote_path,
)
if not dry_run:
davclient.upload_directory(local_path=path, remote_path=remote_path)
+
+
+if __name__ == "__main__":
+
+ import argparse
+
+ parser = argparse.ArgumentParser(
+ description="Deploys documentation from python-only packages"
+ )
+ parser.add_argument(
+ "directory",
+ help="Directory containing the sphinx build to deploy",
+ )
+ parser.add_argument(
+ "-p",
+ "--package",
+ default=os.environ.get("CI_PROJECT_PATH", None),
+ help="The package being built [default: %(default)s]",
+ )
+ parser.add_argument(
+ "-x",
+ "--visibility",
+ default=os.environ.get("CI_PROJECT_VISIBILITY", "private"),
+ help="The visibility of the package being built [default: %(default)s]",
+ )
+ parser.add_argument(
+ "-b",
+ "--branch",
+ default=os.environ.get("CI_COMMIT_REF_NAME", None),
+ help="Name of the branch being built [default: %(default)s]",
+ )
+ parser.add_argument(
+ "-t",
+ "--tag",
+ default=os.environ.get("CI_COMMIT_TAG", None),
+ help="If building a tag, pass it with this flag [default: %(default)s]",
+ )
+ parser.add_argument(
+ "-u",
+ "--username",
+ default=os.environ.get("DOCUSER", None),
+ help="Username for webdav deployment [default: %(default)s]",
+ )
+ parser.add_argument(
+ "-P",
+ "--password",
+ default=os.environ.get("DOCPASS", None),
+ help="Password for webdav deployment [default: %(default)s]",
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ help="Be verbose (enables INFO logging)",
+ action="store_const",
+ dest="loglevel",
+ default=logging.WARNING,
+ const=logging.INFO,
+ )
+
+ args = parser.parse_args()
+ logging.basicConfig(level=args.loglevel)
+
+ deploy_documentation(
+ args.directory,
+ package=args.package,
+ stable=(args.tag is not None),
+ latest=True,
+ public=(args.visibility == "public"),
+ branch=args.branch,
+ tag=args.tag,
+ username=args.username,
+ password=args.password,
+ dry_run=False,
+ )
diff --git a/bob/devtools/webdav3/__init__.py b/bob/devtools/webdav3/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/bob/devtools/webdav3/client.py b/bob/devtools/webdav3/client.py
deleted file mode 100644
index 1b571aa46402e10d3ab1048187403d4d6dbbd073..0000000000000000000000000000000000000000
--- a/bob/devtools/webdav3/client.py
+++ /dev/null
@@ -1,1161 +0,0 @@
-# -*- coding: utf-8
-
-import functools
-import os
-import shutil
-import threading
-
-from io import BytesIO
-from re import sub
-
-import lxml.etree as etree
-import requests
-
-from ..log import get_logger
-from .connection import ProxySettings, WebDAVSettings
-from .exceptions import (
- ConnectionException,
- LocalResourceNotFound,
- MethodNotSupported,
- NoConnection,
- NotEnoughSpace,
- OptionNotValid,
- RemoteParentNotFound,
- RemoteResourceNotFound,
- ResourceTooBig,
- ResponseErrorCode,
-)
-from .urn import Urn
-
-logger = get_logger(__name__)
-
-
-try:
- from urllib.parse import unquote, urlsplit
-except ImportError:
- from urllib import unquote
-
- from urlparse import urlsplit
-
-__version__ = "0.2"
-
-
-def listdir(directory):
- """Returns list of nested files and directories for local directory by
- path.
-
- :param directory: absolute or relative path to local directory
- :return: list nested of file or directory names
- """
- file_names = list()
- for filename in os.listdir(directory):
- file_path = os.path.join(directory, filename)
- if os.path.isdir(file_path):
- filename = "{filename}{separate}".format(
- filename=filename, separate=os.path.sep
- )
- file_names.append(filename)
- return file_names
-
-
-def get_options(option_type, from_options):
- """Extract options for specified option type from all options.
-
- :param option_type: the object of specified type of options
- :param from_options: all options dictionary
- :return: the dictionary of options for specified type, each option can be filled by value from all options
- dictionary or blank in case the option for specified type is not exist in all options dictionary
- """
- _options = dict()
-
- for key in option_type.keys:
- key_with_prefix = "{prefix}{key}".format(
- prefix=option_type.prefix, key=key
- )
- if key not in from_options and key_with_prefix not in from_options:
- _options[key] = ""
- elif key in from_options:
- _options[key] = from_options.get(key)
- else:
- _options[key] = from_options.get(key_with_prefix)
-
- return _options
-
-
-def wrap_connection_error(fn):
- @functools.wraps(fn)
- def _wrapper(self, *args, **kw):
- logger.debug("Requesting %s(%s, %s)", fn, args, kw)
- try:
- res = fn(self, *args, **kw)
- except requests.ConnectionError:
- raise NoConnection(self.webdav.hostname)
- except requests.RequestException as re:
- raise ConnectionException(re)
- else:
- return res
-
- return _wrapper
-
-
-class Client(object):
- """The client for WebDAV servers provides an ability to control files on
- remote WebDAV server."""
-
- # path to root directory of WebDAV
- root = "/"
-
- # Max size of file for uploading
- large_size = 2 * 1024 * 1024 * 1024
-
- # request timeout in seconds
- timeout = 30
-
- # HTTP headers for different actions
- http_header = {
- "list": ["Accept: */*", "Depth: 1"],
- "free": ["Accept: */*", "Depth: 0", "Content-Type: text/xml"],
- "copy": ["Accept: */*"],
- "move": ["Accept: */*"],
- "mkdir": ["Accept: */*", "Connection: Keep-Alive"],
- "clean": ["Accept: */*", "Connection: Keep-Alive"],
- "check": ["Accept: */*"],
- "info": ["Accept: */*", "Depth: 1"],
- "get_property": [
- "Accept: */*",
- "Depth: 1",
- "Content-Type: application/x-www-form-urlencoded",
- ],
- "set_property": [
- "Accept: */*",
- "Depth: 1",
- "Content-Type: application/x-www-form-urlencoded",
- ],
- }
-
- def get_headers(self, action, headers_ext=None):
- """Returns HTTP headers of specified WebDAV actions.
-
- :param action: the identifier of action.
- :param headers_ext: (optional) the addition headers list witch sgould be added to basic HTTP headers for
- the specified action.
- :return: the dictionary of headers for specified action.
- """
- if action in Client.http_header:
- try:
- headers = Client.http_header[action].copy()
- except AttributeError:
- headers = Client.http_header[action][:]
- else:
- headers = list()
-
- if headers_ext:
- headers.extend(headers_ext)
-
- if self.webdav.token:
- webdav_token = "Authorization: OAuth {token}".format(
- token=self.webdav.token
- )
- headers.append(webdav_token)
- return dict([map(lambda s: s.strip(), i.split(":")) for i in headers])
-
- def get_url(self, path):
- """Generates url by uri path.
-
- :param path: uri path.
- :return: the url string.
- """
- url = {
- "hostname": self.webdav.hostname,
- "root": self.webdav.root,
- "path": path,
- }
- return "{hostname}{root}{path}".format(**url)
-
- def get_full_path(self, urn):
- """Generates full path to remote resource exclude hostname.
-
- :param urn: the URN to resource.
- :return: full path to resource with root path.
- """
- return "{root}{path}".format(root=self.webdav.root, path=urn.path())
-
- def execute_request(self, action, path, data=None, headers_ext=None):
- """Generate request to WebDAV server for specified action and path and
- execute it.
-
- :param action: the action for WebDAV server which should be executed.
- :param path: the path to resource for action
- :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes,
- or file-like object to send in the body of the :class:`Request`.
- :param headers_ext: (optional) the addition headers list witch should be added to basic HTTP headers for
- the specified action.
- :return: HTTP response of request.
- """
- response = requests.request(
- method=Client.requests[action],
- url=self.get_url(path),
- auth=(self.webdav.login, self.webdav.password),
- headers=self.get_headers(action, headers_ext),
- timeout=self.timeout,
- data=data,
- )
- if response.status_code == 507:
- raise NotEnoughSpace()
- if response.status_code >= 400:
- raise ResponseErrorCode(
- url=self.get_url(path),
- code=response.status_code,
- message=response.content,
- )
- return response
-
- # mapping of actions to WebDAV methods
- requests = {
- "download": "GET",
- "upload": "PUT",
- "copy": "COPY",
- "move": "MOVE",
- "mkdir": "MKCOL",
- "clean": "DELETE",
- "check": "HEAD",
- "list": "PROPFIND",
- "free": "PROPFIND",
- "info": "PROPFIND",
- "publish": "PROPPATCH",
- "unpublish": "PROPPATCH",
- "published": "PROPPATCH",
- "get_property": "PROPFIND",
- "set_property": "PROPPATCH",
- }
-
- meta_xmlns = {"https://webdav.yandex.ru": "urn:yandex:disk:meta"}
-
- def __init__(self, options):
- """Constructor of WebDAV client.
-
- :param options: the dictionary of connection options to WebDAV can include proxy server options.
- WebDev settings:
- `webdav_hostname`: url for WebDAV server should contain protocol and ip address or domain name.
- Example: `https://webdav.server.com`.
- `webdav_login`: (optional) login name for WebDAV server can be empty in case using of token auth.
- `webdav_password`: (optional) password for WebDAV server can be empty in case using of token auth.
- `webdav_token': (optional) token for WebDAV server can be empty in case using of login/password auth.
- `webdav_root`: (optional) root directory of WebDAV server. Defaults is `/`.
- `webdav_cert_path`: (optional) path to certificate.
- `webdav_key_path`: (optional) path to private key.
- `webdav_recv_speed`: (optional) rate limit data download speed in Bytes per second.
- Defaults to unlimited speed.
- `webdav_send_speed`: (optional) rate limit data upload speed in Bytes per second.
- Defaults to unlimited speed.
- `webdav_verbose`: (optional) set verbose mode on.off. By default verbose mode is off.
- Proxy settings (optional):
- `proxy_hostname`: url to proxy server should contain protocol and ip address or domain name and if needed
- port. Example: `https://proxy.server.com:8383`.
- `proxy_login`: login name for proxy server.
- `proxy_password`: password for proxy server.
- """
- webdav_options = get_options(
- option_type=WebDAVSettings, from_options=options
- )
- proxy_options = get_options(
- option_type=ProxySettings, from_options=options
- )
-
- self.webdav = WebDAVSettings(webdav_options)
- self.proxy = ProxySettings(proxy_options)
- self.default_options = {}
-
- def valid(self):
- """Validates of WebDAV and proxy settings.
-
- :return: True in case settings are valid and False otherwise.
- """
- return True if self.webdav.valid() and self.proxy.valid() else False
-
- @wrap_connection_error
- def list(self, remote_path=root):
- """Returns list of nested files and directories for remote WebDAV
- directory by path. More information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND.
-
- :param remote_path: path to remote directory.
- :return: list of nested file or directory names.
- """
- directory_urn = Urn(remote_path, directory=True)
- if directory_urn.path() != Client.root:
- if not self.check(directory_urn.path()):
- raise RemoteResourceNotFound(directory_urn.path())
-
- response = self.execute_request(
- action="list", path=directory_urn.quote()
- )
- urns = WebDavXmlUtils.parse_get_list_response(response.content)
-
- path = Urn.normalize_path(self.get_full_path(directory_urn))
- return [
- urn.filename()
- for urn in urns
- if Urn.compare_path(path, urn.path()) is False
- ]
-
- @wrap_connection_error
- def free(self):
- """Returns an amount of free space on remote WebDAV server. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND.
-
- :return: an amount of free space in bytes.
- """
- data = WebDavXmlUtils.create_free_space_request_content()
- response = self.execute_request(action="free", path="", data=data)
- return WebDavXmlUtils.parse_free_space_response(
- response.content, self.webdav.hostname
- )
-
- @wrap_connection_error
- def check(self, remote_path=root):
- """Checks an existence of remote resource on WebDAV server by remote
- path. More information you can find by link
- http://webdav.org/specs/rfc4918.html#rfc.section.9.4.
-
- :param remote_path: (optional) path to resource on WebDAV server. Defaults is root directory of WebDAV.
- :return: True if resource is exist or False otherwise
- """
- urn = Urn(remote_path)
- try:
- response = self.execute_request(action="check", path=urn.quote())
- except ResponseErrorCode:
- return False
-
- if int(response.status_code) == 200:
- return True
- return False
-
- @wrap_connection_error
- def mkdir(self, remote_path):
- """Makes new directory on WebDAV server. More information you can find
- by link http://webdav.org/specs/rfc4918.html#METHOD_MKCOL.
-
- :param remote_path: path to directory
- :return: True if request executed with code 200 or 201 and False otherwise.
- """
- directory_urn = Urn(remote_path, directory=True)
- if not self.check(directory_urn.parent()):
- raise RemoteParentNotFound(directory_urn.path())
-
- response = self.execute_request(
- action="mkdir", path=directory_urn.quote()
- )
- return response.status_code in (200, 201)
-
- @wrap_connection_error
- def download_from(self, buff, remote_path):
- """Downloads file from WebDAV and writes it in buffer.
-
- :param buff: buffer object for writing of downloaded file content.
- :param remote_path: path to file on WebDAV server.
- """
- urn = Urn(remote_path)
- if self.is_dir(urn.path()):
- raise OptionNotValid(name="remote_path", value=remote_path)
-
- if not self.check(urn.path()):
- raise RemoteResourceNotFound(urn.path())
-
- response = self.execute_request(action="download", path=urn.quote())
- buff.write(response.content)
-
- def download(self, remote_path, local_path, progress=None):
- """Downloads remote resource from WebDAV and save it in local path.
- More information you can find by link
- http://webdav.org/specs/rfc4918.html#rfc.section.9.4.
-
- :param remote_path: the path to remote resource for downloading can be file and directory.
- :param local_path: the path to save resource locally.
- :param progress: progress function. Not supported now.
- """
- urn = Urn(remote_path)
- if self.is_dir(urn.path()):
- self.download_directory(
- local_path=local_path,
- remote_path=remote_path,
- progress=progress,
- )
- else:
- self.download_file(
- local_path=local_path,
- remote_path=remote_path,
- progress=progress,
- )
-
- def download_directory(self, remote_path, local_path, progress=None):
- """Downloads directory and downloads all nested files and directories
- from remote WebDAV to local. If there is something on local path it
- deletes directories and files then creates new.
-
- :param remote_path: the path to directory for downloading form WebDAV server.
- :param local_path: the path to local directory for saving downloaded files and directories.
- :param progress: Progress function. Not supported now.
- """
- urn = Urn(remote_path, directory=True)
- if not self.is_dir(urn.path()):
- raise OptionNotValid(name="remote_path", value=remote_path)
-
- if os.path.exists(local_path):
- shutil.rmtree(local_path)
-
- os.makedirs(local_path)
-
- for resource_name in self.list(urn.path()):
- _remote_path = "{parent}{name}".format(
- parent=urn.path(), name=resource_name
- )
- _local_path = os.path.join(local_path, resource_name)
- self.download(
- local_path=_local_path,
- remote_path=_remote_path,
- progress=progress,
- )
-
- @wrap_connection_error
- def download_file(self, remote_path, local_path, progress=None):
- """Downloads file from WebDAV server and save it locally. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#rfc.section.9.4.
-
- :param remote_path: the path to remote file for downloading.
- :param local_path: the path to save file locally.
- :param progress: progress function. Not supported now.
- """
- urn = Urn(remote_path)
- if self.is_dir(urn.path()):
- raise OptionNotValid(name="remote_path", value=remote_path)
-
- if os.path.isdir(local_path):
- raise OptionNotValid(name="local_path", value=local_path)
-
- if not self.check(urn.path()):
- raise RemoteResourceNotFound(urn.path())
-
- with open(local_path, "wb") as local_file:
- response = self.execute_request("download", urn.quote())
- for block in response.iter_content(1024):
- local_file.write(block)
-
- def download_sync(self, remote_path, local_path, callback=None):
- """Downloads remote resources from WebDAV server synchronously.
-
- :param remote_path: the path to remote resource on WebDAV server. Can be file and directory.
- :param local_path: the path to save resource locally.
- :param callback: the callback which will be invoked when downloading is complete.
- """
- self.download(local_path=local_path, remote_path=remote_path)
- if callback:
- callback()
-
- def download_async(self, remote_path, local_path, callback=None):
- """Downloads remote resources from WebDAV server asynchronously.
-
- :param remote_path: the path to remote resource on WebDAV server. Can be file and directory.
- :param local_path: the path to save resource locally.
- :param callback: the callback which will be invoked when downloading is complete.
- """
-
- def target():
- return self.download_sync(
- local_path=local_path,
- remote_path=remote_path,
- callback=callback,
- )
-
- threading.Thread(target=target).start()
-
- @wrap_connection_error
- def upload_to(self, buff, remote_path):
- """Uploads file from buffer to remote path on WebDAV server. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PUT.
-
- :param buff: the buffer with content for file.
- :param remote_path: the path to save file remotely on WebDAV server.
- """
- urn = Urn(remote_path)
- if urn.is_dir():
- raise OptionNotValid(name="remote_path", value=remote_path)
-
- if not self.check(urn.parent()):
- raise RemoteParentNotFound(urn.path())
-
- self.execute_request(action="upload", path=urn.quote(), data=buff)
-
- def upload(self, remote_path, local_path, progress=None):
- """Uploads resource to remote path on WebDAV server. In case resource
- is directory it will upload all nested files and directories. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PUT.
-
- :param remote_path: the path for uploading resources on WebDAV server. Can be file and directory.
- :param local_path: the path to local resource for uploading.
- :param progress: Progress function. Not supported now.
- """
- if os.path.isdir(local_path):
- self.upload_directory(
- local_path=local_path,
- remote_path=remote_path,
- progress=progress,
- )
- else:
- self.upload_file(local_path=local_path, remote_path=remote_path)
-
- def upload_directory(self, remote_path, local_path, progress=None):
- """Uploads directory to remote path on WebDAV server. In case directory
- is exist on remote server it will delete it and then upload directory
- with nested files and directories.
-
- :param remote_path: the path to directory for uploading on WebDAV server.
- :param local_path: the path to local directory for uploading.
- :param progress: Progress function. Not supported now.
- """
- urn = Urn(remote_path, directory=True)
- if not urn.is_dir():
- raise OptionNotValid(name="remote_path", value=remote_path)
-
- if not os.path.isdir(local_path):
- raise OptionNotValid(name="local_path", value=local_path)
-
- if not os.path.exists(local_path):
- raise LocalResourceNotFound(local_path)
-
- if self.check(urn.path()):
- self.clean(urn.path())
-
- self.mkdir(remote_path)
-
- for resource_name in listdir(local_path):
- _remote_path = "{parent}{name}".format(
- parent=urn.path(), name=resource_name
- )
- _local_path = os.path.join(local_path, resource_name)
- self.upload(
- local_path=_local_path,
- remote_path=_remote_path,
- progress=progress,
- )
-
- @wrap_connection_error
- def upload_file(self, remote_path, local_path, progress=None):
- """Uploads file to remote path on WebDAV server. File should be 2Gb or
- less. More information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PUT.
-
- :param remote_path: the path to uploading file on WebDAV server.
- :param local_path: the path to local file for uploading.
- :param progress: Progress function. Not supported now.
- """
- if not os.path.exists(local_path):
- raise LocalResourceNotFound(local_path)
-
- urn = Urn(remote_path)
- if urn.is_dir():
- raise OptionNotValid(name="remote_path", value=remote_path)
-
- if os.path.isdir(local_path):
- raise OptionNotValid(name="local_path", value=local_path)
-
- if not self.check(urn.parent()):
- raise RemoteParentNotFound(urn.path())
-
- with open(local_path, "rb") as local_file:
- file_size = os.path.getsize(local_path)
- if file_size > self.large_size:
- raise ResourceTooBig(
- path=local_path, size=file_size, max_size=self.large_size
- )
-
- self.execute_request(
- action="upload", path=urn.quote(), data=local_file
- )
-
- def upload_sync(self, remote_path, local_path, callback=None):
- """Uploads resource to remote path on WebDAV server synchronously. In
- case resource is directory it will upload all nested files and
- directories.
-
- :param remote_path: the path for uploading resources on WebDAV server. Can be file and directory.
- :param local_path: the path to local resource for uploading.
- :param callback: the callback which will be invoked when downloading is complete.
- """
- self.upload(local_path=local_path, remote_path=remote_path)
-
- if callback:
- callback()
-
- def upload_async(self, remote_path, local_path, callback=None):
- """Uploads resource to remote path on WebDAV server asynchronously. In
- case resource is directory it will upload all nested files and
- directories.
-
- :param remote_path: the path for uploading resources on WebDAV server. Can be file and directory.
- :param local_path: the path to local resource for uploading.
- :param callback: the callback which will be invoked when downloading is complete.
- """
-
- def target():
- return self.upload_sync(
- local_path=local_path,
- remote_path=remote_path,
- callback=callback,
- )
-
- threading.Thread(target=target).start()
-
- @wrap_connection_error
- def copy(self, remote_path_from, remote_path_to):
- """Copies resource from one place to another on WebDAV server. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_COPY.
-
- :param remote_path_from: the path to resource which will be copied,
- :param remote_path_to: the path where resource will be copied.
- """
- urn_from = Urn(remote_path_from)
- if not self.check(urn_from.path()):
- raise RemoteResourceNotFound(urn_from.path())
-
- urn_to = Urn(remote_path_to)
- if not self.check(urn_to.parent()):
- raise RemoteParentNotFound(urn_to.path())
-
- header_destination = "Destination: {path}".format(
- path=self.get_full_path(urn_to)
- )
- self.execute_request(
- action="copy",
- path=urn_from.quote(),
- headers_ext=[header_destination],
- )
-
- @wrap_connection_error
- def move(self, remote_path_from, remote_path_to, overwrite=False):
- """Moves resource from one place to another on WebDAV server. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_MOVE.
-
- :param remote_path_from: the path to resource which will be moved,
- :param remote_path_to: the path where resource will be moved.
- :param overwrite: (optional) the flag, overwrite file if it exists. Defaults is False
- """
- urn_from = Urn(remote_path_from)
- if not self.check(urn_from.path()):
- raise RemoteResourceNotFound(urn_from.path())
-
- urn_to = Urn(remote_path_to)
- if not self.check(urn_to.parent()):
- raise RemoteParentNotFound(urn_to.path())
-
- header_destination = "Destination: {path}".format(
- path=self.get_full_path(urn_to)
- )
- header_overwrite = "Overwrite: {flag}".format(
- flag="T" if overwrite else "F"
- )
- self.execute_request(
- action="move",
- path=urn_from.quote(),
- headers_ext=[header_destination, header_overwrite],
- )
-
- @wrap_connection_error
- def clean(self, remote_path):
- """Cleans (Deletes) a remote resource on WebDAV server. The name of
- method is not changed for back compatibility with original library.
- More information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_DELETE.
-
- :param remote_path: the remote resource whisch will be deleted.
- """
- urn = Urn(remote_path)
- self.execute_request(action="clean", path=urn.quote())
-
- @wrap_connection_error
- def info(self, remote_path):
- """Gets information about resource on WebDAV. More information you can
- find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND.
-
- :param remote_path: the path to remote resource.
- :return: a dictionary of information attributes and them values with following keys:
- `created`: date of resource creation,
- `name`: name of resource,
- `size`: size of resource,
- `modified`: date of resource modification.
- """
- urn = Urn(remote_path)
- if not self.check(urn.path()) and not self.check(
- Urn(remote_path, directory=True).path()
- ):
- raise RemoteResourceNotFound(remote_path)
-
- response = self.execute_request(action="info", path=urn.quote())
- path = self.get_full_path(urn)
- return WebDavXmlUtils.parse_info_response(
- content=response.content, path=path, hostname=self.webdav.hostname
- )
-
- @wrap_connection_error
- def is_dir(self, remote_path):
- """Checks is the remote resource directory. More information you can
- find by link http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND.
-
- :param remote_path: the path to remote resource.
- :return: True in case the remote resource is directory and False otherwise.
- """
- urn = Urn(remote_path)
- parent_urn = Urn(urn.parent())
- if not self.check(urn.path()) and not self.check(
- Urn(remote_path, directory=True).path()
- ):
- raise RemoteResourceNotFound(remote_path)
-
- response = self.execute_request(action="info", path=parent_urn.quote())
- path = self.get_full_path(urn)
- return WebDavXmlUtils.parse_is_dir_response(
- content=response.content, path=path, hostname=self.webdav.hostname
- )
-
- @wrap_connection_error
- def get_property(self, remote_path, option):
- """Gets metadata property of remote resource on WebDAV server. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PROPFIND.
-
- :param remote_path: the path to remote resource.
- :param option: the property attribute as dictionary with following keys:
- `namespace`: (optional) the namespace for XML property which will be set,
- `name`: the name of property which will be set.
- :return: the value of property or None if property is not found.
- """
- urn = Urn(remote_path)
- if not self.check(urn.path()):
- raise RemoteResourceNotFound(urn.path())
-
- data = WebDavXmlUtils.create_get_property_request_content(option)
- response = self.execute_request(
- action="get_property", path=urn.quote(), data=data
- )
- return WebDavXmlUtils.parse_get_property_response(
- response.content, option["name"]
- )
-
- @wrap_connection_error
- def set_property(self, remote_path, option):
- """Sets metadata property of remote resource on WebDAV server. More
- information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PROPPATCH.
-
- :param remote_path: the path to remote resource.
- :param option: the property attribute as dictionary with following keys:
- `namespace`: (optional) the namespace for XML property which will be set,
- `name`: the name of property which will be set,
- `value`: (optional) the value of property which will be set. Defaults is empty string.
- """
- self.set_property_batch(remote_path=remote_path, option=[option])
-
- @wrap_connection_error
- def set_property_batch(self, remote_path, option):
- """Sets batch metadata properties of remote resource on WebDAV server
- in batch. More information you can find by link
- http://webdav.org/specs/rfc4918.html#METHOD_PROPPATCH.
-
- :param remote_path: the path to remote resource.
- :param option: the property attributes as list of dictionaries with following keys:
- `namespace`: (optional) the namespace for XML property which will be set,
- `name`: the name of property which will be set,
- `value`: (optional) the value of property which will be set. Defaults is empty string.
- """
- urn = Urn(remote_path)
- if not self.check(urn.path()):
- raise RemoteResourceNotFound(urn.path())
-
- data = WebDavXmlUtils.create_set_property_batch_request_content(option)
- self.execute_request(action="set_property", path=urn.quote(), data=data)
-
- def resource(self, remote_path):
- urn = Urn(remote_path)
- return Resource(self, urn)
-
- def push(self, remote_directory, local_directory):
- def prune(src, exp):
- return [sub(exp, "", item) for item in src]
-
- urn = Urn(remote_directory, directory=True)
-
- if not self.is_dir(urn.path()):
- raise OptionNotValid(name="remote_path", value=remote_directory)
-
- if not os.path.isdir(local_directory):
- raise OptionNotValid(name="local_path", value=local_directory)
-
- if not os.path.exists(local_directory):
- raise LocalResourceNotFound(local_directory)
-
- paths = self.list(urn.path())
- expression = "{begin}{end}".format(begin="^", end=urn.path())
- remote_resource_names = prune(paths, expression)
-
- for local_resource_name in listdir(local_directory):
-
- local_path = os.path.join(local_directory, local_resource_name)
- remote_path = "{remote_directory}{resource_name}".format(
- remote_directory=urn.path(), resource_name=local_resource_name
- )
-
- if os.path.isdir(local_path):
- if not self.check(remote_path=remote_path):
- self.mkdir(remote_path=remote_path)
- self.push(
- remote_directory=remote_path, local_directory=local_path
- )
- else:
- if local_resource_name in remote_resource_names:
- continue
- self.upload_file(remote_path=remote_path, local_path=local_path)
-
- def pull(self, remote_directory, local_directory):
- def prune(src, exp):
- return [sub(exp, "", item) for item in src]
-
- urn = Urn(remote_directory, directory=True)
-
- if not self.is_dir(urn.path()):
- raise OptionNotValid(name="remote_path", value=remote_directory)
-
- if not os.path.exists(local_directory):
- raise LocalResourceNotFound(local_directory)
-
- local_resource_names = listdir(local_directory)
-
- paths = self.list(urn.path())
- expression = "{begin}{end}".format(begin="^", end=remote_directory)
- remote_resource_names = prune(paths, expression)
-
- for remote_resource_name in remote_resource_names:
-
- local_path = os.path.join(local_directory, remote_resource_name)
- remote_path = "{remote_directory}{resource_name}".format(
- remote_directory=urn.path(), resource_name=remote_resource_name
- )
-
- remote_urn = Urn(remote_path)
-
- if self.is_dir(remote_urn.path()):
- if not os.path.exists(local_path):
- os.mkdir(local_path)
- self.pull(
- remote_directory=remote_path, local_directory=local_path
- )
- else:
- if remote_resource_name in local_resource_names:
- continue
- self.download_file(
- remote_path=remote_path, local_path=local_path
- )
-
- def sync(self, remote_directory, local_directory):
-
- self.pull(
- remote_directory=remote_directory, local_directory=local_directory
- )
- self.push(
- remote_directory=remote_directory, local_directory=local_directory
- )
-
-
-class Resource(object):
- def __init__(self, client, urn):
- self.client = client
- self.urn = urn
-
- def __str__(self):
- return "resource {path}".format(path=self.urn.path())
-
- def is_dir(self):
- return self.client.is_dir(self.urn.path())
-
- def rename(self, new_name):
- old_path = self.urn.path()
- parent_path = self.urn.parent()
- new_name = Urn(new_name).filename()
- new_path = "{directory}{filename}".format(
- directory=parent_path, filename=new_name
- )
-
- self.client.move(remote_path_from=old_path, remote_path_to=new_path)
- self.urn = Urn(new_path)
-
- def move(self, remote_path):
- new_urn = Urn(remote_path)
- self.client.move(
- remote_path_from=self.urn.path(), remote_path_to=new_urn.path()
- )
- self.urn = new_urn
-
- def copy(self, remote_path):
- urn = Urn(remote_path)
- self.client.copy(
- remote_path_from=self.urn.path(), remote_path_to=remote_path
- )
- return Resource(self.client, urn)
-
- def info(self, params=None):
- info = self.client.info(self.urn.path())
- if not params:
- return info
-
- return {key: value for (key, value) in info.items() if key in params}
-
- def clean(self):
- return self.client.clean(self.urn.path())
-
- def check(self):
- return self.client.check(self.urn.path())
-
- def read_from(self, buff):
- self.client.upload_to(buff=buff, remote_path=self.urn.path())
-
- def read(self, local_path):
- return self.client.upload_sync(
- local_path=local_path, remote_path=self.urn.path()
- )
-
- def read_async(self, local_path, callback=None):
- return self.client.upload_async(
- local_path=local_path,
- remote_path=self.urn.path(),
- callback=callback,
- )
-
- def write_to(self, buff):
- return self.client.download_from(buff=buff, remote_path=self.urn.path())
-
- def write(self, local_path):
- return self.client.download_sync(
- local_path=local_path, remote_path=self.urn.path()
- )
-
- def write_async(self, local_path, callback=None):
- return self.client.download_async(
- local_path=local_path,
- remote_path=self.urn.path(),
- callback=callback,
- )
-
- def publish(self):
- return self.client.publish(self.urn.path())
-
- def unpublish(self):
- return self.client.unpublish(self.urn.path())
-
- @property
- def property(self, option):
- return self.client.get_property(
- remote_path=self.urn.path(), option=option
- )
-
- @property.setter
- def property(self, option, value):
- option["value"] = value.__str__()
- self.client.set_property(remote_path=self.urn.path(), option=option)
-
-
-class WebDavXmlUtils:
- def __init__(self):
- pass
-
- @staticmethod
- def parse_get_list_response(content):
- """Parses of response content XML from WebDAV server and extract file
- and directory names.
-
- :param content: the XML content of HTTP response from WebDAV server for getting list of files by remote path.
- :return: list of extracted file or directory names.
- """
- try:
- tree = etree.fromstring(content)
- hrees = [
- Urn.separate + unquote(urlsplit(hree.text).path)
- for hree in tree.findall(".//{DAV:}href")
- ]
- return [Urn(hree) for hree in hrees]
- except etree.XMLSyntaxError:
- return list()
-
- @staticmethod
- def create_free_space_request_content():
- """Creates an XML for requesting of free space on remote WebDAV server.
-
- :return: the XML string of request content.
- """
- root = etree.Element("propfind", xmlns="DAV:")
- prop = etree.SubElement(root, "prop")
- etree.SubElement(prop, "quota-available-bytes")
- etree.SubElement(prop, "quota-used-bytes")
- tree = etree.ElementTree(root)
- return WebDavXmlUtils.etree_to_string(tree)
-
- @staticmethod
- def parse_free_space_response(content, hostname):
- """Parses of response content XML from WebDAV server and extract an
- amount of free space.
-
- :param content: the XML content of HTTP response from WebDAV server for getting free space.
- :param hostname: the server hostname.
- :return: an amount of free space in bytes.
- """
- try:
- tree = etree.fromstring(content)
- node = tree.find(".//{DAV:}quota-available-bytes")
- if node is not None:
- return int(node.text)
- else:
- raise MethodNotSupported(name="free", server=hostname)
- except TypeError:
- raise MethodNotSupported(name="free", server=hostname)
- except etree.XMLSyntaxError:
- return str()
-
- @staticmethod
- def parse_info_response(content, path, hostname):
- """Parses of response content XML from WebDAV server and extract an
- information about resource.
-
- :param content: the XML content of HTTP response from WebDAV server.
- :param path: the path to resource.
- :param hostname: the server hostname.
- :return: a dictionary of information attributes and them values with following keys:
- `created`: date of resource creation,
- `name`: name of resource,
- `size`: size of resource,
- `modified`: date of resource modification.
- """
- response = WebDavXmlUtils.extract_response_for_path(
- content=content, path=path, hostname=hostname
- )
- find_attributes = {
- "created": ".//{DAV:}creationdate",
- "name": ".//{DAV:}displayname",
- "size": ".//{DAV:}getcontentlength",
- "modified": ".//{DAV:}getlastmodified",
- }
- info = dict()
- for (name, value) in find_attributes.items():
- info[name] = response.findtext(value)
- return info
-
- @staticmethod
- def parse_is_dir_response(content, path, hostname):
- """Parses of response content XML from WebDAV server and extract an
- information about resource.
-
- :param content: the XML content of HTTP response from WebDAV server.
- :param path: the path to resource.
- :param hostname: the server hostname.
- :return: True in case the remote resource is directory and False otherwise.
- """
- response = WebDavXmlUtils.extract_response_for_path(
- content=content, path=path, hostname=hostname
- )
- resource_type = response.find(".//{DAV:}resourcetype")
- if resource_type is None:
- raise MethodNotSupported(name="is_dir", server=hostname)
- dir_type = resource_type.find("{DAV:}collection")
-
- return True if dir_type is not None else False
-
- @staticmethod
- def create_get_property_request_content(option):
- """Creates an XML for requesting of getting a property value of remote
- WebDAV resource.
-
- :param option: the property attributes as dictionary with following keys:
- `namespace`: (optional) the namespace for XML property which will be get,
- `name`: the name of property which will be get.
- :return: the XML string of request content.
- """
- root = etree.Element("propfind", xmlns="DAV:")
- prop = etree.SubElement(root, "prop")
- etree.SubElement(
- prop, option.get("name", ""), xmlns=option.get("namespace", "")
- )
- tree = etree.ElementTree(root)
- return WebDavXmlUtils.etree_to_string(tree)
-
- @staticmethod
- def parse_get_property_response(content, name):
- """Parses of response content XML from WebDAV server for getting
- metadata property value for some resource.
-
- :param content: the XML content of response as string.
- :param name: the name of property for finding a value in response
- :return: the value of property if it has been found or None otherwise.
- """
- tree = etree.fromstring(content)
- return tree.xpath("//*[local-name() = $name]", name=name)[0].text
-
- @staticmethod
- def create_set_property_batch_request_content(options):
- """Creates an XML for requesting of setting a property values for
- remote WebDAV resource in batch.
-
- :param options: the property attributes as list of dictionaries with following keys:
- `namespace`: (optional) the namespace for XML property which will be set,
- `name`: the name of property which will be set,
- `value`: (optional) the value of property which will be set. Defaults is empty string.
- :return: the XML string of request content.
- """
- root_node = etree.Element("propertyupdate", xmlns="DAV:")
- set_node = etree.SubElement(root_node, "set")
- prop_node = etree.SubElement(set_node, "prop")
- for option in options:
- opt_node = etree.SubElement(
- prop_node, option["name"], xmlns=option.get("namespace", "")
- )
- opt_node.text = option.get("value", "")
- tree = etree.ElementTree(root_node)
- return WebDavXmlUtils.etree_to_string(tree)
-
- @staticmethod
- def etree_to_string(tree):
- """Creates string from lxml.etree.ElementTree with XML declaration and
- UTF-8 encoding.
-
- :param tree: the instance of ElementTree
- :return: the string of XML.
- """
- buff = BytesIO()
- tree.write(buff, xml_declaration=True, encoding="UTF-8")
- return buff.getvalue()
-
- @staticmethod
- def extract_response_for_path(content, path, hostname):
- """Extracts single response for specified remote resource.
-
- :param content: raw content of response as string.
- :param path: the path to needed remote resource.
- :param hostname: the server hostname.
- :return: XML object of response for the remote resource defined by path.
- """
- try:
- tree = etree.fromstring(content)
- responses = tree.findall("{DAV:}response")
-
- n_path = Urn.normalize_path(path)
-
- for resp in responses:
- href = resp.findtext("{DAV:}href")
-
- if Urn.compare_path(n_path, href) is True:
- return resp
- raise RemoteResourceNotFound(path)
- except etree.XMLSyntaxError:
- raise MethodNotSupported(name="is_dir", server=hostname)
diff --git a/bob/devtools/webdav3/connection.py b/bob/devtools/webdav3/connection.py
deleted file mode 100644
index 1825cd486aa01fc77b303cf33f1b834e347b9f5a..0000000000000000000000000000000000000000
--- a/bob/devtools/webdav3/connection.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from os.path import exists
-
-from .exceptions import OptionNotValid
-from .urn import Urn
-
-
-class ConnectionSettings:
- def is_valid(self):
- pass
-
- def valid(self):
-
- try:
- self.is_valid()
- except OptionNotValid:
- return False
- else:
- return True
-
-
-class WebDAVSettings(ConnectionSettings):
- ns = "webdav:"
- prefix = "webdav_"
- keys = {
- "hostname",
- "login",
- "password",
- "token",
- "root",
- "cert_path",
- "key_path",
- "recv_speed",
- "send_speed",
- "verbose",
- }
-
- hostname = None
- login = None
- password = None
- token = None
- root = None
- cert_path = None
- key_path = None
- recv_speed = None
- send_speed = None
- verbose = None
-
- def __init__(self, options):
-
- self.options = dict()
-
- for key in self.keys:
- value = options.get(key, "")
- self.options[key] = value
- self.__dict__[key] = value
-
- self.root = Urn(self.root).quote() if self.root else ""
- self.root = self.root.rstrip(Urn.separate)
-
- def is_valid(self):
-
- if not self.hostname:
- raise OptionNotValid(
- name="hostname", value=self.hostname, ns=self.ns
- )
-
- if self.cert_path and not exists(self.cert_path):
- raise OptionNotValid(
- name="cert_path", value=self.cert_path, ns=self.ns
- )
-
- if self.key_path and not exists(self.key_path):
- raise OptionNotValid(
- name="key_path", value=self.key_path, ns=self.ns
- )
-
- if self.key_path and not self.cert_path:
- raise OptionNotValid(
- name="cert_path", value=self.cert_path, ns=self.ns
- )
-
- if self.password and not self.login:
- raise OptionNotValid(name="login", value=self.login, ns=self.ns)
-
- if not self.token and not self.login:
- raise OptionNotValid(name="login", value=self.login, ns=self.ns)
-
-
-class ProxySettings(ConnectionSettings):
- ns = "proxy:"
- prefix = "proxy_"
- keys = {"hostname", "login", "password"}
-
- hostname = None
- login = None
- password = None
-
- def __init__(self, options):
-
- self.options = dict()
-
- for key in self.keys:
- value = options.get(key, "")
- self.options[key] = value
- self.__dict__[key] = value
-
- def is_valid(self):
-
- if self.password and not self.login:
- raise OptionNotValid(name="login", value=self.login, ns=self.ns)
-
- if self.login or self.password:
- if not self.hostname:
- raise OptionNotValid(
- name="hostname", value=self.hostname, ns=self.ns
- )
diff --git a/bob/devtools/webdav3/exceptions.py b/bob/devtools/webdav3/exceptions.py
deleted file mode 100644
index 4d546bbec053ae6fd09a5e79f962b6562a23a239..0000000000000000000000000000000000000000
--- a/bob/devtools/webdav3/exceptions.py
+++ /dev/null
@@ -1,118 +0,0 @@
-class WebDavException(Exception):
- pass
-
-
-class NotValid(WebDavException):
- pass
-
-
-class OptionNotValid(NotValid):
- def __init__(self, name, value, ns=""):
- self.name = name
- self.value = value
- self.ns = ns
-
- def __str__(self):
- return "Option ({ns}{name}={value}) have invalid name or value".format(
- ns=self.ns, name=self.name, value=self.value
- )
-
-
-class CertificateNotValid(NotValid):
- pass
-
-
-class NotFound(WebDavException):
- pass
-
-
-class LocalResourceNotFound(NotFound):
- def __init__(self, path):
- self.path = path
-
- def __str__(self):
- return "Local file: {path} not found".format(path=self.path)
-
-
-class RemoteResourceNotFound(NotFound):
- def __init__(self, path):
- self.path = path
-
- def __str__(self):
- return "Remote resource: {path} not found".format(path=self.path)
-
-
-class RemoteParentNotFound(NotFound):
- def __init__(self, path):
- self.path = path
-
- def __str__(self):
- return "Remote parent for: {path} not found".format(path=self.path)
-
-
-class ResourceTooBig(WebDavException):
- def __init__(self, path, size, max_size):
- self.path = path
- self.size = size
- self.max_size = max_size
-
- def __str__(self):
- return "Resource {path} is too big, it should be less then {max_size} but actually: {size}".format(
- path=self.path, max_size=self.max_size, size=self.size
- )
-
-
-class MethodNotSupported(WebDavException):
- def __init__(self, name, server):
- self.name = name
- self.server = server
-
- def __str__(self):
- return "Method {name} not supported for {server}".format(
- name=self.name, server=self.server
- )
-
-
-class ConnectionException(WebDavException):
- def __init__(self, exception):
- self.exception = exception
-
- def __str__(self):
- return self.exception.__str__()
-
-
-class NoConnection(WebDavException):
- def __init__(self, hostname):
- self.hostname = hostname
-
- def __str__(self):
- return "Not connection with {hostname}".format(hostname=self.hostname)
-
-
-# This exception left only for supporting original library interface.
-class NotConnection(WebDavException):
- def __init__(self, hostname):
- self.hostname = hostname
-
- def __str__(self):
- return "No connection with {hostname}".format(hostname=self.hostname)
-
-
-class ResponseErrorCode(WebDavException):
- def __init__(self, url, code, message):
- self.url = url
- self.code = code
- self.message = message
-
- def __str__(self):
- return "Request to {url} failed with code {code} and message: {message}".format(
- url=self.url, code=self.code, message=self.message
- )
-
-
-class NotEnoughSpace(WebDavException):
- def __init__(self):
- pass
-
- def __str__(self):
- return "Not enough space on the server"
diff --git a/bob/devtools/webdav3/urn.py b/bob/devtools/webdav3/urn.py
deleted file mode 100644
index 0bdf2709b206ea8147b1b0f8e23f0fbee338ee4f..0000000000000000000000000000000000000000
--- a/bob/devtools/webdav3/urn.py
+++ /dev/null
@@ -1,83 +0,0 @@
-try:
- from urllib.parse import quote, unquote, urlsplit
-except ImportError:
- from urllib import unquote, quote
- from urlparse import urlsplit
-
-from re import sub
-
-
-class Urn(object):
-
- separate = "/"
-
- def __init__(self, path, directory=False):
-
- self._path = quote(path)
- expressions = r"/\.+/", "/+"
- for expression in expressions:
- self._path = sub(expression, Urn.separate, self._path)
-
- if not self._path.startswith(Urn.separate):
- self._path = "{begin}{end}".format(
- begin=Urn.separate, end=self._path
- )
-
- if directory and not self._path.endswith(Urn.separate):
- self._path = "{begin}{end}".format(
- begin=self._path, end=Urn.separate
- )
-
- def __str__(self):
- return self.path()
-
- def path(self):
- return unquote(self._path)
-
- def quote(self):
- return self._path
-
- def filename(self):
-
- path_split = self._path.split(Urn.separate)
- name = (
- path_split[-2] + Urn.separate
- if path_split[-1] == ""
- else path_split[-1]
- )
- return unquote(name)
-
- def parent(self):
-
- path_split = self._path.split(Urn.separate)
- nesting_level = self.nesting_level()
- parent_path_split = path_split[:nesting_level]
- parent = (
- self.separate.join(parent_path_split)
- if nesting_level != 1
- else Urn.separate
- )
- if not parent.endswith(Urn.separate):
- return unquote(parent + Urn.separate)
- else:
- return unquote(parent)
-
- def nesting_level(self):
- return self._path.count(Urn.separate, 0, -1)
-
- def is_dir(self):
- return self._path[-1] == Urn.separate
-
- @staticmethod
- def normalize_path(path):
- result = sub("/{2,}", "/", path)
- return (
- result
- if len(result) < 1 or result[-1] != Urn.separate
- else result[:-1]
- )
-
- @staticmethod
- def compare_path(path_a, href):
- unqouted_path = Urn.separate + unquote(urlsplit(href).path)
- return Urn.normalize_path(path_a) == Urn.normalize_path(unqouted_path)
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 15676a51c02131a8cee35ea5d364a425a1ca4337..176a756af28de080eb27b5ba8fcaec7cbf4b0155 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -53,6 +53,7 @@ requirements:
- tabulate
- python-graphviz
- pip
+ - webdavclient3
test:
requires:
diff --git a/doc/api.rst b/doc/api.rst
index 3132a59ac2f33da4ac8032eea12f4d23eb98aef4..bd9a35f9e3db89f2e021401edf7b94a7cd41ec0d 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -16,7 +16,6 @@
bob.devtools.mirror
bob.devtools.deploy
bob.devtools.graph
- bob.devtools.webdav3.client
Detailed Information
@@ -41,9 +40,3 @@ Detailed Information
.. automodule:: bob.devtools.deploy
.. automodule:: bob.devtools.graph
-
-
-WebDAV Python Client
---------------------
-
-.. automodule:: bob.devtools.webdav3.client
diff --git a/setup.py b/setup.py
index a314d48b7036cecbbcea7d52d706e8bc5c3f8b3d..a05b9e19a18d145615d8b4150dc57e1dad5bcc49 100644
--- a/setup.py
+++ b/setup.py
@@ -29,6 +29,7 @@ requires = [
"termcolor",
"psutil",
"pytz",
+ "webdavclient3",
]
setup(