Merge branch 'python-deploy' into 'master'

Python-only pipelines

Closes #84

See merge request !262

LICENSE

@@ -25,36 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-------------------------------------------------------------------------------
-
-Code from the "webdav3" directory was copied from the Github repository
-https://github.com/ezhov-evgeny/webdav-client-python-3, but later modified and
-repackaged as part of this package.
-
-The authors asked to reproduce the following license text.
-
-
-COPYRIGHT AND PERMISSION NOTICE
--------------------------------
-
-Copyright (c) 2016, The WDC Project, and many contributors, see the THANKS
-file.
-
-All rights reserved.
-
-Permission to use, copy, modify, and distribute this software for any purpose
-with or without fee is hereby granted, provided that the above copyright notice
-and this permission notice appear in all copies.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
-NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
-
-Except as contained in this notice, the name of a copyright holder shall not be
-used in advertising or otherwise to promote the sale, use or other dealings in
-this Software without prior written authorization of the copyright holder.

bob/devtools/constants.py

@@ -7,7 +7,7 @@ import os
 
 import pkg_resources
 
-from . import bootstrap
+from . import bootstrap, deploy
 from .log import get_logger
 
 logger = get_logger(__name__)
@@ -30,38 +30,12 @@ CONDA_RECIPE_APPEND = pkg_resources.resource_filename(
 
 
 SERVER = bootstrap._SERVER
+assert SERVER == deploy._SERVER, "Also change deploy._SERVER to match this!"
 """This is the default server use use to store data and build artifacts"""
 
-
-WEBDAV_PATHS = {
-    True: {  # stable?
-        False: {  # visible?
-            "root": "/private-upload",
-            "conda": "/conda",
-            "docs": "/docs",
-        },
-        True: {  # visible?
-            "root": "/public-upload",
-            "conda": "/conda",
-            "docs": "/docs",
-        },
-    },
-    False: {  # stable?
-        False: {  # visible?
-            "root": "/private-upload",
-            "conda": "/conda/label/beta",
-            "docs": "/docs",
-        },
-        True: {  # visible?
-            "root": "/public-upload",
-            "conda": "/conda/label/beta",
-            "docs": "/docs",
-        },
-    },
-}
+WEBDAV_PATHS = deploy._WEBDAV_PATHS
 """Default locations of our webdav upload paths"""
 
-
 IDIAP_ROOT_CA = b"""
 Idiap Root CA 2016 - for internal use
 =====================================

bob/devtools/data/gitlab-ci/python-package.yaml

+# This YAML file contains descriptions for the CI of python-only packages
+# - do **not** modify it unless you know what you're doing (and up to!)
+
+# Definition of global variables (all stages)
+variables:
+  PYTHONUNBUFFERED: "1"
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  PRE_COMMIT_HOME: "${CI_PROJECT_DIR}/.cache/pre-commit"
+  DEPLOY: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/deploy.py"
+
+
+# Definition of our build pipeline order
+stages:
+  - build
+  - test
+  - deploy
+  - pypi
+
+
+# Build targets
+build:
+  image: python:latest
+  tags:
+    - docker
+  stage: build
+  before_script:
+    - pip install twine pre-commit sphinx sphinx-rtd-theme
+  script:
+    - "[ -r .pre-commit-config.yaml ] && pre-commit run --all-files --show-diff-on-failure --verbose"
+    - python setup.py sdist --formats=zip
+    - twine check dist/*.zip
+    - pip install -e .
+    - "[ -e doc ] && sphinx-build doc html"
+  artifacts:
+    paths:
+      - dist/*.zip
+      - html
+    expire_in: 1 week
+  cache:
+    key: "build-py"
+    paths:
+      - ${PRE_COMMIT_HOME}
+      - ${PIP_CACHE_DIR}
+
+
+# Test targets
+.test_template:
+  tags:
+    - docker
+  stage: test
+  dependencies:
+    - build
+  before_script:
+    - pip install tox
+  cache:
+    key: "test-py"
+    paths:
+      - ${PIP_CACHE_DIR}
+
+
+test_py38:
+  extends: .test_template
+  image: python:3.8
+  script:
+    - tox -e py38
+
+test_py39:
+  extends: .test_template
+  image: python:3.9
+  script:
+    - tox -e py39
+
+test_py310:
+  extends: .test_template
+  image: python:3.10
+  script:
+    - tox -e py310
+
+
+.deploy_template:
+  image: python:latest
+  tags:
+    - docker
+  stage: deploy
+  dependencies:
+    - test_py38
+    - test_py39
+    - test_py310
+    - build
+  before_script:
+    - pip install webdavclient3
+    - curl --silent "${DEPLOY}" --output "deploydocs.py"
+  script:
+    - python ./deploydocs.py -v html
+
+deploy_beta:
+  extends: .deploy_template
+  environment: beta
+  only:
+    - master
+
+deploy_stable:
+  extends: .deploy_template
+  environment: stable
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+
+pypi:
+  image: python:latest
+  tags:
+    - docker
+  stage: pypi
+  environment: pypi
+  only:
+    refs:
+      - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+    variables:
+      - $CI_PROJECT_VISIBILITY == "public"
+  except:
+    - branches
+  dependencies:
+    - test_py38
+    - test_py39
+    - test_py310
+    - build
+  before_script:
+    - pip install twine
+  script:
+    - twine --skip-existing --username=${PYPIUSER} --password=${PYPIPASS} dist/*.zip
+  cache:
+    paths:
+      - ${PIP_CACHE_DIR}

bob/devtools/deploy.py

@@ -4,12 +4,41 @@
 """Deployment utilities for conda packages and documentation via webDAV."""
 
 
+import logging
 import os
 
-from .constants import SERVER, WEBDAV_PATHS
-from .log import get_logger
+logger = logging.getLogger(__name__)
 
-logger = get_logger(__name__)
+# This must be a copy of what is in bootstrap.py.
+# Notice this script is also called independently of bob.devtools!
+_SERVER = "http://www.idiap.ch"
+
+_WEBDAV_PATHS = {
+    True: {  # stable?
+        False: {  # visible?
+            "root": "/private-upload",
+            "conda": "/conda",
+            "docs": "/docs",
+        },
+        True: {  # visible?
+            "root": "/public-upload",
+            "conda": "/conda",
+            "docs": "/docs",
+        },
+    },
+    False: {  # stable?
+        False: {  # visible?
+            "root": "/private-upload",
+            "conda": "/conda/label/beta",
+            "docs": "/docs",
+        },
+        True: {  # visible?
+            "root": "/public-upload",
+            "conda": "/conda/label/beta",
+            "docs": "/docs",
+        },
+    },
+}
 
 
 def _setup_webdav_client(server, root, username, password):
@@ -23,7 +52,7 @@ def _setup_webdav_client(server, root, username, password):
         webdav_password=password,
     )
 
-    from .webdav3 import client as webdav
+    from webdav3 import client as webdav
 
     retval = webdav.Client(webdav_options)
     assert retval.valid()
@@ -56,9 +85,9 @@ def deploy_conda_package(
         messages.
     """
 
-    server_info = WEBDAV_PATHS[stable][public]
+    server_info = _WEBDAV_PATHS[stable][public]
     davclient = _setup_webdav_client(
-        SERVER, server_info["root"], username, password
+        _SERVER, server_info["root"], username, password
     )
 
     basename = os.path.basename(package)
@@ -71,18 +100,18 @@ def deploy_conda_package(
                 "The file %s/%s already exists on the server "
                 "- this can be due to more than one build with deployment "
                 "running at the same time.  Re-running the broken builds "
-                "normally fixes it" % (SERVER, remote_path)
+                "normally fixes it" % (_SERVER, remote_path)
             )
 
         else:
             logger.info(
-                "[dav] rm -f %s%s%s", SERVER, server_info["root"], remote_path
+                "[dav] rm -f %s%s%s", _SERVER, server_info["root"], remote_path
             )
             if not dry_run:
                 davclient.clean(remote_path)
 
     logger.info(
-        "[dav] %s -> %s%s%s", package, SERVER, server_info["root"], remote_path
+        "[dav] %s -> %s%s%s", package, _SERVER, server_info["root"], remote_path
     )
     if not dry_run:
         davclient.upload(local_path=package, remote_path=remote_path)
@@ -132,9 +161,9 @@ def deploy_documentation(
             "ensure documentation is being produced for your project!" % path
         )
 
-    server_info = WEBDAV_PATHS[stable][public]
+    server_info = _WEBDAV_PATHS[stable][public]
     davclient = _setup_webdav_client(
-        SERVER, server_info["root"], username, password
+        _SERVER, server_info["root"], username, password
     )
 
     remote_path_prefix = "%s/%s" % (server_info["docs"], package)
@@ -161,7 +190,85 @@ def deploy_documentation(
                 davclient.mkdir(remote_path_prefix)
         remote_path = "%s/%s" % (remote_path_prefix, k)
         logger.info(
-            "[dav] %s -> %s%s%s", path, SERVER, server_info["root"], remote_path
+            "[dav] %s -> %s%s%s",
+            path,
+            _SERVER,
+            server_info["root"],
+            remote_path,
         )
         if not dry_run:
             davclient.upload_directory(local_path=path, remote_path=remote_path)
+
+
+if __name__ == "__main__":
+
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Deploys documentation from python-only packages"
+    )
+    parser.add_argument(
+        "directory",
+        help="Directory containing the sphinx build to deploy",
+    )
+    parser.add_argument(
+        "-p",
+        "--package",
+        default=os.environ.get("CI_PROJECT_PATH", None),
+        help="The package being built [default: %(default)s]",
+    )
+    parser.add_argument(
+        "-x",
+        "--visibility",
+        default=os.environ.get("CI_PROJECT_VISIBILITY", "private"),
+        help="The visibility of the package being built [default: %(default)s]",
+    )
+    parser.add_argument(
+        "-b",
+        "--branch",
+        default=os.environ.get("CI_COMMIT_REF_NAME", None),
+        help="Name of the branch being built [default: %(default)s]",
+    )
+    parser.add_argument(
+        "-t",
+        "--tag",
+        default=os.environ.get("CI_COMMIT_TAG", None),
+        help="If building a tag, pass it with this flag [default: %(default)s]",
+    )
+    parser.add_argument(
+        "-u",
+        "--username",
+        default=os.environ.get("DOCUSER", None),
+        help="Username for webdav deployment [default: %(default)s]",
+    )
+    parser.add_argument(
+        "-P",
+        "--password",
+        default=os.environ.get("DOCPASS", None),
+        help="Password for webdav deployment [default: %(default)s]",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Be verbose (enables INFO logging)",
+        action="store_const",
+        dest="loglevel",
+        default=logging.WARNING,
+        const=logging.INFO,
+    )
+
+    args = parser.parse_args()
+    logging.basicConfig(level=args.loglevel)
+
+    deploy_documentation(
+        args.directory,
+        package=args.package,
+        stable=(args.tag is not None),
+        latest=True,
+        public=(args.visibility == "public"),
+        branch=args.branch,
+        tag=args.tag,
+        username=args.username,
+        password=args.password,
+        dry_run=False,
+    )

bob/devtools/webdav3/connection.py

-from os.path import exists
-
-from .exceptions import OptionNotValid
-from .urn import Urn
-
-
-class ConnectionSettings:
-    def is_valid(self):
-        pass
-
-    def valid(self):
-
-        try:
-            self.is_valid()
-        except OptionNotValid:
-            return False
-        else:
-            return True
-
-
-class WebDAVSettings(ConnectionSettings):
-    ns = "webdav:"
-    prefix = "webdav_"
-    keys = {
-        "hostname",
-        "login",
-        "password",
-        "token",
-        "root",
-        "cert_path",
-        "key_path",
-        "recv_speed",
-        "send_speed",
-        "verbose",
-    }
-
-    hostname = None
-    login = None
-    password = None
-    token = None
-    root = None
-    cert_path = None
-    key_path = None
-    recv_speed = None
-    send_speed = None
-    verbose = None
-
-    def __init__(self, options):
-
-        self.options = dict()
-
-        for key in self.keys:
-            value = options.get(key, "")
-            self.options[key] = value
-            self.__dict__[key] = value
-
-        self.root = Urn(self.root).quote() if self.root else ""
-        self.root = self.root.rstrip(Urn.separate)
-
-    def is_valid(self):
-
-        if not self.hostname:
-            raise OptionNotValid(
-                name="hostname", value=self.hostname, ns=self.ns
-            )
-
-        if self.cert_path and not exists(self.cert_path):
-            raise OptionNotValid(
-                name="cert_path", value=self.cert_path, ns=self.ns
-            )
-
-        if self.key_path and not exists(self.key_path):
-            raise OptionNotValid(
-                name="key_path", value=self.key_path, ns=self.ns
-            )
-
-        if self.key_path and not self.cert_path:
-            raise OptionNotValid(
-                name="cert_path", value=self.cert_path, ns=self.ns
-            )
-
-        if self.password and not self.login:
-            raise OptionNotValid(name="login", value=self.login, ns=self.ns)
-
-        if not self.token and not self.login:
-            raise OptionNotValid(name="login", value=self.login, ns=self.ns)
-
-
-class ProxySettings(ConnectionSettings):
-    ns = "proxy:"
-    prefix = "proxy_"
-    keys = {"hostname", "login", "password"}
-
-    hostname = None
-    login = None
-    password = None
-
-    def __init__(self, options):
-
-        self.options = dict()
-
-        for key in self.keys:
-            value = options.get(key, "")
-            self.options[key] = value
-            self.__dict__[key] = value
-
-    def is_valid(self):
-
-        if self.password and not self.login:
-            raise OptionNotValid(name="login", value=self.login, ns=self.ns)
-
-        if self.login or self.password:
-            if not self.hostname:
-                raise OptionNotValid(
-                    name="hostname", value=self.hostname, ns=self.ns
-                )

bob/devtools/webdav3/exceptions.py

-class WebDavException(Exception):
-    pass
-
-
-class NotValid(WebDavException):
-    pass
-
-
-class OptionNotValid(NotValid):
-    def __init__(self, name, value, ns=""):
-        self.name = name
-        self.value = value
-        self.ns = ns
-
-    def __str__(self):
-        return "Option ({ns}{name}={value}) have invalid name or value".format(
-            ns=self.ns, name=self.name, value=self.value
-        )
-
-
-class CertificateNotValid(NotValid):
-    pass
-
-
-class NotFound(WebDavException):
-    pass
-
-
-class LocalResourceNotFound(NotFound):
-    def __init__(self, path):
-        self.path = path
-
-    def __str__(self):
-        return "Local file: {path} not found".format(path=self.path)
-
-
-class RemoteResourceNotFound(NotFound):
-    def __init__(self, path):
-        self.path = path
-
-    def __str__(self):
-        return "Remote resource: {path} not found".format(path=self.path)
-
-
-class RemoteParentNotFound(NotFound):
-    def __init__(self, path):
-        self.path = path
-
-    def __str__(self):
-        return "Remote parent for: {path} not found".format(path=self.path)
-
-
-class ResourceTooBig(WebDavException):
-    def __init__(self, path, size, max_size):
-        self.path = path
-        self.size = size
-        self.max_size = max_size
-
-    def __str__(self):
-        return "Resource {path} is too big, it should be less then {max_size} but actually: {size}".format(
-            path=self.path, max_size=self.max_size, size=self.size
-        )
-
-
-class MethodNotSupported(WebDavException):
-    def __init__(self, name, server):
-        self.name = name
-        self.server = server
-
-    def __str__(self):
-        return "Method {name} not supported for {server}".format(
-            name=self.name, server=self.server
-        )
-
-
-class ConnectionException(WebDavException):
-    def __init__(self, exception):
-        self.exception = exception
-
-    def __str__(self):
-        return self.exception.__str__()
-
-
-class NoConnection(WebDavException):
-    def __init__(self, hostname):
-        self.hostname = hostname
-
-    def __str__(self):
-        return "Not connection with {hostname}".format(hostname=self.hostname)
-
-
-# This exception left only for supporting original library interface.
-class NotConnection(WebDavException):
-    def __init__(self, hostname):
-        self.hostname = hostname
-
-    def __str__(self):
-        return "No connection with {hostname}".format(hostname=self.hostname)
-
-
-class ResponseErrorCode(WebDavException):
-    def __init__(self, url, code, message):
-        self.url = url
-        self.code = code
-        self.message = message
-
-    def __str__(self):
-        return "Request to {url} failed with code {code} and message: {message}".format(
-            url=self.url, code=self.code, message=self.message
-        )
-
-
-class NotEnoughSpace(WebDavException):
-    def __init__(self):
-        pass
-
-    def __str__(self):
-        return "Not enough space on the server"

bob/devtools/webdav3/urn.py

-try:
-    from urllib.parse import quote, unquote, urlsplit
-except ImportError:
-    from urllib import unquote, quote
-    from urlparse import urlsplit
-
-from re import sub
-
-
-class Urn(object):
-
-    separate = "/"
-
-    def __init__(self, path, directory=False):
-
-        self._path = quote(path)
-        expressions = r"/\.+/", "/+"
-        for expression in expressions:
-            self._path = sub(expression, Urn.separate, self._path)
-
-        if not self._path.startswith(Urn.separate):
-            self._path = "{begin}{end}".format(
-                begin=Urn.separate, end=self._path
-            )
-
-        if directory and not self._path.endswith(Urn.separate):
-            self._path = "{begin}{end}".format(
-                begin=self._path, end=Urn.separate
-            )
-
-    def __str__(self):
-        return self.path()
-
-    def path(self):
-        return unquote(self._path)
-
-    def quote(self):
-        return self._path
-
-    def filename(self):
-
-        path_split = self._path.split(Urn.separate)
-        name = (
-            path_split[-2] + Urn.separate
-            if path_split[-1] == ""
-            else path_split[-1]
-        )
-        return unquote(name)
-
-    def parent(self):
-
-        path_split = self._path.split(Urn.separate)
-        nesting_level = self.nesting_level()
-        parent_path_split = path_split[:nesting_level]
-        parent = (
-            self.separate.join(parent_path_split)
-            if nesting_level != 1
-            else Urn.separate
-        )
-        if not parent.endswith(Urn.separate):
-            return unquote(parent + Urn.separate)
-        else:
-            return unquote(parent)
-
-    def nesting_level(self):
-        return self._path.count(Urn.separate, 0, -1)
-
-    def is_dir(self):
-        return self._path[-1] == Urn.separate
-
-    @staticmethod
-    def normalize_path(path):
-        result = sub("/{2,}", "/", path)
-        return (
-            result
-            if len(result) < 1 or result[-1] != Urn.separate
-            else result[:-1]
-        )
-
-    @staticmethod
-    def compare_path(path_a, href):
-        unqouted_path = Urn.separate + unquote(urlsplit(href).path)
-        return Urn.normalize_path(path_a) == Urn.normalize_path(unqouted_path)

conda/meta.yaml

@@ -53,6 +53,7 @@ requirements:
     - tabulate
     - python-graphviz
     - pip
+    - webdavclient3
 
 test:
   requires:

doc/api.rst

@@ -16,7 +16,6 @@
    bob.devtools.mirror
    bob.devtools.deploy
    bob.devtools.graph
-   bob.devtools.webdav3.client
 
 
 Detailed Information
@@ -41,9 +40,3 @@ Detailed Information
 .. automodule:: bob.devtools.deploy
 
 .. automodule:: bob.devtools.graph
-
-
-WebDAV Python Client
---------------------
-
-.. automodule:: bob.devtools.webdav3.client

setup.py

@@ -29,6 +29,7 @@ requires = [
     "termcolor",
     "psutil",
     "pytz",
+    "webdavclient3",
 ]
 
 setup(