Merge branch 'support-base-py37' into 'master'

Independent builds and python support

Closes #13

See merge request !11

.gitlab-ci.yml

@@ -56,6 +56,12 @@ build_linux_36:
   <<: *linux_build_job
   variables:
     PYTHON_VERSION: "3.6"
+
+
+build_linux_37:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "3.7"
     BUILD_EGG: "true"
   script:
     - python3 ./bob/devtools/bootstrap.py -vv build
@@ -76,18 +82,27 @@ build_macosx_36:
     PYTHON_VERSION: "3.6"
 
 
+build_macosx_37:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "3.7"
+
+
 # Deploy targets
 .deploy_template: &deploy_job
   stage: deploy
   script:
-    - python3 ./bob/devtools/bootstrap.py -vv local bdt
+    - python3 ./bob/devtools/bootstrap.py -vv local base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
+    - bdt ci base-deploy -vv
     - bdt ci deploy -vv
     - bdt ci clean -vv
   dependencies:
     - build_linux_36
+    - build_linux_37
     - build_macosx_36
+    - build_macosx_37
   tags:
     - docker
   cache: &build_caches
@@ -121,14 +136,16 @@ pypi:
   except:
     - branches
   script:
-    - python3 ./bob/devtools/bootstrap.py -vv local bdt
+    - python3 ./bob/devtools/bootstrap.py -vv local base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
     - bdt ci pypi -vv dist/*.zip
     - bdt ci clean -vv
   dependencies:
     - build_linux_36
+    - build_linux_37
     - build_macosx_36
+    - build_macosx_37
   tags:
     - docker
   cache: &build_caches

MANIFEST.in

 include LICENSE README.rst buildout.cfg version.txt
 recursive-include doc conf.py *.rst *.sh *.png *.ico
 recursive-include bob/devtools/data *.md *.yaml *.pem matplotlibrc
-recursive-include bob/devtools/templates conf.py *.rst *.png *.ico LICENSE COPYING .gitlab-ci.yml .gitignore *.cfg *.txt *.py
+recursive-include bob/devtools/templates conf.py *.rst *.png *.ico LICENSE COPYING MANIFEST.in .gitlab-ci.yml .gitignore *.cfg *.txt *.py

bob/devtools/bootstrap.py

@@ -383,7 +383,7 @@ if __name__ == '__main__':
     f.write(_BASE_CONDARC)
 
   conda_version = '4'
-  conda_build_version = '3'
+  conda_build_version = '3.16'
 
   conda_verbosity = []
   #if args.verbose >= 2:

bob/devtools/build.py

@@ -154,6 +154,61 @@ def get_parsed_recipe(metadata):
   return yaml.load(output)
 
 
+def exists_on_channel(channel_url, name, version, build_number,
+    python_version):
+  """Checks on the given channel if a package with the specs exist
+
+  Args:
+
+    channel_url: The URL where to look for packages clashes (normally a beta
+      channel)
+    name: The name of the package
+    version: The version of the package
+    build_number: The build number of the package
+    python_version: The current version of python we're building for
+
+  Returns: A complete package name, version and build string, if the package
+  already exists in the channel or ``None`` otherwise.
+
+  """
+
+  from conda.exports import get_index
+
+  # no dot in py_ver
+  py_ver = python_version.replace('.', '')
+
+  # get the channel index
+  logger.debug('Downloading channel index from %s', channel_url)
+  index = get_index(channel_urls=[channel_url], prepend=False)
+
+  logger.info('Checking for %s-%s-%s...', name, version, build_number)
+
+  for dist in index:
+
+    if dist.name == name and dist.version == version and \
+        dist.build_string.endswith('_%s' % build_number):
+
+      # two possible options must be checked - (i) the package build_string
+      # starts with ``py``, which means it is a python specific package so we
+      # must also check for the matching python version.  (ii) the package is
+      # not a python-specific package and a simple match will do
+      if dist.build_string.startswith('py'):
+        match = re.match('py[2-9][0-9]+', dist.build_string)
+        if match and match.group() == 'py{}'.format(py_ver):
+          logger.debug('Found matching package (%s-%s-%s)', dist.name,
+              dist.version, dist.build_string)
+          return (dist.name, dist.version, dist.build_string)
+
+      else:
+        logger.debug('Found matching package (%s-%s-%s)', dist.name,
+            dist.version, dist.build_string)
+        return (dist.name, dist.version, dist.build_string)
+
+  logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
+      name, version, py_ver, build_number, len(index))
+  return
+
+
 def remove_pins(deps):
   return [l.split()[0] for l in deps]
 
@@ -407,6 +462,69 @@ def git_clean_build(runner, verbose):
       ['--exclude=%s' % k for k in exclude_from_cleanup])
 
 
+def base_build(server, intranet, recipe_dir, conda_build_config,
+    python_version, condarc_options):
+  '''Builds a non-beat/bob software dependence that does not exist on defaults
+
+  This function will build a software dependence that is required for our
+  software stack, but does not (yet) exist on the defaults channels.  It first
+  check if the build should run for the current architecture, checks if the
+  package is not already built on our public channel and, if that is true, then
+  proceeds with the build of the dependence.
+
+
+  Args:
+
+    server: The base address of the server containing our conda channels
+    intranet: Boolean indicating if we should add "private"/"public" prefixes
+      on the returned paths
+    recipe_dir: The directory containing the recipe's ``meta.yaml`` file
+    conda_build_config: Path to the ``conda_build_config.yaml`` file to use
+    python_version: String with the python version to build for, in the format
+      ``x.y`` (should be passed even if not building a python package)
+    condarc_options: Pre-parsed condarc options loaded from the respective YAML
+      file
+
+  '''
+
+  # if you get to this point, tries to build the package
+  public_channels = bootstrap.get_channels(public=True, stable=True,
+    server=server, intranet=intranet)
+
+  logger.info('Using the following channels during (potential) build:\n  - %s',
+      '\n  - '.join(public_channels + ['defaults']))
+  condarc_options['channels'] = public_channels + ['defaults']
+
+  logger.info('Merging conda configuration files...')
+  conda_config = make_conda_config(conda_build_config, python_version,
+      None, condarc_options)
+
+  metadata = get_rendered_metadata(recipe_dir, conda_config)
+  recipe = get_parsed_recipe(metadata)
+
+  if recipe is None:
+    logger.info('Skipping build for %s - rendering returned None', recipe_dir)
+    return
+
+  # no dot in py_ver
+  py_ver = python_version.replace('.', '')
+  arch = conda_arch()
+
+  candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
+      recipe['package']['version'], recipe['build']['number'],
+      python_version)
+  if candidate is not None:
+    logger.info('Skipping build for %s-%s-(py%s_?)%s for %s - exists ' \
+        'on channel', candidate[0], candidate[1], candidate[2], py_ver, arch)
+    return
+
+  # if you get to this point, just builds the package
+    logger.info('Building %s-%s-(py%s_?)%s for %s',
+      recipe['package']['name'], recipe['package']['version'],
+      recipe['build']['number'], py_ver, arch)
+  conda_build.api.build(recipe_dir, config=conda_config)
+
+
 if __name__ == '__main__':
 
   import argparse
@@ -476,8 +594,22 @@ if __name__ == '__main__':
   with open(condarc, 'rb') as f:
     condarc_options = yaml.load(f)
 
+  # dump packages at conda_root
+  condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
+
+  # builds all dependencies in the 'deps' subdirectory - or at least checks
+  # these dependencies are already available; these dependencies go directly to
+  # the public channel once built
+  for recipe in glob.glob(os.path.join('deps', '*')):
+    if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
+      # ignore - not a conda package
+      continue
+    base_build(server, not args.internet, recipe, conda_build_config,
+        args.python_version, condarc_options)
+
   # notice this condarc typically will only contain the defaults channel - we
-  # need to boost this up with more channels to get it right.
+  # need to boost this up with more channels to get it right for this package's
+  # build
   public = ( args.visibility == 'public' )
   channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
       server=server, intranet=(not args.internet))
@@ -485,18 +617,15 @@ if __name__ == '__main__':
       '\n  - '.join(channels + ['defaults']))
   condarc_options['channels'] = channels + ['defaults']
 
-  # dump packages at conda_root
-  condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
-
-  logger.info('Merging conda configuration files...')
-  conda_config = make_conda_config(conda_build_config, args.python_version,
-      recipe_append, condarc_options)
-
   # retrieve the current build number for this build
   build_number, _ = next_build_number(channels[0], args.name, version,
       args.python_version)
   bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
 
+  logger.info('Merging conda configuration files...')
+  conda_config = make_conda_config(conda_build_config, args.python_version,
+      recipe_append, condarc_options)
+
   # runs the build using the conda-build API
   arch = conda_arch()
   logger.info('Building %s-%s-py%s (build: %d) for %s',
@@ -517,4 +646,4 @@ if __name__ == '__main__':
     else:
       logger.info('twine check (a.k.a. readme check) %s: OK', package[0])
 
-  git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2))
+  git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))

bob/devtools/data/conda_build_config.yaml

@@ -32,6 +32,7 @@ pin_run_as_build:
 ## the dependencies that we build against multiple versions
 python:
   - 3.6
+  - 3.7
 
 zip_keys:
   -                             # [win]

bob/devtools/data/gitlab-ci/single-package.yaml

@@ -20,9 +20,9 @@ stages:
   stage: build
   script:
     - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-    - python3 bootstrap.py -vv channel bdt
+    - python3 bootstrap.py -vv channel base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
     - bdt ci build -vv
     - bdt ci clean -vv
   cache: &build_caches
@@ -66,9 +66,9 @@ build_linux_36:
     BUILD_EGG: "true"
   script:
     - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-    - python3 bootstrap.py -vv channel bdt
+    - python3 bootstrap.py -vv channel base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
     - bdt ci build -vv
     - bdt ci readme -vv dist/*.zip
     - bdt ci clean -vv
@@ -91,9 +91,9 @@ build_macosx_36:
   stage: deploy
   script:
     - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-    - python3 bootstrap.py channel bdt
+    - python3 bootstrap.py channel base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
     - bdt ci deploy -vv
     - bdt ci clean -vv
   dependencies:
@@ -133,9 +133,9 @@ pypi:
     - branches
   script:
     - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-    - python3 bootstrap.py -vv channel bdt
+    - python3 bootstrap.py -vv channel base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
     - bdt ci pypi -vv dist/*.zip
     - bdt ci clean -vv
   dependencies:

bob/devtools/scripts/ci.py

@@ -30,6 +30,76 @@ def ci():
   pass
 
 
+@ci.command(epilog='''
+Examples:
+
+  1. Deploys base build artifacts (dependencies) to the appropriate channels:
+
+     $ bdt ci base-deploy -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def base_deploy(dry_run):
+    """Deploys dependencies not available at the defaults channel
+
+    Deployment happens to our public channel directly, as these are
+    dependencies are required for proper bob/beat package runtime environments.
+    """
+
+    if dry_run:
+        logger.warn('!!!! DRY RUN MODE !!!!')
+        logger.warn('Nothing is being deployed to server')
+
+    package = os.environ['CI_PROJECT_PATH']
+
+    from ..constants import WEBDAV_PATHS
+    server_info = WEBDAV_PATHS[True][True]  #stable=True, visible=True
+
+    logger.info('Deploying dependence packages to %s%s%s...', SERVER,
+        server_info['root'], server_info['conda'])
+
+    # setup webdav connection
+    webdav_options = {
+        'webdav_hostname': SERVER,
+        'webdav_root': server_info['root'],
+        'webdav_login': os.environ['DOCUSER'],
+        'webdav_password': os.environ['DOCPASS'],
+        }
+    from ..webdav3 import client as webdav
+    davclient = webdav.Client(webdav_options)
+    assert davclient.valid()
+
+    group, name = package.split('/')
+
+    # uploads conda package artificats
+    for arch in ('linux-64', 'osx-64', 'noarch'):
+      # finds conda dependencies and uploads what we can find
+      package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
+          '*.tar.bz2')
+      deploy_packages = glob.glob(package_path)
+      for k in deploy_packages:
+        basename = os.path.basename(k)
+        if basename.startswith(name):
+          logger.debug('Skipping deploying of %s - not a base package', k)
+          continue
+
+        remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
+        if davclient.check(remote_path):
+          raise RuntimeError('The file %s/%s already exists on the server ' \
+              '- this can be due to more than one build with deployment ' \
+              'running at the same time.  Re-running the broken builds ' \
+              'normally fixes it' % (SERVER, remote_path))
+        logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
+            remote_path)
+        if not dry_run:
+          davclient.upload(local_path=k, remote_path=remote_path)
+
+
 @ci.command(epilog='''
 Examples:
 
@@ -291,4 +361,4 @@ def clean(ctx):
   from ..build import git_clean_build
   from ..bootstrap import run_cmdline
 
-  git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2))
+  git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))

conda/meta.yaml

@@ -68,6 +68,7 @@ test:
     - bdt new --help
     - bdt ci --help
     - bdt ci build --help
+    - bdt ci base-deploy --help
     - bdt ci deploy --help
     - bdt ci pypi --help
     - bdt ci readme --help

deps/python-gitlab/meta.yaml

+{% set name = "python-gitlab" %}
+{% set version = "1.7.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
+  sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
+
+build:
+  number: 0
+  script: "{{ PYTHON }} -m pip install . -vv"
+  entry_points:
+    - gitlab = gitlab.cli:main
+
+requirements:
+  host:
+    - python
+    - pip
+  run:
+    - python
+    - requests
+    - six
+
+test:
+  imports:
+    - gitlab
+
+about:
+  home: https://github.com/python-gitlab/python-gitlab
+  license: LGPL-3.0
+  license_family: LGPL
+  license_file: COPYING
+  summary: 'Python wrapper for the GitLab API'
+  description: |
+    python-gitlab is a Python package providing access to the GitLab
+    server API. It supports the v4 API of GitLab, and provides a CLI
+    tool (gitlab).
+  doc_url: https://python-gitlab.readthedocs.io/
+  dev_url: https://github.com/python-gitlab/python-gitlab
+
+extra:
+  recipe-maintainers:
+    - anjos

setup.py

@@ -59,6 +59,7 @@ setup(
         'bdt.ci.cli': [
           'build = bob.devtools.scripts.ci:build',
           'clean = bob.devtools.scripts.ci:clean',
+          'base-deploy = bob.devtools.scripts.ci:base_deploy',
           'deploy = bob.devtools.scripts.ci:deploy',
           'readme = bob.devtools.scripts.ci:readme',
           'pypi = bob.devtools.scripts.ci:pypi',