Support for python 3.7 and installation on the "base" conda environment

.gitlab-ci.yml

@@ -56,6 +56,12 @@ build_linux_36:
   <<: *linux_build_job
   variables:
     PYTHON_VERSION: "3.6"
+
+
+build_linux_37:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "3.7"
     BUILD_EGG: "true"
   script:
     - python3 ./bob/devtools/bootstrap.py -vv build
@@ -76,18 +82,27 @@ build_macosx_36:
     PYTHON_VERSION: "3.6"
 
 
+build_macosx_37:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "3.7"
+
+
 # Deploy targets
 .deploy_template: &deploy_job
   stage: deploy
   script:
-    - python3 ./bob/devtools/bootstrap.py -vv local bdt
+    - python3 ./bob/devtools/bootstrap.py -vv local base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
+    - bdt ci base-deploy -vv
     - bdt ci deploy -vv
     - bdt ci clean -vv
   dependencies:
     - build_linux_36
+    - build_linux_37
     - build_macosx_36
+    - build_macosx_37
   tags:
     - docker
   cache: &build_caches
@@ -121,14 +136,16 @@ pypi:
   except:
     - branches
   script:
-    - python3 ./bob/devtools/bootstrap.py -vv local bdt
+    - python3 ./bob/devtools/bootstrap.py -vv local base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate bdt
+    - conda activate base
     - bdt ci pypi -vv dist/*.zip
     - bdt ci clean -vv
   dependencies:
     - build_linux_36
+    - build_linux_37
     - build_macosx_36
+    - build_macosx_37
   tags:
     - docker
   cache: &build_caches

bob/devtools/build.py

@@ -154,6 +154,43 @@ def get_parsed_recipe(metadata):
   return yaml.load(output)
 
 
+def exists_on_channel(channel_url, name, version, build_number):
+  """Checks on the given channel if a package with the specs exist
+
+  Args:
+
+    channel_url: The URL where to look for packages clashes (normally a beta
+      channel)
+    name: The name of the package
+    version: The version of the package
+    build_number: The build number of the package
+
+  Returns: ``True``, if the package already exists in the channel or ``False``
+  otherwise
+
+  """
+
+  from conda.exports import get_index
+
+  # get the channel index
+  logger.debug('Downloading channel index from %s', channel_url)
+  index = get_index(channel_urls=[channel_url], prepend=False)
+
+  logger.info('Checking for %s-%s_*_%s...', name, version, build_number)
+  for dist in index:
+
+    if dist.name == name and dist.version == version and \
+        dist.build_string.endswith('_%s' % build_number):
+      match = re.match('py[2-9][0-9]+', dist.build_string)
+      logger.info('Found matching package (%s-%s_%s)', dist.name, dist.version,
+          dist.build_string)
+      return True
+
+  logger.info('No matches for %s-%s_%s found among %d packages',
+      name, version, build_number, len(index))
+  return False
+
+
 def remove_pins(deps):
   return [l.split()[0] for l in deps]
 
@@ -407,6 +444,52 @@ def git_clean_build(runner, verbose):
       ['--exclude=%s' % k for k in exclude_from_cleanup])
 
 
+def base_build(server, intranet, recipe_dir, config):
+  '''Builds a non-beat/bob software dependence that does not exist on defaults
+
+  This function will build a software dependence that is required for our
+  software stack, but does not (yet) exist on the defaults channels.  It first
+  check if the build should run for the current architecture, checks if the
+  package is not already built on our public channel and, if that is true, then
+  proceeds with the build of the dependence.
+
+
+  Args:
+
+    server: The base address of the server containing our conda channels
+    intranet: Boolean indicating if we should add "private"/"public" prefixes
+      on the returned paths
+    recipe_dir: The directory containing the recipe's ``meta.yaml`` file
+    config: A dictionary containing the merged configuration, as produced by
+      conda-build API's ``get_or_merge_config()`` function
+
+  '''
+
+  # if you get to this point, tries to build the package
+  public_channel = bootstrap.get_channels(public=True, stable=True,
+    server=server, intranet=intranet)[0]
+  metadata = get_rendered_metadata(recipe_dir, config)
+  recipe = get_parsed_recipe(metadata)
+
+  if recipe is None:
+    logger.warn('Skipping build for %s - rendering returned None', recipe_dir)
+    continue
+
+  if exists_on_channel(public_channel, recipe['package']['name'],
+      recipe['package']['version'], recipe['build']['number']):
+    logger.warn('Skipping build for %s-%s_%s - exists on channel already',
+        recipe['package']['name'], recipe['package']['version'],
+        recipe['build']['number'])
+    continue
+
+  # if you get to this point, just builds the package
+  arch = conda_arch()
+  logger.info('Building %s-%s (build: %d) for %s',
+      recipe['package']['name'], recipe['package']['version'],
+      recipe['build']['number'], arch)
+  conda_build.api.build(recipe_dir, config=conda_config)
+
+
 if __name__ == '__main__':
 
   import argparse
@@ -477,7 +560,8 @@ if __name__ == '__main__':
     condarc_options = yaml.load(f)
 
   # notice this condarc typically will only contain the defaults channel - we
-  # need to boost this up with more channels to get it right.
+  # need to boost this up with more channels to get it right for this package's
+  # build
   public = ( args.visibility == 'public' )
   channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
       server=server, intranet=(not args.internet))
@@ -492,6 +576,15 @@ if __name__ == '__main__':
   conda_config = make_conda_config(conda_build_config, args.python_version,
       recipe_append, condarc_options)
 
+  # builds all dependencies in the 'deps' subdirectory - or at least checks
+  # these dependencies are already available; these dependencies go directly to
+  # the public channel once built
+  for recipe in glob.glob(os.path.join('deps', '*')):
+    if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
+      # ignore - not a conda package
+      continue
+    base_build(server, not args.internet, recipe, conda_config)
+
   # retrieve the current build number for this build
   build_number, _ = next_build_number(channels[0], args.name, version,
       args.python_version)
@@ -517,4 +610,4 @@ if __name__ == '__main__':
     else:
       logger.info('twine check (a.k.a. readme check) %s: OK', package[0])
 
-  git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2))
+  git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))

bob/devtools/scripts/ci.py

@@ -30,6 +30,76 @@ def ci():
   pass
 
 
+@ci.command(epilog='''
+Examples:
+
+  1. Deploys base build artifacts (dependencies) to the appropriate channels:
+
+     $ bdt ci base-deploy -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def base_deploy(dry_run):
+    """Deploys dependencies not available at the defaults channel
+
+    Deployment happens to our public channel directly, as these are
+    dependencies are required for proper bob/beat package runtime environments.
+    """
+
+    if dry_run:
+        logger.warn('!!!! DRY RUN MODE !!!!')
+        logger.warn('Nothing is being deployed to server')
+
+    package = os.environ['CI_PROJECT_PATH']
+
+    from ..constants import WEBDAV_PATHS
+    server_info = WEBDAV_PATHS[True][True]  #stable=True, visible=True
+
+    logger.info('Deploying dependence packages to %s%s%s...', SERVER,
+        server_info['root'], server_info['conda'])
+
+    # setup webdav connection
+    webdav_options = {
+        'webdav_hostname': SERVER,
+        'webdav_root': server_info['root'],
+        'webdav_login': os.environ['DOCUSER'],
+        'webdav_password': os.environ['DOCPASS'],
+        }
+    from ..webdav3 import client as webdav
+    davclient = webdav.Client(webdav_options)
+    assert davclient.valid()
+
+    group, name = package.split('/')
+
+    # uploads conda package artificats
+    for arch in ('linux-64', 'osx-64', 'noarch'):
+      # finds conda dependencies and uploads what we can find
+      package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
+          '*.tar.bz2')
+      deploy_packages = glob.glob(package_path)
+      for k in deploy_packages:
+        basename = os.path.basename(k)
+        if basename.startswith(name):
+          logger.debug('Skipping deploying of %s - not a base package', k)
+          continue
+
+        remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
+        if davclient.check(remote_path):
+          raise RuntimeError('The file %s/%s already exists on the server ' \
+              '- this can be due to more than one build with deployment ' \
+              'running at the same time.  Re-running the broken builds ' \
+              'normally fixes it' % (SERVER, remote_path))
+        logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
+            remote_path)
+        if not dry_run:
+          davclient.upload(local_path=k, remote_path=remote_path)
+
+
 @ci.command(epilog='''
 Examples:
 
@@ -291,4 +361,4 @@ def clean(ctx):
   from ..build import git_clean_build
   from ..bootstrap import run_cmdline
 
-  git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2))
+  git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))

conda/meta.yaml

@@ -68,6 +68,7 @@ test:
     - bdt new --help
     - bdt ci --help
     - bdt ci build --help
+    - bdt ci base-deploy --help
     - bdt ci deploy --help
     - bdt ci pypi --help
     - bdt ci readme --help

deps/python-gitlab/meta.yaml

+{% set name = "python-gitlab" %}
+{% set version = "1.7.0" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
+  sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
+
+build:
+  number: 0
+  script: "{{ PYTHON }} -m pip install . -vv"
+  entry_points:
+    - gitlab = gitlab.cli:main
+
+requirements:
+  host:
+    - python
+    - pip
+  run:
+    - python
+    - requests
+    - six
+
+test:
+  imports:
+    - gitlab
+
+about:
+  home: https://github.com/python-gitlab/python-gitlab
+  license: LGPL-3.0
+  license_family: LGPL
+  license_file: COPYING
+  summary: 'Python wrapper for the GitLab API'
+  description: |
+    python-gitlab is a Python package providing access to the GitLab
+    server API. It supports the v4 API of GitLab, and provides a CLI
+    tool (gitlab).
+  doc_url: https://python-gitlab.readthedocs.io/
+  dev_url: https://github.com/python-gitlab/python-gitlab
+
+extra:
+  recipe-maintainers:
+    - anjos

setup.py

@@ -59,6 +59,7 @@ setup(
         'bdt.ci.cli': [
           'build = bob.devtools.scripts.ci:build',
           'clean = bob.devtools.scripts.ci:clean',
+          'base-deploy = bob.devtools.scripts.ci:base_deploy',
           'deploy = bob.devtools.scripts.ci:deploy',
           'readme = bob.devtools.scripts.ci:readme',
           'pypi = bob.devtools.scripts.ci:pypi',