Commit e7ac23c9 authored by André Anjos's avatar André Anjos 💬

Support for python 3.7 and installation on the "base" conda environment

parent 0c6cff16
Pipeline #26792 failed with stage
in 2 minutes and 45 seconds
......@@ -56,6 +56,12 @@ build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
build_linux_37:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.7"
BUILD_EGG: "true"
script:
- python3 ./bob/devtools/bootstrap.py -vv build
......@@ -76,18 +82,27 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
build_macosx_37:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.7"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
script:
- python3 ./bob/devtools/bootstrap.py -vv local bdt
- python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci base-deploy -vv
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
- build_linux_37
- build_macosx_36
- build_macosx_37
tags:
- docker
cache: &build_caches
......@@ -121,14 +136,16 @@ pypi:
except:
- branches
script:
- python3 ./bob/devtools/bootstrap.py -vv local bdt
- python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
- build_linux_36
- build_linux_37
- build_macosx_36
- build_macosx_37
tags:
- docker
cache: &build_caches
......
......@@ -154,6 +154,43 @@ def get_parsed_recipe(metadata):
return yaml.load(output)
def exists_on_channel(channel_url, name, version, build_number):
"""Checks on the given channel if a package with the specs exist
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
name: The name of the package
version: The version of the package
build_number: The build number of the package
Returns: ``True``, if the package already exists in the channel or ``False``
otherwise
"""
from conda.exports import get_index
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
logger.info('Checking for %s-%s_*_%s...', name, version, build_number)
for dist in index:
if dist.name == name and dist.version == version and \
dist.build_string.endswith('_%s' % build_number):
match = re.match('py[2-9][0-9]+', dist.build_string)
logger.info('Found matching package (%s-%s_%s)', dist.name, dist.version,
dist.build_string)
return True
logger.info('No matches for %s-%s_%s found among %d packages',
name, version, build_number, len(index))
return False
def remove_pins(deps):
return [l.split()[0] for l in deps]
......@@ -407,6 +444,52 @@ def git_clean_build(runner, verbose):
['--exclude=%s' % k for k in exclude_from_cleanup])
def base_build(server, intranet, recipe_dir, config):
'''Builds a non-beat/bob software dependence that does not exist on defaults
This function will build a software dependence that is required for our
software stack, but does not (yet) exist on the defaults channels. It first
check if the build should run for the current architecture, checks if the
package is not already built on our public channel and, if that is true, then
proceeds with the build of the dependence.
Args:
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the returned paths
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
config: A dictionary containing the merged configuration, as produced by
conda-build API's ``get_or_merge_config()`` function
'''
# if you get to this point, tries to build the package
public_channel = bootstrap.get_channels(public=True, stable=True,
server=server, intranet=intranet)[0]
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
if recipe is None:
logger.warn('Skipping build for %s - rendering returned None', recipe_dir)
continue
if exists_on_channel(public_channel, recipe['package']['name'],
recipe['package']['version'], recipe['build']['number']):
logger.warn('Skipping build for %s-%s_%s - exists on channel already',
recipe['package']['name'], recipe['package']['version'],
recipe['build']['number'])
continue
# if you get to this point, just builds the package
arch = conda_arch()
logger.info('Building %s-%s (build: %d) for %s',
recipe['package']['name'], recipe['package']['version'],
recipe['build']['number'], arch)
conda_build.api.build(recipe_dir, config=conda_config)
if __name__ == '__main__':
import argparse
......@@ -477,7 +560,8 @@ if __name__ == '__main__':
condarc_options = yaml.load(f)
# notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right.
# need to boost this up with more channels to get it right for this package's
# build
public = ( args.visibility == 'public' )
channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
server=server, intranet=(not args.internet))
......@@ -492,6 +576,15 @@ if __name__ == '__main__':
conda_config = make_conda_config(conda_build_config, args.python_version,
recipe_append, condarc_options)
# builds all dependencies in the 'deps' subdirectory - or at least checks
# these dependencies are already available; these dependencies go directly to
# the public channel once built
for recipe in glob.glob(os.path.join('deps', '*')):
if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
# ignore - not a conda package
continue
base_build(server, not args.internet, recipe, conda_config)
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0], args.name, version,
args.python_version)
......@@ -517,4 +610,4 @@ if __name__ == '__main__':
else:
logger.info('twine check (a.k.a. readme check) %s: OK', package[0])
git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2))
git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))
......@@ -30,6 +30,76 @@ def ci():
pass
@ci.command(epilog='''
Examples:
1. Deploys base build artifacts (dependencies) to the appropriate channels:
$ bdt ci base-deploy -vv
''')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def base_deploy(dry_run):
"""Deploys dependencies not available at the defaults channel
Deployment happens to our public channel directly, as these are
dependencies are required for proper bob/beat package runtime environments.
"""
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing is being deployed to server')
package = os.environ['CI_PROJECT_PATH']
from ..constants import WEBDAV_PATHS
server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True
logger.info('Deploying dependence packages to %s%s%s...', SERVER,
server_info['root'], server_info['conda'])
# setup webdav connection
webdav_options = {
'webdav_hostname': SERVER,
'webdav_root': server_info['root'],
'webdav_login': os.environ['DOCUSER'],
'webdav_password': os.environ['DOCPASS'],
}
from ..webdav3 import client as webdav
davclient = webdav.Client(webdav_options)
assert davclient.valid()
group, name = package.split('/')
# uploads conda package artificats
for arch in ('linux-64', 'osx-64', 'noarch'):
# finds conda dependencies and uploads what we can find
package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
'*.tar.bz2')
deploy_packages = glob.glob(package_path)
for k in deploy_packages:
basename = os.path.basename(k)
if basename.startswith(name):
logger.debug('Skipping deploying of %s - not a base package', k)
continue
remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
if davclient.check(remote_path):
raise RuntimeError('The file %s/%s already exists on the server ' \
'- this can be due to more than one build with deployment ' \
'running at the same time. Re-running the broken builds ' \
'normally fixes it' % (SERVER, remote_path))
logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
remote_path)
if not dry_run:
davclient.upload(local_path=k, remote_path=remote_path)
@ci.command(epilog='''
Examples:
......@@ -291,4 +361,4 @@ def clean(ctx):
from ..build import git_clean_build
from ..bootstrap import run_cmdline
git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2))
git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))
......@@ -68,6 +68,7 @@ test:
- bdt new --help
- bdt ci --help
- bdt ci build --help
- bdt ci base-deploy --help
- bdt ci deploy --help
- bdt ci pypi --help
- bdt ci readme --help
......
{% set name = "python-gitlab" %}
{% set version = "1.7.0" %}
package:
name: {{ name|lower }}
version: {{ version }}
source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
build:
number: 0
script: "{{ PYTHON }} -m pip install . -vv"
entry_points:
- gitlab = gitlab.cli:main
requirements:
host:
- python
- pip
run:
- python
- requests
- six
test:
imports:
- gitlab
about:
home: https://github.com/python-gitlab/python-gitlab
license: LGPL-3.0
license_family: LGPL
license_file: COPYING
summary: 'Python wrapper for the GitLab API'
description: |
python-gitlab is a Python package providing access to the GitLab
server API. It supports the v4 API of GitLab, and provides a CLI
tool (gitlab).
doc_url: https://python-gitlab.readthedocs.io/
dev_url: https://github.com/python-gitlab/python-gitlab
extra:
recipe-maintainers:
- anjos
......@@ -59,6 +59,7 @@ setup(
'bdt.ci.cli': [
'build = bob.devtools.scripts.ci:build',
'clean = bob.devtools.scripts.ci:clean',
'base-deploy = bob.devtools.scripts.ci:base_deploy',
'deploy = bob.devtools.scripts.ci:deploy',
'readme = bob.devtools.scripts.ci:readme',
'pypi = bob.devtools.scripts.ci:pypi',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment