Commit d3f9d468 authored by André Anjos's avatar André Anjos

Merge branch 'support-base-py37' into 'master'

Independent builds and python support

Closes #13

See merge request !11
parents 0c6cff16 bb84e88c
Pipeline #26808 canceled with stages
in 3 minutes and 30 seconds
......@@ -56,6 +56,12 @@ build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
build_linux_37:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.7"
BUILD_EGG: "true"
script:
- python3 ./bob/devtools/bootstrap.py -vv build
......@@ -76,18 +82,27 @@ build_macosx_36:
PYTHON_VERSION: "3.6"
build_macosx_37:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.7"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
script:
- python3 ./bob/devtools/bootstrap.py -vv local bdt
- python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci base-deploy -vv
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
- build_linux_37
- build_macosx_36
- build_macosx_37
tags:
- docker
cache: &build_caches
......@@ -121,14 +136,16 @@ pypi:
except:
- branches
script:
- python3 ./bob/devtools/bootstrap.py -vv local bdt
- python3 ./bob/devtools/bootstrap.py -vv local base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
- build_linux_36
- build_linux_37
- build_macosx_36
- build_macosx_37
tags:
- docker
cache: &build_caches
......
include LICENSE README.rst buildout.cfg version.txt
recursive-include doc conf.py *.rst *.sh *.png *.ico
recursive-include bob/devtools/data *.md *.yaml *.pem matplotlibrc
recursive-include bob/devtools/templates conf.py *.rst *.png *.ico LICENSE COPYING .gitlab-ci.yml .gitignore *.cfg *.txt *.py
recursive-include bob/devtools/templates conf.py *.rst *.png *.ico LICENSE COPYING MANIFEST.in .gitlab-ci.yml .gitignore *.cfg *.txt *.py
......@@ -383,7 +383,7 @@ if __name__ == '__main__':
f.write(_BASE_CONDARC)
conda_version = '4'
conda_build_version = '3'
conda_build_version = '3.16'
conda_verbosity = []
#if args.verbose >= 2:
......
......@@ -154,6 +154,61 @@ def get_parsed_recipe(metadata):
return yaml.load(output)
def exists_on_channel(channel_url, name, version, build_number,
python_version):
"""Checks on the given channel if a package with the specs exist
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
name: The name of the package
version: The version of the package
build_number: The build number of the package
python_version: The current version of python we're building for
Returns: A complete package name, version and build string, if the package
already exists in the channel or ``None`` otherwise.
"""
from conda.exports import get_index
# no dot in py_ver
py_ver = python_version.replace('.', '')
# get the channel index
logger.debug('Downloading channel index from %s', channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
logger.info('Checking for %s-%s-%s...', name, version, build_number)
for dist in index:
if dist.name == name and dist.version == version and \
dist.build_string.endswith('_%s' % build_number):
# two possible options must be checked - (i) the package build_string
# starts with ``py``, which means it is a python specific package so we
# must also check for the matching python version. (ii) the package is
# not a python-specific package and a simple match will do
if dist.build_string.startswith('py'):
match = re.match('py[2-9][0-9]+', dist.build_string)
if match and match.group() == 'py{}'.format(py_ver):
logger.debug('Found matching package (%s-%s-%s)', dist.name,
dist.version, dist.build_string)
return (dist.name, dist.version, dist.build_string)
else:
logger.debug('Found matching package (%s-%s-%s)', dist.name,
dist.version, dist.build_string)
return (dist.name, dist.version, dist.build_string)
logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
name, version, py_ver, build_number, len(index))
return
def remove_pins(deps):
return [l.split()[0] for l in deps]
......@@ -407,6 +462,69 @@ def git_clean_build(runner, verbose):
['--exclude=%s' % k for k in exclude_from_cleanup])
def base_build(server, intranet, recipe_dir, conda_build_config,
python_version, condarc_options):
'''Builds a non-beat/bob software dependence that does not exist on defaults
This function will build a software dependence that is required for our
software stack, but does not (yet) exist on the defaults channels. It first
check if the build should run for the current architecture, checks if the
package is not already built on our public channel and, if that is true, then
proceeds with the build of the dependence.
Args:
server: The base address of the server containing our conda channels
intranet: Boolean indicating if we should add "private"/"public" prefixes
on the returned paths
recipe_dir: The directory containing the recipe's ``meta.yaml`` file
conda_build_config: Path to the ``conda_build_config.yaml`` file to use
python_version: String with the python version to build for, in the format
``x.y`` (should be passed even if not building a python package)
condarc_options: Pre-parsed condarc options loaded from the respective YAML
file
'''
# if you get to this point, tries to build the package
public_channels = bootstrap.get_channels(public=True, stable=True,
server=server, intranet=intranet)
logger.info('Using the following channels during (potential) build:\n - %s',
'\n - '.join(public_channels + ['defaults']))
condarc_options['channels'] = public_channels + ['defaults']
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, python_version,
None, condarc_options)
metadata = get_rendered_metadata(recipe_dir, conda_config)
recipe = get_parsed_recipe(metadata)
if recipe is None:
logger.info('Skipping build for %s - rendering returned None', recipe_dir)
return
# no dot in py_ver
py_ver = python_version.replace('.', '')
arch = conda_arch()
candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
recipe['package']['version'], recipe['build']['number'],
python_version)
if candidate is not None:
logger.info('Skipping build for %s-%s-(py%s_?)%s for %s - exists ' \
'on channel', candidate[0], candidate[1], candidate[2], py_ver, arch)
return
# if you get to this point, just builds the package
logger.info('Building %s-%s-(py%s_?)%s for %s',
recipe['package']['name'], recipe['package']['version'],
recipe['build']['number'], py_ver, arch)
conda_build.api.build(recipe_dir, config=conda_config)
if __name__ == '__main__':
import argparse
......@@ -476,8 +594,22 @@ if __name__ == '__main__':
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
# dump packages at conda_root
condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
# builds all dependencies in the 'deps' subdirectory - or at least checks
# these dependencies are already available; these dependencies go directly to
# the public channel once built
for recipe in glob.glob(os.path.join('deps', '*')):
if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
# ignore - not a conda package
continue
base_build(server, not args.internet, recipe, conda_build_config,
args.python_version, condarc_options)
# notice this condarc typically will only contain the defaults channel - we
# need to boost this up with more channels to get it right.
# need to boost this up with more channels to get it right for this package's
# build
public = ( args.visibility == 'public' )
channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
server=server, intranet=(not args.internet))
......@@ -485,18 +617,15 @@ if __name__ == '__main__':
'\n - '.join(channels + ['defaults']))
condarc_options['channels'] = channels + ['defaults']
# dump packages at conda_root
condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, args.python_version,
recipe_append, condarc_options)
# retrieve the current build number for this build
build_number, _ = next_build_number(channels[0], args.name, version,
args.python_version)
bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
logger.info('Merging conda configuration files...')
conda_config = make_conda_config(conda_build_config, args.python_version,
recipe_append, condarc_options)
# runs the build using the conda-build API
arch = conda_arch()
logger.info('Building %s-%s-py%s (build: %d) for %s',
......@@ -517,4 +646,4 @@ if __name__ == '__main__':
else:
logger.info('twine check (a.k.a. readme check) %s: OK', package[0])
git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 2))
git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))
......@@ -32,6 +32,7 @@ pin_run_as_build:
## the dependencies that we build against multiple versions
python:
- 3.6
- 3.7
zip_keys:
- # [win]
......
......@@ -20,9 +20,9 @@ stages:
stage: build
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel bdt
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci build -vv
- bdt ci clean -vv
cache: &build_caches
......@@ -66,9 +66,9 @@ build_linux_36:
BUILD_EGG: "true"
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel bdt
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci build -vv
- bdt ci readme -vv dist/*.zip
- bdt ci clean -vv
......@@ -91,9 +91,9 @@ build_macosx_36:
stage: deploy
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py channel bdt
- python3 bootstrap.py channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
......@@ -133,9 +133,9 @@ pypi:
- branches
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel bdt
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate bdt
- conda activate base
- bdt ci pypi -vv dist/*.zip
- bdt ci clean -vv
dependencies:
......
......@@ -30,6 +30,76 @@ def ci():
pass
@ci.command(epilog='''
Examples:
1. Deploys base build artifacts (dependencies) to the appropriate channels:
$ bdt ci base-deploy -vv
''')
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
def base_deploy(dry_run):
"""Deploys dependencies not available at the defaults channel
Deployment happens to our public channel directly, as these are
dependencies are required for proper bob/beat package runtime environments.
"""
if dry_run:
logger.warn('!!!! DRY RUN MODE !!!!')
logger.warn('Nothing is being deployed to server')
package = os.environ['CI_PROJECT_PATH']
from ..constants import WEBDAV_PATHS
server_info = WEBDAV_PATHS[True][True] #stable=True, visible=True
logger.info('Deploying dependence packages to %s%s%s...', SERVER,
server_info['root'], server_info['conda'])
# setup webdav connection
webdav_options = {
'webdav_hostname': SERVER,
'webdav_root': server_info['root'],
'webdav_login': os.environ['DOCUSER'],
'webdav_password': os.environ['DOCPASS'],
}
from ..webdav3 import client as webdav
davclient = webdav.Client(webdav_options)
assert davclient.valid()
group, name = package.split('/')
# uploads conda package artificats
for arch in ('linux-64', 'osx-64', 'noarch'):
# finds conda dependencies and uploads what we can find
package_path = os.path.join(os.environ['CONDA_ROOT'], 'conda-bld', arch,
'*.tar.bz2')
deploy_packages = glob.glob(package_path)
for k in deploy_packages:
basename = os.path.basename(k)
if basename.startswith(name):
logger.debug('Skipping deploying of %s - not a base package', k)
continue
remote_path = '%s/%s/%s' % (server_info['conda'], arch, basename)
if davclient.check(remote_path):
raise RuntimeError('The file %s/%s already exists on the server ' \
'- this can be due to more than one build with deployment ' \
'running at the same time. Re-running the broken builds ' \
'normally fixes it' % (SERVER, remote_path))
logger.info('[dav] %s -> %s%s%s', k, SERVER, server_info['root'],
remote_path)
if not dry_run:
davclient.upload(local_path=k, remote_path=remote_path)
@ci.command(epilog='''
Examples:
......@@ -291,4 +361,4 @@ def clean(ctx):
from ..build import git_clean_build
from ..bootstrap import run_cmdline
git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=2))
git_clean_build(run_cmdline, verbose=(ctx.meta['verbosity']>=3))
......@@ -68,6 +68,7 @@ test:
- bdt new --help
- bdt ci --help
- bdt ci build --help
- bdt ci base-deploy --help
- bdt ci deploy --help
- bdt ci pypi --help
- bdt ci readme --help
......
{% set name = "python-gitlab" %}
{% set version = "1.7.0" %}
package:
name: {{ name|lower }}
version: {{ version }}
source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
build:
number: 0
script: "{{ PYTHON }} -m pip install . -vv"
entry_points:
- gitlab = gitlab.cli:main
requirements:
host:
- python
- pip
run:
- python
- requests
- six
test:
imports:
- gitlab
about:
home: https://github.com/python-gitlab/python-gitlab
license: LGPL-3.0
license_family: LGPL
license_file: COPYING
summary: 'Python wrapper for the GitLab API'
description: |
python-gitlab is a Python package providing access to the GitLab
server API. It supports the v4 API of GitLab, and provides a CLI
tool (gitlab).
doc_url: https://python-gitlab.readthedocs.io/
dev_url: https://github.com/python-gitlab/python-gitlab
extra:
recipe-maintainers:
- anjos
......@@ -59,6 +59,7 @@ setup(
'bdt.ci.cli': [
'build = bob.devtools.scripts.ci:build',
'clean = bob.devtools.scripts.ci:clean',
'base-deploy = bob.devtools.scripts.ci:base_deploy',
'deploy = bob.devtools.scripts.ci:deploy',
'readme = bob.devtools.scripts.ci:readme',
'pypi = bob.devtools.scripts.ci:pypi',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment