Skip to content
Snippets Groups Projects
Commit 80b4a8d1 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

DRY

parent 1808f37b
No related branches found
No related tags found
No related merge requests found
Pipeline #26000 failed
Stage: build
Stage: deploy
Hide whitespace changes
Inline Side-by-side
Showing
with 503 additions and 658 deletions
.gitlab-ci.yml
+ 3
− 3
View file @ 80b4a8d1
......@@ -14,7 +14,7 @@ stages:
.build_template: &build_job
stage: build
before_script:
- python3 ./ci/bootstrap.py build
- python3 ./bob/devtools/bootstrap.py build
script:
- ./ci/build.sh
cache: &build_caches
......@@ -74,7 +74,7 @@ build_macosx_36:
.deploy_template: &deploy_job
stage: deploy
before_script:
- python3 ./ci/bootstrap.py local myenv
- python3 ./bob/devtools/bootstrap.py local myenv
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
......@@ -115,7 +115,7 @@ pypi:
except:
- branches
before_script:
- python3 ./ci/bootstrap.py local myenv
- python3 ./bob/devtools/bootstrap.py local myenv
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
......
bob/devtools/bootstrap.py
+ 356
− 95
View file @ 80b4a8d1
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''Bootstraps a new miniconda installation and prepares it for development
This command uses a bare-minimum python3 installation (with SSL support) to
bootstrap a new miniconda installation preset for the defined activity. It is
primarily intended for CI operation and prefixes build and deployment steps.
Usage: python3 %s <cmd> build|local|beta|stable [<name>]
Arguments:
<cmd> How to prepare the current environment. Use:
build to build bob.devtools
local to bootstrap deploy|pypi stages for bob.devtools builds
beta to bootstrap CI environment for beta builds
stable to bootstrap CI environment for stable builds
test to locally test this bootstrap script
<name> (optional) if command is one of ``local|beta|stable`` provide the
name of env for bob.devtools installation')
'''
BASE_CONDARC = '''\
default_channels:
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
'''
'''Methods to bootstrap working environments based on conda-packages'''
import os
import json
import sys
import glob
import time
import shutil
import platform
import subprocess
import logging
logger = logging.getLogger(__name__)
logger = logging.getLogger('bootstrap')
import yaml
_INTERVALS = (
('weeks', 604800), # 60 * 60 * 24 * 7
('days', 86400), # 60 * 60 * 24
('hours', 3600), # 60 * 60
('minutes', 60),
('seconds', 1),
)
def human_time(seconds, granularity=2):
'''Returns a human readable time string like "1 day, 2 hours"'''
def make_conda_config(config, python, append_file, condarc):
result = []
from conda_build.api import get_or_merge_config
from conda_build.conda_interface import url_path
for name, count in _INTERVALS:
value = seconds // count
if value:
seconds -= value * count
if value == 1:
name = name.rstrip('s')
result.append("{} {}".format(int(value), name))
else:
# Add a blank if we're in the middle of other values
if len(result) > 0:
result.append(None)
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
if not result:
if seconds < 1.0:
return '%.2f seconds' % seconds
else:
if seconds == 1:
return '1 second'
else:
return '%d seconds' % seconds
retval = get_or_merge_config(None, variant_config_files=config,
python=python, append_sections_file=append_file, **condarc_options)
return ', '.join([x for x in result[:granularity] if x is not None])
retval.channel_urls = []
for url in condarc_options['channels']:
# allow people to specify relative or absolute paths to local channels
# These channels still must follow conda rules - they must have the
# appropriate platform-specific subdir (e.g. win-64)
if os.path.isdir(url):
if not os.path.isabs(url):
url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
url = url_path(url)
retval.channel_urls.append(url)
def run_cmdline(cmd, env=None):
'''Runs a command on a environment, logs output and reports status
return retval
Parameters:
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
cmd (list): The command to run, with parameters separated on a list
from conda_build.api import render
return render(recipe_dir, config=config)
env (dict, Optional): Environment to use for running the program on. If not
set, use :py:obj:`os.environ`.
'''
def get_parsed_recipe(metadata):
'''Renders the recipe and returns the interpreted YAML file'''
if env is None: env = os.environ
from conda_build.api import output_yaml
output = output_yaml(metadata[0][0])
return yaml.load(output)
logger.info('(system) %s' % ' '.join(cmd))
start = time.time()
out = b''
def remove_pins(deps):
return [l.split()[0] for l in deps]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
env=env)
for line in iter(p.stdout.readline, b''):
sys.stdout.write(line.decode(sys.stdout.encoding))
sys.stdout.flush()
def parse_dependencies(recipe_dir, config):
if p.wait() != 0:
raise RuntimeError("command `%s' exited with error state (%d)" % \
(' '.join(cmd), p.returncode))
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
recipe.get('test', {}).get('requires', []) + \
['bob.buildout', 'mr.developer', 'ipdb']
# by last, packages required for local dev
total = time.time() - start
logger.info('command took %s' % human_time(total))
def get_env_directory(conda, name):
cmd = [conda, 'env', 'list', '--json']
output = subprocess.check_output(cmd)
data = json.loads(output)
retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
if retval:
return retval[0]
return None
def touch(path):
'''Python-implementation of the "touch" command-line application'''
def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
with open(path, 'a'):
os.utime(path, None)
specs = []
for k in packages:
k = ' '.join(k.split()[:2]) # remove eventual build string
if any(elem in k for elem in '><|'):
specs.append(k.replace(' ', ''))
else:
specs.append(k.replace(' ', '='))
# if the current environment exists, delete it first
envdir = get_env_directory(conda, name)
if envdir is not None:
if overwrite:
cmd = [conda, 'env', 'remove', '--yes', '--name', name]
logger.debug('$ ' + ' '.join(cmd))
if not dry_run:
status = subprocess.call(cmd)
if status != 0:
return status
else:
raise RuntimeError('environment `%s\' exists in `%s\' - use '
'--overwrite to overwrite' % (name, envdir))
cmd = [conda, 'create', '--yes', '--name', name]
if dry_run:
cmd.append('--dry-run')
if use_local:
cmd.append('--use-local')
cmd.extend(sorted(specs))
logger.debug('$ ' + ' '.join(cmd))
status = subprocess.call(cmd)
if status != 0:
return status
# copy the used condarc file to the just created environment
if not dry_run:
# get envdir again - it may just be created!
envdir = get_env_directory(conda, name)
destrc = os.path.join(envdir, '.condarc')
logger.debug('$ cp %s -> %s' % (condarc, destrc))
shutil.copy2(condarc, destrc)
return status
def merge_conda_cache(cache, prefix):
'''Merges conda pkg caches and conda-bld folders'''
pkgs_dir = os.path.join(prefix, 'pkgs')
pkgs_urls_txt = os.path.join(pkgs_dir, 'urls.txt')
if not os.path.exists(pkgs_dir):
logger.info('mkdir -p %s', pkgs_dir)
os.makedirs(pkgs_dir)
logger.info('touch %s', pkgs_urls_txt)
touch(pkgs_urls_txt)
# move packages on cache/pkgs to pkgs_dir
cached_pkgs_dir = os.path.join(cache, 'pkgs')
cached_packages = glob.glob(os.path.join(cached_pkgs_dir, '*.tar.bz2'))
cached_packages = [k for k in cached_packages if not \
k.startswith(os.environ['CI_PROJECT_NAME'] + '-')]
logger.info('Merging %d cached conda packages', len(cached_packages))
for k in cached_packages:
dst = os.path.join(pkgs_dir, os.path.basename(k))
logger.debug('(move) %s -> %s', k, dst)
os.rename(k, dst)
# merge urls.txt files
logger.info('Merging urls.txt files from cache...')
urls = []
cached_pkgs_urls_txt = os.path.join(cached_pkgs_dir, 'urls.txt')
with open(pkgs_urls_txt, 'rb') as f1, \
open(cached_pkgs_urls_txt, 'rb') as f2:
data = set(f1.readlines() + f2.readlines())
data = sorted(list(data))
with open(pkgs_urls_txt, 'wb') as f:
f.writelines(data)
pkgs_urls = os.path.join(pkgs_dir, 'urls')
touch(pkgs_urls)
# move conda-bld build results
cached_conda_bld = os.path.join(cache, 'conda-bld')
if os.path.exists(cached_conda_bld):
dst = os.path.join(prefix, 'conda-bld')
logger.info('(move) %s -> %s', cached_conda_bld, dst)
os.rename(cached_conda_bld, dst)
def get_miniconda_sh():
'''Retrieves the miniconda3 installer for the current system'''
import http.client
server = 'repo.continuum.io' #https
path = '/miniconda/Miniconda3-latest-%s-x86_64.sh'
if platform.system() == 'Darwin':
path = path % 'MacOSX'
else:
path = path % 'Linux'
logger.info('Connecting to https://%s...', server)
conn = http.client.HTTPSConnection(server)
conn.request("GET", path)
r1 = conn.getresponse()
assert r1.status == 200, 'Request for https://%s%s - returned status %d ' \
'(%s)' % (server, path, r1.status, r1.reason)
dst = 'miniconda.sh'
logger.info('(download) https://%s%s -> %s...', server, path, dst)
with open(dst, 'wb') as f:
f.write(r1.read())
def install_miniconda(prefix):
'''Creates a new miniconda installation'''
logger.info("Installing miniconda in %s...", prefix)
if not os.path.exists('miniconda.sh'): #re-downloads installer
get_miniconda_sh()
else:
logger.info("Re-using cached miniconda3 installer")
cached = None
if os.path.exists(prefix): #this is the previous cache, move it
cached = prefix + '.cached'
if os.path.exists(cached):
logger.info('(rmtree) %s', cached)
shutil.rmtree(cached)
logger.info('(move) %s -> %s', prefix, cached)
os.rename(prefix, cached)
run_cmdline(['bash', 'miniconda.sh', '-b', '-p', prefix])
if cached is not None:
merge_conda_cache(cached, prefix)
shutil.rmtree(cached)
def get_channels(public, stable):
'''Returns the relevant conda channels to consider if building project
The subset of channels to be returned depends on the visibility and stability
of the package being built. Here are the rules:
* public and stable: only returns the public stable channel(s)
* public and not stable: returns both public stable and beta channels
* not public and stable: returns both public and private stable channels
* not public and not stable: returns all channels
Beta channels have priority over stable channels, if returned. Private
channels have priority over public channles, if turned.
Args:
public: Boolean indicating if we're supposed to include only public
channels
stable: Boolean indicating if we're supposed to include only stable
channels
Returns: a list of channels that need to be considered.
'''
server = "http://www.idiap.ch"
channels = []
if not public:
if not stable: #allowed private channels
channels += [server + '/private/conda/label/beta'] #allowed betas
channels += [server + '/private/conda']
if not stable:
channels += [server + '/public/conda/label/beta'] #allowed betas
channels += [server + '/public/conda']
return channels
def add_channels_condarc(channels, condarc):
'''Appends passed channel list to condarc file, print contents'''
with open(condarc, 'at') as f:
f.write('channels:\n')
for k in channels:
f.write(' - %s\n' % k)
with open(condarc, 'rt') as f:
logger.info('Contents of $CONDARC:\n%s', f.read())
def setup_logger():
'''Sets-up the logging for this command at level ``INFO``'''
warn_err = logging.StreamHandler(sys.stderr)
warn_err.setLevel(logging.WARNING)
logger.addHandler(warn_err)
# debug and info messages are written to sys.stdout
class _InfoFilter:
def filter(self, record):
return record.levelno <= logging.INFO
debug_info = logging.StreamHandler(sys.stdout)
debug_info.setLevel(logging.DEBUG)
debug_info.addFilter(_InfoFilter())
logger.addHandler(debug_info)
formatter = logging.Formatter('%(levelname)s@%(asctime)s: %(message)s')
for handler in logger.handlers:
handler.setFormatter(formatter)
logger.setLevel(logging.INFO)
if __name__ == '__main__':
if len(sys.argv) == 1:
print(__doc__ % sys.argv[0])
sys.exit(1)
setup_logger()
if sys.argv[1] == 'test':
# sets up local variables for testing
os.environ['CI_PROJECT_DIR'] = os.path.realpath(os.curdir)
os.environ['CI_PROJECT_NAME'] = 'bob.devtools'
os.environ['CONDA_ROOT'] = os.path.join(os.environ['CI_PROJECT_DIR'],
'miniconda')
prefix = os.environ['CONDA_ROOT']
logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
workdir = os.environ['CI_PROJECT_DIR']
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
install_miniconda(prefix)
conda_bin = os.path.join(prefix, 'bin', 'conda')
# creates the condarc file
condarc = os.path.join(prefix, 'condarc')
logger.info('(create) %s', condarc)
with open(condarc, 'wt') as f:
f.write(BASE_CONDARC)
os.environ['CONDARC'] = condarc
logger.info('os.environ["%s"] = %s', 'CONDARC', condarc)
conda_version = '4'
conda_build_version = '3'
if sys.argv[1] in ('build', 'test'):
# simple - just use the defaults channels when self building
add_channels_condarc(['defaults'], condarc)
run_cmdline([conda_bin, 'install', '-n', 'base',
'python',
'conda=%s' % conda_version,
'conda-build=%s' % conda_build_version,
])
elif sys.argv[1] == 'local':
# index the locally built packages
run_cmdline([conda_bin, 'install', '-n', 'base',
'python',
'conda=%s' % conda_version,
'conda-build=%s' % conda_build_version,
])
conda_bld_path = os.path.join(prefix, 'conda-bld')
run_cmdline([conda_bin, 'index', conda_bld_path])
# add the locally build directory before defaults, boot from there
channels = get_channels(public=True, stable=True)
add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
elif sys.argv[1] in ('beta', 'stable'):
# installs from channel
channels = get_channels(
public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
stable=os.environ.get('CI_COMMIT_TAG') is not None)
add_channels_condarc(channels + ['defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
else:
logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'")
logger.error("The value '%s' is not currently supported", sys.argv[1])
sys.exit(1)
# clean up
run_cmdline([conda_bin, 'clean', '--lock'])
# print conda information for debugging purposes
run_cmdline([conda_bin, 'info'])
ci/nextbuild.py → bob/devtools/build.py
+ 0
− 0
View file @ 80b4a8d1
File moved
bob/devtools/create.py 0 → 100644
+ 128
− 0
View file @ 80b4a8d1
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
'''Methods to create working environments based on conda-packages'''
import os
import json
import shutil
import subprocess
import logging
logger = logging.getLogger(__name__)
import yaml
def make_conda_config(config, python, append_file, condarc):
from conda_build.api import get_or_merge_config
from conda_build.conda_interface import url_path
with open(condarc, 'rb') as f:
condarc_options = yaml.load(f)
retval = get_or_merge_config(None, variant_config_files=config,
python=python, append_sections_file=append_file, **condarc_options)
retval.channel_urls = []
for url in condarc_options['channels']:
# allow people to specify relative or absolute paths to local channels
# These channels still must follow conda rules - they must have the
# appropriate platform-specific subdir (e.g. win-64)
if os.path.isdir(url):
if not os.path.isabs(url):
url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
url = url_path(url)
retval.channel_urls.append(url)
return retval
def get_rendered_metadata(recipe_dir, config):
'''Renders the recipe and returns the interpreted YAML file'''
from conda_build.api import render
return render(recipe_dir, config=config)
def get_parsed_recipe(metadata):
'''Renders the recipe and returns the interpreted YAML file'''
from conda_build.api import output_yaml
output = output_yaml(metadata[0][0])
return yaml.load(output)
def remove_pins(deps):
return [l.split()[0] for l in deps]
def parse_dependencies(recipe_dir, config):
metadata = get_rendered_metadata(recipe_dir, config)
recipe = get_parsed_recipe(metadata)
return remove_pins(recipe['requirements'].get('build', [])) + \
remove_pins(recipe['requirements'].get('host', [])) + \
recipe['requirements'].get('run', []) + \
recipe.get('test', {}).get('requires', []) + \
['bob.buildout', 'mr.developer', 'ipdb']
# by last, packages required for local dev
def get_env_directory(conda, name):
cmd = [conda, 'env', 'list', '--json']
output = subprocess.check_output(cmd)
data = json.loads(output)
retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
if retval:
return retval[0]
return None
def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
specs = []
for k in packages:
k = ' '.join(k.split()[:2]) # remove eventual build string
if any(elem in k for elem in '><|'):
specs.append(k.replace(' ', ''))
else:
specs.append(k.replace(' ', '='))
# if the current environment exists, delete it first
envdir = get_env_directory(conda, name)
if envdir is not None:
if overwrite:
cmd = [conda, 'env', 'remove', '--yes', '--name', name]
logger.debug('$ ' + ' '.join(cmd))
if not dry_run:
status = subprocess.call(cmd)
if status != 0:
return status
else:
raise RuntimeError('environment `%s\' exists in `%s\' - use '
'--overwrite to overwrite' % (name, envdir))
cmd = [conda, 'create', '--yes', '--name', name]
if dry_run:
cmd.append('--dry-run')
if use_local:
cmd.append('--use-local')
cmd.extend(sorted(specs))
logger.debug('$ ' + ' '.join(cmd))
status = subprocess.call(cmd)
if status != 0:
return status
# copy the used condarc file to the just created environment
if not dry_run:
# get envdir again - it may just be created!
envdir = get_env_directory(conda, name)
destrc = os.path.join(envdir, '.condarc')
logger.debug('$ cp %s -> %s' % (condarc, destrc))
shutil.copy2(condarc, destrc)
return status
bob/devtools/scripts/build.py
+ 2
− 2
View file @ 80b4a8d1
......@@ -12,7 +12,8 @@ import click
from . import bdt
from ..log import verbosity_option
from ..conda import next_build_number, osname
from ..bootstrap import get_rendered_metadata, get_parsed_recipe
from ..create import get_rendered_metadata, get_parsed_recipe, \
make_conda_config
from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER, MATPLOTLIB_RCDIR, set_environment
......@@ -85,7 +86,6 @@ def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
logger.debug("CONDARC=%s", condarc)
from ..bootstrap import make_conda_config
conda_config = make_conda_config(config, python, append_file, condarc)
set_environment('LANG', 'en_US.UTF-8', os.environ)
......
bob/devtools/scripts/bootstrap.py → bob/devtools/scripts/create.py
+ 6
− 6
View file @ 80b4a8d1
......@@ -12,7 +12,7 @@ import yaml
from . import bdt
from ..log import verbosity_option
from ..bootstrap import parse_dependencies, conda_create, make_conda_config
from ..create import parse_dependencies, conda_create, make_conda_config
from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER
......@@ -24,7 +24,7 @@ Examples:
\b
$ cd bob.package.foo
$ bdt bootstrap -vv myenv
$ bdt create -vv myenv
The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
......@@ -33,12 +33,12 @@ Examples:
2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
$ bdt bootstrap -vv --python=3.6 myenv
$ bdt create -vv --python=3.6 myenv
3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
$ bdt bootstrap -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
$ bdt create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
Notice the condarc file **must** end in `condarc', or conda will complain.
......@@ -49,7 +49,7 @@ Examples:
shell will be printed:
$ bdt bootstrap -vvv --dry-run myenv
$ bdt create -vvv --dry-run myenv
''')
@click.argument('name')
@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
......@@ -80,7 +80,7 @@ Examples:
help='Allow the use of local channels for package retrieval')
@verbosity_option()
@bdt.raise_on_error
def bootstrap(name, recipe_dir, python, overwrite, condarc, config,
def create(name, recipe_dir, python, overwrite, condarc, config,
append_file, docserver, dry_run, use_local):
"""Creates a development environment for a recipe
......
ci/bootstrap.py deleted 100755 → 0
+ 0
− 389
View file @ 1808f37b
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''Bootstraps a new miniconda installation and prepares it for development
This command uses a bare-minimum python3 installation (with SSL support) to
bootstrap a new miniconda installation preset for the defined activity. It is
primarily intended for CI operation and prefixes build and deployment steps.
Usage: python3 %s <cmd> build|local|beta|stable [<name>]
Arguments:
<cmd> How to prepare the current environment. Use:
build to build bob.devtools
local to bootstrap deploy|pypi stages for bob.devtools builds
beta to bootstrap CI environment for beta builds
stable to bootstrap CI environment for stable builds
test to locally test this bootstrap script
<name> (optional) if command is one of ``local|beta|stable`` provide the
name of env for bob.devtools installation')
'''
BASE_CONDARC = '''\
default_channels:
- https://repo.anaconda.com/pkgs/main
- https://repo.anaconda.com/pkgs/free
- https://repo.anaconda.com/pkgs/r
- https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
'''
import os
import sys
import glob
import time
import shutil
import platform
import subprocess
import logging
logger = logging.getLogger('bootstrap')
_INTERVALS = (
('weeks', 604800), # 60 * 60 * 24 * 7
('days', 86400), # 60 * 60 * 24
('hours', 3600), # 60 * 60
('minutes', 60),
('seconds', 1),
)
def human_time(seconds, granularity=2):
'''Returns a human readable time string like "1 day, 2 hours"'''
result = []
for name, count in _INTERVALS:
value = seconds // count
if value:
seconds -= value * count
if value == 1:
name = name.rstrip('s')
result.append("{} {}".format(int(value), name))
else:
# Add a blank if we're in the middle of other values
if len(result) > 0:
result.append(None)
if not result:
if seconds < 1.0:
return '%.2f seconds' % seconds
else:
if seconds == 1:
return '1 second'
else:
return '%d seconds' % seconds
return ', '.join([x for x in result[:granularity] if x is not None])
def run_cmdline(cmd, env=None):
'''Runs a command on a environment, logs output and reports status
Parameters:
cmd (list): The command to run, with parameters separated on a list
env (dict, Optional): Environment to use for running the program on. If not
set, use :py:obj:`os.environ`.
'''
if env is None: env = os.environ
logger.info('(system) %s' % ' '.join(cmd))
start = time.time()
out = b''
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
env=env)
for line in iter(p.stdout.readline, b''):
sys.stdout.write(line.decode(sys.stdout.encoding))
sys.stdout.flush()
if p.wait() != 0:
raise RuntimeError("command `%s' exited with error state (%d)" % \
(' '.join(cmd), p.returncode))
total = time.time() - start
logger.info('command took %s' % human_time(total))
def touch(path):
'''Python-implementation of the "touch" command-line application'''
with open(path, 'a'):
os.utime(path, None)
def merge_conda_cache(cache, prefix):
'''Merges conda pkg caches and conda-bld folders'''
pkgs_dir = os.path.join(prefix, 'pkgs')
pkgs_urls_txt = os.path.join(pkgs_dir, 'urls.txt')
if not os.path.exists(pkgs_dir):
logger.info('mkdir -p %s', pkgs_dir)
os.makedirs(pkgs_dir)
logger.info('touch %s', pkgs_urls_txt)
touch(pkgs_urls_txt)
# move packages on cache/pkgs to pkgs_dir
cached_pkgs_dir = os.path.join(cache, 'pkgs')
cached_packages = glob.glob(os.path.join(cached_pkgs_dir, '*.tar.bz2'))
cached_packages = [k for k in cached_packages if not \
k.startswith(os.environ['CI_PROJECT_NAME'] + '-')]
logger.info('Merging %d cached conda packages', len(cached_packages))
for k in cached_packages:
dst = os.path.join(pkgs_dir, os.path.basename(k))
logger.debug('(move) %s -> %s', k, dst)
os.rename(k, dst)
# merge urls.txt files
logger.info('Merging urls.txt files from cache...')
urls = []
cached_pkgs_urls_txt = os.path.join(cached_pkgs_dir, 'urls.txt')
with open(pkgs_urls_txt, 'rb') as f1, \
open(cached_pkgs_urls_txt, 'rb') as f2:
data = set(f1.readlines() + f2.readlines())
data = sorted(list(data))
with open(pkgs_urls_txt, 'wb') as f:
f.writelines(data)
pkgs_urls = os.path.join(pkgs_dir, 'urls')
touch(pkgs_urls)
# move conda-bld build results
cached_conda_bld = os.path.join(cache, 'conda-bld')
if os.path.exists(cached_conda_bld):
dst = os.path.join(prefix, 'conda-bld')
logger.info('(move) %s -> %s', cached_conda_bld, dst)
os.rename(cached_conda_bld, dst)
def get_miniconda_sh():
'''Retrieves the miniconda3 installer for the current system'''
import http.client
server = 'repo.continuum.io' #https
path = '/miniconda/Miniconda3-latest-%s-x86_64.sh'
if platform.system() == 'Darwin':
path = path % 'MacOSX'
else:
path = path % 'Linux'
logger.info('Connecting to https://%s...', server)
conn = http.client.HTTPSConnection(server)
conn.request("GET", path)
r1 = conn.getresponse()
assert r1.status == 200, 'Request for https://%s%s - returned status %d ' \
'(%s)' % (server, path, r1.status, r1.reason)
dst = 'miniconda.sh'
logger.info('(download) https://%s%s -> %s...', server, path, dst)
with open(dst, 'wb') as f:
f.write(r1.read())
def install_miniconda(prefix):
'''Creates a new miniconda installation'''
logger.info("Installing miniconda in %s...", prefix)
if not os.path.exists('miniconda.sh'): #re-downloads installer
get_miniconda_sh()
else:
logger.info("Re-using cached miniconda3 installer")
cached = None
if os.path.exists(prefix): #this is the previous cache, move it
cached = prefix + '.cached'
if os.path.exists(cached):
logger.info('(rmtree) %s', cached)
shutil.rmtree(cached)
logger.info('(move) %s -> %s', prefix, cached)
os.rename(prefix, cached)
run_cmdline(['bash', 'miniconda.sh', '-b', '-p', prefix])
if cached is not None:
merge_conda_cache(cached, prefix)
shutil.rmtree(cached)
def get_channels(public, stable):
'''Returns the relevant conda channels to consider if building project
The subset of channels to be returned depends on the visibility and stability
of the package being built. Here are the rules:
* public and stable: only returns the public stable channel(s)
* public and not stable: returns both public stable and beta channels
* not public and stable: returns both public and private stable channels
* not public and not stable: returns all channels
Beta channels have priority over stable channels, if returned. Private
channels have priority over public channles, if turned.
Args:
public: Boolean indicating if we're supposed to include only public
channels
stable: Boolean indicating if we're supposed to include only stable
channels
Returns: a list of channels that need to be considered.
'''
server = "http://www.idiap.ch"
channels = []
if not public:
if not stable: #allowed private channels
channels += [server + '/private/conda/label/beta'] #allowed betas
channels += [server + '/private/conda']
if not stable:
channels += [server + '/public/conda/label/beta'] #allowed betas
channels += [server + '/public/conda']
return channels
def add_channels_condarc(channels, condarc):
'''Appends passed channel list to condarc file, print contents'''
with open(condarc, 'at') as f:
f.write('channels:\n')
for k in channels:
f.write(' - %s\n' % k)
with open(condarc, 'rt') as f:
logger.info('Contents of $CONDARC:\n%s', f.read())
def setup_logger():
'''Sets-up the logging for this command at level ``INFO``'''
warn_err = logging.StreamHandler(sys.stderr)
warn_err.setLevel(logging.WARNING)
logger.addHandler(warn_err)
# debug and info messages are written to sys.stdout
class _InfoFilter:
def filter(self, record):
return record.levelno <= logging.INFO
debug_info = logging.StreamHandler(sys.stdout)
debug_info.setLevel(logging.DEBUG)
debug_info.addFilter(_InfoFilter())
logger.addHandler(debug_info)
formatter = logging.Formatter('%(levelname)s@%(asctime)s: %(message)s')
for handler in logger.handlers:
handler.setFormatter(formatter)
logger.setLevel(logging.INFO)
if __name__ == '__main__':
if len(sys.argv) == 1:
print(__doc__ % sys.argv[0])
sys.exit(1)
setup_logger()
if sys.argv[1] == 'test':
# sets up local variables for testing
os.environ['CI_PROJECT_DIR'] = os.path.realpath(os.curdir)
os.environ['CI_PROJECT_NAME'] = 'bob.devtools'
os.environ['CONDA_ROOT'] = os.path.join(os.environ['CI_PROJECT_DIR'],
'miniconda')
prefix = os.environ['CONDA_ROOT']
logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
workdir = os.environ['CI_PROJECT_DIR']
logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
install_miniconda(prefix)
conda_bin = os.path.join(prefix, 'bin', 'conda')
# creates the condarc file
condarc = os.path.join(prefix, 'condarc')
logger.info('(create) %s', condarc)
with open(condarc, 'wt') as f:
f.write(BASE_CONDARC)
os.environ['CONDARC'] = condarc
logger.info('os.environ["%s"] = %s', 'CONDARC', condarc)
conda_version = '4'
conda_build_version = '3'
if sys.argv[1] in ('build', 'test'):
# simple - just use the defaults channels when self building
add_channels_condarc(['defaults'], condarc)
run_cmdline([conda_bin, 'install', '-n', 'base',
'python',
'conda=%s' % conda_version,
'conda-build=%s' % conda_build_version,
])
elif sys.argv[1] == 'local':
# index the locally built packages
run_cmdline([conda_bin, 'install', '-n', 'base',
'python',
'conda=%s' % conda_version,
'conda-build=%s' % conda_build_version,
])
conda_bld_path = os.path.join(prefix, 'conda-bld')
run_cmdline([conda_bin, 'index', conda_bld_path])
# add the locally build directory before defaults, boot from there
channels = get_channels(public=True, stable=True)
add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
elif sys.argv[1] in ('beta', 'stable'):
# installs from channel
channels = get_channels(
public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
stable=os.environ.get('CI_COMMIT_TAG') is not None)
add_channels_condarc(channels + ['defaults'], condarc)
run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
else:
logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'")
logger.error("The value '%s' is not currently supported", sys.argv[1])
sys.exit(1)
# clean up
run_cmdline([conda_bin, 'clean', '--lock'])
# print conda information for debugging purposes
run_cmdline([conda_bin, 'info'])
ci/bootstrap.sh deleted 100755 → 0
+ 0
− 161
View file @ 1808f37b
#!/usr/bin/env bash
# Bootstraps a new conda installation and prepares base environment
# if "local" is passed as parameter, then self installs an already built
# version of bob.devtools available on your conda-bld directory. If you pass
# "beta", then it bootstraps from the package installed on our conda beta
# channel. If you pass "stable", then it bootstraps installing the package
# available on the stable channel.
#
# If bootstrapping anything else than "build", then provide a second argument
# with the name of the environment that one wants to create with an
# installation of bob.devtools.
# datetime prefix for logging
log_datetime() {
echo "($(date +%T.%3N))"
}
# Functions for coloring echo commands
log_info() {
echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
}
log_error() {
echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
}
# Function for running command and echoing results
run_cmd() {
log_info "$ ${@}"
${@}
local status=$?
if [ ${status} != 0 ]; then
log_error "Command Failed \"${@}\""
exit ${status}
fi
}
# Checks just if the variable is defined and has non-zero length
check_defined() {
if [ -z "${!1+abc}" ]; then
log_error "Variable ${1} is undefined - aborting...";
exit 1
elif [ -z "${!1}" ]; then
log_error "Variable ${1} is zero-length - aborting...";
exit 1
fi
log_info "${1}=${!1}"
}
# merges conda cache folders
# $1: Path to the main cache to keep. The directory must exist.
# $2: Path to the extra cache to be merged into main cache
merge_conda_cache() {
if [ -e ${1} ]; then
_cached_urlstxt="${2}/pkgs/urls.txt"
_urlstxt="${1}/pkgs/urls.txt"
if [ -e ${2}/pkgs ]; then
log_info "Merging urls.txt and packages with cached files..."
mv ${2}/pkgs/*.tar.bz2 ${1}/pkgs
run_cmd rm -f ${1}/pkgs/${CI_PROJECT_NAME}-*-*_*.tar.bz2
cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt}
else
run_cmd mkdir -p ${1}/pkgs
run_cmd touch ${1}/pkgs/urls.txt
fi
run_cmd touch ${1}/pkgs/urls
if [ -d ${2}/conda-bld ]; then
log_info "Moving conda-bld packages (artifacts)..."
run_cmd mv ${2}/conda-bld ${1}
fi
fi
}
# installs a miniconda installation.
# $1: Path to where to install miniconda.
install_miniconda() {
log_info "Installing miniconda in ${1} ..."
# checks if miniconda.sh exists
if [ ! -e miniconda.sh ]; then
log_info "Downloading latest miniconda3 installer..."
# downloads the latest conda installation script
if [ "$(uname -s)" == "Linux" ]; then
_os="Linux"
else
_os="MacOSX"
fi
obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh
run_cmd curl --silent --output miniconda.sh ${obj}
else
log_info "Re-using cached miniconda3 installer..."
ls -l miniconda.sh
fi
# move cache to a different folder if it exists
if [ -e ${1} ]; then
run_cmd mv ${1} ${1}.cached
fi
# install miniconda
run_cmd bash miniconda.sh -b -p ${1}
# Put back cache and merge urls.txt
merge_conda_cache ${1} ${1}.cached
# remove the backup cache folder
rm -rf ${1}.cached
# List currently available packages on cache
# run_cmd ls -l ${1}/pkgs/
# run_cmd cat ${1}/pkgs/urls.txt
hash -r
}
check_defined CONDA_ROOT
check_defined CI_PROJECT_DIR
export CONDARC="${CONDA_ROOT}/condarc"
check_defined CONDARC
# checks if a conda installation exists. Otherwise, installs one
if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
install_miniconda ${CONDA_ROOT}
fi
run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
echo "Contents of \`${CONDARC}':"
cat ${CONDARC}
# setup conda-channels
CONDA_CHANNEL_ROOT="http://www.idiap.ch/public/conda"
check_defined CONDA_CHANNEL_ROOT
CONDA_CLI_CHANNELS="-c ${CONDA_CHANNEL_ROOT} -c defaults"
# creates a base installation depending on the purpose
if [ "${1}" == "build" ]; then
run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
elif [ "${1}" == "local" ]; then
# updates the base installation, installs conda-build
run_cmd ls -l ${CONDA_ROOT}/conda-bld
run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
CONDA_CLI_CHANNELS="-c ${CONDA_ROOT}/conda-bld ${CONDA_CLI_CHANNELS}"
run_cmd ${CONDA_ROOT}/bin/conda index ${CONDA_ROOT}/conda-bld
run_cmd ls -l ${CONDA_ROOT}/conda-bld
run_cmd ls -l ${CONDA_ROOT}/conda-bld/noarch/
run_cmd ${CONDA_ROOT}/bin/conda create -n "${2}" --override-channels ${CONDA_CLI_CHANNELS} bob.devtools
elif [ "${1}" == "beta" ] || [ "${1}" == "stable" ]; then
run_cmd ${CONDA_ROOT}/bin/conda create -n "${2}" --override-channels ${CONDA_CLI_CHANNELS} bob.devtools
else
log_error "Bootstrap with 'build', or 'local|beta|stable <name>'"
log_error "The value '${1}' is not currently supported"
exit 1
fi
# cleans up
run_cmd ${CONDA_ROOT}/bin/conda clean --lock
# print conda information for debugging purposes
run_cmd ${CONDA_ROOT}/bin/conda info
conda/meta.yaml
+ 1
− 1
View file @ 80b4a8d1
......@@ -61,7 +61,7 @@ test:
- bdt release --help
- bdt visibility --help
- bdt dumpsphinx --help
- bdt bootstrap --help
- bdt create --help
- bdt build --help
- bdt getpath --help
- bdt caupdate --help
......
doc/api.rst
+ 6
− 0
View file @ 80b4a8d1
......@@ -12,7 +12,9 @@
bob.devtools.constants
bob.devtools.release
bob.devtools.changelog
bob.devtools.create
bob.devtools.bootstrap
bob.devtools.build
bob.devtools.webdav3.client
......@@ -31,8 +33,12 @@ Detailed Information
.. automodule:: bob.devtools.changelog
.. automodule:: bob.devtools.create
.. automodule:: bob.devtools.bootstrap
.. automodule:: bob.devtools.build
WebDAV Python Client
--------------------
......
setup.py
+ 1
− 1
View file @ 80b4a8d1
......@@ -48,7 +48,7 @@ setup(
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
'create = bob.devtools.scripts.create:create',
'build = bob.devtools.scripts.build:build',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment