diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1ecd818b2caa5ac57cedb8d102abd18d5709ec68..5bbc8a4042f064ff021711925c56c4a266d663a5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ stages:
.build_template: &build_job
stage: build
before_script:
- - python3 ./ci/bootstrap.py build
+ - python3 ./bob/devtools/bootstrap.py build
script:
- ./ci/build.sh
cache: &build_caches
@@ -74,7 +74,7 @@ build_macosx_36:
.deploy_template: &deploy_job
stage: deploy
before_script:
- - python3 ./ci/bootstrap.py local myenv
+ - python3 ./bob/devtools/bootstrap.py local myenv
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
@@ -115,7 +115,7 @@ pypi:
except:
- branches
before_script:
- - python3 ./ci/bootstrap.py local myenv
+ - python3 ./bob/devtools/bootstrap.py local myenv
script:
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate myenv
diff --git a/bob/devtools/bootstrap.py b/bob/devtools/bootstrap.py
index 5ea04ae97cd686f3034421d670f031b5ab8e97da..930ffafe3e1ef571c884f4b08b40af2c8afedb62 100644
--- a/bob/devtools/bootstrap.py
+++ b/bob/devtools/bootstrap.py
@@ -1,128 +1,389 @@
-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+'''Bootstraps a new miniconda installation and prepares it for development
+
+This command uses a bare-minimum python3 installation (with SSL support) to
+bootstrap a new miniconda installation preset for the defined activity. It is
+primarily intended for CI operation and prefixes build and deployment steps.
+
+Usage: python3 %s <cmd> build|local|beta|stable [<name>]
+
+Arguments:
+
+ <cmd> How to prepare the current environment. Use:
+
+ build to build bob.devtools
+ local to bootstrap deploy|pypi stages for bob.devtools builds
+ beta to bootstrap CI environment for beta builds
+ stable to bootstrap CI environment for stable builds
+ test to locally test this bootstrap script
+
+ <name> (optional) if command is one of ``local|beta|stable`` provide the
+ name of env for bob.devtools installation')
+'''
+
+
+BASE_CONDARC = '''\
+default_channels:
+ - https://repo.anaconda.com/pkgs/main
+ - https://repo.anaconda.com/pkgs/free
+ - https://repo.anaconda.com/pkgs/r
+ - https://repo.anaconda.com/pkgs/pro
+add_pip_as_python_dependency: false #!final
+changeps1: false #!final
+always_yes: true #!final
+quiet: true #!final
+show_channel_urls: true #!final
+anaconda_upload: false #!final
+ssl_verify: false #!final
+'''
-'''Methods to bootstrap working environments based on conda-packages'''
import os
-import json
+import sys
+import glob
+import time
import shutil
+import platform
import subprocess
import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('bootstrap')
+
-import yaml
+_INTERVALS = (
+ ('weeks', 604800), # 60 * 60 * 24 * 7
+ ('days', 86400), # 60 * 60 * 24
+ ('hours', 3600), # 60 * 60
+ ('minutes', 60),
+ ('seconds', 1),
+ )
+def human_time(seconds, granularity=2):
+ '''Returns a human readable time string like "1 day, 2 hours"'''
-def make_conda_config(config, python, append_file, condarc):
+ result = []
- from conda_build.api import get_or_merge_config
- from conda_build.conda_interface import url_path
+ for name, count in _INTERVALS:
+ value = seconds // count
+ if value:
+ seconds -= value * count
+ if value == 1:
+ name = name.rstrip('s')
+ result.append("{} {}".format(int(value), name))
+ else:
+ # Add a blank if we're in the middle of other values
+ if len(result) > 0:
+ result.append(None)
- with open(condarc, 'rb') as f:
- condarc_options = yaml.load(f)
+ if not result:
+ if seconds < 1.0:
+ return '%.2f seconds' % seconds
+ else:
+ if seconds == 1:
+ return '1 second'
+ else:
+ return '%d seconds' % seconds
- retval = get_or_merge_config(None, variant_config_files=config,
- python=python, append_sections_file=append_file, **condarc_options)
+ return ', '.join([x for x in result[:granularity] if x is not None])
- retval.channel_urls = []
- for url in condarc_options['channels']:
- # allow people to specify relative or absolute paths to local channels
- # These channels still must follow conda rules - they must have the
- # appropriate platform-specific subdir (e.g. win-64)
- if os.path.isdir(url):
- if not os.path.isabs(url):
- url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
- url = url_path(url)
- retval.channel_urls.append(url)
+def run_cmdline(cmd, env=None):
+ '''Runs a command on a environment, logs output and reports status
- return retval
+ Parameters:
-def get_rendered_metadata(recipe_dir, config):
- '''Renders the recipe and returns the interpreted YAML file'''
+ cmd (list): The command to run, with parameters separated on a list
- from conda_build.api import render
- return render(recipe_dir, config=config)
+ env (dict, Optional): Environment to use for running the program on. If not
+ set, use :py:obj:`os.environ`.
+ '''
-def get_parsed_recipe(metadata):
- '''Renders the recipe and returns the interpreted YAML file'''
+ if env is None: env = os.environ
- from conda_build.api import output_yaml
- output = output_yaml(metadata[0][0])
- return yaml.load(output)
+ logger.info('(system) %s' % ' '.join(cmd))
+ start = time.time()
+ out = b''
-def remove_pins(deps):
- return [l.split()[0] for l in deps]
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ env=env)
+ for line in iter(p.stdout.readline, b''):
+ sys.stdout.write(line.decode(sys.stdout.encoding))
+ sys.stdout.flush()
-def parse_dependencies(recipe_dir, config):
+ if p.wait() != 0:
+ raise RuntimeError("command `%s' exited with error state (%d)" % \
+ (' '.join(cmd), p.returncode))
- metadata = get_rendered_metadata(recipe_dir, config)
- recipe = get_parsed_recipe(metadata)
- return remove_pins(recipe['requirements'].get('build', [])) + \
- remove_pins(recipe['requirements'].get('host', [])) + \
- recipe['requirements'].get('run', []) + \
- recipe.get('test', {}).get('requires', []) + \
- ['bob.buildout', 'mr.developer', 'ipdb']
- # by last, packages required for local dev
+ total = time.time() - start
+ logger.info('command took %s' % human_time(total))
-def get_env_directory(conda, name):
- cmd = [conda, 'env', 'list', '--json']
- output = subprocess.check_output(cmd)
- data = json.loads(output)
- retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
- if retval:
- return retval[0]
- return None
+def touch(path):
+ '''Python-implementation of the "touch" command-line application'''
-def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
+ with open(path, 'a'):
+ os.utime(path, None)
- specs = []
- for k in packages:
- k = ' '.join(k.split()[:2]) # remove eventual build string
- if any(elem in k for elem in '><|'):
- specs.append(k.replace(' ', ''))
- else:
- specs.append(k.replace(' ', '='))
-
- # if the current environment exists, delete it first
- envdir = get_env_directory(conda, name)
- if envdir is not None:
- if overwrite:
- cmd = [conda, 'env', 'remove', '--yes', '--name', name]
- logger.debug('$ ' + ' '.join(cmd))
- if not dry_run:
- status = subprocess.call(cmd)
- if status != 0:
- return status
- else:
- raise RuntimeError('environment `%s\' exists in `%s\' - use '
- '--overwrite to overwrite' % (name, envdir))
-
- cmd = [conda, 'create', '--yes', '--name', name]
- if dry_run:
- cmd.append('--dry-run')
- if use_local:
- cmd.append('--use-local')
- cmd.extend(sorted(specs))
- logger.debug('$ ' + ' '.join(cmd))
- status = subprocess.call(cmd)
- if status != 0:
- return status
-
- # copy the used condarc file to the just created environment
- if not dry_run:
- # get envdir again - it may just be created!
- envdir = get_env_directory(conda, name)
- destrc = os.path.join(envdir, '.condarc')
- logger.debug('$ cp %s -> %s' % (condarc, destrc))
- shutil.copy2(condarc, destrc)
-
- return status
+
+def merge_conda_cache(cache, prefix):
+ '''Merges conda pkg caches and conda-bld folders'''
+
+ pkgs_dir = os.path.join(prefix, 'pkgs')
+ pkgs_urls_txt = os.path.join(pkgs_dir, 'urls.txt')
+ if not os.path.exists(pkgs_dir):
+ logger.info('mkdir -p %s', pkgs_dir)
+ os.makedirs(pkgs_dir)
+ logger.info('touch %s', pkgs_urls_txt)
+ touch(pkgs_urls_txt)
+
+ # move packages on cache/pkgs to pkgs_dir
+ cached_pkgs_dir = os.path.join(cache, 'pkgs')
+ cached_packages = glob.glob(os.path.join(cached_pkgs_dir, '*.tar.bz2'))
+ cached_packages = [k for k in cached_packages if not \
+ k.startswith(os.environ['CI_PROJECT_NAME'] + '-')]
+ logger.info('Merging %d cached conda packages', len(cached_packages))
+ for k in cached_packages:
+ dst = os.path.join(pkgs_dir, os.path.basename(k))
+ logger.debug('(move) %s -> %s', k, dst)
+ os.rename(k, dst)
+
+ # merge urls.txt files
+ logger.info('Merging urls.txt files from cache...')
+ urls = []
+ cached_pkgs_urls_txt = os.path.join(cached_pkgs_dir, 'urls.txt')
+ with open(pkgs_urls_txt, 'rb') as f1, \
+ open(cached_pkgs_urls_txt, 'rb') as f2:
+ data = set(f1.readlines() + f2.readlines())
+ data = sorted(list(data))
+ with open(pkgs_urls_txt, 'wb') as f:
+ f.writelines(data)
+
+ pkgs_urls = os.path.join(pkgs_dir, 'urls')
+ touch(pkgs_urls)
+
+ # move conda-bld build results
+ cached_conda_bld = os.path.join(cache, 'conda-bld')
+ if os.path.exists(cached_conda_bld):
+ dst = os.path.join(prefix, 'conda-bld')
+ logger.info('(move) %s -> %s', cached_conda_bld, dst)
+ os.rename(cached_conda_bld, dst)
+
+
+def get_miniconda_sh():
+ '''Retrieves the miniconda3 installer for the current system'''
+
+ import http.client
+
+ server = 'repo.continuum.io' #https
+ path = '/miniconda/Miniconda3-latest-%s-x86_64.sh'
+ if platform.system() == 'Darwin':
+ path = path % 'MacOSX'
+ else:
+ path = path % 'Linux'
+
+ logger.info('Connecting to https://%s...', server)
+ conn = http.client.HTTPSConnection(server)
+ conn.request("GET", path)
+ r1 = conn.getresponse()
+
+ assert r1.status == 200, 'Request for https://%s%s - returned status %d ' \
+ '(%s)' % (server, path, r1.status, r1.reason)
+
+ dst = 'miniconda.sh'
+ logger.info('(download) https://%s%s -> %s...', server, path, dst)
+ with open(dst, 'wb') as f:
+ f.write(r1.read())
+
+
+def install_miniconda(prefix):
+ '''Creates a new miniconda installation'''
+
+ logger.info("Installing miniconda in %s...", prefix)
+
+ if not os.path.exists('miniconda.sh'): #re-downloads installer
+ get_miniconda_sh()
+ else:
+ logger.info("Re-using cached miniconda3 installer")
+
+ cached = None
+ if os.path.exists(prefix): #this is the previous cache, move it
+ cached = prefix + '.cached'
+ if os.path.exists(cached):
+ logger.info('(rmtree) %s', cached)
+ shutil.rmtree(cached)
+ logger.info('(move) %s -> %s', prefix, cached)
+ os.rename(prefix, cached)
+
+ run_cmdline(['bash', 'miniconda.sh', '-b', '-p', prefix])
+ if cached is not None:
+ merge_conda_cache(cached, prefix)
+ shutil.rmtree(cached)
+
+
+def get_channels(public, stable):
+ '''Returns the relevant conda channels to consider if building project
+
+ The subset of channels to be returned depends on the visibility and stability
+ of the package being built. Here are the rules:
+
+ * public and stable: only returns the public stable channel(s)
+ * public and not stable: returns both public stable and beta channels
+ * not public and stable: returns both public and private stable channels
+ * not public and not stable: returns all channels
+
+ Beta channels have priority over stable channels, if returned. Private
+ channels have priority over public channles, if turned.
+
+
+ Args:
+
+ public: Boolean indicating if we're supposed to include only public
+ channels
+ stable: Boolean indicating if we're supposed to include only stable
+ channels
+
+
+ Returns: a list of channels that need to be considered.
+
+ '''
+
+ server = "http://www.idiap.ch"
+ channels = []
+
+ if not public:
+ if not stable: #allowed private channels
+ channels += [server + '/private/conda/label/beta'] #allowed betas
+ channels += [server + '/private/conda']
+ if not stable:
+ channels += [server + '/public/conda/label/beta'] #allowed betas
+ channels += [server + '/public/conda']
+
+ return channels
+
+
+def add_channels_condarc(channels, condarc):
+ '''Appends passed channel list to condarc file, print contents'''
+
+ with open(condarc, 'at') as f:
+ f.write('channels:\n')
+ for k in channels:
+ f.write(' - %s\n' % k)
+
+ with open(condarc, 'rt') as f:
+ logger.info('Contents of $CONDARC:\n%s', f.read())
+
+
+def setup_logger():
+ '''Sets-up the logging for this command at level ``INFO``'''
+
+ warn_err = logging.StreamHandler(sys.stderr)
+ warn_err.setLevel(logging.WARNING)
+ logger.addHandler(warn_err)
+
+ # debug and info messages are written to sys.stdout
+
+ class _InfoFilter:
+ def filter(self, record):
+ return record.levelno <= logging.INFO
+
+ debug_info = logging.StreamHandler(sys.stdout)
+ debug_info.setLevel(logging.DEBUG)
+ debug_info.addFilter(_InfoFilter())
+ logger.addHandler(debug_info)
+
+ formatter = logging.Formatter('%(levelname)s@%(asctime)s: %(message)s')
+
+ for handler in logger.handlers:
+ handler.setFormatter(formatter)
+
+ logger.setLevel(logging.INFO)
+
+
+if __name__ == '__main__':
+
+ if len(sys.argv) == 1:
+ print(__doc__ % sys.argv[0])
+ sys.exit(1)
+
+ setup_logger()
+
+ if sys.argv[1] == 'test':
+ # sets up local variables for testing
+ os.environ['CI_PROJECT_DIR'] = os.path.realpath(os.curdir)
+ os.environ['CI_PROJECT_NAME'] = 'bob.devtools'
+ os.environ['CONDA_ROOT'] = os.path.join(os.environ['CI_PROJECT_DIR'],
+ 'miniconda')
+
+ prefix = os.environ['CONDA_ROOT']
+ logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
+
+ workdir = os.environ['CI_PROJECT_DIR']
+ logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
+
+ install_miniconda(prefix)
+ conda_bin = os.path.join(prefix, 'bin', 'conda')
+
+ # creates the condarc file
+ condarc = os.path.join(prefix, 'condarc')
+ logger.info('(create) %s', condarc)
+ with open(condarc, 'wt') as f:
+ f.write(BASE_CONDARC)
+ os.environ['CONDARC'] = condarc
+ logger.info('os.environ["%s"] = %s', 'CONDARC', condarc)
+
+ conda_version = '4'
+ conda_build_version = '3'
+
+ if sys.argv[1] in ('build', 'test'):
+
+ # simple - just use the defaults channels when self building
+ add_channels_condarc(['defaults'], condarc)
+ run_cmdline([conda_bin, 'install', '-n', 'base',
+ 'python',
+ 'conda=%s' % conda_version,
+ 'conda-build=%s' % conda_build_version,
+ ])
+
+ elif sys.argv[1] == 'local':
+
+ # index the locally built packages
+ run_cmdline([conda_bin, 'install', '-n', 'base',
+ 'python',
+ 'conda=%s' % conda_version,
+ 'conda-build=%s' % conda_build_version,
+ ])
+ conda_bld_path = os.path.join(prefix, 'conda-bld')
+ run_cmdline([conda_bin, 'index', conda_bld_path])
+ # add the locally build directory before defaults, boot from there
+ channels = get_channels(public=True, stable=True)
+ add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
+ run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
+
+ elif sys.argv[1] in ('beta', 'stable'):
+
+ # installs from channel
+ channels = get_channels(
+ public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
+ stable=os.environ.get('CI_COMMIT_TAG') is not None)
+ add_channels_condarc(channels + ['defaults'], condarc)
+ run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
+
+ else:
+
+ logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'")
+ logger.error("The value '%s' is not currently supported", sys.argv[1])
+ sys.exit(1)
+
+ # clean up
+ run_cmdline([conda_bin, 'clean', '--lock'])
+
+ # print conda information for debugging purposes
+ run_cmdline([conda_bin, 'info'])
diff --git a/ci/nextbuild.py b/bob/devtools/build.py
similarity index 100%
rename from ci/nextbuild.py
rename to bob/devtools/build.py
diff --git a/bob/devtools/create.py b/bob/devtools/create.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed3077ebb3c9e564a0b57911f48d118593d45f09
--- /dev/null
+++ b/bob/devtools/create.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+'''Methods to create working environments based on conda-packages'''
+
+import os
+import json
+import shutil
+import subprocess
+
+import logging
+logger = logging.getLogger(__name__)
+
+import yaml
+
+
+def make_conda_config(config, python, append_file, condarc):
+
+ from conda_build.api import get_or_merge_config
+ from conda_build.conda_interface import url_path
+
+ with open(condarc, 'rb') as f:
+ condarc_options = yaml.load(f)
+
+ retval = get_or_merge_config(None, variant_config_files=config,
+ python=python, append_sections_file=append_file, **condarc_options)
+
+ retval.channel_urls = []
+
+ for url in condarc_options['channels']:
+ # allow people to specify relative or absolute paths to local channels
+ # These channels still must follow conda rules - they must have the
+ # appropriate platform-specific subdir (e.g. win-64)
+ if os.path.isdir(url):
+ if not os.path.isabs(url):
+ url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
+ url = url_path(url)
+ retval.channel_urls.append(url)
+
+ return retval
+
+
+def get_rendered_metadata(recipe_dir, config):
+ '''Renders the recipe and returns the interpreted YAML file'''
+
+ from conda_build.api import render
+ return render(recipe_dir, config=config)
+
+
+def get_parsed_recipe(metadata):
+ '''Renders the recipe and returns the interpreted YAML file'''
+
+ from conda_build.api import output_yaml
+ output = output_yaml(metadata[0][0])
+ return yaml.load(output)
+
+
+def remove_pins(deps):
+ return [l.split()[0] for l in deps]
+
+
+def parse_dependencies(recipe_dir, config):
+
+ metadata = get_rendered_metadata(recipe_dir, config)
+ recipe = get_parsed_recipe(metadata)
+ return remove_pins(recipe['requirements'].get('build', [])) + \
+ remove_pins(recipe['requirements'].get('host', [])) + \
+ recipe['requirements'].get('run', []) + \
+ recipe.get('test', {}).get('requires', []) + \
+ ['bob.buildout', 'mr.developer', 'ipdb']
+ # by last, packages required for local dev
+
+
+def get_env_directory(conda, name):
+
+ cmd = [conda, 'env', 'list', '--json']
+ output = subprocess.check_output(cmd)
+ data = json.loads(output)
+ retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
+ if retval:
+ return retval[0]
+ return None
+
+
+def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
+
+ specs = []
+ for k in packages:
+ k = ' '.join(k.split()[:2]) # remove eventual build string
+ if any(elem in k for elem in '><|'):
+ specs.append(k.replace(' ', ''))
+ else:
+ specs.append(k.replace(' ', '='))
+
+ # if the current environment exists, delete it first
+ envdir = get_env_directory(conda, name)
+ if envdir is not None:
+ if overwrite:
+ cmd = [conda, 'env', 'remove', '--yes', '--name', name]
+ logger.debug('$ ' + ' '.join(cmd))
+ if not dry_run:
+ status = subprocess.call(cmd)
+ if status != 0:
+ return status
+ else:
+ raise RuntimeError('environment `%s\' exists in `%s\' - use '
+ '--overwrite to overwrite' % (name, envdir))
+
+ cmd = [conda, 'create', '--yes', '--name', name]
+ if dry_run:
+ cmd.append('--dry-run')
+ if use_local:
+ cmd.append('--use-local')
+ cmd.extend(sorted(specs))
+ logger.debug('$ ' + ' '.join(cmd))
+ status = subprocess.call(cmd)
+ if status != 0:
+ return status
+
+ # copy the used condarc file to the just created environment
+ if not dry_run:
+ # get envdir again - it may just be created!
+ envdir = get_env_directory(conda, name)
+ destrc = os.path.join(envdir, '.condarc')
+ logger.debug('$ cp %s -> %s' % (condarc, destrc))
+ shutil.copy2(condarc, destrc)
+
+ return status
diff --git a/bob/devtools/scripts/build.py b/bob/devtools/scripts/build.py
index 53c6e35c69122417b234cc290c5df34835ec3d23..8aabd9157b5d0c2b4c31f80b083ecd032f1a2fc1 100644
--- a/bob/devtools/scripts/build.py
+++ b/bob/devtools/scripts/build.py
@@ -12,7 +12,8 @@ import click
from . import bdt
from ..log import verbosity_option
from ..conda import next_build_number, osname
-from ..bootstrap import get_rendered_metadata, get_parsed_recipe
+from ..create import get_rendered_metadata, get_parsed_recipe, \
+ make_conda_config
from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER, MATPLOTLIB_RCDIR, set_environment
@@ -85,7 +86,6 @@ def build(recipe_dir, python, condarc, config, channel, no_test, append_file,
logger.debug("CONDARC=%s", condarc)
- from ..bootstrap import make_conda_config
conda_config = make_conda_config(config, python, append_file, condarc)
set_environment('LANG', 'en_US.UTF-8', os.environ)
diff --git a/bob/devtools/scripts/bootstrap.py b/bob/devtools/scripts/create.py
similarity index 93%
rename from bob/devtools/scripts/bootstrap.py
rename to bob/devtools/scripts/create.py
index 07d9f14477e21639170cfb0b04ae2ff6b8718f88..e09bca5355f46d910a87e16b74539fc1905786c8 100644
--- a/bob/devtools/scripts/bootstrap.py
+++ b/bob/devtools/scripts/create.py
@@ -12,7 +12,7 @@ import yaml
from . import bdt
from ..log import verbosity_option
-from ..bootstrap import parse_dependencies, conda_create, make_conda_config
+from ..create import parse_dependencies, conda_create, make_conda_config
from ..constants import CONDARC, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
SERVER
@@ -24,7 +24,7 @@ Examples:
\b
$ cd bob.package.foo
- $ bdt bootstrap -vv myenv
+ $ bdt create -vv myenv
The above command assumes the directory `conda' exists on the current directory and that it contains a file called `meta.yaml' containing the recipe for the package you want to create a development environment for.
@@ -33,12 +33,12 @@ Examples:
2. By default, we use the native python version of your conda installation as the default python version to use for the newly created environment. You may select a different one with `--python=X.Y':
- $ bdt bootstrap -vv --python=3.6 myenv
+ $ bdt create -vv --python=3.6 myenv
3. By default, we use our own condarc and `conda_build_config.yaml` files that are used in creating packages for our CI/CD system. If you wish to use your own, specify them on the command line:
- $ bdt bootstrap -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
+ $ bdt create -vv --python=3.6 --config=config.yaml --condarc=~/.condarc myenv
Notice the condarc file **must** end in `condarc', or conda will complain.
@@ -49,7 +49,7 @@ Examples:
shell will be printed:
- $ bdt bootstrap -vvv --dry-run myenv
+ $ bdt create -vvv --dry-run myenv
''')
@click.argument('name')
@click.argument('recipe-dir', required=False, type=click.Path(file_okay=False,
@@ -80,7 +80,7 @@ Examples:
help='Allow the use of local channels for package retrieval')
@verbosity_option()
@bdt.raise_on_error
-def bootstrap(name, recipe_dir, python, overwrite, condarc, config,
+def create(name, recipe_dir, python, overwrite, condarc, config,
append_file, docserver, dry_run, use_local):
"""Creates a development environment for a recipe
diff --git a/ci/bootstrap.py b/ci/bootstrap.py
deleted file mode 100755
index 930ffafe3e1ef571c884f4b08b40af2c8afedb62..0000000000000000000000000000000000000000
--- a/ci/bootstrap.py
+++ /dev/null
@@ -1,389 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-'''Bootstraps a new miniconda installation and prepares it for development
-
-This command uses a bare-minimum python3 installation (with SSL support) to
-bootstrap a new miniconda installation preset for the defined activity. It is
-primarily intended for CI operation and prefixes build and deployment steps.
-
-Usage: python3 %s <cmd> build|local|beta|stable [<name>]
-
-Arguments:
-
- <cmd> How to prepare the current environment. Use:
-
- build to build bob.devtools
- local to bootstrap deploy|pypi stages for bob.devtools builds
- beta to bootstrap CI environment for beta builds
- stable to bootstrap CI environment for stable builds
- test to locally test this bootstrap script
-
- <name> (optional) if command is one of ``local|beta|stable`` provide the
- name of env for bob.devtools installation')
-'''
-
-
-BASE_CONDARC = '''\
-default_channels:
- - https://repo.anaconda.com/pkgs/main
- - https://repo.anaconda.com/pkgs/free
- - https://repo.anaconda.com/pkgs/r
- - https://repo.anaconda.com/pkgs/pro
-add_pip_as_python_dependency: false #!final
-changeps1: false #!final
-always_yes: true #!final
-quiet: true #!final
-show_channel_urls: true #!final
-anaconda_upload: false #!final
-ssl_verify: false #!final
-'''
-
-
-import os
-import sys
-import glob
-import time
-import shutil
-import platform
-import subprocess
-
-import logging
-logger = logging.getLogger('bootstrap')
-
-
-_INTERVALS = (
- ('weeks', 604800), # 60 * 60 * 24 * 7
- ('days', 86400), # 60 * 60 * 24
- ('hours', 3600), # 60 * 60
- ('minutes', 60),
- ('seconds', 1),
- )
-
-def human_time(seconds, granularity=2):
- '''Returns a human readable time string like "1 day, 2 hours"'''
-
- result = []
-
- for name, count in _INTERVALS:
- value = seconds // count
- if value:
- seconds -= value * count
- if value == 1:
- name = name.rstrip('s')
- result.append("{} {}".format(int(value), name))
- else:
- # Add a blank if we're in the middle of other values
- if len(result) > 0:
- result.append(None)
-
- if not result:
- if seconds < 1.0:
- return '%.2f seconds' % seconds
- else:
- if seconds == 1:
- return '1 second'
- else:
- return '%d seconds' % seconds
-
- return ', '.join([x for x in result[:granularity] if x is not None])
-
-
-def run_cmdline(cmd, env=None):
- '''Runs a command on a environment, logs output and reports status
-
-
- Parameters:
-
- cmd (list): The command to run, with parameters separated on a list
-
- env (dict, Optional): Environment to use for running the program on. If not
- set, use :py:obj:`os.environ`.
-
- '''
-
- if env is None: env = os.environ
-
- logger.info('(system) %s' % ' '.join(cmd))
-
- start = time.time()
- out = b''
-
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- env=env)
-
- for line in iter(p.stdout.readline, b''):
- sys.stdout.write(line.decode(sys.stdout.encoding))
- sys.stdout.flush()
-
- if p.wait() != 0:
- raise RuntimeError("command `%s' exited with error state (%d)" % \
- (' '.join(cmd), p.returncode))
-
- total = time.time() - start
-
- logger.info('command took %s' % human_time(total))
-
-
-
-def touch(path):
- '''Python-implementation of the "touch" command-line application'''
-
- with open(path, 'a'):
- os.utime(path, None)
-
-
-def merge_conda_cache(cache, prefix):
- '''Merges conda pkg caches and conda-bld folders'''
-
- pkgs_dir = os.path.join(prefix, 'pkgs')
- pkgs_urls_txt = os.path.join(pkgs_dir, 'urls.txt')
- if not os.path.exists(pkgs_dir):
- logger.info('mkdir -p %s', pkgs_dir)
- os.makedirs(pkgs_dir)
- logger.info('touch %s', pkgs_urls_txt)
- touch(pkgs_urls_txt)
-
- # move packages on cache/pkgs to pkgs_dir
- cached_pkgs_dir = os.path.join(cache, 'pkgs')
- cached_packages = glob.glob(os.path.join(cached_pkgs_dir, '*.tar.bz2'))
- cached_packages = [k for k in cached_packages if not \
- k.startswith(os.environ['CI_PROJECT_NAME'] + '-')]
- logger.info('Merging %d cached conda packages', len(cached_packages))
- for k in cached_packages:
- dst = os.path.join(pkgs_dir, os.path.basename(k))
- logger.debug('(move) %s -> %s', k, dst)
- os.rename(k, dst)
-
- # merge urls.txt files
- logger.info('Merging urls.txt files from cache...')
- urls = []
- cached_pkgs_urls_txt = os.path.join(cached_pkgs_dir, 'urls.txt')
- with open(pkgs_urls_txt, 'rb') as f1, \
- open(cached_pkgs_urls_txt, 'rb') as f2:
- data = set(f1.readlines() + f2.readlines())
- data = sorted(list(data))
- with open(pkgs_urls_txt, 'wb') as f:
- f.writelines(data)
-
- pkgs_urls = os.path.join(pkgs_dir, 'urls')
- touch(pkgs_urls)
-
- # move conda-bld build results
- cached_conda_bld = os.path.join(cache, 'conda-bld')
- if os.path.exists(cached_conda_bld):
- dst = os.path.join(prefix, 'conda-bld')
- logger.info('(move) %s -> %s', cached_conda_bld, dst)
- os.rename(cached_conda_bld, dst)
-
-
-def get_miniconda_sh():
- '''Retrieves the miniconda3 installer for the current system'''
-
- import http.client
-
- server = 'repo.continuum.io' #https
- path = '/miniconda/Miniconda3-latest-%s-x86_64.sh'
- if platform.system() == 'Darwin':
- path = path % 'MacOSX'
- else:
- path = path % 'Linux'
-
- logger.info('Connecting to https://%s...', server)
- conn = http.client.HTTPSConnection(server)
- conn.request("GET", path)
- r1 = conn.getresponse()
-
- assert r1.status == 200, 'Request for https://%s%s - returned status %d ' \
- '(%s)' % (server, path, r1.status, r1.reason)
-
- dst = 'miniconda.sh'
- logger.info('(download) https://%s%s -> %s...', server, path, dst)
- with open(dst, 'wb') as f:
- f.write(r1.read())
-
-
-def install_miniconda(prefix):
- '''Creates a new miniconda installation'''
-
- logger.info("Installing miniconda in %s...", prefix)
-
- if not os.path.exists('miniconda.sh'): #re-downloads installer
- get_miniconda_sh()
- else:
- logger.info("Re-using cached miniconda3 installer")
-
- cached = None
- if os.path.exists(prefix): #this is the previous cache, move it
- cached = prefix + '.cached'
- if os.path.exists(cached):
- logger.info('(rmtree) %s', cached)
- shutil.rmtree(cached)
- logger.info('(move) %s -> %s', prefix, cached)
- os.rename(prefix, cached)
-
- run_cmdline(['bash', 'miniconda.sh', '-b', '-p', prefix])
- if cached is not None:
- merge_conda_cache(cached, prefix)
- shutil.rmtree(cached)
-
-
-def get_channels(public, stable):
- '''Returns the relevant conda channels to consider if building project
-
- The subset of channels to be returned depends on the visibility and stability
- of the package being built. Here are the rules:
-
- * public and stable: only returns the public stable channel(s)
- * public and not stable: returns both public stable and beta channels
- * not public and stable: returns both public and private stable channels
- * not public and not stable: returns all channels
-
- Beta channels have priority over stable channels, if returned. Private
- channels have priority over public channles, if turned.
-
-
- Args:
-
- public: Boolean indicating if we're supposed to include only public
- channels
- stable: Boolean indicating if we're supposed to include only stable
- channels
-
-
- Returns: a list of channels that need to be considered.
-
- '''
-
- server = "http://www.idiap.ch"
- channels = []
-
- if not public:
- if not stable: #allowed private channels
- channels += [server + '/private/conda/label/beta'] #allowed betas
- channels += [server + '/private/conda']
- if not stable:
- channels += [server + '/public/conda/label/beta'] #allowed betas
- channels += [server + '/public/conda']
-
- return channels
-
-
-def add_channels_condarc(channels, condarc):
- '''Appends passed channel list to condarc file, print contents'''
-
- with open(condarc, 'at') as f:
- f.write('channels:\n')
- for k in channels:
- f.write(' - %s\n' % k)
-
- with open(condarc, 'rt') as f:
- logger.info('Contents of $CONDARC:\n%s', f.read())
-
-
-def setup_logger():
- '''Sets-up the logging for this command at level ``INFO``'''
-
- warn_err = logging.StreamHandler(sys.stderr)
- warn_err.setLevel(logging.WARNING)
- logger.addHandler(warn_err)
-
- # debug and info messages are written to sys.stdout
-
- class _InfoFilter:
- def filter(self, record):
- return record.levelno <= logging.INFO
-
- debug_info = logging.StreamHandler(sys.stdout)
- debug_info.setLevel(logging.DEBUG)
- debug_info.addFilter(_InfoFilter())
- logger.addHandler(debug_info)
-
- formatter = logging.Formatter('%(levelname)s@%(asctime)s: %(message)s')
-
- for handler in logger.handlers:
- handler.setFormatter(formatter)
-
- logger.setLevel(logging.INFO)
-
-
-if __name__ == '__main__':
-
- if len(sys.argv) == 1:
- print(__doc__ % sys.argv[0])
- sys.exit(1)
-
- setup_logger()
-
- if sys.argv[1] == 'test':
- # sets up local variables for testing
- os.environ['CI_PROJECT_DIR'] = os.path.realpath(os.curdir)
- os.environ['CI_PROJECT_NAME'] = 'bob.devtools'
- os.environ['CONDA_ROOT'] = os.path.join(os.environ['CI_PROJECT_DIR'],
- 'miniconda')
-
- prefix = os.environ['CONDA_ROOT']
- logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
-
- workdir = os.environ['CI_PROJECT_DIR']
- logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
-
- install_miniconda(prefix)
- conda_bin = os.path.join(prefix, 'bin', 'conda')
-
- # creates the condarc file
- condarc = os.path.join(prefix, 'condarc')
- logger.info('(create) %s', condarc)
- with open(condarc, 'wt') as f:
- f.write(BASE_CONDARC)
- os.environ['CONDARC'] = condarc
- logger.info('os.environ["%s"] = %s', 'CONDARC', condarc)
-
- conda_version = '4'
- conda_build_version = '3'
-
- if sys.argv[1] in ('build', 'test'):
-
- # simple - just use the defaults channels when self building
- add_channels_condarc(['defaults'], condarc)
- run_cmdline([conda_bin, 'install', '-n', 'base',
- 'python',
- 'conda=%s' % conda_version,
- 'conda-build=%s' % conda_build_version,
- ])
-
- elif sys.argv[1] == 'local':
-
- # index the locally built packages
- run_cmdline([conda_bin, 'install', '-n', 'base',
- 'python',
- 'conda=%s' % conda_version,
- 'conda-build=%s' % conda_build_version,
- ])
- conda_bld_path = os.path.join(prefix, 'conda-bld')
- run_cmdline([conda_bin, 'index', conda_bld_path])
- # add the locally build directory before defaults, boot from there
- channels = get_channels(public=True, stable=True)
- add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
- run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
-
- elif sys.argv[1] in ('beta', 'stable'):
-
- # installs from channel
- channels = get_channels(
- public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
- stable=os.environ.get('CI_COMMIT_TAG') is not None)
- add_channels_condarc(channels + ['defaults'], condarc)
- run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
-
- else:
-
- logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'")
- logger.error("The value '%s' is not currently supported", sys.argv[1])
- sys.exit(1)
-
- # clean up
- run_cmdline([conda_bin, 'clean', '--lock'])
-
- # print conda information for debugging purposes
- run_cmdline([conda_bin, 'info'])
diff --git a/ci/bootstrap.sh b/ci/bootstrap.sh
deleted file mode 100755
index 3805cfce4a52431ed300f4457a435ca209cb9346..0000000000000000000000000000000000000000
--- a/ci/bootstrap.sh
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/usr/bin/env bash
-
-# Bootstraps a new conda installation and prepares base environment
-# if "local" is passed as parameter, then self installs an already built
-# version of bob.devtools available on your conda-bld directory. If you pass
-# "beta", then it bootstraps from the package installed on our conda beta
-# channel. If you pass "stable", then it bootstraps installing the package
-# available on the stable channel.
-#
-# If bootstrapping anything else than "build", then provide a second argument
-# with the name of the environment that one wants to create with an
-# installation of bob.devtools.
-
-# datetime prefix for logging
-log_datetime() {
- echo "($(date +%T.%3N))"
-}
-
-# Functions for coloring echo commands
-log_info() {
- echo -e "$(log_datetime) \033[1;34m${@}\033[0m"
-}
-
-log_error() {
- echo -e "$(log_datetime) \033[1;31mError: ${@}\033[0m" >&2
-}
-
-# Function for running command and echoing results
-run_cmd() {
- log_info "$ ${@}"
- ${@}
- local status=$?
- if [ ${status} != 0 ]; then
- log_error "Command Failed \"${@}\""
- exit ${status}
- fi
-}
-
-# Checks just if the variable is defined and has non-zero length
-check_defined() {
- if [ -z "${!1+abc}" ]; then
- log_error "Variable ${1} is undefined - aborting...";
- exit 1
- elif [ -z "${!1}" ]; then
- log_error "Variable ${1} is zero-length - aborting...";
- exit 1
- fi
- log_info "${1}=${!1}"
-}
-
-# merges conda cache folders
-# $1: Path to the main cache to keep. The directory must exist.
-# $2: Path to the extra cache to be merged into main cache
-merge_conda_cache() {
- if [ -e ${1} ]; then
- _cached_urlstxt="${2}/pkgs/urls.txt"
- _urlstxt="${1}/pkgs/urls.txt"
- if [ -e ${2}/pkgs ]; then
- log_info "Merging urls.txt and packages with cached files..."
- mv ${2}/pkgs/*.tar.bz2 ${1}/pkgs
- run_cmd rm -f ${1}/pkgs/${CI_PROJECT_NAME}-*-*_*.tar.bz2
- cat ${_urlstxt} ${_cached_urlstxt} | sort | uniq > ${_urlstxt}
- else
- run_cmd mkdir -p ${1}/pkgs
- run_cmd touch ${1}/pkgs/urls.txt
- fi
- run_cmd touch ${1}/pkgs/urls
- if [ -d ${2}/conda-bld ]; then
- log_info "Moving conda-bld packages (artifacts)..."
- run_cmd mv ${2}/conda-bld ${1}
- fi
- fi
-}
-
-# installs a miniconda installation.
-# $1: Path to where to install miniconda.
-install_miniconda() {
- log_info "Installing miniconda in ${1} ..."
-
- # checks if miniconda.sh exists
- if [ ! -e miniconda.sh ]; then
- log_info "Downloading latest miniconda3 installer..."
- # downloads the latest conda installation script
- if [ "$(uname -s)" == "Linux" ]; then
- _os="Linux"
- else
- _os="MacOSX"
- fi
- obj=https://repo.continuum.io/miniconda/Miniconda3-latest-${_os}-x86_64.sh
- run_cmd curl --silent --output miniconda.sh ${obj}
- else
- log_info "Re-using cached miniconda3 installer..."
- ls -l miniconda.sh
- fi
-
- # move cache to a different folder if it exists
- if [ -e ${1} ]; then
- run_cmd mv ${1} ${1}.cached
- fi
-
- # install miniconda
- run_cmd bash miniconda.sh -b -p ${1}
-
- # Put back cache and merge urls.txt
- merge_conda_cache ${1} ${1}.cached
- # remove the backup cache folder
- rm -rf ${1}.cached
-
- # List currently available packages on cache
- # run_cmd ls -l ${1}/pkgs/
- # run_cmd cat ${1}/pkgs/urls.txt
-
- hash -r
-}
-
-
-check_defined CONDA_ROOT
-check_defined CI_PROJECT_DIR
-
-export CONDARC="${CONDA_ROOT}/condarc"
-check_defined CONDARC
-
-# checks if a conda installation exists. Otherwise, installs one
-if [ ! -e ${CONDA_ROOT}/bin/conda ]; then
- install_miniconda ${CONDA_ROOT}
-fi
-
-run_cmd cp -fv ${CI_PROJECT_DIR}/bob/devtools/data/base-condarc ${CONDARC}
-echo "Contents of \`${CONDARC}':"
-cat ${CONDARC}
-
-# setup conda-channels
-CONDA_CHANNEL_ROOT="http://www.idiap.ch/public/conda"
-check_defined CONDA_CHANNEL_ROOT
-CONDA_CLI_CHANNELS="-c ${CONDA_CHANNEL_ROOT} -c defaults"
-
-# creates a base installation depending on the purpose
-if [ "${1}" == "build" ]; then
- run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
-elif [ "${1}" == "local" ]; then
- # updates the base installation, installs conda-build
- run_cmd ls -l ${CONDA_ROOT}/conda-bld
- run_cmd ${CONDA_ROOT}/bin/conda install -n base python conda=4 conda-build=3
- CONDA_CLI_CHANNELS="-c ${CONDA_ROOT}/conda-bld ${CONDA_CLI_CHANNELS}"
- run_cmd ${CONDA_ROOT}/bin/conda index ${CONDA_ROOT}/conda-bld
- run_cmd ls -l ${CONDA_ROOT}/conda-bld
- run_cmd ls -l ${CONDA_ROOT}/conda-bld/noarch/
- run_cmd ${CONDA_ROOT}/bin/conda create -n "${2}" --override-channels ${CONDA_CLI_CHANNELS} bob.devtools
-elif [ "${1}" == "beta" ] || [ "${1}" == "stable" ]; then
- run_cmd ${CONDA_ROOT}/bin/conda create -n "${2}" --override-channels ${CONDA_CLI_CHANNELS} bob.devtools
-else
- log_error "Bootstrap with 'build', or 'local|beta|stable <name>'"
- log_error "The value '${1}' is not currently supported"
- exit 1
-fi
-
-# cleans up
-run_cmd ${CONDA_ROOT}/bin/conda clean --lock
-
-# print conda information for debugging purposes
-run_cmd ${CONDA_ROOT}/bin/conda info
diff --git a/conda/meta.yaml b/conda/meta.yaml
index 23e147e3aeebac36bf64a5e4e2ddb58526940d0b..c0d16c240c054436698701ed9d461ebfaf95539e 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -61,7 +61,7 @@ test:
- bdt release --help
- bdt visibility --help
- bdt dumpsphinx --help
- - bdt bootstrap --help
+ - bdt create --help
- bdt build --help
- bdt getpath --help
- bdt caupdate --help
diff --git a/doc/api.rst b/doc/api.rst
index b3762d86a4338517137e4b419fc0b330911a850e..cf688d1784c666b1cb1ae01abe3538ed213081fb 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -12,7 +12,9 @@
bob.devtools.constants
bob.devtools.release
bob.devtools.changelog
+ bob.devtools.create
bob.devtools.bootstrap
+ bob.devtools.build
bob.devtools.webdav3.client
@@ -31,8 +33,12 @@ Detailed Information
.. automodule:: bob.devtools.changelog
+.. automodule:: bob.devtools.create
+
.. automodule:: bob.devtools.bootstrap
+.. automodule:: bob.devtools.build
+
WebDAV Python Client
--------------------
diff --git a/setup.py b/setup.py
index 04415d1829e9d28c429350eae84c6db9a4de2379..c9f5cf3ce9caf702a8542865a124696487657ea9 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ setup(
'lasttag = bob.devtools.scripts.lasttag:lasttag',
'visibility = bob.devtools.scripts.visibility:visibility',
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx',
- 'bootstrap = bob.devtools.scripts.bootstrap:bootstrap',
+ 'create = bob.devtools.scripts.create:create',
'build = bob.devtools.scripts.build:build',
'getpath = bob.devtools.scripts.getpath:getpath',
'caupdate = bob.devtools.scripts.caupdate:caupdate',