Merge branch 'base-build' into 'master'

Base build

See merge request !14

bob/devtools/data/gitlab-ci/base-build.yaml

+# This YAML file contains descriptions for the CI of most of our Bob/BEAT/BATL
+# packages - do **not** modify it unless you know what you're doing (and up
+# to!)
+
+# Definition of global variables (all stages)
+variables:
+  CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+  BOOTSTRAP: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/bootstrap.py"
+
+
+# Definition of our build pipeline order
+stages:
+  - build
+
+
+# Build targets
+.build_template: &build_job
+  stage: build
+  script:
+    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
+    - python3 bootstrap.py -vv channel base
+    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
+    - conda activate base
+    - bdt ci base-build -vv order.txt
+    - '[ "${CI_COMMIT_REF_NAME}" = "master" ] && bdt ci base-deploy -vv'
+    - bdt ci clean -vv
+  cache: &build_caches
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
+build_linux:
+  <<: *build_job
+  tags:
+    - docker
+  image: continuumio/conda-concourse-ci
+  cache:
+    <<: *build_caches
+    key: "linux-cache"
+
+build_macosx:
+  <<: *build_job
+  tags:
+    - macosx
+  cache:
+    <<: *build_caches
+    key: "macosx-cache"

bob/devtools/scripts/ci.py

@@ -300,6 +300,57 @@ def pypi(package, dry_run):
         logger.info('%s: Deployed to PyPI - OK', k)
 
 
+@ci.command(epilog='''
+Examples:
+
+  1. Builds a list of packages (base dependencies) defined in a text file
+
+     $ bdt ci base-build -vv order.txt
+
+''')
+@click.argument('order', required=True, type=click.Path(file_okay=True,
+  dir_okay=False, exists=True), nargs=1)
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def base_build(order, dry_run):
+  """Builds base (dependence) packages
+
+  This command builds dependence packages (packages that are not Bob/BEAT
+  packages) in the CI infrastructure.  It is **not** meant to be used outside
+  this context.
+  """
+
+  from ..constants import CONDA_BUILD_CONFIG
+  from ..build import base_build as _build
+
+  condarc = os.path.join(os.environ['CONDA_ROOT'], 'condarc')
+  logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
+  with open(condarc, 'rb') as f:
+    condarc_options = yaml.load(f)
+
+  # dump packages at conda_root
+  condarc_options['croot'] = os.path.join(os.environ['CONDA_ROOT'],
+      'conda-bld')
+
+  # loads dirnames from order file (accepts # comments and empty lines)
+  recipes = []
+  with open(order, 'rt') as f:
+    for line in f:
+      line = line.partition('#')[0].strip()
+      if line: recipes.append(line)
+
+  for recipe in recipes:
+    if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
+      # ignore - not a conda package
+      continue
+    base_build(SERVER, True, recipe, CONDA_BUILD_CONFIG,
+        os.environ['PYTHON_VERSION'], args.python_version, condarc_options)
+
+
 @ci.command(epilog='''
 Examples:
 

conda/meta.yaml

@@ -58,18 +58,25 @@ test:
     - bdt -?
     - bdt --help
     - bdt lasttag --help
+      #- bdt lasttag -vv bob/bob.devtools
     - bdt changelog --help
+      #- bdt changelog -vv bob/bob.devtools changelog.md
     - bdt release --help
+      #- bdt release -vvd changelog.md
     - bdt visibility --help
+      #- bdt visibility -vv bob/bob.devtools
     - bdt dumpsphinx --help
     - bdt dumpsphinx https://docs.python.org/3/objects.inv > /dev/null
     - bdt create --help
     - bdt build --help
     - bdt getpath --help
+      #- bdt getpath -vv bob/bob.devtools .gitignore
     - bdt caupdate --help
     - bdt new --help
     - bdt new -t "New package" -o bob.foobar bob/bob.foobar "John Doe" "joe.doe@example.com"
+    - bdt new -t "New package" -o beat.foobar beat/beat.foobar "John Doe" "joe.doe@example.com"
     - bdt ci --help
+    - bdt ci base-build --help
     - bdt ci build --help
     - bdt ci base-deploy --help
     - bdt ci deploy --help

setup.py

@@ -59,6 +59,7 @@ setup(
           ],
 
         'bdt.ci.cli': [
+          'base-build = bob.devtools.scripts.ci:base_build',
           'build = bob.devtools.scripts.ci:build',
           'clean = bob.devtools.scripts.ci:clean',
           'base-deploy = bob.devtools.scripts.ci:base_deploy',