[ci] Add build instructions

bob/devtools/build.py

@@ -314,6 +314,77 @@ def get_docserver_setup(public, stable, server, intranet):
   return '|'.join(entries)
 
 
+def check_version(workdir, envtag):
+  '''Checks if the version being built and the value reported match
+
+  This method will read the contents of the file ``version.txt`` and compare it
+  to the potentially set ``envtag`` (may be ``None``).  If the value of
+  ``envtag`` is different than ``None``, ensure it matches the value in
+  ``version.txt`` or raises an exception.
+
+
+  Args:
+
+    workdir: The work directory where the repo of the package being built was
+      checked-out
+    envtag: The output of os.environ.get('CI_COMMIT_TAG') (may be ``None``)
+
+
+  Returns: A tuple with the version of the package that we're currently
+  building and a boolean flag indicating if the version number represents a
+  pre-release or a stable release.
+  '''
+
+  version = open(os.path.join(workdir, "version.txt"), 'rt').read().rstrip()
+
+  # if we're building a stable release, ensure a tag is set
+  parsed_version = distutils.version.LooseVersion(version).version
+  is_prerelease = any([isinstance(k, str) for k in parsed_version])
+  if is_prerelease:
+    if envtag is not None:
+      raise EnvironmentError('"version.txt" indicates version is a ' \
+          'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
+          'which indicates this is a **stable** build. ' \
+          'Have you created the tag using ``bdt release``?' % (version,
+          envtag))
+  else:  #it is a stable build
+    if envtag is None:
+      raise EnvironmentError('"version.txt" indicates version is a ' \
+          'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
+          'variable defined, which indicates this is **not** ' \
+          'a tagged build. Use ``bdt release`` to create stable releases' % \
+          (version,))
+    if envtag[1:] != version:
+      raise EnvironmentError('"version.txt" and the value of ' \
+          'os.environ["CI_COMMIT_TAG"] do **NOT** agree - the former ' \
+          'reports version %s, the latter, %s' % (version, envtag[1:]))
+
+  return version, is_prerelease
+
+
+def git_clean_build(runner, arch):
+  '''Runs git-clean to clean-up build products
+
+  Args:
+
+    runner: A pointer to the ``run_cmdline()`` function
+
+  '''
+
+  # runs git clean to clean everything that is not needed. This helps to keep
+  # the disk usage on CI machines to a minimum.
+  exclude_from_cleanup = [
+      "miniconda.sh",   #the installer, cached
+      "miniconda/pkgs/*.tar.bz2",  #downloaded packages, cached
+      "miniconda/pkgs/urls.txt",  #download index, cached
+      "miniconda/conda-bld/%s/*.tar.bz2" % (arch,),  #build artifact -- conda
+      "dist/*.zip",  #build artifact -- pypi package
+      "sphinx",  #build artifact -- documentation
+      ]
+  runner(['git', 'clean', '-qffdx'] + \
+      ['--exclude=%s' % k for k in exclude_from_cleanup])
+
+
 if __name__ == '__main__':
 
   # loads the "adjacent" bootstrap module
@@ -338,40 +409,18 @@ if __name__ == '__main__':
   pyver = os.environ['PYTHON_VERSION']
   logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
 
-  bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, os.environ,
-      verbose=True)
-  bootstrap.set_environment('LANG', 'en_US.UTF-8', os.environ,
-      verbose=True)
-  bootstrap.set_environment('LC_ALL', os.environ['LANG'], os.environ,
-      verbose=True)
+  bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, verbose=True)
+  bootstrap.set_environment('LANG', 'en_US.UTF-8', verbose=True)
+  bootstrap.set_environment('LC_ALL', os.environ['LANG'], verbose=True)
 
   # get information about the version of the package being built
-  version = open("version.txt").read().rstrip()
-  os.environ['BOB_PACKAGE_VERSION'] = version
-  logger.info('os.environ["%s"] = %s', 'BOB_PACKAGE_VERSION', version)
-
-  # if we're building a stable release, ensure a tag is set
-  parsed_version = distutils.version.LooseVersion(version).version
-  is_prerelease = any([isinstance(k, str) for k in parsed_version])
-  if is_prerelease:
-    if os.environ.get('CI_COMMIT_TAG') is not None:
-      raise EnvironmentError('"version.txt" indicates version is a ' \
-          'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
-          'which indicates this is a **stable** build. ' \
-          'Have you created the tag using ``bdt release``?', version,
-          os.environ['CI_COMMIT_TAG'])
-  else:  #it is a stable build
-    if os.environ.get('CI_COMMIT_TAG') is None:
-      raise EnvironmentError('"version.txt" indicates version is a ' \
-          'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
-          'variable defined, which indicates this is **not** ' \
-          'a tagged build. Use ``bdt release`` to create stable releases',
-          version)
+  version, is_prerelease = check_version(workdir,
+      os.environ.get('CI_COMMIT_TAG'))
+  bootstrap.set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
 
   # create the build configuration
   conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
   recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')
-  logger.info('Merging conda configuration files...')
 
   condarc = os.path.join(prefix, 'condarc')
   logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
@@ -380,13 +429,14 @@ if __name__ == '__main__':
 
   # notice this condarc typically will only contain the defaults channel - we
   # need to boost this up with more channels to get it right.
-  channels = bootstrap.get_channels(
-      public=(os.environ['CI_PROJECT_VISIBILITY']=='public'),
-      stable=(not is_prerelease), server=bootstrap._SERVER, intranet=True)
+  public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
+  channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
+      server=bootstrap._SERVER, intranet=True)
   logger.info('Using the following channels during build:\n  - %s',
       '\n  - '.join(channels + ['defaults']))
   condarc_options['channels'] = channels + ['defaults']
 
+  logger.info('Merging conda configuration files...')
   conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
       condarc_options)
 
@@ -401,15 +451,4 @@ if __name__ == '__main__':
       name, version, pyver.replace('.',''), build_number, arch)
   conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
 
-  # runs git clean to clean everything that is not needed. This helps to keep
-  # the disk usage on CI machines to a minimum.
-  exclude_from_cleanup = [
-      "miniconda.sh",   #the installer, cached
-      "miniconda/pkgs/*.tar.bz2",  #downloaded packages, cached
-      "miniconda/pkgs/urls.txt",  #download index, cached
-      "miniconda/conda-bld/%s/*.tar.bz2" % (arch,),  #build artifact -- conda
-      "dist/*.zip",  #build artifact -- pypi package
-      "sphinx",  #build artifact -- documentation
-      ]
-  bootstrap.run_cmdline(['git', 'clean', '-qffdx'] + \
-      ['--exclude=%s' % k for k in exclude_from_cleanup])
+  git_clean_build(bootstrap.run_cmdline, arch)

bob/devtools/scripts/build.py

@@ -6,9 +6,10 @@ import sys
 import logging
 logger = logging.getLogger(__name__)
 
-import pkg_resources
-import click
 import yaml
+import click
+import pkg_resources
+import conda_build.api
 
 from . import bdt
 from ..log import verbosity_option
@@ -107,16 +108,16 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
 
   conda_config = make_conda_config(config, python, append_file, condarc_options)
 
-  set_environment('LANG', 'en_US.UTF-8', os.environ)
-  set_environment('LC_ALL', os.environ['LANG'], os.environ)
-  set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, os.environ)
+  set_environment('LANG', 'en_US.UTF-8', verbose=True)
+  set_environment('LC_ALL', os.environ['LANG'], verbose=True)
+  set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
 
   # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
   # and derived documentation building via Sphinx)
-  set_environment('DOCSERVER', server, os.environ)
+  set_environment('DOCSERVER', server, verbose=True)
   doc_urls = get_docserver_setup(public=(not private), stable=stable,
       server=server, intranet=private)
-  set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, server=server)
+  set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
 
   for d in recipe_dir:
 
@@ -126,7 +127,7 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
     version_candidate = os.path.join(d, '..', 'version.txt')
     if os.path.exists(version_candidate):
       version = open(version_candidate).read().rstrip()
-      set_environment('BOB_PACKAGE_VERSION', version, os.environ)
+      set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
 
     # pre-renders the recipe - figures out package name and version
     metadata = get_rendered_metadata(d, conda_config)
@@ -146,12 +147,11 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
         rendered_recipe['package']['name'],
         rendered_recipe['package']['version'], python)
 
-    set_environment('BOB_BUILD_NUMBER', str(build_number), os.environ)
+    set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
 
     logger.info('Building %s-%s-py%s (build: %d) for %s',
         rendered_recipe['package']['name'],
         rendered_recipe['package']['version'], python.replace('.',''),
         build_number, arch)
     if not dry_run:
-      from conda_build.api import build
-      build(d, config=conda_config, notest=no_test)
+      conda_build.api.build(d, config=conda_config, notest=no_test)

bob/devtools/scripts/ci.py

@@ -13,9 +13,15 @@ from click_plugins import with_plugins
 from . import bdt
 from ..log import verbosity_option
 from ..ci import is_stable, is_visible_outside
-from ..constants import SERVER, WEBDAV_PATHS, CACERT
 from ..webdav3 import client as webdav
 
+from ..constants import SERVER, WEBDAV_PATHS, CACERT, CONDA_BUILD_CONFIG, \
+    CONDA_RECIPE_APPEND, MATPLOTLIB_RCDIR, BASE_CONDARC
+from ..build import next_build_number, conda_arch, should_skip_build, \
+    get_rendered_metadata, get_parsed_recipe, make_conda_config, \
+    get_docserver_setup, check_version, git_clean_build
+from ..bootstrap import set_environment, get_channels, run_cmdline
+
 
 @with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
 @click.group(cls=bdt.AliasedGroup)
@@ -201,3 +207,90 @@ def pypi(dry_run):
       from twine.commands.upload import upload
       upload(settings, zip_files)
       logger.info('Deployment to PyPI successful')
+
+
+@ci.command(epilog='''
+Examples:
+
+  1. Builds the current package
+
+     $ bdt ci build -vv
+
+''')
+@click.option('-d', '--dry-run/--no-dry-run', default=False,
+    help='Only goes through the actions, but does not execute them ' \
+        '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
+        'printing to help you understand what will be done')
+@verbosity_option()
+@bdt.raise_on_error
+def build(dry_run):
+  """Builds packages
+
+  This command builds packages in the CI infrastructure.  It is **not** meant
+  to be used outside this context.
+  """
+
+  if dry_run:
+      logger.warn('!!!! DRY RUN MODE !!!!')
+      logger.warn('Nothing is being built')
+
+  prefix = os.environ['CONDA_ROOT']
+  logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
+
+  workdir = os.environ['CI_PROJECT_DIR']
+  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
+
+  name = os.environ['CI_PROJECT_NAME']
+  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)
+
+  pyver = os.environ['PYTHON_VERSION']
+  logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
+
+  set_environment('LANG', 'en_US.UTF-8', os.environ, verbose=True)
+  set_environment('LC_ALL', os.environ['LANG'], os.environ, verbose=True)
+  set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
+
+  # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
+  # and derived documentation building via Sphinx)
+  set_environment('DOCSERVER', SERVER, os.environ, verbose=True)
+  public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
+  doc_urls = get_docserver_setup(public=public, stable=(not is_prerelease),
+      server=SERVER, intranet=True)
+  set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
+
+  # get information about the version of the package being built
+  version, is_prerelease = check_version(workdir,
+      os.environ.get('CI_COMMIT_TAG'))
+  set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
+
+  condarc = os.path.join(prefix, 'condarc')
+  logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
+  with open(condarc, 'rb') as f:
+    condarc_options = yaml.load(f)
+
+  # notice this condarc typically will only contain the defaults channel - we
+  # need to boost this up with more channels to get it right.
+  channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
+      server=SERVER, intranet=True)
+  logger.info('Using the following channels during build:\n  - %s',
+      '\n  - '.join(channels + ['defaults']))
+  condarc_options['channels'] = channels + ['defaults']
+
+  # create the build configuration
+  logger.info('Merging conda configuration files...')
+  conda_config = make_conda_config(CONDA_BUILD_CONFIG, pyver,
+      CONDA_RECIPE_APPEND, condarc_options)
+
+  # retrieve the current build number for this build
+  build_number, _ = next_build_number(channels[0], name, version, pyver)
+  set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
+
+  # runs the build using the conda-build API
+  arch = conda_arch()
+  logger.info('Building %s-%s-py%s (build: %d) for %s',
+      name, version, pyver.replace('.',''), build_number, arch)
+
+  if not dry_run:
+    conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
+
+  git_clean_build(run_cmdline, arch)