A lot of DRY, but mostly for the ci biuld

bob/devtools/bootstrap.py

@@ -7,7 +7,7 @@ This command uses a bare-minimum python3 installation (with SSL support) to
 bootstrap a new miniconda installation preset for the defined activity.  It is
 primarily intended for CI operation and prefixes build and deployment steps.
 
-Usage: python3 %s <cmd> build|local|beta|stable [<name>]
+Usage: python3 %s <cmd> build|local|channel [<name>]
 
 Arguments:
 
@@ -15,8 +15,7 @@ Arguments:
 
          build   to build bob.devtools
          local   to bootstrap deploy|pypi stages for bob.devtools builds
-         beta    to bootstrap CI environment for beta builds
-         stable  to bootstrap CI environment for stable builds
+         channel to bootstrap CI environment for beta/stable builds
          test    to locally test this bootstrap script
 
   <name>  (optional) if command is one of ``local|beta|stable`` provide the
@@ -363,6 +362,8 @@ if __name__ == '__main__':
   logger.info('(create) %s', condarc)
   with open(condarc, 'wt') as f:
     f.write(_BASE_CONDARC)
+  # we just add the "defaults" channels to the stock condarc
+  add_channels_condarc(['defaults'], condarc)
 
   conda_version = '4'
   conda_build_version = '3'
@@ -370,7 +371,6 @@ if __name__ == '__main__':
   if sys.argv[1] in ('build', 'test'):
 
     # simple - just use the defaults channels when self building
-    add_channels_condarc(['defaults'], condarc)
     run_cmdline([conda_bin, 'install', '-n', 'base',
       'python',
       'conda=%s' % conda_version,
@@ -389,23 +389,29 @@ if __name__ == '__main__':
     run_cmdline([conda_bin, 'index', conda_bld_path])
     # add the locally build directory before defaults, boot from there
     channels = get_channels(public=True, stable=True, server=_SERVER,
-        intranet=True)
-    add_channels_condarc(channels + [conda_bld_path, 'defaults'], condarc)
-    run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
+        intranet=True) + ['defaults']
+    channels = ['--override-channels'] + \
+        ['--channel=' + conda_bld_path] + \
+        ['--channel=%s' % k for k in channels]
+    run_cmdline([conda_bin, 'create'] + channels + \
+        ['-n', sys.argv[2], 'bob.devtools'])
 
-  elif sys.argv[1] in ('beta', 'stable'):
+  elif sys.argv[1] == 'channel':
 
     # installs from channel
     channels = get_channels(
-        public=os.environ['CI_PROJECT_VISIBILITY'] == 'public',
-        stable=os.environ.get('CI_COMMIT_TAG') is not None,
+        public=(os.environ['CI_PROJECT_VISIBILITY'] == 'public'),
+        stable=('CI_COMMIT_TAG' in os.environ),
         server=_SERVER, intranet=True)
+    channels = ['--override-channels'] + \
+        ['--channel=%s' % k for k in channels]
     add_channels_condarc(channels + ['defaults'], condarc)
-    run_cmdline([conda_bin, 'create', '-n', sys.argv[2], 'bob.devtools'])
+    run_cmdline([conda_bin, 'create'] + channels + \
+        ['-n', sys.argv[2], 'bob.devtools'])
 
   else:
 
-    logger.error("Bootstrap with 'build', or 'local|beta|stable <name>'")
+    logger.error("Bootstrap with 'build', or 'local|channel <name>'")
     logger.error("The value '%s' is not currently supported", sys.argv[1])
     sys.exit(1)
 

bob/devtools/ci.py

@@ -38,20 +38,6 @@ def is_master(refname, tag):
   return refname == 'master'
 
 
-def is_visible_outside(package, visibility):
-  '''Determines if the project is visible outside Idiap'''
-
-  logger.info('Project %s visibility is "%s"', package, visibility)
-
-  if visibility == 'internal':
-    visibility = 'private' #the same thing for this command
-    logger.warn('Project %s visibility switched to "%s".  ' \
-        'For this command, it all boils down to the same...', package,
-        visibility)
-
-  return visibility == 'public'
-
-
 def is_stable(package, refname, tag):
   '''Determines if the package being published is stable
 

bob/devtools/scripts/build.py

@@ -109,7 +109,8 @@ def build(recipe_dir, python, condarc, config, no_test, append_file,
   prefix = get_env_directory(os.environ['CONDA_EXE'], 'base')
   condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
 
-  conda_config = make_conda_config(config, python, append_file, condarc_options)
+  conda_config = make_conda_config(config, python, append_file,
+      condarc_options)
 
   set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
 

bob/devtools/scripts/ci.py

@@ -14,15 +14,9 @@ from click_plugins import with_plugins
 
 from . import bdt
 from ..log import verbosity_option
-from ..ci import is_stable, is_visible_outside
 from ..webdav3 import client as webdav
-
-from ..constants import SERVER, WEBDAV_PATHS, CACERT, CONDA_BUILD_CONFIG, \
-    CONDA_RECIPE_APPEND, MATPLOTLIB_RCDIR, BASE_CONDARC
-from ..build import next_build_number, conda_arch, should_skip_build, \
-    get_rendered_metadata, get_parsed_recipe, make_conda_config, \
-    get_docserver_setup, check_version, git_clean_build
-from ..bootstrap import set_environment, get_channels, run_cmdline
+from ..constants import SERVER, WEBDAV_PATHS, CACERT
+from ..bootstrap import set_environment, run_cmdline
 
 
 @with_plugins(pkg_resources.iter_entry_points('bdt.ci.cli'))
@@ -72,11 +66,12 @@ def deploy(dry_run):
     package = os.environ['CI_PROJECT_PATH']
 
     # determine project visibility
-    visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
+    visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
 
     # determine if building master branch or tag - and if tag is on master
-    tag = os.environ.get('CI_COMMIT_TAG')
-    stable = is_stable(package, os.environ['CI_COMMIT_REF_NAME'], tag)
+    stable = is_stable(package,
+        os.environ['CI_COMMIT_REF_NAME'],
+        os.environ['CI_COMMIT_TAG'])
 
     server_info = WEBDAV_PATHS[stable][visible]
 
@@ -174,7 +169,7 @@ def pypi(dry_run):
     package = os.environ['CI_PROJECT_PATH']
 
     # determine project visibility
-    visible = is_visible_outside(package, os.environ['CI_PROJECT_VISIBILITY'])
+    visible = (os.environ['CI_PROJECT_VISIBILITY'] == 'public')
 
     if not visible:
       raise RuntimeError('The repository %s is not public - a package ' \
@@ -233,84 +228,23 @@ def build(dry_run):
   to be used outside this context.
   """
 
-  if dry_run:
-      logger.warn('!!!! DRY RUN MODE !!!!')
-      logger.warn('Nothing is being built')
-
-  prefix = os.environ['CONDA_ROOT']
-  logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)
-
-  workdir = os.environ['CI_PROJECT_DIR']
-  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)
-
-  pyver = os.environ['PYTHON_VERSION']
-  logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)
-
-  set_environment('MATPLOTLIBRC', MATPLOTLIB_RCDIR, verbose=True)
-
-  # get information about the version of the package being built
-  version, is_prerelease = check_version(workdir,
-      os.environ.get('CI_COMMIT_TAG'))
-  set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
-
-  # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
-  # and derived documentation building via Sphinx)
-  set_environment('DOCSERVER', SERVER, os.environ, verbose=True)
-  public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
-  doc_urls = get_docserver_setup(public=public, stable=(not is_prerelease),
-      server=SERVER, intranet=True)
-  set_environment('BOB_DOCUMENTATION_SERVER', doc_urls, verbose=True)
-
-  condarc = os.path.join(prefix, 'condarc')
-  logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
-  with open(condarc, 'rb') as f:
-    condarc_options = yaml.load(f)
-
-  # notice this condarc typically will only contain the defaults channel - we
-  # need to boost this up with more channels to get it right.
-  channels = get_channels(public=public, stable=(not is_prerelease),
-      server=SERVER, intranet=True)
-  logger.info('Using the following channels during build:\n  - %s',
-      '\n  - '.join(channels + ['defaults']))
-  condarc_options['channels'] = channels + ['defaults']
-
-  # dump packages at conda_root
-  condarc_options['croot'] = os.path.join(prefix, 'conda-bld')
-
-  # create the build configuration
-  logger.info('Merging conda configuration files...')
-  conda_config = make_conda_config(CONDA_BUILD_CONFIG, pyver,
-      CONDA_RECIPE_APPEND, condarc_options)
-
-  recipe_dir = os.path.join(workdir, 'conda')
-  if not os.path.exists(recipe_dir):
-    raise RuntimeError("The directory %s does not exist" % recipe_dir)
-
-  # pre-renders the recipe - figures out package name and version
-  metadata = get_rendered_metadata(recipe_dir, conda_config)
-
-  arch = conda_arch()
-  if should_skip_build(metadata):
-    logger.warn('Skipping UNSUPPORTED build of "%s" for py%s on %s',
-        d, python.replace('.',''), arch)
-    return 0
-
-  # converts the metadata output into parsed yaml and continues the process
-  rendered_recipe = get_parsed_recipe(metadata)
-
-  # retrieve the current build number for this build
-  build_number, _ = next_build_number(channels[0],
-      rendered_recipe['package']['name'],
-      rendered_recipe['package']['version'], python)
-  set_environment('BOB_BUILD_NUMBER', str(build_number), verbose=True)
-
-  # runs the build using the conda-build API
-  logger.info('Building %s-%s-py%s (build: %d) for %s',
-      rendered_recipe['package']['name'],
-      rendered_recipe['package']['version'], python.replace('.',''),
-      build_number, arch)
-
-  if not dry_run:
-    conda_build.api.build(recipe_dir, config=conda_config)
+  from ..constants import CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND
+  from ..build import git_clean_build
+  from ..bootstrap import run_cmdline
+
+  from .build import build
+  build(
+      recipe_dir=[os.path.join(os.path.realpath(os.curdir), 'conda')],
+      python=os.environ['PYTHON_VERSION'],  #python version
+      condarc=None,  #custom build configuration
+      config=CONDA_BUILD_CONFIG,
+      no_test=False,
+      append_file=CONDA_RECIPE_APPEND,
+      server=SERVER,
+      private=(os.environ['CI_PROJECT_VISIBILITY'] != 'public'),
+      stable='CI_COMMIT_TAG' in os.environ,
+      dry_run=dry_run,
+      verbosity=verbosity,
+      )
 
   git_clean_build(run_cmdline, arch)