[build] Support multi-python base builds

bob/devtools/build.py

@@ -86,7 +86,10 @@ def next_build_number(channel_url, name, version, python):
   for dist in index:
 
     if dist.name == name and dist.version == version:
-      match = re.match('py[2-9][0-9]+', dist.build_string)
+      if py_ver:
+        match = re.match('py[2-9][0-9]+', dist.build_string)
+      else:
+        match = re.match('py', dist.build_string)
 
       if match and match.group() == 'py{}'.format(py_ver):
         logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
@@ -165,7 +168,9 @@ def exists_on_channel(channel_url, name, version, build_number,
     name: The name of the package
     version: The version of the package
     build_number: The build number of the package
-    python_version: The current version of python we're building for
+    python_version: The current version of python we're building for.  May be
+      ``noarch``, to check for "noarch" packages or ``None``, in which case we
+      don't check for the python version
 
   Returns: A complete package name, version and build string, if the package
   already exists in the channel or ``None`` otherwise.
@@ -174,8 +179,10 @@ def exists_on_channel(channel_url, name, version, build_number,
 
   from conda.exports import get_index
 
-  # no dot in py_ver
-  py_ver = python_version.replace('.', '')
+  # handles different cases as explained on the description of
+  # ``python_version``
+  py_ver = python_version.replace('.', '') if python_version else None
+  if py_ver == 'noarch': py_ver = ''
 
   # get the channel index
   logger.debug('Downloading channel index from %s', channel_url)
@@ -204,8 +211,12 @@ def exists_on_channel(channel_url, name, version, build_number,
             dist.version, dist.build_string)
         return (dist.name, dist.version, dist.build_string)
 
-  logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
-      name, version, py_ver, build_number, len(index))
+  if py_ver is None:
+    logger.info('No matches for %s-%s-%s found among %d packages',
+        name, version, build_number, len(index))
+  else:
+    logger.info('No matches for %s-%s-py%s_%s found among %d packages',
+        name, version, py_ver, build_number, len(index))
   return
 
 
@@ -483,7 +494,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
     recipe_dir: The directory containing the recipe's ``meta.yaml`` file
     conda_build_config: Path to the ``conda_build_config.yaml`` file to use
     python_version: String with the python version to build for, in the format
-      ``x.y`` (should be passed even if not building a python package)
+      ``x.y`` (should be passed even if not building a python package).  It
+      can also be set to ``noarch``, or ``None``.  If set to ``None``, then we
+      don't assume there is a python-specific version being built.  If set to
+      ``noarch``, then it is a python package without a specific build.
     condarc_options: Pre-parsed condarc options loaded from the respective YAML
       file
 
@@ -498,8 +512,12 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
   condarc_options['channels'] = public_channels + ['defaults']
 
   logger.info('Merging conda configuration files...')
-  conda_config = make_conda_config(conda_build_config, python_version,
-      None, condarc_options)
+  if python_version not in ('noarch', None):
+    conda_config = make_conda_config(conda_build_config, python_version,
+        None, condarc_options)
+  else:
+    conda_config = make_conda_config(conda_build_config, None, None,
+        condarc_options)
 
   metadata = get_rendered_metadata(recipe_dir, conda_config)
   recipe = get_parsed_recipe(metadata)
@@ -508,8 +526,10 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
     logger.info('Skipping build for %s - rendering returned None', recipe_dir)
     return
 
-  # no dot in py_ver
-  py_ver = python_version.replace('.', '')
+  # handles different cases as explained on the description of
+  # ``python_version``
+  py_ver = python_version.replace('.', '') if python_version else None
+  if py_ver == 'noarch': py_ver = ''
   arch = conda_arch()
 
   candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
@@ -521,9 +541,14 @@ def base_build(bootstrap, server, intranet, recipe_dir, conda_build_config,
     return
 
   # if you get to this point, just builds the package
-    logger.info('Building %s-%s-(py%s_?)%s for %s',
+  if py_ver is None:
+    logger.info('Building %s-%s-%s for %s',
       recipe['package']['name'], recipe['package']['version'],
-      recipe['build']['number'], py_ver, arch)
+      recipe['build']['number'], arch)
+  else:
+    logger.info('Building %s-%s-py%s_%s for %s',
+      recipe['package']['name'], recipe['package']['version'], py_ver,
+      recipe['build']['number'], arch)
   conda_build.api.build(recipe_dir, config=conda_config)
 
 

bob/devtools/data/gitlab-ci/base-build.yaml

@@ -21,7 +21,8 @@ stages:
     - python3 bootstrap.py -vv channel base
     - source ${CONDA_ROOT}/etc/profile.d/conda.sh
     - conda activate base
-    - bdt ci base-build -vv order.txt
+    - bdt ci base-build -vv nopython.txt
+    - bdt ci base-build -vv --python=3.6 python.txt
     - '[ "${CI_COMMIT_REF_NAME}" = "master" ] && bdt ci base-deploy -vv'
     - bdt ci clean -vv
   cache: &build_caches
@@ -31,10 +32,8 @@ stages:
       - ${CONDA_ROOT}/pkgs/urls.txt
 
 
-build_linux_36:
+build_linux:
   <<: *build_job
-  variables:
-    PYTHON_VERSION: "3.6"
   tags:
     - docker
   image: continuumio/conda-concourse-ci
@@ -42,10 +41,9 @@ build_linux_36:
     <<: *build_caches
     key: "linux-cache"
 
-build_macosx_36:
+
+build_macosx:
   <<: *build_job
-  variables:
-    PYTHON_VERSION: "3.6"
   tags:
     - macosx
   cache:

bob/devtools/scripts/ci.py

@@ -310,13 +310,16 @@ Examples:
 ''')
 @click.argument('order', required=True, type=click.Path(file_okay=True,
   dir_okay=False, exists=True), nargs=1)
+@click.option('-p', '--python', multiple=True,
+    help='Versions of python in the format "x.y" we should build for.  Pass ' \
+        'various times this option to build for multiple python versions')
 @click.option('-d', '--dry-run/--no-dry-run', default=False,
     help='Only goes through the actions, but does not execute them ' \
         '(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
         'printing to help you understand what will be done')
 @verbosity_option()
 @bdt.raise_on_error
-def base_build(order, dry_run):
+def base_build(order, python, dry_run):
   """Builds base (dependence) packages
 
   This command builds dependence packages (packages that are not Bob/BEAT
@@ -343,17 +346,26 @@ def base_build(order, dry_run):
       line = line.partition('#')[0].strip()
       if line: recipes.append(line)
 
+  import itertools
   from .. import bootstrap
 
-  for order, recipe in enumerate(recipes):
-    click.echo('\n' + (60*'='))
-    click.echo('Building "%s" (%d/%d)' % (recipe, order+1, len(recipes)))
-    click.echo((60*'=') + '\n')
+  # combine all versions of python with recipes
+  if python:
+    recipes = list(itertools.product(python, recipes))
+  else:
+    recipes = list(itertools.product([None], recipes))
+
+  for order, (pyver, recipe) in enumerate(recipes):
+    click.echo('\n' + (80*'='))
+    pytext = 'for python-%s' % pyver if pyver is not None else ''
+    click.echo('Building "%s" %s(%d/%d)' % \
+        (recipe, pytext, order+1, total_recipes))
+    click.echo((80*'=') + '\n')
     if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
       logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
       continue
-    _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG,
-        os.environ['PYTHON_VERSION'], condarc_options)
+    _build(bootstrap, SERVER, True, recipe, CONDA_BUILD_CONFIG, pyver,
+        condarc_options)
 
 
 @ci.command(epilog='''