[build] Check specific version of python builds (if needed); Be more...

[build] Check specific version of python builds (if needed); Be more consistent when outputing package names

bob/devtools/build.py

@@ -154,7 +154,8 @@ def get_parsed_recipe(metadata):
   return yaml.load(output)
 
 
-def exists_on_channel(channel_url, name, version, build_number):
+def exists_on_channel(channel_url, name, version, build_number,
+    python_version):
   """Checks on the given channel if a package with the specs exist
 
   Args:
@@ -164,31 +165,48 @@ def exists_on_channel(channel_url, name, version, build_number):
     name: The name of the package
     version: The version of the package
     build_number: The build number of the package
+    python_version: The current version of python we're building for
 
-  Returns: ``True``, if the package already exists in the channel or ``False``
-  otherwise
+  Returns: A complete package name, version and build string, if the package
+  already exists in the channel or ``None`` otherwise.
 
   """
 
   from conda.exports import get_index
 
+  # no dot in py_ver
+  py_ver = python_version.replace('.', '')
+
   # get the channel index
   logger.debug('Downloading channel index from %s', channel_url)
   index = get_index(channel_urls=[channel_url], prepend=False)
 
-  logger.info('Checking for %s-%s_*_%s...', name, version, build_number)
+  logger.info('Checking for %s-%s-%s...', name, version, build_number)
+
   for dist in index:
 
     if dist.name == name and dist.version == version and \
         dist.build_string.endswith('_%s' % build_number):
-      match = re.match('py[2-9][0-9]+', dist.build_string)
-      logger.info('Found matching package (%s-%s_%s)', dist.name, dist.version,
-          dist.build_string)
-      return True
 
-  logger.info('No matches for %s-%s_%s found among %d packages',
-      name, version, build_number, len(index))
-  return False
+      # two possible options must be checked - (i) the package build_string
+      # starts with ``py``, which means it is a python specific package so we
+      # must also check for the matching python version.  (ii) the package is
+      # not a python-specific package and a simple match will do
+      if dist.build_string.startswith('py'):
+        match = re.match('py[2-9][0-9]+', dist.build_string)
+        if match and match.group() == 'py{}'.format(py_ver):
+          logger.debug('Found matching package (%s-%s-%s)', dist.name,
+              dist.version, dist.build_string)
+          return (dist.name, dist.version, dist.build_string)
+
+      else:
+        logger.debug('Found matching package (%s-%s-%s)', dist.name,
+            dist.version, dist.build_string)
+        return (dist.name, dist.version, dist.build_string)
+
+  logger.info('No matches for %s-%s-(py%s_?)%s found among %d packages',
+      name, version, py_ver, build_number, len(index))
+  return
 
 
 def remove_pins(deps):
@@ -444,7 +462,8 @@ def git_clean_build(runner, verbose):
       ['--exclude=%s' % k for k in exclude_from_cleanup])
 
 
-def base_build(server, intranet, recipe_dir, config):
+def base_build(server, intranet, recipe_dir, conda_build_config,
+    python_version, condarc_options):
   '''Builds a non-beat/bob software dependence that does not exist on defaults
 
   This function will build a software dependence that is required for our
@@ -460,33 +479,49 @@ def base_build(server, intranet, recipe_dir, config):
     intranet: Boolean indicating if we should add "private"/"public" prefixes
       on the returned paths
     recipe_dir: The directory containing the recipe's ``meta.yaml`` file
-    config: A dictionary containing the merged configuration, as produced by
-      conda-build API's ``get_or_merge_config()`` function
+    conda_build_config: Path to the ``conda_build_config.yaml`` file to use
+    python_version: String with the python version to build for, in the format
+      ``x.y`` (should be passed even if not building a python package)
+    condarc_options: Pre-parsed condarc options loaded from the respective YAML
+      file
 
   '''
 
   # if you get to this point, tries to build the package
-  public_channel = bootstrap.get_channels(public=True, stable=True,
-    server=server, intranet=intranet)[0]
-  metadata = get_rendered_metadata(recipe_dir, config)
+  public_channels = bootstrap.get_channels(public=True, stable=True,
+    server=server, intranet=intranet)
+
+  logger.info('Using the following channels during (potential) build:\n  - %s',
+      '\n  - '.join(public_channels + ['defaults']))
+  condarc_options['channels'] = public_channels + ['defaults']
+
+  logger.info('Merging conda configuration files...')
+  conda_config = make_conda_config(conda_build_config, python_version,
+      None, condarc_options)
+
+  metadata = get_rendered_metadata(recipe_dir, conda_config)
   recipe = get_parsed_recipe(metadata)
 
   if recipe is None:
-    logger.warn('Skipping build for %s - rendering returned None', recipe_dir)
+    logger.info('Skipping build for %s - rendering returned None', recipe_dir)
     return
 
-  if exists_on_channel(public_channel, recipe['package']['name'],
-      recipe['package']['version'], recipe['build']['number']):
-    logger.warn('Skipping build for %s-%s_%s - exists on channel already',
-        recipe['package']['name'], recipe['package']['version'],
-        recipe['build']['number'])
+  # no dot in py_ver
+  py_ver = python_version.replace('.', '')
+  arch = conda_arch()
+
+  candidate = exists_on_channel(public_channels[0], recipe['package']['name'],
+      recipe['package']['version'], recipe['build']['number'],
+      python_version)
+  if candidate is not None:
+    logger.info('Skipping build for %s-%s-(py%s_?)%s for %s - exists ' \
+        'on channel', candidate[0], candidate[1], candidate[2], py_ver)
     return
 
   # if you get to this point, just builds the package
-  arch = conda_arch()
-  logger.info('Building %s-%s (build: %d) for %s',
+    logger.info('Building %s-%s-(py%s_?)%s for %s',
       recipe['package']['name'], recipe['package']['version'],
-      recipe['build']['number'], arch)
+      recipe['build']['number'], py_ver, arch)
   conda_build.api.build(recipe_dir, config=conda_config)
 
 
@@ -559,23 +594,9 @@ if __name__ == '__main__':
   with open(condarc, 'rb') as f:
     condarc_options = yaml.load(f)
 
-  # notice this condarc typically will only contain the defaults channel - we
-  # need to boost this up with more channels to get it right for this package's
-  # build
-  public = ( args.visibility == 'public' )
-  channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
-      server=server, intranet=(not args.internet))
-  logger.info('Using the following channels during build:\n  - %s',
-      '\n  - '.join(channels + ['defaults']))
-  condarc_options['channels'] = channels + ['defaults']
-
   # dump packages at conda_root
   condarc_options['croot'] = os.path.join(args.conda_root, 'conda-bld')
 
-  logger.info('Merging conda configuration files...')
-  conda_config = make_conda_config(conda_build_config, args.python_version,
-      recipe_append, condarc_options)
-
   # builds all dependencies in the 'deps' subdirectory - or at least checks
   # these dependencies are already available; these dependencies go directly to
   # the public channel once built
@@ -583,13 +604,28 @@ if __name__ == '__main__':
     if not os.path.exists(os.path.join(recipe, 'meta.yaml')):
       # ignore - not a conda package
       continue
-    base_build(server, not args.internet, recipe, conda_config)
+    base_build(server, not args.internet, recipe, conda_build_config,
+        args.python_version, condarc_options)
+
+  # notice this condarc typically will only contain the defaults channel - we
+  # need to boost this up with more channels to get it right for this package's
+  # build
+  public = ( args.visibility == 'public' )
+  channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
+      server=server, intranet=(not args.internet))
+  logger.info('Using the following channels during build:\n  - %s',
+      '\n  - '.join(channels + ['defaults']))
+  condarc_options['channels'] = channels + ['defaults']
 
   # retrieve the current build number for this build
   build_number, _ = next_build_number(channels[0], args.name, version,
       args.python_version)
   bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number))
 
+  logger.info('Merging conda configuration files...')
+  conda_config = make_conda_config(conda_build_config, args.python_version,
+      recipe_append, condarc_options)
+
   # runs the build using the conda-build API
   arch = conda_arch()
   logger.info('Building %s-%s-py%s (build: %d) for %s',