build.py 11.9 KB
Newer Older
1
2
3
#!/usr/bin/env python
# -*- coding: utf-8 -*-

4
5
6
7
8
9
10
11
'''Tools for self-building and other utilities

This script, if called in standalone format, can be used to build the current
package.  It contains various functions and utilities that can be used by
modules inside the package itself.  It assumes a base installation for the
build is operational (i.e., the python package for ``conda-build`` is
installed).
'''
12
13
14
15

import os
import re
import sys
16
17
18
19
import json
import shutil
import platform
import subprocess
20

21
22
import logging
logger = logging.getLogger(__name__)
23

24
import yaml
25
import distutils.version
26
import conda_build.api
27
28


29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def osname():
  """Returns the current OS name as recognized by conda"""

  r = 'unknown'
  if platform.system().lower() == 'linux':
    r = 'linux'
  elif platform.system().lower() == 'darwin':
    r = 'osx'
  else:
    raise RuntimeError('Unsupported system "%s"' % platform.system())

  if platform.machine().lower() == 'x86_64':
    r += '-64'
  else:
    raise RuntimeError('Unsupported machine type "%s"' % platform.machine())

  return r


def should_skip_build(metadata_tuples):
  """Takes the output of render_recipe as input and evaluates if this
  recipe's build should be skipped.
  """

  return all(m[0].skip() for m in metadata_tuples)


def next_build_number(channel_url, name, version, python):
  """Calculates the next build number of a package given the channel

  This function returns the next build number (integer) for a package given its
  recipe, dependencies, name, version and python version.  It looks on the
  channel URL provided and figures out if any clash would happen and what would
  be the highest build number available for that configuration.


  Args:

    channel_url: The URL where to look for packages clashes (normally a beta
      channel)
    name: The name of the package
    version: The version of the package
    python: The version of python as 2 digits (e.g.: "2.7" or "3.6")

  Returns: The next build number with the current configuration.  Zero (0) is
  returned if no match is found.  Also returns the URLs of the packages it
  finds with matches on the name, version and python-version.

  """

  from conda.exports import get_index
80
81
82
83
84

  # no dot in py_ver
  py_ver = python.replace('.', '')

  # get the channel index
85
  logger.debug('Downloading channel index from %s', channel_url)
86
87
88
89
90
91
92
93
94
95
96
  index = get_index(channel_urls=[channel_url], prepend=False)

  # search if package with the same version exists
  build_number = 0
  urls = []
  for dist in index:

    if dist.name == name and dist.version == version:
      match = re.match('py[2-9][0-9]+', dist.build_string)

      if match and match.group() == 'py{}'.format(py_ver):
97
98
        logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
            name, version, py_ver)
99
100
101
102
103
        build_number = max(build_number, dist.build_number + 1)
        urls.append(index[dist].url)

  urls = [url.replace(channel_url, '') for url in urls]

104
105
106
107
  return build_number, urls


def make_conda_config(config, python, append_file, condarc_options):
André Anjos's avatar
André Anjos committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  '''Creates a conda configuration for a build merging various sources

  This function will use the conda-build API to construct a configuration by
  merging different sources of information.

  Args:

    config: Path leading to the ``conda_build_config.yaml`` to use
    python: The version of python to use for the build as ``x.y`` (e.g.
      ``3.6``)
    append_file: Path leading to the ``recipe_append.yaml`` file to use
    condarc_options: A dictionary (typically read from a condarc YAML file)
      that contains build and channel options

  Returns: A dictionary containing the merged configuration, as produced by
  conda-build API's ``get_or_merge_config()`` function.
  '''
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273

  from conda_build.api import get_or_merge_config
  from conda_build.conda_interface import url_path

  retval = get_or_merge_config(None, variant_config_files=config,
      python=python, append_sections_file=append_file, **condarc_options)

  retval.channel_urls = []

  for url in condarc_options['channels']:
    # allow people to specify relative or absolute paths to local channels
    #    These channels still must follow conda rules - they must have the
    #    appropriate platform-specific subdir (e.g. win-64)
    if os.path.isdir(url):
      if not os.path.isabs(url):
        url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
      url = url_path(url)
    retval.channel_urls.append(url)

  return retval


def get_rendered_metadata(recipe_dir, config):
  '''Renders the recipe and returns the interpreted YAML file'''

  from conda_build.api import render
  return render(recipe_dir, config=config)


def get_parsed_recipe(metadata):
  '''Renders the recipe and returns the interpreted YAML file'''

  from conda_build.api import output_yaml
  output = output_yaml(metadata[0][0])
  return yaml.load(output)


def remove_pins(deps):
  return [l.split()[0] for l in deps]


def parse_dependencies(recipe_dir, config):

  metadata = get_rendered_metadata(recipe_dir, config)
  recipe = get_parsed_recipe(metadata)
  return remove_pins(recipe['requirements'].get('build', [])) + \
      remove_pins(recipe['requirements'].get('host', [])) + \
      recipe['requirements'].get('run', []) + \
      recipe.get('test', {}).get('requires', []) + \
      ['bob.buildout', 'mr.developer', 'ipdb']
      # by last, packages required for local dev


def get_env_directory(conda, name):

  cmd = [conda, 'env', 'list', '--json']
  output = subprocess.check_output(cmd)
  data = json.loads(output)
  retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
  if retval:
    return retval[0]
  return None


def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
  '''Creates a new conda environment following package specifications

  This command can create a new conda environment following the list of input
  packages.  It will overwrite an existing environment if indicated.

  Args:
    conda: path to the main conda executable of the installation
    name: the name of the environment to create or overwrite
    overwrite: if set to ```True``, overwrite potentially existing environments
      with the same name
    condarc: a dictionary of options for conda, including channel urls
    packages: the package list specification
    dry_run: if set, then don't execute anything, just print stuff
    use_local: include the local conda-bld directory as a possible installation
      channel (useful for testing multiple interdependent recipes that are
      built locally)
  '''

  from .bootstrap import run_cmdline

  specs = []
  for k in packages:
    k = ' '.join(k.split()[:2])  # remove eventual build string
    if any(elem in k for elem in '><|'):
      specs.append(k.replace(' ', ''))
    else:
      specs.append(k.replace(' ', '='))

  # if the current environment exists, delete it first
  envdir = get_env_directory(conda, name)
  if envdir is not None:
    if overwrite:
      cmd = [conda, 'env', 'remove', '--yes', '--name', name]
      logger.debug('$ ' + ' '.join(cmd))
      if not dry_run:
        run_cmdline(cmd)
    else:
      raise RuntimeError('environment `%s\' exists in `%s\' - use '
                         '--overwrite to overwrite' % (name, envdir))

  cmdline_channels = ['--channel=%s' % k for k in condarc['channels']]
  cmd = [conda, 'create', '--yes', '--name', name, '--override-channels'] + \
      cmdline_channels
  if dry_run:
    cmd.append('--dry-run')
  if use_local:
     cmd.append('--use-local')
  cmd.extend(sorted(specs))
  run_cmdline(cmd)

  # creates a .condarc file to sediment the just created environment
  if not dry_run:
    # get envdir again - it may just be created!
    envdir = get_env_directory(conda, name)
    destrc = os.path.join(envdir, '.condarc')
    logger.info('Creating %s...', destrc)
    with open(destrc, 'w') as f:
      yaml.dump(condarc, f, indent=2)


if __name__ == '__main__':

  # loads the "adjacent" bootstrap module
  import importlib.util
  mydir = os.path.dirname(os.path.realpath(sys.argv[0]))
  bootstrap_file = os.path.join(mydir, 'bootstrap.py')
  spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
  bootstrap = importlib.util.module_from_spec(spec)
  spec.loader.exec_module(bootstrap)

  bootstrap.setup_logger(logger)

  prefix = os.environ['CONDA_ROOT']
  logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)

  workdir = os.environ['CI_PROJECT_DIR']
  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)

  name = os.environ['CI_PROJECT_NAME']
  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)

  pyver = os.environ['PYTHON_VERSION']
  logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)

André Anjos's avatar
André Anjos committed
274
275
276
277
278
279
  bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, os.environ,
      verbose=True)
  bootstrap.set_environment('LANG', 'en_US.UTF-8', os.environ,
      verbose=True)
  bootstrap.set_environment('LC_ALL', os.environ['LANG'], os.environ,
      verbose=True)
280
281
282
283
284
285
286
287
288
289
290

  # create the build configuration
  conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
  recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')
  logger.info('Merging conda configuration files...')

  condarc = os.path.join(prefix, 'condarc')
  logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
  with open(condarc, 'rb') as f:
    condarc_options = yaml.load(f)

André Anjos's avatar
André Anjos committed
291
292
293
294
295
296
297
  # notice this condarc typically will only contain the defaults channel - we
  # need to boost this up with more channels to get it right.
  channels = bootstrap.get_channels(
      public=(os.environ['CI_PROJECT_VISIBILITY']=='public'),
      stable=(not is_prerelease), server=bootstrap._SERVER, intranet=True)
  condarc_options['channels'] = channels + ['defaults']

298
299
300
301
302
303
304
305
  conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
      condarc_options)

  version = open("version.txt").read().rstrip()
  os.environ['BOB_PACKAGE_VERSION'] = version
  logger.info('os.environ["%s"] = %s', 'BOB_PACKAGE_VERSION', version)

  # if we're build a stable release, ensure a tag is set
306
  parsed_version = distutils.version.LooseVersion(version).version
307
  is_prerelease = any([isinstance(k, str) for k in parsed_version])
308
  if is_prerelease:
309
310
311
312
313
314
315
316
317
318
319
320
321
322
    if os.environ.get('CI_COMMIT_TAG') is not None:
      raise EnvironmentError('"version.txt" indicates version is a ' \
          'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
          'which indicates this is a **stable** build. ' \
          'Have you created the tag using ``bdt release``?', version,
          os.environ['CI_COMMIT_TAG'])
  else:  #it is a stable build
    if os.environ.get('CI_COMMIT_TAG') is None:
      raise EnvironmentError('"version.txt" indicates version is a ' \
          'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
          'variable defined, which indicates this is **not** ' \
          'a tagged build. Use ``bdt release`` to create stable releases',
          version)

André Anjos's avatar
André Anjos committed
323
  # retrieve the current build number for this build
324
  build_number, _ = next_build_number(channels[0], name, version, pyver)
André Anjos's avatar
André Anjos committed
325
326
  bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number),
      verbose=True)
327
328
329

  # runs the build using the conda-build API
  logger.info('Building %s-%s-py%s (build: %d) for %s',
André Anjos's avatar
André Anjos committed
330
331
      name, version, pyver.replace('.',''), build_number, osname())
  conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
332
333
334
335
336
337
338
339
340
341
342
343
344

  # runs git clean to clean everything that is not needed. This helps to keep
  # the disk usage on CI machines to a minimum.
  exclude_from_cleanup = [
      "miniconda.sh",   #the installer, cached
      "miniconda/pkgs/*.tar.bz2",  #downloaded packages, cached
      "miniconda/pkgs/urls.txt",  #download index, cached
      "miniconda/conda-bld/${_os}-64/*.tar.bz2",  #build artifact -- conda
      "dist/*.zip",  #build artifact -- pypi package
      "sphinx",  #build artifact -- documentation
      ]
  bootstrap.run_cmdline(['git', 'clean', '-ffdx'] + \
      ['--exclude=%s' % k for k in exclude_from_cleanup])