build.py 15.3 KB
Newer Older
1
2
3
#!/usr/bin/env python
# -*- coding: utf-8 -*-

4
5
6
7
8
9
10
11
'''Tools for self-building and other utilities

This script, if called in standalone format, can be used to build the current
package.  It contains various functions and utilities that can be used by
modules inside the package itself.  It assumes a base installation for the
build is operational (i.e., the python package for ``conda-build`` is
installed).
'''
12
13
14
15

import os
import re
import sys
16
17
18
19
import json
import shutil
import platform
import subprocess
20

21
22
import logging
logger = logging.getLogger(__name__)
23

24
import yaml
25
import distutils.version
26
import conda_build.api
27
28


29
30
def conda_arch():
  """Returns the current OS name and architecture as recognized by conda"""
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

  r = 'unknown'
  if platform.system().lower() == 'linux':
    r = 'linux'
  elif platform.system().lower() == 'darwin':
    r = 'osx'
  else:
    raise RuntimeError('Unsupported system "%s"' % platform.system())

  if platform.machine().lower() == 'x86_64':
    r += '-64'
  else:
    raise RuntimeError('Unsupported machine type "%s"' % platform.machine())

  return r


def should_skip_build(metadata_tuples):
  """Takes the output of render_recipe as input and evaluates if this
  recipe's build should be skipped.
  """

  return all(m[0].skip() for m in metadata_tuples)


def next_build_number(channel_url, name, version, python):
  """Calculates the next build number of a package given the channel

  This function returns the next build number (integer) for a package given its
  recipe, dependencies, name, version and python version.  It looks on the
  channel URL provided and figures out if any clash would happen and what would
  be the highest build number available for that configuration.


  Args:

    channel_url: The URL where to look for packages clashes (normally a beta
      channel)
    name: The name of the package
    version: The version of the package
    python: The version of python as 2 digits (e.g.: "2.7" or "3.6")

  Returns: The next build number with the current configuration.  Zero (0) is
  returned if no match is found.  Also returns the URLs of the packages it
  finds with matches on the name, version and python-version.

  """

  from conda.exports import get_index
80
81
82
83
84

  # no dot in py_ver
  py_ver = python.replace('.', '')

  # get the channel index
85
  logger.debug('Downloading channel index from %s', channel_url)
86
87
88
89
90
91
92
93
94
95
96
  index = get_index(channel_urls=[channel_url], prepend=False)

  # search if package with the same version exists
  build_number = 0
  urls = []
  for dist in index:

    if dist.name == name and dist.version == version:
      match = re.match('py[2-9][0-9]+', dist.build_string)

      if match and match.group() == 'py{}'.format(py_ver):
97
98
        logger.debug("Found match at %s for %s-%s-py%s", index[dist].url,
            name, version, py_ver)
99
100
101
102
103
        build_number = max(build_number, dist.build_number + 1)
        urls.append(index[dist].url)

  urls = [url.replace(channel_url, '') for url in urls]

104
105
106
107
  return build_number, urls


def make_conda_config(config, python, append_file, condarc_options):
André Anjos's avatar
André Anjos committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  '''Creates a conda configuration for a build merging various sources

  This function will use the conda-build API to construct a configuration by
  merging different sources of information.

  Args:

    config: Path leading to the ``conda_build_config.yaml`` to use
    python: The version of python to use for the build as ``x.y`` (e.g.
      ``3.6``)
    append_file: Path leading to the ``recipe_append.yaml`` file to use
    condarc_options: A dictionary (typically read from a condarc YAML file)
      that contains build and channel options

  Returns: A dictionary containing the merged configuration, as produced by
  conda-build API's ``get_or_merge_config()`` function.
  '''
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249

  from conda_build.api import get_or_merge_config
  from conda_build.conda_interface import url_path

  retval = get_or_merge_config(None, variant_config_files=config,
      python=python, append_sections_file=append_file, **condarc_options)

  retval.channel_urls = []

  for url in condarc_options['channels']:
    # allow people to specify relative or absolute paths to local channels
    #    These channels still must follow conda rules - they must have the
    #    appropriate platform-specific subdir (e.g. win-64)
    if os.path.isdir(url):
      if not os.path.isabs(url):
        url = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), url)))
      url = url_path(url)
    retval.channel_urls.append(url)

  return retval


def get_rendered_metadata(recipe_dir, config):
  '''Renders the recipe and returns the interpreted YAML file'''

  from conda_build.api import render
  return render(recipe_dir, config=config)


def get_parsed_recipe(metadata):
  '''Renders the recipe and returns the interpreted YAML file'''

  from conda_build.api import output_yaml
  output = output_yaml(metadata[0][0])
  return yaml.load(output)


def remove_pins(deps):
  return [l.split()[0] for l in deps]


def parse_dependencies(recipe_dir, config):

  metadata = get_rendered_metadata(recipe_dir, config)
  recipe = get_parsed_recipe(metadata)
  return remove_pins(recipe['requirements'].get('build', [])) + \
      remove_pins(recipe['requirements'].get('host', [])) + \
      recipe['requirements'].get('run', []) + \
      recipe.get('test', {}).get('requires', []) + \
      ['bob.buildout', 'mr.developer', 'ipdb']
      # by last, packages required for local dev


def get_env_directory(conda, name):

  cmd = [conda, 'env', 'list', '--json']
  output = subprocess.check_output(cmd)
  data = json.loads(output)
  retval = [k for k in data.get('envs', []) if k.endswith(os.sep + name)]
  if retval:
    return retval[0]
  return None


def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
  '''Creates a new conda environment following package specifications

  This command can create a new conda environment following the list of input
  packages.  It will overwrite an existing environment if indicated.

  Args:
    conda: path to the main conda executable of the installation
    name: the name of the environment to create or overwrite
    overwrite: if set to ```True``, overwrite potentially existing environments
      with the same name
    condarc: a dictionary of options for conda, including channel urls
    packages: the package list specification
    dry_run: if set, then don't execute anything, just print stuff
    use_local: include the local conda-bld directory as a possible installation
      channel (useful for testing multiple interdependent recipes that are
      built locally)
  '''

  from .bootstrap import run_cmdline

  specs = []
  for k in packages:
    k = ' '.join(k.split()[:2])  # remove eventual build string
    if any(elem in k for elem in '><|'):
      specs.append(k.replace(' ', ''))
    else:
      specs.append(k.replace(' ', '='))

  # if the current environment exists, delete it first
  envdir = get_env_directory(conda, name)
  if envdir is not None:
    if overwrite:
      cmd = [conda, 'env', 'remove', '--yes', '--name', name]
      logger.debug('$ ' + ' '.join(cmd))
      if not dry_run:
        run_cmdline(cmd)
    else:
      raise RuntimeError('environment `%s\' exists in `%s\' - use '
                         '--overwrite to overwrite' % (name, envdir))

  cmdline_channels = ['--channel=%s' % k for k in condarc['channels']]
  cmd = [conda, 'create', '--yes', '--name', name, '--override-channels'] + \
      cmdline_channels
  if dry_run:
    cmd.append('--dry-run')
  if use_local:
     cmd.append('--use-local')
  cmd.extend(sorted(specs))
  run_cmdline(cmd)

  # creates a .condarc file to sediment the just created environment
  if not dry_run:
    # get envdir again - it may just be created!
    envdir = get_env_directory(conda, name)
    destrc = os.path.join(envdir, '.condarc')
    logger.info('Creating %s...', destrc)
    with open(destrc, 'w') as f:
      yaml.dump(condarc, f, indent=2)


250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def get_docserver_setup(public, stable, server, intranet):
  '''Returns a setup for BOB_DOCUMENTATION_SERVER

  What is available to build the documentation depends on the setup of
  ``public`` and ``stable``:

  * public and stable: only returns the public stable channel(s)
  * public and not stable: returns both public stable and beta channels
  * not public and stable: returns both public and private stable channels
  * not public and not stable: returns all channels

  Beta channels have priority over stable channels, if returned.  Private
  channels have priority over public channles, if turned.


  Args:

    public: Boolean indicating if we're supposed to include only public
      channels
    stable: Boolean indicating if we're supposed to include only stable
      channels
    server: The base address of the server containing our conda channels
    intranet: Boolean indicating if we should add "private"/"public" prefixes
      on the returned paths


  Returns: a string to be used by bob.extension to find dependent
  documentation projects.

  '''

  if (not public) and (not intranet):
    raise RuntimeError('You cannot request for private channels and set' \
        ' intranet=False (server=%s) - these are conflicting options' % server)

  entries = []

  # public documentation: always can access
  prefix = '/software/bob'
  if server.endswith(prefix):  # don't repeat yourself...
    prefix = ''
  if stable:
    entries += [
        server + prefix + '/docs/bob/%(name)s/%(version)s/',
        server + prefix + '/docs/bob/%(name)s/stable/',
        ]
  else:
    entries += [
        server + prefix + '/docs/bob/%(name)s/master/',
        ]

  if private:
    # add private channels, (notice they are not accessible outside idiap)
    prefix = '/private' if intranet else ''
    if stable:
      entries += [
          server + prefix + '/docs/bob/%(name)s/%(version)s/',
          server + prefix + '/docs/bob/%(name)s/stable/',
          ]
    else:
      entries += [
          server + prefix + '/docs/bob/%(name)s/master/',
          ]

  return '|'.join(entries)


André Anjos's avatar
André Anjos committed
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
def check_version(workdir, envtag):
  '''Checks if the version being built and the value reported match

  This method will read the contents of the file ``version.txt`` and compare it
  to the potentially set ``envtag`` (may be ``None``).  If the value of
  ``envtag`` is different than ``None``, ensure it matches the value in
  ``version.txt`` or raises an exception.


  Args:

    workdir: The work directory where the repo of the package being built was
      checked-out
    envtag: The output of os.environ.get('CI_COMMIT_TAG') (may be ``None``)


  Returns: A tuple with the version of the package that we're currently
  building and a boolean flag indicating if the version number represents a
  pre-release or a stable release.
  '''

  version = open(os.path.join(workdir, "version.txt"), 'rt').read().rstrip()

  # if we're building a stable release, ensure a tag is set
  parsed_version = distutils.version.LooseVersion(version).version
  is_prerelease = any([isinstance(k, str) for k in parsed_version])
  if is_prerelease:
    if envtag is not None:
      raise EnvironmentError('"version.txt" indicates version is a ' \
          'pre-release (v%s) - but os.environ["CI_COMMIT_TAG"]="%s", ' \
          'which indicates this is a **stable** build. ' \
          'Have you created the tag using ``bdt release``?' % (version,
          envtag))
  else:  #it is a stable build
    if envtag is None:
      raise EnvironmentError('"version.txt" indicates version is a ' \
          'stable build (v%s) - but there is no os.environ["CI_COMMIT_TAG"] ' \
          'variable defined, which indicates this is **not** ' \
          'a tagged build. Use ``bdt release`` to create stable releases' % \
          (version,))
    if envtag[1:] != version:
      raise EnvironmentError('"version.txt" and the value of ' \
          'os.environ["CI_COMMIT_TAG"] do **NOT** agree - the former ' \
          'reports version %s, the latter, %s' % (version, envtag[1:]))

  return version, is_prerelease


def git_clean_build(runner, arch):
  '''Runs git-clean to clean-up build products

  Args:

    runner: A pointer to the ``run_cmdline()`` function

  '''

  # runs git clean to clean everything that is not needed. This helps to keep
  # the disk usage on CI machines to a minimum.
  exclude_from_cleanup = [
      "miniconda.sh",   #the installer, cached
      "miniconda/pkgs/*.tar.bz2",  #downloaded packages, cached
      "miniconda/pkgs/urls.txt",  #download index, cached
      "miniconda/conda-bld/%s/*.tar.bz2" % (arch,),  #build artifact -- conda
      "dist/*.zip",  #build artifact -- pypi package
      "sphinx",  #build artifact -- documentation
      ]
  runner(['git', 'clean', '-qffdx'] + \
      ['--exclude=%s' % k for k in exclude_from_cleanup])


388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
if __name__ == '__main__':

  # loads the "adjacent" bootstrap module
  import importlib.util
  mydir = os.path.dirname(os.path.realpath(sys.argv[0]))
  bootstrap_file = os.path.join(mydir, 'bootstrap.py')
  spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
  bootstrap = importlib.util.module_from_spec(spec)
  spec.loader.exec_module(bootstrap)

  bootstrap.setup_logger(logger)

  prefix = os.environ['CONDA_ROOT']
  logger.info('os.environ["%s"] = %s', 'CONDA_ROOT', prefix)

  workdir = os.environ['CI_PROJECT_DIR']
  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_DIR', workdir)

  name = os.environ['CI_PROJECT_NAME']
  logger.info('os.environ["%s"] = %s', 'CI_PROJECT_NAME', name)

  pyver = os.environ['PYTHON_VERSION']
  logger.info('os.environ["%s"] = %s', 'PYTHON_VERSION', pyver)

André Anjos's avatar
André Anjos committed
412
413
414
  bootstrap.set_environment('DOCSERVER', bootstrap._SERVER, verbose=True)
  bootstrap.set_environment('LANG', 'en_US.UTF-8', verbose=True)
  bootstrap.set_environment('LC_ALL', os.environ['LANG'], verbose=True)
415

416
  # get information about the version of the package being built
André Anjos's avatar
André Anjos committed
417
418
419
  version, is_prerelease = check_version(workdir,
      os.environ.get('CI_COMMIT_TAG'))
  bootstrap.set_environment('BOB_PACKAGE_VERSION', version, verbose=True)
420

421
422
423
424
425
426
427
428
429
  # create the build configuration
  conda_build_config = os.path.join(mydir, 'data', 'conda_build_config.yaml')
  recipe_append = os.path.join(mydir, 'data', 'recipe_append.yaml')

  condarc = os.path.join(prefix, 'condarc')
  logger.info('Loading (this build\'s) CONDARC file from %s...', condarc)
  with open(condarc, 'rb') as f:
    condarc_options = yaml.load(f)

André Anjos's avatar
André Anjos committed
430
431
  # notice this condarc typically will only contain the defaults channel - we
  # need to boost this up with more channels to get it right.
André Anjos's avatar
André Anjos committed
432
433
434
  public = ( os.environ['CI_PROJECT_VISIBILITY']=='public' )
  channels = bootstrap.get_channels(public=public, stable=(not is_prerelease),
      server=bootstrap._SERVER, intranet=True)
André Anjos's avatar
André Anjos committed
435
436
  logger.info('Using the following channels during build:\n  - %s',
      '\n  - '.join(channels + ['defaults']))
André Anjos's avatar
André Anjos committed
437
438
  condarc_options['channels'] = channels + ['defaults']

André Anjos's avatar
André Anjos committed
439
  logger.info('Merging conda configuration files...')
440
441
442
  conda_config = make_conda_config(conda_build_config, pyver, recipe_append,
      condarc_options)

André Anjos's avatar
André Anjos committed
443
  # retrieve the current build number for this build
444
  build_number, _ = next_build_number(channels[0], name, version, pyver)
André Anjos's avatar
André Anjos committed
445
446
  bootstrap.set_environment('BOB_BUILD_NUMBER', str(build_number),
      verbose=True)
447
448

  # runs the build using the conda-build API
449
  arch = conda_arch()
450
  logger.info('Building %s-%s-py%s (build: %d) for %s',
451
      name, version, pyver.replace('.',''), build_number, arch)
André Anjos's avatar
André Anjos committed
452
  conda_build.api.build(os.path.join(workdir, 'conda'), config=conda_config)
453

André Anjos's avatar
André Anjos committed
454
  git_clean_build(bootstrap.run_cmdline, arch)