bootstrap.py 13.2 KB
Newer Older
André Anjos's avatar
André Anjos committed
1 2 3 4
#!/usr/bin/env python3
# -*- coding: utf-8 -*-


5
'''Bootstraps a new miniconda installation and prepares it for development'''
André Anjos's avatar
André Anjos committed
6 7


8
_BASE_CONDARC = '''\
André Anjos's avatar
André Anjos committed
9 10 11 12 13 14 15 16 17 18 19 20
default_channels:
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/free
  - https://repo.anaconda.com/pkgs/r
  - https://repo.anaconda.com/pkgs/pro
add_pip_as_python_dependency: false #!final
changeps1: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
21 22
channels:
  - defaults
André Anjos's avatar
André Anjos committed
23
'''
24

25 26 27 28 29 30 31 32 33 34 35
_SERVER = 'http://www.idiap.ch'

_INTERVALS = (
    ('weeks', 604800),  # 60 * 60 * 24 * 7
    ('days', 86400),    # 60 * 60 * 24
    ('hours', 3600),    # 60 * 60
    ('minutes', 60),
    ('seconds', 1),
    )
'''Time intervals that make up human readable time slots'''

36

37
import os
André Anjos's avatar
André Anjos committed
38 39 40
import sys
import glob
import time
41
import shutil
André Anjos's avatar
André Anjos committed
42
import platform
43 44 45
import subprocess

import logging
46
logger = logging.getLogger(__name__)
André Anjos's avatar
André Anjos committed
47

48

49
def set_environment(name, value, env=os.environ):
50 51 52 53 54 55 56 57 58 59
    '''Function to setup the environment variable and print debug message

    Args:

      name: The name of the environment variable to set
      value: The value to set the environment variable to
      env: Optional environment (dictionary) where to set the variable at
    '''

    env[name] = value
60
    logger.info('environ["%s"] = %s', name, value)
61
    return value
62

63

André Anjos's avatar
André Anjos committed
64 65
def human_time(seconds, granularity=2):
  '''Returns a human readable time string like "1 day, 2 hours"'''
66

André Anjos's avatar
André Anjos committed
67
  result = []
68

André Anjos's avatar
André Anjos committed
69 70 71 72 73 74 75 76 77 78 79
  for name, count in _INTERVALS:
    value = seconds // count
    if value:
      seconds -= value * count
      if value == 1:
        name = name.rstrip('s')
      result.append("{} {}".format(int(value), name))
    else:
      # Add a blank if we're in the middle of other values
      if len(result) > 0:
        result.append(None)
80

André Anjos's avatar
André Anjos committed
81 82 83 84 85 86 87 88
  if not result:
    if seconds < 1.0:
      return '%.2f seconds' % seconds
    else:
      if seconds == 1:
        return '1 second'
      else:
        return '%d seconds' % seconds
89

André Anjos's avatar
André Anjos committed
90
  return ', '.join([x for x in result[:granularity] if x is not None])
91 92


André Anjos's avatar
André Anjos committed
93 94
def run_cmdline(cmd, env=None):
  '''Runs a command on a environment, logs output and reports status
95 96


André Anjos's avatar
André Anjos committed
97
  Parameters:
98

André Anjos's avatar
André Anjos committed
99
    cmd (list): The command to run, with parameters separated on a list
100

André Anjos's avatar
André Anjos committed
101 102
    env (dict, Optional): Environment to use for running the program on. If not
      set, use :py:obj:`os.environ`.
103

André Anjos's avatar
André Anjos committed
104
  '''
105

André Anjos's avatar
André Anjos committed
106
  if env is None: env = os.environ
107

André Anjos's avatar
André Anjos committed
108
  logger.info('(system) %s' % ' '.join(cmd))
109

André Anjos's avatar
André Anjos committed
110
  start = time.time()
111

112 113 114 115 116 117 118 119
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
      env=env, bufsize=1, universal_newlines=True)

  for line in iter(p.stdout.readline, ''):
    sys.stdout.write(line)
    sys.stdout.flush()

  if p.wait() != 0:
André Anjos's avatar
André Anjos committed
120 121
    raise RuntimeError("command `%s' exited with error state (%d)" % \
        (' '.join(cmd), p.returncode))
122

André Anjos's avatar
André Anjos committed
123
  total = time.time() - start
124

André Anjos's avatar
André Anjos committed
125
  logger.info('command took %s' % human_time(total))
126 127


128

André Anjos's avatar
André Anjos committed
129 130
def touch(path):
  '''Python-implementation of the "touch" command-line application'''
131

André Anjos's avatar
André Anjos committed
132 133
  with open(path, 'a'):
    os.utime(path, None)
134

André Anjos's avatar
André Anjos committed
135

136 137 138 139 140 141 142 143 144
def merge_conda_cache(cache, prefix, name):
  '''Merges conda pkg caches and conda-bld folders

  Args:

    cache: The cached directory (from previous builds)
    prefix: The current prefix (root of conda installation)
    name: The name of the current package
  '''
André Anjos's avatar
André Anjos committed
145 146 147 148 149 150 151 152 153 154 155 156 157

  pkgs_dir = os.path.join(prefix, 'pkgs')
  pkgs_urls_txt = os.path.join(pkgs_dir, 'urls.txt')
  if not os.path.exists(pkgs_dir):
    logger.info('mkdir -p %s', pkgs_dir)
    os.makedirs(pkgs_dir)
    logger.info('touch %s', pkgs_urls_txt)
    touch(pkgs_urls_txt)

  # move packages on cache/pkgs to pkgs_dir
  cached_pkgs_dir = os.path.join(cache, 'pkgs')
  cached_packages = glob.glob(os.path.join(cached_pkgs_dir, '*.tar.bz2'))
  cached_packages = [k for k in cached_packages if not \
158
      k.startswith(name + '-')]
André Anjos's avatar
André Anjos committed
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
  logger.info('Merging %d cached conda packages', len(cached_packages))
  for k in cached_packages:
    dst = os.path.join(pkgs_dir, os.path.basename(k))
    logger.debug('(move) %s -> %s', k, dst)
    os.rename(k, dst)

  # merge urls.txt files
  logger.info('Merging urls.txt files from cache...')
  urls = []
  cached_pkgs_urls_txt = os.path.join(cached_pkgs_dir, 'urls.txt')
  with open(pkgs_urls_txt, 'rb') as f1, \
      open(cached_pkgs_urls_txt, 'rb') as f2:
    data = set(f1.readlines() + f2.readlines())
    data = sorted(list(data))
  with open(pkgs_urls_txt, 'wb') as f:
    f.writelines(data)

  pkgs_urls = os.path.join(pkgs_dir, 'urls')
  touch(pkgs_urls)

  # move conda-bld build results
  cached_conda_bld = os.path.join(cache, 'conda-bld')
  if os.path.exists(cached_conda_bld):
    dst = os.path.join(prefix, 'conda-bld')
    logger.info('(move) %s -> %s', cached_conda_bld, dst)
    os.rename(cached_conda_bld, dst)


def get_miniconda_sh():
  '''Retrieves the miniconda3 installer for the current system'''

  import http.client

  server = 'repo.continuum.io'  #https
  path = '/miniconda/Miniconda3-latest-%s-x86_64.sh'
  if platform.system() == 'Darwin':
    path = path % 'MacOSX'
  else:
    path = path % 'Linux'

  logger.info('Connecting to https://%s...', server)
  conn = http.client.HTTPSConnection(server)
  conn.request("GET", path)
  r1 = conn.getresponse()

  assert r1.status == 200, 'Request for https://%s%s - returned status %d ' \
      '(%s)' % (server, path, r1.status, r1.reason)

  dst = 'miniconda.sh'
  logger.info('(download) https://%s%s -> %s...', server, path, dst)
  with open(dst, 'wb') as f:
    f.write(r1.read())


213 214 215 216 217 218 219 220 221
def install_miniconda(prefix, name):
  '''Creates a new miniconda installation

  Args:

    prefix: The path leading to the (new) root of the miniconda installation
    name: The name of this package

  '''
André Anjos's avatar
André Anjos committed
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240

  logger.info("Installing miniconda in %s...", prefix)

  if not os.path.exists('miniconda.sh'):  #re-downloads installer
    get_miniconda_sh()
  else:
    logger.info("Re-using cached miniconda3 installer")

  cached = None
  if os.path.exists(prefix):  #this is the previous cache, move it
    cached = prefix + '.cached'
    if os.path.exists(cached):
      logger.info('(rmtree) %s', cached)
      shutil.rmtree(cached)
    logger.info('(move) %s -> %s', prefix, cached)
    os.rename(prefix, cached)

  run_cmdline(['bash', 'miniconda.sh', '-b', '-p', prefix])
  if cached is not None:
241
    merge_conda_cache(cached, prefix, name)
André Anjos's avatar
André Anjos committed
242 243 244
    shutil.rmtree(cached)


245
def get_channels(public, stable, server, intranet):
André Anjos's avatar
André Anjos committed
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
  '''Returns the relevant conda channels to consider if building project

  The subset of channels to be returned depends on the visibility and stability
  of the package being built.  Here are the rules:

  * public and stable: only returns the public stable channel(s)
  * public and not stable: returns both public stable and beta channels
  * not public and stable: returns both public and private stable channels
  * not public and not stable: returns all channels

  Beta channels have priority over stable channels, if returned.  Private
  channels have priority over public channles, if turned.


  Args:

    public: Boolean indicating if we're supposed to include only public
      channels
    stable: Boolean indicating if we're supposed to include only stable
      channels
266 267 268
    server: The base address of the server containing our conda channels
    intranet: Boolean indicating if we should add "private"/"public" prefixes
      on the conda paths
André Anjos's avatar
André Anjos committed
269 270 271 272 273 274


  Returns: a list of channels that need to be considered.

  '''

275 276 277 278
  if (not public) and (not intranet):
    raise RuntimeError('You cannot request for private channels and set' \
        ' intranet=False (server=%s) - these are conflicting options' % server)

279 280
  channels = []

André Anjos's avatar
André Anjos committed
281
  if not public:
282
    prefix = '/private' if intranet else ''
André Anjos's avatar
André Anjos committed
283
    if not stable:  #allowed private channels
284 285 286 287
      channels += [server + prefix + '/conda/label/beta']  #allowed betas
    channels += [server + prefix + '/conda']

  prefix = '/public' if intranet else ''
André Anjos's avatar
André Anjos committed
288
  if not stable:
289 290
    channels += [server + prefix + '/conda/label/beta']  #allowed betas
  channels += [server + prefix + '/conda']
André Anjos's avatar
André Anjos committed
291 292 293 294

  return channels


295
def setup_logger(logger, level):
André Anjos's avatar
André Anjos committed
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
  '''Sets-up the logging for this command at level ``INFO``'''

  warn_err = logging.StreamHandler(sys.stderr)
  warn_err.setLevel(logging.WARNING)
  logger.addHandler(warn_err)

  # debug and info messages are written to sys.stdout

  class _InfoFilter:
    def filter(self, record):
      return record.levelno <= logging.INFO

  debug_info = logging.StreamHandler(sys.stdout)
  debug_info.setLevel(logging.DEBUG)
  debug_info.addFilter(_InfoFilter())
  logger.addHandler(debug_info)

  formatter = logging.Formatter('%(levelname)s@%(asctime)s: %(message)s')

  for handler in logger.handlers:
    handler.setFormatter(formatter)

318 319 320 321 322 323 324 325 326 327 328
  if level not in range(0, 4):
    raise ValueError(
        "The verbosity level %d does not exist. Please reduce the number of "
        "'--verbose' parameters in your command line" % level)
  # set up the verbosity level of the logging system
  log_level = {
      0: logging.ERROR,
      1: logging.WARNING,
      2: logging.INFO,
      3: logging.DEBUG
  }[level]
André Anjos's avatar
André Anjos committed
329

330
  logger.setLevel(log_level)
André Anjos's avatar
André Anjos committed
331 332


333
if __name__ == '__main__':
André Anjos's avatar
André Anjos committed
334

335 336 337 338 339 340 341 342 343 344 345 346 347 348
  import argparse

  parser = argparse.ArgumentParser(description='Bootstraps a new miniconda ' \
      'installation and prepares it for development')
  parser.add_argument('command', choices=['build', 'local', 'channel'],
      help='How to prepare the current environment. Use: ``build``, to ' \
          'build bob.devtools, ``local``, to bootstrap deploy or pypi ' \
          'stages for bob.devtools builds, ``channel`` channel to bootstrap ' \
          'CI environment for beta/stable builds')
  parser.add_argument('envname', nargs='?', default='bdt',
      help='The name of the conda environment that will host bdt ' \
          '[default: %(default)s]')
  parser.add_argument('-n', '--name',
      default=os.environ.get('CI_PROJECT_NAME', 'bob.devtools'),
349
      help='The name of the project being built [default: %(default)s]')
350 351 352 353
  parser.add_argument('-c', '--conda-root',
      default=os.environ.get('CONDA_ROOT',
        os.path.realpath(os.path.join(os.curdir, 'miniconda'))),
      help='The location where we should install miniconda ' \
354
          '[default: %(default)s]')
355
  parser.add_argument('-V', '--visibility',
356
      choices=['public', 'internal', 'private'],
357
      default=os.environ.get('CI_PROJECT_VISIBILITY', 'public'),
358
      help='The visibility level for this project [default: %(default)s]')
359 360
  parser.add_argument('-t', '--tag',
      default=os.environ.get('CI_COMMIT_TAG', None),
361
      help='If building a tag, pass it with this flag [default: %(default)s]')
362
  parser.add_argument('--verbose', '-v', action='count', default=0,
363 364 365
      help='Increases the verbosity level.  We always prints error and ' \
          'critical messages. Use a single ``-v`` to enable warnings, ' \
          'two ``-vv`` to enable information messages and three ``-vvv`` ' \
366
          'to enable debug messages [default: %(default)s]')
367 368 369 370 371 372 373

  args = parser.parse_args()

  setup_logger(logger, args.verbose)

  install_miniconda(args.conda_root, args.name)
  conda_bin = os.path.join(args.conda_root, 'bin', 'conda')
André Anjos's avatar
André Anjos committed
374 375

  # creates the condarc file
376
  condarc = os.path.join(args.conda_root, 'condarc')
André Anjos's avatar
André Anjos committed
377 378
  logger.info('(create) %s', condarc)
  with open(condarc, 'wt') as f:
379
    f.write(_BASE_CONDARC)
André Anjos's avatar
André Anjos committed
380 381 382 383

  conda_version = '4'
  conda_build_version = '3'

384
  conda_verbosity = []
385
  #if args.verbose >= 2:
386
  #  conda_verbosity = ['-v']
387
  if args.verbose >= 3:
388
    conda_verbosity = ['-vv']
389 390

  if args.command == 'build':
André Anjos's avatar
André Anjos committed
391 392

    # simple - just use the defaults channels when self building
393
    run_cmdline([conda_bin, 'install'] + conda_verbosity + ['-n', 'base',
André Anjos's avatar
André Anjos committed
394 395 396
      'python',
      'conda=%s' % conda_version,
      'conda-build=%s' % conda_build_version,
397
      'twine',  #required for checking readme of python (zip) distro
André Anjos's avatar
André Anjos committed
398 399
      ])

400
  elif args.command == 'local':
André Anjos's avatar
André Anjos committed
401 402

    # index the locally built packages
403
    run_cmdline([conda_bin, 'install'] + conda_verbosity + ['-n', 'base',
André Anjos's avatar
André Anjos committed
404 405 406
      'python',
      'conda=%s' % conda_version,
      'conda-build=%s' % conda_build_version,
407
      'twine',  #required for checking readme of python (zip) distro
André Anjos's avatar
André Anjos committed
408
      ])
409
    conda_bld_path = os.path.join(args.conda_root, 'conda-bld')
André Anjos's avatar
André Anjos committed
410 411
    run_cmdline([conda_bin, 'index', conda_bld_path])
    # add the locally build directory before defaults, boot from there
412
    channels = get_channels(public=True, stable=True, server=_SERVER,
413 414 415 416
        intranet=True) + ['defaults']
    channels = ['--override-channels'] + \
        ['--channel=' + conda_bld_path] + \
        ['--channel=%s' % k for k in channels]
417
    run_cmdline([conda_bin, 'create'] + conda_verbosity + channels + \
418
        ['-n', args.envname, 'bob.devtools'])
André Anjos's avatar
André Anjos committed
419

420
  elif args.command == 'channel':
André Anjos's avatar
André Anjos committed
421 422 423

    # installs from channel
    channels = get_channels(
424 425
        public=(args.visibility == 'public'),
        stable=(args.tag is not None),
426
        server=_SERVER, intranet=True) + ['defaults']
427 428
    channels = ['--override-channels'] + \
        ['--channel=%s' % k for k in channels]
429
    run_cmdline([conda_bin, 'create'] + conda_verbosity + channels + \
430
        ['-n', args.envname, 'bob.devtools'])
André Anjos's avatar
André Anjos committed
431 432

  # clean up
433
  run_cmdline([conda_bin, 'clean'] + conda_verbosity + ['--lock'])
André Anjos's avatar
André Anjos committed
434 435

  # print conda information for debugging purposes
436
  run_cmdline([conda_bin, 'info'] + conda_verbosity)