bootstrap.py 18.4 KB
Newer Older
André Anjos's avatar
DRY    
André Anjos committed
1
2
3
4
#!/usr/bin/env python3
# -*- coding: utf-8 -*-


5
"""Bootstraps a new miniconda installation and prepares it for development."""
André Anjos's avatar
DRY    
André Anjos committed
6
7


André Anjos's avatar
André Anjos committed
8
_BASE_CONDARC = """\
9
default_channels: #!final
André Anjos's avatar
DRY    
André Anjos committed
10
11
12
13
14
15
16
  - https://repo.anaconda.com/pkgs/main
add_pip_as_python_dependency: false #!final
always_yes: true #!final
quiet: true #!final
show_channel_urls: true #!final
anaconda_upload: false #!final
ssl_verify: false #!final
17
18
19
remote_connect_timeout_secs: 120.0 #!final
remote_max_retries: 50 #!final
remote_read_timeout_secs: 180.0 #!final
20
21
channels:
  - defaults
André Anjos's avatar
André Anjos committed
22
"""
André Anjos's avatar
André Anjos committed
23

André Anjos's avatar
André Anjos committed
24
_SERVER = "http://www.idiap.ch"
25
26

_INTERVALS = (
André Anjos's avatar
André Anjos committed
27
28
29
30
31
32
33
    ("weeks", 604800),  # 60 * 60 * 24 * 7
    ("days", 86400),  # 60 * 60 * 24
    ("hours", 3600),  # 60 * 60
    ("minutes", 60),
    ("seconds", 1),
)
"""Time intervals that make up human readable time slots"""
34

35

André Anjos's avatar
André Anjos committed
36
import os
André Anjos's avatar
DRY    
André Anjos committed
37
38
39
import sys
import glob
import time
André Anjos's avatar
André Anjos committed
40
import shutil
André Anjos's avatar
DRY    
André Anjos committed
41
import platform
André Anjos's avatar
André Anjos committed
42
43
44
import subprocess

import logging
André Anjos's avatar
André Anjos committed
45

46
logger = logging.getLogger(__name__)
André Anjos's avatar
DRY    
André Anjos committed
47

48

49
50
51
def do_hack(project_dir):
    """
    This function is supposed to be for temporary usage.
52

53
54
    It implements hacks for the issues: https://gitlab.idiap.ch/bob/bob.devtools/merge_requests/112
    and https://github.com/conda/conda-build/issues/3767)
55

56
    """
57

58
    #### HACK to avoid ripgrep ignoring bin/ directories in our checkouts
59
60
61
    import shutil

    git_ignore_file = os.path.join(project_dir, ".gitignore")
62
    if os.path.exists(git_ignore_file):
63
64
65
66
        logger.warning('Removing ".gitignore" to overcome issues with ripgrep')
        logger.warning(
            "See https://gitlab.idiap.ch/bob/bob.devtools/merge_requests/112"
        )
67
68
        os.unlink(git_ignore_file)
    #### END OF HACK
69

70
    #### HACK that avoids this issue: https://github.com/conda/conda-build/issues/3767
71
    license_file = os.path.join(project_dir, "LICENSE")
72
    if not os.path.exists(license_file):
73
74
75
        license_file = os.path.join(project_dir, "LICENSE.AGPL")

    recipe_dir = os.path.join(project_dir, "conda")
76
    if os.path.exists(license_file) and os.path.exists(recipe_dir):
77
78
79
80
81
82
83
84
85
86
87
        logger.warning(
            "Copying LICENSE file to `./conda` dir to avoid issue with conda build (https://github.com/conda/conda-build/issues/3767)"
        )
        logger.warning(
            "Replacing ../LICENSE to LICENSE (https://github.com/conda/conda-build/issues/3767)"
        )
        shutil.copyfile(
            license_file,
            os.path.join(recipe_dir, os.path.basename(license_file)),
        )

88
        # Checking COPYING file just in case
89
90
91
92
93
        copying_file = os.path.join(project_dir, "COPYING")
        if os.path.exists(copying_file):
            shutil.copyfile(copying_file, os.path.join(recipe_dir, "COPYING"))

        meta_file = os.path.join(recipe_dir, "meta.yaml")
94
        recipe = open(meta_file).readlines()
95
96
97
98
99
100
101
        recipe = [
            l.replace("../COPYING", "COPYING")
            .replace("../LICENSE", "LICENSE")
            .replace("../LICENSE.AGPL", "LICENSE.AGPL")
            for l in recipe
        ]
        open(meta_file, "wt").write("".join(recipe))
102
103
104
    #### END OF HACK


105
def set_environment(name, value, env=os.environ):
106
    """Function to setup the environment variable and print debug message.
107
108
109
110
111
112

    Args:

      name: The name of the environment variable to set
      value: The value to set the environment variable to
      env: Optional environment (dictionary) where to set the variable at
André Anjos's avatar
André Anjos committed
113
    """
114
115

    env[name] = value
116
    logger.info('environ["%s"] = %s', name, value)
117
    return value
118

André Anjos's avatar
André Anjos committed
119

André Anjos's avatar
DRY    
André Anjos committed
120
def human_time(seconds, granularity=2):
121
    """Returns a human readable time string like "1 day, 2 hours"."""
André Anjos's avatar
André Anjos committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

    result = []

    for name, count in _INTERVALS:
        value = seconds // count
        if value:
            seconds -= value * count
            if value == 1:
                name = name.rstrip("s")
            result.append("{} {}".format(int(value), name))
        else:
            # Add a blank if we're in the middle of other values
            if len(result) > 0:
                result.append(None)

    if not result:
        if seconds < 1.0:
            return "%.2f seconds" % seconds
        else:
            if seconds == 1:
                return "1 second"
            else:
                return "%d seconds" % seconds

    return ", ".join([x for x in result[:granularity] if x is not None])
147
148


André Anjos's avatar
DRY    
André Anjos committed
149
def run_cmdline(cmd, env=None):
150
    """Runs a command on a environment, logs output and reports status.
151

152
    Parameters:
153

154
      cmd (list): The command to run, with parameters separated on a list
155

156
157
158
      env (dict, Optional): Environment to use for running the program on. If not
        set, use :py:obj:`os.environ`.
    """
159

André Anjos's avatar
André Anjos committed
160
161
    if env is None:
        env = os.environ
André Anjos's avatar
André Anjos committed
162

André Anjos's avatar
André Anjos committed
163
    logger.info("(system) %s" % " ".join(cmd))
André Anjos's avatar
André Anjos committed
164

André Anjos's avatar
André Anjos committed
165
    start = time.time()
André Anjos's avatar
André Anjos committed
166

André Anjos's avatar
André Anjos committed
167
168
169
170
171
172
173
174
    p = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        env=env,
        bufsize=1,
        universal_newlines=True,
    )
175

André Anjos's avatar
André Anjos committed
176
177
178
    for line in iter(p.stdout.readline, ""):
        sys.stdout.write(line)
        sys.stdout.flush()
André Anjos's avatar
André Anjos committed
179

André Anjos's avatar
André Anjos committed
180
181
182
183
184
    if p.wait() != 0:
        raise RuntimeError(
            "command `%s' exited with error state (%d)"
            % (" ".join(cmd), p.returncode)
        )
André Anjos's avatar
André Anjos committed
185

André Anjos's avatar
André Anjos committed
186
    total = time.time() - start
André Anjos's avatar
André Anjos committed
187

André Anjos's avatar
André Anjos committed
188
    logger.info("command took %s" % human_time(total))
André Anjos's avatar
André Anjos committed
189

190

André Anjos's avatar
DRY    
André Anjos committed
191
def touch(path):
192
    """Python-implementation of the "touch" command-line application."""
André Anjos's avatar
André Anjos committed
193

André Anjos's avatar
André Anjos committed
194
195
    with open(path, "a"):
        os.utime(path, None)
André Anjos's avatar
André Anjos committed
196

André Anjos's avatar
DRY    
André Anjos committed
197

198
def merge_conda_cache(cache, prefix, name):
199
    """Merges conda pkg caches and conda-bld folders.
200

201
    Args:
202

203
204
205
206
      cache: The cached directory (from previous builds)
      prefix: The current prefix (root of conda installation)
      name: The name of the current package
    """
André Anjos's avatar
André Anjos committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256

    pkgs_dir = os.path.join(prefix, "pkgs")
    pkgs_urls_txt = os.path.join(pkgs_dir, "urls.txt")
    if not os.path.exists(pkgs_dir):
        logger.info("mkdir -p %s", pkgs_dir)
        os.makedirs(pkgs_dir)
        logger.info("touch %s", pkgs_urls_txt)
        touch(pkgs_urls_txt)

    # move packages on cache/pkgs to pkgs_dir
    cached_pkgs_dir = os.path.join(cache, "pkgs")
    cached_packages = glob.glob(os.path.join(cached_pkgs_dir, "*.tar.bz2"))
    cached_packages = [
        k for k in cached_packages if not k.startswith(name + "-")
    ]
    logger.info("Merging %d cached conda packages", len(cached_packages))
    for k in cached_packages:
        dst = os.path.join(pkgs_dir, os.path.basename(k))
        logger.debug("(move) %s -> %s", k, dst)
        os.rename(k, dst)

    # merge urls.txt files
    logger.info("Merging urls.txt files from cache...")
    urls = []
    cached_pkgs_urls_txt = os.path.join(cached_pkgs_dir, "urls.txt")

    if not os.path.exists(cached_pkgs_urls_txt):
        with open(pkgs_urls_txt, "rb") as f1:
            data = set(f1.readlines())
            data = sorted(list(data))
    else:
        # use both cached and actual conda package caches
        with open(pkgs_urls_txt, "rb") as f1, open(
            cached_pkgs_urls_txt, "rb"
        ) as f2:
            data = set(f1.readlines() + f2.readlines())
            data = sorted(list(data))

    with open(pkgs_urls_txt, "wb") as f:
        f.writelines(data)

    pkgs_urls = os.path.join(pkgs_dir, "urls")
    touch(pkgs_urls)

    # move conda-bld build results
    cached_conda_bld = os.path.join(cache, "conda-bld")
    if os.path.exists(cached_conda_bld):
        dst = os.path.join(prefix, "conda-bld")
        logger.info("(move) %s -> %s", cached_conda_bld, dst)
        os.rename(cached_conda_bld, dst)
André Anjos's avatar
DRY    
André Anjos committed
257
258


259
260
def ensure_miniconda_sh():
    """Retrieves the miniconda3 installer for the current system.
André Anjos's avatar
DRY    
André Anjos committed
261

262
263
264
265
    Checks the hash of the miniconda3 installer against the expected version,
    if that does not match, erase existing installer and re-downloads new
    installer.
    """
André Anjos's avatar
DRY    
André Anjos committed
266

267
    # WARNING: if you update this version, remember to update hahes below
268
    # AND our "mirror" in the internal webserver
269
    path = "/miniconda/Miniconda3-4.7.12-%s-x86_64.sh"
André Anjos's avatar
André Anjos committed
270
    if platform.system() == "Darwin":
271
        md5sum = "677f38d5ab7e1ce4fef134068e3bd76a"
André Anjos's avatar
André Anjos committed
272
273
        path = path % "MacOSX"
    else:
274
        md5sum = "0dba759b8ecfc8948f626fa18785e3d8"
André Anjos's avatar
André Anjos committed
275
        path = path % "Linux"
André Anjos's avatar
DRY    
André Anjos committed
276

277
278
279
    if os.path.exists("miniconda.sh"):
        logger.info("(check) miniconda.sh md5sum (== %s?)", md5sum)
        import hashlib
280

281
282
283
284
285
        actual_md5 = hashlib.md5(open("miniconda.sh", "rb").read()).hexdigest()
        if actual_md5 == md5sum:
            logger.info("Re-using cached miniconda3 installer (hash matches)")
            return
        else:
286
287
288
289
            logger.info(
                "Erasing cached miniconda3 installer (%s does NOT " "match)",
                actual_md5,
            )
290
291
292
293
294
            os.unlink("miniconda.sh")

    # re-downloads installer
    import http.client

André Anjos's avatar
André Anjos committed
295
296
297
298
    server = "www.idiap.ch"  # http

    logger.info("Connecting to http://%s...", server)
    conn = http.client.HTTPConnection(server)
André Anjos's avatar
André Anjos committed
299
300
    conn.request("GET", path)
    r1 = conn.getresponse()
André Anjos's avatar
DRY    
André Anjos committed
301

André Anjos's avatar
André Anjos committed
302
    assert r1.status == 200, (
André Anjos's avatar
André Anjos committed
303
        "Request for http://%s%s - returned status %d "
André Anjos's avatar
André Anjos committed
304
305
        "(%s)" % (server, path, r1.status, r1.reason)
    )
André Anjos's avatar
DRY    
André Anjos committed
306

André Anjos's avatar
André Anjos committed
307
    dst = "miniconda.sh"
André Anjos's avatar
André Anjos committed
308
    logger.info("(download) http://%s%s -> %s...", server, path, dst)
André Anjos's avatar
André Anjos committed
309
310
    with open(dst, "wb") as f:
        f.write(r1.read())
André Anjos's avatar
DRY    
André Anjos committed
311
312


313
def install_miniconda(prefix, name):
314
    """Creates a new miniconda installation.
315

316
    Args:
317

318
319
320
      prefix: The path leading to the (new) root of the miniconda installation
      name: The name of this package
    """
André Anjos's avatar
DRY    
André Anjos committed
321

André Anjos's avatar
André Anjos committed
322
    logger.info("Installing miniconda in %s...", prefix)
323
    ensure_miniconda_sh()
André Anjos's avatar
DRY    
André Anjos committed
324

André Anjos's avatar
André Anjos committed
325
326
327
328
329
330
331
332
    cached = None
    if os.path.exists(prefix):  # this is the previous cache, move it
        cached = prefix + ".cached"
        if os.path.exists(cached):
            logger.info("(rmtree) %s", cached)
            shutil.rmtree(cached)
        logger.info("(move) %s -> %s", prefix, cached)
        os.rename(prefix, cached)
André Anjos's avatar
DRY    
André Anjos committed
333

André Anjos's avatar
André Anjos committed
334
335
336
337
    run_cmdline(["bash", "miniconda.sh", "-b", "-p", prefix])
    if cached is not None:
        merge_conda_cache(cached, prefix, name)
        shutil.rmtree(cached)
André Anjos's avatar
DRY    
André Anjos committed
338
339


340
def get_channels(public, stable, server, intranet, group):
341
    """Returns the relevant conda channels to consider if building project.
André Anjos's avatar
DRY    
André Anjos committed
342

André Anjos's avatar
André Anjos committed
343
344
    The subset of channels to be returned depends on the visibility and
    stability of the package being built.  Here are the rules:
André Anjos's avatar
DRY    
André Anjos committed
345

346
347
348
349
    * public and stable: only returns the public stable channel(s)
    * public and not stable: returns both public stable and beta channels
    * not public and stable: returns both public and private stable channels
    * not public and not stable: returns all channels
André Anjos's avatar
DRY    
André Anjos committed
350

351
352
    Beta channels have priority over stable channels, if returned.  Private
    channels have priority over public channles, if turned.
André Anjos's avatar
DRY    
André Anjos committed
353
354


355
    Args:
André Anjos's avatar
DRY    
André Anjos committed
356

357
358
359
360
361
362
363
      public: Boolean indicating if we're supposed to include only public
        channels
      stable: Boolean indicating if we're supposed to include only stable
        channels
      server: The base address of the server containing our conda channels
      intranet: Boolean indicating if we should add "private"/"public" prefixes
        on the conda paths
André Anjos's avatar
André Anjos committed
364
365
366
367
      group: The group of packages (gitlab namespace) the package we're
        compiling is part of.  Values should match URL namespaces currently
        available on our internal webserver.  Currently, only "bob" or "beat"
        will work.
André Anjos's avatar
DRY    
André Anjos committed
368
369


370
371
    Returns: a list of channels that need to be considered.
    """
André Anjos's avatar
DRY    
André Anjos committed
372

André Anjos's avatar
André Anjos committed
373
374
375
376
377
378
    if (not public) and (not intranet):
        raise RuntimeError(
            "You cannot request for private channels and set"
            " intranet=False (server=%s) - these are conflicting options"
            % server
        )
379

André Anjos's avatar
André Anjos committed
380
    channels = []
381

André Anjos's avatar
André Anjos committed
382
383
384
385
386
    if not public:
        prefix = "/private"
        if not stable:  # allowed private channels
            channels += [server + prefix + "/conda/label/beta"]  # allowed betas
        channels += [server + prefix + "/conda"]
387

André Anjos's avatar
André Anjos committed
388
389
390
391
392
    # do not use '/public' versions here
    prefix = "/software/" + group
    if not stable:
        channels += [server + prefix + "/conda/label/beta"]  # allowed betas
    channels += [server + prefix + "/conda"]
André Anjos's avatar
DRY    
André Anjos committed
393

André Anjos's avatar
André Anjos committed
394
    return channels
André Anjos's avatar
DRY    
André Anjos committed
395
396


397
def setup_logger(logger, level):
André Anjos's avatar
André Anjos committed
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
    """Sets-up the logging for this command at level ``INFO``"""

    warn_err = logging.StreamHandler(sys.stderr)
    warn_err.setLevel(logging.WARNING)
    logger.addHandler(warn_err)

    # debug and info messages are written to sys.stdout

    class _InfoFilter:
        def filter(self, record):
            return record.levelno <= logging.INFO

    debug_info = logging.StreamHandler(sys.stdout)
    debug_info.setLevel(logging.DEBUG)
    debug_info.addFilter(_InfoFilter())
    logger.addHandler(debug_info)

    formatter = logging.Formatter("%(levelname)s@%(asctime)s: %(message)s")

    for handler in logger.handlers:
        handler.setFormatter(formatter)

    if level not in range(0, 4):
        raise ValueError(
            "The verbosity level %d does not exist. Please reduce the number of "
            "'--verbose' parameters in your command line" % level
        )
    # set up the verbosity level of the logging system
    log_level = {
        0: logging.ERROR,
        1: logging.WARNING,
        2: logging.INFO,
        3: logging.DEBUG,
    }[level]

    logger.setLevel(log_level)


if __name__ == "__main__":

    import argparse

    parser = argparse.ArgumentParser(
        description="Bootstraps a new miniconda "
        "installation and prepares it for development"
    )
    parser.add_argument(
        "command",
        choices=["build", "local", "channel"],
        help="How to prepare the current environment. Use: ``build``, to "
        "build bob.devtools, ``local``, to bootstrap deploy or pypi "
        "stages for bob.devtools builds, ``channel`` channel to bootstrap "
        "CI environment for beta/stable builds",
    )
    parser.add_argument(
        "envname",
        nargs="?",
        default="bdt",
        help="The name of the conda environment that will host bdt "
        "[default: %(default)s]",
    )
    parser.add_argument(
        "-n",
        "--name",
        default=os.environ.get("CI_PROJECT_NAME", "bob.devtools"),
        help="The name of the project being built [default: %(default)s]",
    )
    parser.add_argument(
        "-c",
        "--conda-root",
        default=os.environ.get(
            "CONDA_ROOT", os.path.realpath(os.path.join(os.curdir, "miniconda"))
        ),
        help="The location where we should install miniconda "
        "[default: %(default)s]",
    )
    parser.add_argument(
        "-t",
        "--tag",
        default=os.environ.get("CI_COMMIT_TAG", None),
        help="If building a tag, pass it with this flag [default: %(default)s]",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        action="count",
        default=0,
        help="Increases the verbosity level.  We always prints error and "
        "critical messages. Use a single ``-v`` to enable warnings, "
        "two ``-vv`` to enable information messages and three ``-vvv`` "
        "to enable debug messages [default: %(default)s]",
    )

    args = parser.parse_args()

    setup_logger(logger, args.verbose)

495
496
497
    # Run conda-build hacks
    # TODO: Remove this hack as soon as possible
    do_hack(".")
498
499
500

    condarc = os.path.join(args.conda_root, "condarc")

André Anjos's avatar
André Anjos committed
501
502
503
504
505
506
507
    install_miniconda(args.conda_root, args.name)
    conda_bin = os.path.join(args.conda_root, "bin", "conda")

    # creates the condarc file
    condarc = os.path.join(args.conda_root, "condarc")
    logger.info("(create) %s", condarc)
    with open(condarc, "wt") as f:
508
509
510
511
        # Replaces https://repo.anaconda.com/pkgs/main by
        # http://www.idiap.ch/software/bob/defaults with so it is optimized for
        # a CI build.  Notice we consider this script is only executed in this
        # context.  The URL should NOT work outside of Idiap's network.
512
513
514
515
516
517
        f.write(
            _BASE_CONDARC.replace(
                "https://repo.anaconda.com/pkgs/main",
                "http://www.idiap.ch/defaults",
            )
        )
André Anjos's avatar
André Anjos committed
518

519
    conda_version = "4"
520
    conda_build_version = "3"
André Anjos's avatar
André Anjos committed
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
    conda_verify_version = "3"

    conda_verbosity = []
    # if args.verbose >= 2:
    #  conda_verbosity = ['-v']
    if args.verbose >= 3:
        conda_verbosity = ["-vv"]

    if args.command == "build":

        # simple - just use the defaults channels when self building
        run_cmdline(
            [conda_bin, "install"]
            + conda_verbosity
            + [
                "-n",
                "base",
                "python",
                "conda=%s" % conda_version,
                "conda-build=%s" % conda_build_version,
                "conda-verify=%s" % conda_verify_version,
                "twine",  # required for checking readme of python (zip) distro
            ]
        )

    elif args.command == "local":

        # index the locally built packages
        run_cmdline(
            [conda_bin, "install"]
            + conda_verbosity
            + [
                "-n",
                "base",
                "python",
                "conda=%s" % conda_version,
                "conda-build=%s" % conda_build_version,
                "conda-verify=%s" % conda_verify_version,
                "twine",  # required for checking readme of python (zip) distro
            ]
        )
        conda_bld_path = os.path.join(args.conda_root, "conda-bld")
        run_cmdline([conda_bin, "index", conda_bld_path])
        channels = get_channels(
            public=True, stable=True, server=_SERVER, intranet=True, group="bob"
        ) + ["defaults"]
        channels = (
            ["--override-channels"]
            + ["--channel=" + conda_bld_path]
            + ["--channel=%s" % k for k in channels]
        )
        conda_cmd = "install" if args.envname in ("base", "root") else "create"
        run_cmdline(
            [conda_bin, conda_cmd]
            + conda_verbosity
            + channels
            + ["-n", args.envname, "bob.devtools"]
        )

    elif args.command == "channel":

        # installs from channel
        channels = get_channels(
            public=True,
            stable=(args.tag is not None),
            server=_SERVER,
            intranet=True,
            group="bob",
        ) + ["defaults"]
        channels = ["--override-channels"] + [
            "--channel=%s" % k for k in channels
        ]
        conda_cmd = "install" if args.envname in ("base", "root") else "create"
        run_cmdline(
            [conda_bin, conda_cmd]
            + conda_verbosity
            + channels
            + ["-n", args.envname, "bob.devtools"]
        )

    # print conda information for debugging purposes
    run_cmdline([conda_bin, "info"] + conda_verbosity)