build.py 28.7 KB
Newer Older
1 2 3
#!/usr/bin/env python
# -*- coding: utf-8 -*-

4
"""Tools for self-building and other utilities."""
5

6 7 8 9

import os
import re
import sys
10
import glob
11 12 13 14
import json
import shutil
import platform
import subprocess
15

16
import logging
17

18
logger = logging.getLogger(__name__)
19

20
import yaml
21
import distutils.version
22 23 24 25 26


def remove_conda_loggers():
    """Cleans-up conda API logger handlers to avoid logging repetition"""

27
    z = logging.getLogger()  # conda places their handlers inside root
28 29 30 31 32
    if z.handlers:
        handler = z.handlers[0]
        z.removeHandler(handler)
        logger.debug("Removed conda logger handler at %s", handler)

33

34
import conda_build.api
35

36
remove_conda_loggers()
37 38


39
def comment_cleanup(lines):
40
    """Cleans-up comments and empty lines from textual data read from files."""
41 42 43 44 45 46

    no_comments = [k.partition("#")[0].strip() for k in lines]
    return [k for k in no_comments if k]


def load_order_file(path):
47
    """Loads an order.txt style file, removes empty lines and comments."""
48 49 50 51 52

    with open(path, "rt") as f:
        return comment_cleanup(f.readlines())


53
def conda_arch():
54
    """Returns the current OS name and architecture as recognized by conda."""
55

André Anjos's avatar
André Anjos committed
56 57 58 59 60 61 62
    r = "unknown"
    if platform.system().lower() == "linux":
        r = "linux"
    elif platform.system().lower() == "darwin":
        r = "osx"
    else:
        raise RuntimeError('Unsupported system "%s"' % platform.system())
63

André Anjos's avatar
André Anjos committed
64 65 66 67
    if platform.machine().lower() == "x86_64":
        r += "-64"
    else:
        raise RuntimeError('Unsupported machine type "%s"' % platform.machine())
68

André Anjos's avatar
André Anjos committed
69
    return r
70 71 72


def should_skip_build(metadata_tuples):
André Anjos's avatar
André Anjos committed
73
    """Takes the output of render_recipe as input and evaluates if this
74
    recipe's build should be skipped."""
75

André Anjos's avatar
André Anjos committed
76
    return all(m[0].skip() for m in metadata_tuples)
77 78


79
def next_build_number(channel_url, basename):
80
    """Calculates the next build number of a package given the channel.
81

82 83
    This function returns the next build number (integer) for a package given
    its resulting tarball base filename (can be obtained with
84
    :py:func:`get_output_path`).
85 86


87
    Args:
88

89 90 91
      channel_url: The URL where to look for packages clashes (normally a beta
        channel)
      basename: The tarball basename to check on the channel
92

93 94 95 96 97
    Returns: The next build number with the current configuration.  Zero (0) is
    returned if no match is found.  Also returns the URLs of the packages it
    finds with matches on the name, version and python-version, ordered by
    (reversed) build-number.
    """
98

99 100
    from conda.exports import fetch_index
    from conda.core.index import calculate_channel_urls
101

102
    remove_conda_loggers()
André Anjos's avatar
André Anjos committed
103 104

    # get the channel index
105 106 107
    channel_urls = calculate_channel_urls(
        [channel_url], prepend=False, use_local=False
    )
108 109
    logger.debug("Downloading channel index from %s", channel_urls)
    index = fetch_index(channel_urls=channel_urls)
André Anjos's avatar
André Anjos committed
110

111
    # remove .tar.bz2/.conda from name, then split from the end twice, on '-'
112
    if basename.endswith(".tar.bz2"):
113
        name, version, build = basename[:-8].rsplit("-", 2)
114
    elif basename.endswith(".conda"):
115 116
        name, version, build = basename[:-6].rsplit("-", 2)
    else:
117 118 119 120
        raise RuntimeError(
            "Package name %s does not end in either "
            ".tar.bz2 or .conda" % (basename,)
        )
André Anjos's avatar
André Anjos committed
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166

    # remove the build number as we're looking for the next value
    # examples to be coped with:
    # vlfeat-0.9.20-0 -> '0'
    # vlfeat-0.9.21-h18fa195_0 -> 'h18fa195_0'
    # tqdm-4.11.1-py36_0 -> 'py36_0'
    # websocket-client-0.47.0-py27haf68d3b_0 -> 'py27haf68d3b_0'
    # websocket-client-0.47.0-py36haf68d3b_0 -> 'py36haf68d3b_0'
    build_variant = build.rsplit("_", 1)[0]
    # vlfeat-0.9.20-0 -> '0'
    # vlfeat-0.9.21-h18fa195_0 -> 'h18fa195'
    # tqdm-4.11.1-py36_0 -> 'py36'
    # websocket-client-0.47.0-py27haf68d3b_0 -> 'py27haf68d3b'
    # websocket-client-0.47.0-py36haf68d3b_0 -> 'py36haf68d3b'
    build_variant = build_variant.split("h", 1)[0]
    # vlfeat-0.9.20-0 -> '0'
    # vlfeat-0.9.21-h18fa195_0 -> ''
    # tqdm-4.11.1-py36_0 -> 'py36'
    # websocket-client-0.47.0-py27haf68d3b_0 -> 'py27'
    # websocket-client-0.47.0-py36haf68d3b_0 -> 'py36'
    if re.match("^[0-9]+$", build_variant) is not None:
        build_variant = ""

    # search if package with the same characteristics
    urls = {}
    build_number = 0
    for dist in index:
        if (
            dist.name == name
            and dist.version == version
            and dist.build_string.startswith(build_variant)
        ):  # match!
            url = index[dist].url
            logger.debug(
                "Found match at %s for %s-%s-%s",
                url,
                name,
                version,
                build_variant,
            )
            build_number = max(build_number, dist.build_number + 1)
            urls[index[dist].timestamp] = url.replace(channel_url, "")

    sorted_urls = [urls[k] for k in reversed(list(urls.keys()))]

    return build_number, sorted_urls
167 168 169


def make_conda_config(config, python, append_file, condarc_options):
170
    """Creates a conda configuration for a build merging various sources.
André Anjos's avatar
André Anjos committed
171

172 173
    This function will use the conda-build API to construct a configuration by
    merging different sources of information.
André Anjos's avatar
André Anjos committed
174

175
    Args:
André Anjos's avatar
André Anjos committed
176

177 178 179 180 181 182
      config: Path leading to the ``conda_build_config.yaml`` to use
      python: The version of python to use for the build as ``x.y`` (e.g.
        ``3.6``)
      append_file: Path leading to the ``recipe_append.yaml`` file to use
      condarc_options: A dictionary (typically read from a condarc YAML file)
        that contains build and channel options
André Anjos's avatar
André Anjos committed
183

184 185 186
    Returns: A dictionary containing the merged configuration, as produced by
    conda-build API's ``get_or_merge_config()`` function.
    """
187

André Anjos's avatar
André Anjos committed
188
    from conda_build.conda_interface import url_path
189

190
    remove_conda_loggers()
191

André Anjos's avatar
André Anjos committed
192 193 194 195 196 197 198
    retval = conda_build.api.get_or_merge_config(
        None,
        variant_config_files=config,
        python=python,
        append_sections_file=append_file,
        **condarc_options,
    )
199

André Anjos's avatar
André Anjos committed
200
    retval.channel_urls = []
201

André Anjos's avatar
André Anjos committed
202 203 204 205 206 207 208 209 210 211 212
    for url in condarc_options["channels"]:
        # allow people to specify relative or absolute paths to local channels
        #    These channels still must follow conda rules - they must have the
        #    appropriate platform-specific subdir (e.g. win-64)
        if os.path.isdir(url):
            if not os.path.isabs(url):
                url = os.path.normpath(
                    os.path.abspath(os.path.join(os.getcwd(), url))
                )
            url = url_path(url)
        retval.channel_urls.append(url)
213

André Anjos's avatar
André Anjos committed
214
    return retval
215 216


217
def get_output_path(metadata, config):
218
    """Renders the recipe and returns the name of the output file."""
219

220
    return conda_build.api.get_output_file_paths(metadata, config=config)
221 222


223
def get_rendered_metadata(recipe_dir, config):
224
    """Renders the recipe and returns the interpreted YAML file."""
225

André Anjos's avatar
André Anjos committed
226
    return conda_build.api.render(recipe_dir, config=config)
227 228 229


def get_parsed_recipe(metadata):
230
    """Renders the recipe and returns the interpreted YAML file."""
231

André Anjos's avatar
André Anjos committed
232 233
    output = conda_build.api.output_yaml(metadata[0][0])
    return yaml.load(output, Loader=yaml.FullLoader)
234 235


236
def exists_on_channel(channel_url, basename):
237
    """Checks on the given channel if a package with the specs exist.
238

239 240
    This procedure always ignores the package hash code, if one is set.  It
    differentiates between `.conda` and `.tar.bz2` packages.
241

242
    Args:
243

244 245 246
      channel_url: The URL where to look for packages clashes (normally a beta
        channel)
      basename: The basename of the tarball to search for
247

248 249
    Returns: A complete package url, if the package already exists in the
    channel or ``None`` otherwise.
250
    """
251

André Anjos's avatar
André Anjos committed
252
    build_number, urls = next_build_number(channel_url, basename)
253

André Anjos's avatar
André Anjos committed
254
    def _get_build_number(name):
255

256 257
        # remove .tar.bz2/.conda from name, then split from the end twice, on
        # '-'
258
        if name.endswith(".conda"):
259
            name, version, build = name[:-6].rsplit("-", 2)
260
        elif name.endswith(".tar.bz2"):
261
            name, version, build = name[:-8].rsplit("-", 2)
262
        else:
263 264 265 266
            raise RuntimeError(
                "Package name %s does not end in either "
                ".tar.bz2 or .conda" % (name,)
            )
267

André Anjos's avatar
André Anjos committed
268 269 270 271 272
        # remove the build number as we're looking for the next value
        # examples to be coped with:
        # vlfeat-0.9.20-0 -> '0'
        # vlfeat-0.9.21-h18fa195_0 -> 'h18fa195_0'
        # tqdm-4.11.1-py36_0 -> 'py36_0'
273
        # untokenize-0.1.1-py_0.conda -> 'py_0'
André Anjos's avatar
André Anjos committed
274 275 276 277
        # websocket-client-0.47.0-py27haf68d3b_0 -> 'py27haf68d3b_0'
        # websocket-client-0.47.0-py36haf68d3b_0 -> 'py36haf68d3b_0'
        s = build.rsplit("_", 1)
        return s[1] if len(s) == 2 else s[0]
278

André Anjos's avatar
André Anjos committed
279
    self_build_number = _get_build_number(basename)
280
    other_build_numbers = dict(
281 282
        [(k, _get_build_number(os.path.basename(k))) for k in urls]
    )
283

284
    if self_build_number in other_build_numbers.values():
285
        pkg_type = ".conda" if basename.endswith(".conda") else ".tar.bz2"
286
        for k, v in other_build_numbers.items():
287
            if k.endswith(pkg_type):  # match
288
                return "".join((channel_url, k))
289 290


291
def remove_pins(deps):
André Anjos's avatar
André Anjos committed
292
    return [l.split()[0] for l in deps]
293 294 295 296


def parse_dependencies(recipe_dir, config):

André Anjos's avatar
André Anjos committed
297 298 299 300 301 302 303 304 305 306
    metadata = get_rendered_metadata(recipe_dir, config)
    recipe = get_parsed_recipe(metadata)
    return (
        remove_pins(recipe["requirements"].get("build", []))
        + remove_pins(recipe["requirements"].get("host", []))
        + recipe["requirements"].get("run", [])
        + recipe.get("test", {}).get("requires", [])
        + ["bob.buildout", "mr.developer", "ipdb"]
    )
    # by last, packages required for local dev
307 308 309


def get_env_directory(conda, name):
310
    """Get the directory of a particular conda environment or fail silently."""
311

André Anjos's avatar
André Anjos committed
312 313 314 315
    cmd = [conda, "env", "list", "--json"]
    output = subprocess.check_output(cmd)
    data = json.loads(output)
    paths = data.get("envs", [])
316

André Anjos's avatar
André Anjos committed
317 318 319 320
    if not paths:
        # real error condition, reports it at least, but no exception raising...
        logger.error("No environments in conda (%s) installation?", conda)
        return None
321

André Anjos's avatar
André Anjos committed
322 323
    if name in ("base", "root"):
        return paths[0]  # first environment is base
324

André Anjos's avatar
André Anjos committed
325 326 327 328
    # else, must search for the path ending in ``/name``
    retval = [k for k in paths if k.endswith(os.sep + name)]
    if retval:
        return retval[0]
329

André Anjos's avatar
André Anjos committed
330 331
    # if no environment with said name is found, return ``None``
    return None
332 333 334


def conda_create(conda, name, overwrite, condarc, packages, dry_run, use_local):
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
    """Creates a new conda environment following package specifications.

    This command can create a new conda environment following the list of input
    packages.  It will overwrite an existing environment if indicated.

    Args:
      conda: path to the main conda executable of the installation
      name: the name of the environment to create or overwrite
      overwrite: if set to ```True``, overwrite potentially existing environments
        with the same name
      condarc: a dictionary of options for conda, including channel urls
      packages: the package list specification
      dry_run: if set, then don't execute anything, just print stuff
      use_local: include the local conda-bld directory as a possible installation
        channel (useful for testing multiple interdependent recipes that are
        built locally)
    """
352

André Anjos's avatar
André Anjos committed
353
    from .bootstrap import run_cmdline
354

André Anjos's avatar
André Anjos committed
355 356 357 358 359 360 361 362 363
    specs = []
    for k in packages:
        k = " ".join(k.split()[:2])  # remove eventual build string
        if any(elem in k for elem in "><|"):
            specs.append(k.replace(" ", ""))
        else:
            specs.append(k.replace(" ", "="))

    # if the current environment exists, delete it first
364
    envdir = get_env_directory(conda, name)
André Anjos's avatar
André Anjos committed
365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
    if envdir is not None:
        if overwrite:
            cmd = [conda, "env", "remove", "--yes", "--name", name]
            logger.debug("$ " + " ".join(cmd))
            if not dry_run:
                run_cmdline(cmd)
        else:
            raise RuntimeError(
                "environment `%s' exists in `%s' - use "
                "--overwrite to overwrite" % (name, envdir)
            )

    cmdline_channels = ["--channel=%s" % k for k in condarc["channels"]]
    cmd = [
        conda,
        "create",
        "--yes",
        "--name",
        name,
        "--override-channels",
    ] + cmdline_channels
    if dry_run:
        cmd.append("--dry-run")
    if use_local:
        cmd.append("--use-local")
    cmd.extend(sorted(specs))
    run_cmdline(cmd)

    # creates a .condarc file to sediment the just created environment
    if not dry_run:
        # get envdir again - it may just be created!
        envdir = get_env_directory(conda, name)
        destrc = os.path.join(envdir, "condarc")
        logger.info("Creating %s...", destrc)
        with open(destrc, "w") as f:
            yaml.dump(condarc, f, indent=2)
401 402


403
def get_docserver_setup(public, stable, server, intranet, group):
404
    """Returns a setup for BOB_DOCUMENTATION_SERVER.
405

406 407
    What is available to build the documentation depends on the setup of
    ``public`` and ``stable``:
408

409 410 411 412
    * public and stable: only returns the public stable channel(s)
    * public and not stable: returns both public stable and beta channels
    * not public and stable: returns both public and private stable channels
    * not public and not stable: returns all channels
413

414 415
    Beta channels have priority over stable channels, if returned.  Private
    channels have priority over public channles, if turned.
416 417


418
    Args:
419

420 421 422 423 424 425 426 427 428 429
      public: Boolean indicating if we're supposed to include only public
        channels
      stable: Boolean indicating if we're supposed to include only stable
        channels
      server: The base address of the server containing our conda channels
      intranet: Boolean indicating if we should add "private"/"public" prefixes
        on the returned paths
      group: The group of packages (gitlab namespace) the package we're compiling
        is part of.  Values should match URL namespaces currently available on
        our internal webserver.  Currently, only "bob" or "beat" will work.
430 431


432 433 434
    Returns: a string to be used by bob.extension to find dependent
    documentation projects.
    """
435

André Anjos's avatar
André Anjos committed
436 437 438 439 440 441
    if (not public) and (not intranet):
        raise RuntimeError(
            "You cannot request for private channels and set"
            " intranet=False (server=%s) - these are conflicting options"
            % server
        )
442

André Anjos's avatar
André Anjos committed
443
    entries = []
444

André Anjos's avatar
André Anjos committed
445 446
    # public documentation: always can access
    prefix = "/software/%s" % group
447
    if stable:
André Anjos's avatar
André Anjos committed
448 449 450 451
        entries += [
            server + prefix + "/docs/" + group + "/%(name)s/%(version)s/",
            server + prefix + "/docs/" + group + "/%(name)s/stable/",
        ]
452
    else:
André Anjos's avatar
André Anjos committed
453 454 455 456 457 458 459 460 461 462 463 464 465 466
        entries += [server + prefix + "/docs/" + group + "/%(name)s/master/"]

    if not public:
        # add private channels, (notice they are not accessible outside idiap)
        prefix = "/private"
        if stable:
            entries += [
                server + prefix + "/docs/" + group + "/%(name)s/%(version)s/",
                server + prefix + "/docs/" + group + "/%(name)s/stable/",
            ]
        else:
            entries += [
                server + prefix + "/docs/" + group + "/%(name)s/master/"
            ]
467

André Anjos's avatar
André Anjos committed
468
    return "|".join(entries)
469 470


André Anjos's avatar
André Anjos committed
471
def check_version(workdir, envtag):
472
    """Checks if the version being built and the value reported match.
André Anjos's avatar
André Anjos committed
473

474 475 476 477
    This method will read the contents of the file ``version.txt`` and compare it
    to the potentially set ``envtag`` (may be ``None``).  If the value of
    ``envtag`` is different than ``None``, ensure it matches the value in
    ``version.txt`` or raises an exception.
André Anjos's avatar
André Anjos committed
478 479


480
    Args:
André Anjos's avatar
André Anjos committed
481

482 483 484
      workdir: The work directory where the repo of the package being built was
        checked-out
      envtag: (optional) tag provided by the environment
André Anjos's avatar
André Anjos committed
485 486


487 488 489 490
    Returns: A tuple with the version of the package that we're currently
    building and a boolean flag indicating if the version number represents a
    pre-release or a stable release.
    """
André Anjos's avatar
André Anjos committed
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522

    version = open(os.path.join(workdir, "version.txt"), "rt").read().rstrip()

    # if we're building a stable release, ensure a tag is set
    parsed_version = distutils.version.LooseVersion(version).version
    is_prerelease = any([isinstance(k, str) for k in parsed_version])
    if is_prerelease:
        if envtag is not None:
            raise EnvironmentError(
                '"version.txt" indicates version is a '
                'pre-release (v%s) - but environment provided tag "%s", '
                "which indicates this is a **stable** build. "
                "Have you created the tag using ``bdt release``?"
                % (version, envtag)
            )
    else:  # it is a stable build
        if envtag is None:
            raise EnvironmentError(
                '"version.txt" indicates version is a '
                "stable build (v%s) - but there is **NO** tag environment "
                "variable defined, which indicates this is **not** "
                "a tagged build. Use ``bdt release`` to create stable releases"
                % (version,)
            )
        if envtag[1:] != version:
            raise EnvironmentError(
                '"version.txt" and the value of '
                "the provided tag do **NOT** agree - the former "
                "reports version %s, the latter, %s" % (version, envtag[1:])
            )

    return version, is_prerelease
André Anjos's avatar
André Anjos committed
523 524


525
def git_clean_build(runner, verbose):
526
    """Runs git-clean to clean-up build products.
André Anjos's avatar
André Anjos committed
527

528
    Args:
André Anjos's avatar
André Anjos committed
529

530 531 532 533
      runner: A pointer to the ``run_cmdline()`` function
      verbose: A boolean flag indicating if the git command should report erased
        files or not
    """
André Anjos's avatar
André Anjos committed
534

André Anjos's avatar
André Anjos committed
535 536 537
    # glob wild card entries we'd like to keep
    exclude_from_cleanup = [
        "miniconda.sh",  # the installer, cached
538
        "torch",  # eventual pytorch caches
André Anjos's avatar
André Anjos committed
539
        "sphinx",  # build artifact -- documentation
540
        "coverage.xml",  # build artifact -- coverage report
André Anjos's avatar
André Anjos committed
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
    ]

    # artifacts
    exclude_from_cleanup += ["miniconda/conda-bld/"]
    exclude_from_cleanup += glob.glob("dist/*.zip")

    logger.debug(
        "Excluding the following paths from git-clean:\n  - %s",
        "  - ".join(exclude_from_cleanup),
    )

    # decide on verbosity
    flags = "-ffdx"
    if not verbose:
        flags += "q"

    runner(
        ["git", "clean", flags]
        + ["--exclude=%s" % k for k in exclude_from_cleanup]
    )


def base_build(
    bootstrap,
    server,
    intranet,
    group,
    recipe_dir,
    conda_build_config,
    condarc_options,
):
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
    """Builds a non-beat/non-bob software dependence that doesn't exist on
    defaults.

    This function will build a software dependence that is required for our
    software stack, but does not (yet) exist on the defaults channels.  It first
    check if the build should run for the current architecture, checks if the
    package is not already built on our public channel and, if that is true, then
    proceeds with the build of the dependence.


    Args:

      bootstrap: Module that should be pre-loaded so this function can be used
        in a pre-bdt build
      server: The base address of the server containing our conda channels
      intranet: Boolean indicating if we should add "private"/"public" prefixes
        on the returned paths
      group: The group of packages (gitlab namespace) the package we're compiling
        is part of.  Values should match URL namespaces currently available on
        our internal webserver.  Currently, only "bob" or "beat" will work.
      recipe_dir: The directory containing the recipe's ``meta.yaml`` file
      conda_build_config: Path to the ``conda_build_config.yaml`` file to use
      condarc_options: Pre-parsed condarc options loaded from the respective YAML
        file


    Returns:

      list: The list of built packages, as returned by
      ``conda_build.api.build()``
    """
603

André Anjos's avatar
André Anjos committed
604
    # if you get to this point, tries to build the package
605
    channels = bootstrap.get_channels(
606
        public=True, stable=True, server=server, intranet=intranet, group=group
André Anjos's avatar
André Anjos committed
607 608
    )

609 610 611
    if "channels" not in condarc_options:
        condarc_options["channels"] = channels + ["defaults"]

André Anjos's avatar
André Anjos committed
612 613
    logger.info(
        "Using the following channels during (potential) build:\n  - %s",
614
        "\n  - ".join(condarc_options["channels"]),
André Anjos's avatar
André Anjos committed
615 616
    )
    logger.info("Merging conda configuration files...")
617 618 619
    conda_config = make_conda_config(
        conda_build_config, None, None, condarc_options
    )
André Anjos's avatar
André Anjos committed
620 621 622 623 624 625

    metadata = get_rendered_metadata(recipe_dir, conda_config)
    arch = conda_arch()

    # checks we should actually build this recipe
    if should_skip_build(metadata):
626 627 628
        logger.warn(
            'Skipping UNSUPPORTED build of "%s" on %s', recipe_dir, arch
        )
André Anjos's avatar
André Anjos committed
629 630
        return

631 632 633 634
    paths = get_output_path(metadata, conda_config)
    urls = [exists_on_channel(channels[0], os.path.basename(k)) for k in paths]

    if all(urls):
635
        logger.info(
André Anjos's avatar
André Anjos committed
636
            "Skipping build(s) for recipe at '%s' as packages with matching "
637
            "characteristics exist (%s)",
André Anjos's avatar
André Anjos committed
638
            recipe_dir,
639 640
            ", ".join(urls),
        )
641
        return
André Anjos's avatar
André Anjos committed
642

643
    if any(urls):
André Anjos's avatar
André Anjos committed
644 645 646 647
        raise RuntimeError(
            "One or more packages for recipe at '%s' already exist (%s). "
            "Change the package build number to trigger a build." % \
            (recipe_dir, ", ".join(urls)),
648
        )
André Anjos's avatar
André Anjos committed
649

650
    # if you get to this point, just builds the package(s)
André Anjos's avatar
André Anjos committed
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
    logger.info("Building %s", path)
    return conda_build.api.build(recipe_dir, config=conda_config)


if __name__ == "__main__":

    import argparse

    parser = argparse.ArgumentParser(
        description="Builds bob.devtools on the CI"
    )
    parser.add_argument(
        "-g",
        "--group",
        default=os.environ.get("CI_PROJECT_NAMESPACE", "bob"),
        help="The namespace of the project being built [default: %(default)s]",
    )
    parser.add_argument(
        "-n",
        "--name",
        default=os.environ.get("CI_PROJECT_NAME", "bob.devtools"),
        help="The name of the project being built [default: %(default)s]",
    )
    parser.add_argument(
        "-c",
        "--conda-root",
        default=os.environ.get(
            "CONDA_ROOT", os.path.realpath(os.path.join(os.curdir, "miniconda"))
        ),
        help="The location where we should install miniconda "
        "[default: %(default)s]",
    )
    parser.add_argument(
        "-V",
        "--visibility",
        choices=["public", "internal", "private"],
        default=os.environ.get("CI_PROJECT_VISIBILITY", "public"),
        help="The visibility level for this project [default: %(default)s]",
    )
    parser.add_argument(
        "-t",
        "--tag",
        default=os.environ.get("CI_COMMIT_TAG", None),
        help="If building a tag, pass it with this flag [default: %(default)s]",
    )
    parser.add_argument(
        "-w",
        "--work-dir",
        default=os.environ.get("CI_PROJECT_DIR", os.path.realpath(os.curdir)),
        help="The directory where the repo was cloned [default: %(default)s]",
    )
    parser.add_argument(
        "-T",
        "--twine-check",
        action="store_true",
        default=False,
        help="If set, then performs the equivalent of a "
        '"twine check" on the generated python package (zip file)',
    )
    parser.add_argument(
        "--internet",
        "-i",
        default=False,
        action="store_true",
        help="If executing on an internet-connected server, unset this flag",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        action="count",
        default=0,
        help="Increases the verbosity level.  We always prints error and "
        "critical messages. Use a single ``-v`` to enable warnings, "
        "two ``-vv`` to enable information messages and three ``-vvv`` "
        "to enable debug messages [default: %(default)s]",
    )
727 728 729 730 731 732 733
    parser.add_argument(
        "--nose-eval-attr",
        "-A",
        default="",
        help="Use this flag to avoid running certain tests during the build.  "
        "It forwards all settings to ``nosetests --eval-attr=<settings>``",
    )
André Anjos's avatar
André Anjos committed
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751

    args = parser.parse_args()

    # loads the "adjacent" bootstrap module
    import importlib.util

    mydir = os.path.dirname(os.path.realpath(sys.argv[0]))
    bootstrap_file = os.path.join(mydir, "bootstrap.py")
    spec = importlib.util.spec_from_file_location("bootstrap", bootstrap_file)
    bootstrap = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(bootstrap)
    server = bootstrap._SERVER

    bootstrap.setup_logger(logger, args.verbose)

    bootstrap.set_environment("DOCSERVER", server)
    bootstrap.set_environment("LANG", "en_US.UTF-8")
    bootstrap.set_environment("LC_ALL", os.environ["LANG"])
752
    bootstrap.set_environment("NOSE_EVAL_ATTR", args.nose_eval_attr)
André Anjos's avatar
André Anjos committed
753 754 755 756 757 758

    # get information about the version of the package being built
    version, is_prerelease = check_version(args.work_dir, args.tag)
    bootstrap.set_environment("BOB_PACKAGE_VERSION", version)

    # create the build configuration
759 760 761
    conda_build_config = os.path.join(args.work_dir, "conda",
            "conda_build_config.yaml")
    recipe_append = os.path.join(args.work_dir, "data", "recipe_append.yaml")
André Anjos's avatar
André Anjos committed
762 763 764 765 766 767 768 769 770 771 772 773

    condarc = os.path.join(args.conda_root, "condarc")
    logger.info("Loading (this build's) CONDARC file from %s...", condarc)
    with open(condarc, "rb") as f:
        condarc_options = yaml.load(f, Loader=yaml.FullLoader)

    # dump packages at conda_root
    prefix = get_env_directory(os.environ["CONDA_EXE"], "base")
    if condarc_options.get("conda-build", {}).get("root-dir") is None:
        condarc_options["croot"] = os.path.join(prefix, "conda-bld")

    # builds all dependencies in the 'deps' subdirectory - or at least checks
774 775
    # these dependencies are already available; these dependencies go directly
    # to the public channel once built
776 777 778
    recipes = load_order_file(os.path.join("deps", "order.txt"))
    for k, recipe in enumerate([os.path.join("deps", k) for k in recipes]):

André Anjos's avatar
André Anjos committed
779 780 781 782 783 784 785 786 787 788 789 790 791
        if not os.path.exists(os.path.join(recipe, "meta.yaml")):
            # ignore - not a conda package
            continue
        base_build(
            bootstrap,
            server,
            not args.internet,
            args.group,
            recipe,
            conda_build_config,
            condarc_options,
        )

792
    public = args.visibility == "public"
André Anjos's avatar
André Anjos committed
793 794 795 796 797 798 799
    channels = bootstrap.get_channels(
        public=public,
        stable=(not is_prerelease),
        server=server,
        intranet=(not args.internet),
        group=args.group,
    )
800 801 802 803

    if "channels" not in condarc_options:
        condarc_options["channels"] = channels + ["defaults"]

André Anjos's avatar
André Anjos committed
804 805
    logger.info(
        "Using the following channels during build:\n  - %s",
806
        "\n  - ".join(condarc_options["channels"]),
André Anjos's avatar
André Anjos committed
807 808 809
    )
    logger.info("Merging conda configuration files...")
    conda_config = make_conda_config(
810
        conda_build_config, None, recipe_append, condarc_options
André Anjos's avatar
André Anjos committed
811 812 813 814
    )

    recipe_dir = os.path.join(args.work_dir, "conda")
    metadata = get_rendered_metadata(recipe_dir, conda_config)
815
    paths = get_output_path(metadata, conda_config)
André Anjos's avatar
André Anjos committed
816 817

    # asserts we're building at the right location
André Anjos's avatar
André Anjos committed
818 819 820 821 822 823 824 825 826
    for path in paths:
        assert path.startswith(os.path.join(args.conda_root, "conda-bld")), (
            'Output path for build (%s) does not start with "%s" - this '
            "typically means this build is running on a shared builder and "
            "the file ~/.conda/environments.txt is polluted with other "
            "environment paths.  To fix, empty that file and set its mode "
            "to read-only for all."
            % (path, os.path.join(args.conda_root, "conda-bld"))
        )
André Anjos's avatar
André Anjos committed
827

828 829 830 831 832 833 834
    # retrieve the current build number(s) for this build
    build_numbers = [
        next_build_number(channels[0], os.path.basename(k))[0] for k in paths
    ]

    # homogenize to the largest build number
    build_number = max([int(k) for k in build_numbers])
André Anjos's avatar
André Anjos committed
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859

    # runs the build using the conda-build API
    arch = conda_arch()

    # notice we cannot build from the pre-parsed metadata because it has already
    # resolved the "wrong" build number.  We'll have to reparse after setting the
    # environment variable BOB_BUILD_NUMBER.
    bootstrap.set_environment("BOB_BUILD_NUMBER", str(build_number))
    conda_build.api.build(recipe_dir, config=conda_config)

    # checks if long_description of python package renders fine
    if args.twine_check:
        from twine.commands.check import check

        package = glob.glob("dist/*.zip")
        failed = check(package)

        if failed:
            raise RuntimeError(
                "twine check (a.k.a. readme check) %s: FAILED" % package[0]
            )
        else:
            logger.info("twine check (a.k.a. readme check) %s: OK", package[0])

    git_clean_build(bootstrap.run_cmdline, verbose=(args.verbose >= 3))