build.py 7.67 KB
Newer Older
1
2
3
4
5
6
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys

7
import yaml
André Anjos's avatar
André Anjos committed
8
9
10
import click
import pkg_resources
import conda_build.api
11
12

from . import bdt
André Anjos's avatar
André Anjos committed
13
14
15
16
17
18
19
20
21
22
from ..build import (
    next_build_number,
    conda_arch,
    should_skip_build,
    get_rendered_metadata,
    get_parsed_recipe,
    make_conda_config,
    get_docserver_setup,
    get_env_directory,
    get_output_path,
23
    remove_conda_loggers,
André Anjos's avatar
André Anjos committed
24
)
25
26
remove_conda_loggers()

André Anjos's avatar
André Anjos committed
27
28
29
30
31
32
33
from ..constants import (
    CONDA_BUILD_CONFIG,
    CONDA_RECIPE_APPEND,
    SERVER,
    MATPLOTLIB_RCDIR,
    BASE_CONDARC,
)
34
from ..bootstrap import set_environment, get_channels
35

36
from ..log import verbosity_option, get_logger, echo_info
André Anjos's avatar
André Anjos committed
37

38
39
logger = get_logger(__name__)

40

André Anjos's avatar
André Anjos committed
41
42
@click.command(
    epilog="""
43
44
45
46
Examples:

  1. Builds recipe from one of our build dependencies (inside bob.conda):

47
\b
48
49
50
51
     $ cd bob.conda
     $ bdt build -vv conda/libblitz


52
  2. Builds recipe from one of our packages, for Python 3.6 (if that is not already the default for you):
53
54
55
56
57
58

     $ bdt build --python=3.6 -vv path/to/conda/dir


  3. To build multiple recipes, just pass the paths to them:

59
     $ bdt build --python=3.6 -vv path/to/recipe-dir1 path/to/recipe-dir2
André Anjos's avatar
André Anjos committed
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
"""
)
@click.argument(
    "recipe-dir",
    required=False,
    type=click.Path(file_okay=False, dir_okay=True, exists=True),
    nargs=-1,
)
@click.option(
    "-p",
    "--python",
    default=("%d.%d" % sys.version_info[:2]),
    show_default=True,
    help="Version of python to build the environment for",
)
@click.option(
    "-r",
    "--condarc",
    help="Use custom conda configuration file instead of our own",
)
@click.option(
    "-m",
    "--config",
    "--variant-config-files",
    show_default=True,
    default=CONDA_BUILD_CONFIG,
    help="overwrites the path leading to " "variant configuration file to use",
)
@click.option(
    "-n",
    "--no-test",
    is_flag=True,
    help="Do not test the package, only builds it",
)
@click.option(
    "-a",
    "--append-file",
    show_default=True,
    default=CONDA_RECIPE_APPEND,
    help="overwrites the path leading to " "appended configuration file to use",
)
@click.option(
    "-S",
    "--server",
    show_default=True,
    default=SERVER,
    help="Server used for downloading conda packages and documentation "
    "indexes of required packages",
)
@click.option(
    "-g",
    "--group",
    show_default=True,
    default="bob",
    help="Group of packages (gitlab namespace) this package belongs to",
)
@click.option(
    "-P",
    "--private/--no-private",
    default=False,
    help="Set this to **include** private channels on your build - "
    "you **must** be at Idiap to execute this build in this case - "
    "you **must** also use the correct server name through --server - "
    "notice this option has no effect to conda if you also pass --condarc",
)
@click.option(
    "-X",
    "--stable/--no-stable",
    default=False,
    help="Set this to **exclude** beta channels from your build - "
    "notice this option has no effect if you also pass --condarc",
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
@click.option(
    "-C",
    "--ci/--no-ci",
    default=False,
    hidden=True,
    help="Use this flag to indicate the build will be running on the CI",
)
147
148
@verbosity_option()
@bdt.raise_on_error
André Anjos's avatar
André Anjos committed
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def build(
    recipe_dir,
    python,
    condarc,
    config,
    no_test,
    append_file,
    server,
    group,
    private,
    stable,
    dry_run,
    ci,
):
163
    """Builds package through conda-build with stock configuration.
164

165
166
167
168
    This command wraps the execution of conda-build so that you use the
    same conda configuration we use for our CI.  It always set ``--no-
    anaconda-upload``.
    """
169

André Anjos's avatar
André Anjos committed
170
171
172
173
174
175
176
177
178
179
180
181
182
    # if we are in a dry-run mode, let's let it be known
    if dry_run:
        logger.warn("!!!! DRY RUN MODE !!!!")
        logger.warn("Nothing will be really built")

    recipe_dir = recipe_dir or [os.path.join(os.path.realpath("."), "conda")]

    logger.debug(
        'This package is considered part of group "%s" - tunning '
        "conda package and documentation URLs for this...",
        group,
    )

183
    #### HACK to avoid ripgrep ignoring bin/ directories in our checkouts
184
185
    # TODO: Remove this hack as soon as possible
    from bob.devtools.bootstrap import do_hack
186
    project_dir = os.path.dirname(recipe_dir[0])
187
    do_hack(project_dir)
188

189

André Anjos's avatar
André Anjos committed
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    # get potential channel upload and other auxiliary channels
    channels = get_channels(
        public=(not private),
        stable=stable,
        server=server,
        intranet=ci,
        group=group,
    )

    if condarc is not None:
        logger.info("Loading CONDARC file from %s...", condarc)
        with open(condarc, "rb") as f:
            condarc_options = yaml.load(f, Loader=yaml.FullLoader)
    else:
        # use default and add channels
        all_channels = []
        all_channels += channels + ["defaults"]
        condarc_options = yaml.load(BASE_CONDARC, Loader=yaml.FullLoader)
        logger.info(
            "Using the following channels during build:\n  - %s",
            "\n  - ".join(all_channels),
        )
        condarc_options["channels"] = all_channels

    # dump packages at base environment
    prefix = get_env_directory(os.environ["CONDA_EXE"], "base")
    condarc_options["croot"] = os.path.join(prefix, "conda-bld")

    conda_config = make_conda_config(
        config, python, append_file, condarc_options
    )

    set_environment("MATPLOTLIBRC", MATPLOTLIB_RCDIR)

    # setup BOB_DOCUMENTATION_SERVER environment variable (used for bob.extension
    # and derived documentation building via Sphinx)
    set_environment("DOCSERVER", server)
    doc_urls = get_docserver_setup(
        public=(not private),
        stable=stable,
        server=server,
        intranet=ci,
        group=group,
    )
    set_environment("BOB_DOCUMENTATION_SERVER", doc_urls)

    arch = conda_arch()

    for d in recipe_dir:

        if not os.path.exists(d):
241
            raise RuntimeError("The directory %s does not exist" % d)
André Anjos's avatar
André Anjos committed
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282

        version_candidate = os.path.join(d, "..", "version.txt")
        if os.path.exists(version_candidate):
            version = open(version_candidate).read().rstrip()
            set_environment("BOB_PACKAGE_VERSION", version)

        # pre-renders the recipe - figures out the destination
        metadata = get_rendered_metadata(d, conda_config)

        # checks if we should actually build this recipe
        if should_skip_build(metadata):
            logger.info(
                "Skipping UNSUPPORTED build of %s for %s", recipe_dir, arch
            )
            continue

        rendered_recipe = get_parsed_recipe(metadata)
        path = get_output_path(metadata, conda_config)

        # gets the next build number
        build_number, _ = next_build_number(channels[0], os.path.basename(path))

        logger.info(
            "Building %s-%s-py%s (build: %d) for %s",
            rendered_recipe["package"]["name"],
            rendered_recipe["package"]["version"],
            python.replace(".", ""),
            build_number,
            arch,
        )

        if not dry_run:
            # set $BOB_BUILD_NUMBER and force conda_build to reparse recipe to get it
            # right
            set_environment("BOB_BUILD_NUMBER", str(build_number))
            paths = conda_build.api.build(
                d, config=conda_config, notest=no_test
            )
            # if you get to this point, the package was successfully rebuilt
            # set environment to signal caller we may dispose of it
            os.environ["BDT_BUILD"] = ":".join(paths)