ci.py 29.5 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
#!/usr/bin/env python

import os
import re
import glob
6
import shutil
André Anjos's avatar
André Anjos committed
7

André Anjos's avatar
André Anjos committed
8
import yaml
André Anjos's avatar
André Anjos committed
9
10
11
12
import click
import pkg_resources
from click_plugins import with_plugins

13
from . import bdt
14
from ..constants import SERVER, WEBDAV_PATHS, BASE_CONDARC
15
from ..deploy import deploy_conda_package, deploy_documentation
16
from ..build import comment_cleanup, load_order_file
André Anjos's avatar
André Anjos committed
17
18
19
20
21
22
from ..ci import (
    read_packages,
    uniq,
    select_conda_build_config,
    select_conda_recipe_append,
    select_user_condarc,
23
    cleanup,
André Anjos's avatar
André Anjos committed
24
)
André Anjos's avatar
André Anjos committed
25

26
from ..log import verbosity_option, get_logger, echo_normal
André Anjos's avatar
André Anjos committed
27

28
29
logger = get_logger(__name__)

André Anjos's avatar
André Anjos committed
30

André Anjos's avatar
André Anjos committed
31
@with_plugins(pkg_resources.iter_entry_points("bdt.ci.cli"))
André Anjos's avatar
André Anjos committed
32
33
@click.group(cls=bdt.AliasedGroup)
def ci():
34
    """Commands for building packages and handling CI activities.
André Anjos's avatar
André Anjos committed
35

36
37
38
39
40
    Commands defined here are supposed to run on our CI, where a number
    of variables that define their behavior is correctly defined.  Do
    **NOT** attempt to run these commands in your own installation.
    Unexpected errors may occur.
    """
André Anjos's avatar
André Anjos committed
41
    pass
André Anjos's avatar
André Anjos committed
42
43


André Anjos's avatar
André Anjos committed
44
45
@ci.command(
    epilog="""
46
47
48
49
50
51
Examples:

  1. Deploys base build artifacts (dependencies) to the appropriate channels:

     $ bdt ci base-deploy -vv

André Anjos's avatar
André Anjos committed
52
53
54
55
56
57
58
59
60
61
"""
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
62
63
64
@verbosity_option()
@bdt.raise_on_error
def base_deploy(dry_run):
65
    """Deploys dependencies not available at the defaults channel.
66
67

    Deployment happens to our public channel directly, as these are
68
69
    dependencies are required for proper bob/beat package runtime
    environments.
70
71
72
    """

    if dry_run:
André Anjos's avatar
André Anjos committed
73
74
        logger.warn("!!!! DRY RUN MODE !!!!")
        logger.warn("Nothing is being deployed to server")
75

André Anjos's avatar
André Anjos committed
76
77
    package = os.environ["CI_PROJECT_PATH"]
    group, name = package.split("/")
78

79
80
    # deploys all conda package artefacts currently available (erases them
    # afterwards)
André Anjos's avatar
André Anjos committed
81
82
    for arch in ("linux-64", "osx-64", "noarch"):
        # finds conda dependencies and uploads what we can find
83
84
85
86
        base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
        conda_paths = os.path.join(base_path, "*.conda")
        tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
        deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)
87

André Anjos's avatar
André Anjos committed
88
        for k in deploy_packages:
89

André Anjos's avatar
André Anjos committed
90
91
92
            if os.path.basename(k).startswith(name):
                logger.debug("Skipping deploying of %s - not a base package", k)
                continue
93

André Anjos's avatar
André Anjos committed
94
95
96
97
98
99
100
101
102
103
            deploy_conda_package(
                k,
                arch=arch,
                stable=True,
                public=True,
                username=os.environ["DOCUSER"],
                password=os.environ["DOCPASS"],
                overwrite=False,
                dry_run=dry_run,
            )
104
105


André Anjos's avatar
André Anjos committed
106
107
@ci.command(
    epilog="""
André Anjos's avatar
André Anjos committed
108
109
110
111
112
113
Examples:

  1. Deploys current build artifacts to the appropriate channels:

     $ bdt ci deploy -vv

114
115
116
117
118

  2. Deploys stable release from non-master branch (e.g. you're releasing a patch release for an older version of a package):

     $ bdt ci deploy -vv --no-latest

André Anjos's avatar
André Anjos committed
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
"""
)
@click.option(
    "-n",
    "--latest/--no-latest",
    default=True,
    help="If set (the default), for stable builds, deploy documentation "
    'to both "stable" and "master" branches, besides "<branch>" and '
    '"<tag>" - otherwise, only deploys documentation to "<branch>" '
    'and "<tag>".  This option is useful if you are publishing '
    "corrections of a release from a stable branch which is **NOT** "
    "the master branch, so you would not like to overwrite "
    'documentation deployments for "stable" and "master"',
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
André Anjos's avatar
André Anjos committed
141
142
@verbosity_option()
@bdt.raise_on_error
143
def deploy(latest, dry_run):
André Anjos's avatar
André Anjos committed
144
145
146
147
148
149
150
151
152
153
154
155
    """Deploys build artifacts (conda packages and sphinx documentation)

    Deployment happens at the "right" locations - conda packages which do not
    represent stable releases are deployed to our conda "beta" channel, while
    stable packages to our root channel.  Sphinx documentation from unstable
    builds (typically the master branch) is deployed to the documentation
    server in a subdirectory named after the current branch name, while stable
    documentation is deployed to a special subdirectory named "stable" and to
    the respective tag name.
    """

    if dry_run:
André Anjos's avatar
André Anjos committed
156
157
        logger.warn("!!!! DRY RUN MODE !!!!")
        logger.warn("Nothing is being deployed to server")
André Anjos's avatar
André Anjos committed
158

André Anjos's avatar
André Anjos committed
159
160
    package = os.environ["CI_PROJECT_PATH"]
    group, name = package.split("/")
André Anjos's avatar
André Anjos committed
161

162
    # determine if building branch or tag, and project visibility
André Anjos's avatar
André Anjos committed
163
164
    stable = "CI_COMMIT_TAG" in os.environ
    public = os.environ["CI_PROJECT_VISIBILITY"] == "public"
André Anjos's avatar
André Anjos committed
165

166
167
    # deploys all conda package artefacts currently available (erases them
    # afterwards)
André Anjos's avatar
André Anjos committed
168
169
    for arch in ("linux-64", "osx-64", "noarch"):
        # finds conda packages and uploads what we can find
170
171
172
173
174
        base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", arch)
        conda_paths = os.path.join(base_path, "*.conda")
        tarbz2_paths = os.path.join(base_path, "*.tar.bz2")
        deploy_packages = glob.glob(conda_paths) + glob.glob(tarbz2_paths)

André Anjos's avatar
André Anjos committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
        for k in deploy_packages:
            deploy_conda_package(
                k,
                arch=arch,
                stable=stable,
                public=public,
                username=os.environ["DOCUSER"],
                password=os.environ["DOCPASS"],
                overwrite=False,
                dry_run=dry_run,
            )

    local_docs = os.path.join(os.environ["CI_PROJECT_DIR"], "sphinx")
    deploy_documentation(
        local_docs,
        package,
        stable=stable,
        latest=latest,
        public=public,
        branch=os.environ["CI_COMMIT_REF_NAME"],
        tag=os.environ.get("CI_COMMIT_TAG"),
        username=os.environ["DOCUSER"],
        password=os.environ["DOCPASS"],
        dry_run=dry_run,
    )


@ci.command(
    epilog="""
204
205
206
207
208
209
210
211
Examples:

  1. Checks the long description of setup.py (correctly parseable and will
     display nicely at PyPI).  Notice this step requires the zip python
     packages:

     $ bdt ci readme -vv dist/*.zip

André Anjos's avatar
André Anjos committed
212
213
214
215
216
217
218
219
"""
)
@click.argument(
    "package",
    required=True,
    type=click.Path(file_okay=True, dir_okay=False, exists=True),
    nargs=-1,
)
220
221
222
@verbosity_option()
@bdt.raise_on_error
def readme(package):
223
    """Checks setup.py's ``long_description`` syntax.
224

225
226
227
    This program checks the syntax of the contents of the
    ``long_description`` field at the package's ``setup()`` function.
    It verifies it will be correctly displayed at PyPI.
228
229
230
231
    """

    for k in package:

André Anjos's avatar
André Anjos committed
232
233
234
235
236
237
        logger.info("Checking python package %s", k)
        # twine check dist/*.zip

        from twine.commands.check import check

        failed = check([k])
238

André Anjos's avatar
André Anjos committed
239
240
241
242
243
244
        if failed:
            raise RuntimeError(
                "twine check (a.k.a. readme check) %s: FAILED" % k
            )
        else:
            logger.info("twine check (a.k.a. readme check) %s: OK", k)
245
246


André Anjos's avatar
André Anjos committed
247
248
@ci.command(
    epilog="""
249
250
251
252
Examples:

  1. Deploys current build artifacts to the Python Package Index (PyPI):

253
     $ bdt ci pypi -vv dist/*.zip
254

André Anjos's avatar
André Anjos committed
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
"""
)
@click.argument(
    "package",
    required=True,
    type=click.Path(file_okay=True, dir_okay=False, exists=True),
    nargs=-1,
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
271
272
@verbosity_option()
@bdt.raise_on_error
André Anjos's avatar
André Anjos committed
273
def pypi(package, dry_run):
274
275
276
    """Deploys build artifacts (python packages to PyPI)

    Deployment is only allowed for packages in which the visibility is
277
278
    "public".  This check prevents publishing of private resources to
    the (public) PyPI webserver.
279
280
281
    """

    if dry_run:
André Anjos's avatar
André Anjos committed
282
283
        logger.warn("!!!! DRY RUN MODE !!!!")
        logger.warn("Nothing is being deployed to server")
284
285

    # determine project visibility
André Anjos's avatar
André Anjos committed
286
    public = os.environ["CI_PROJECT_VISIBILITY"] == "public"
287

288
    if not public:
André Anjos's avatar
André Anjos committed
289
290
291
292
293
294
295
        raise RuntimeError(
            "The repository %s is not public - a package "
            "deriving from it therefore, CANNOT be published to PyPI. "
            "You must follow the relevant software disclosure procedures "
            'and set this repository to "public" before trying again.'
            % os.environ["CI_PROJECT_PATH"]
        )
296

André Anjos's avatar
André Anjos committed
297
    from ..constants import CACERT
298
299
300
    from twine.settings import Settings

    settings = Settings(
André Anjos's avatar
André Anjos committed
301
302
        username=os.environ["PYPIUSER"],
        password=os.environ["PYPIPASS"],
303
304
        skip_existing=True,
        cacert=CACERT,
André Anjos's avatar
André Anjos committed
305
    )
306
307

    if not dry_run:
André Anjos's avatar
André Anjos committed
308
        from twine.commands.upload import upload
309

André Anjos's avatar
André Anjos committed
310
        for k in package:
311

André Anjos's avatar
André Anjos committed
312
313
314
            logger.info("Deploying python package %s to PyPI", k)
            upload(settings, [k])
            logger.info("%s: Deployed to PyPI - OK", k)
André Anjos's avatar
André Anjos committed
315
316


André Anjos's avatar
André Anjos committed
317
318
@ci.command(
    epilog="""
319
320
Examples:

321
322
  1. Builds a list of non-python packages (base dependencies) defined in a text
     file:
323
324
325

     $ bdt ci base-build -vv order.txt

326
327
328
329
330
331

  2. Builds a list of python-dependent packages (base dependencies) defined in
     a text file, for python 3.6 and 3.7:

     $ bdt ci base-build -vv --python=3.6 --python=3.7 order.txt

André Anjos's avatar
André Anjos committed
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
"""
)
@click.argument(
    "order",
    required=True,
    type=click.Path(file_okay=True, dir_okay=False, exists=True),
    nargs=1,
)
@click.option(
    "-g",
    "--group",
    show_default=True,
    default="bob",
    help="Group of packages (gitlab namespace) this package belongs to",
)
@click.option(
    "-p",
    "--python",
    multiple=True,
    help='Versions of python in the format "x.y" we should build for.  Pass '
    "various times this option to build for multiple python versions",
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
362
363
@verbosity_option()
@bdt.raise_on_error
364
def base_build(order, group, python, dry_run):
365
    """Builds base (dependence) packages.
366

367
368
369
370
    This command builds dependence packages (packages that are not
    Bob/BEAT packages) in the CI infrastructure.  It is **not** meant to
    be used outside this context.
    """
371

André Anjos's avatar
André Anjos committed
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
    condarc = select_user_condarc(
        paths=[os.curdir], branch=os.environ.get("CI_COMMIT_REF_NAME")
    )

    condarc = condarc or os.path.join(os.environ["CONDA_ROOT"], "condarc")

    if os.path.exists(condarc):
        logger.info("Loading (this build's) CONDARC file from %s...", condarc)
        with open(condarc, "rb") as f:
            condarc_options = yaml.load(f, Loader=yaml.FullLoader)

    else:  # not building on the CI? - use defaults
        from ..bootstrap import get_channels

        # get potential channel upload and other auxiliary channels
        channels = get_channels(
            public=True,
            stable=True,
            server=SERVER,
            intranet="True",
            group="bob",
        )

        # use default and add channels
        condarc_options = yaml.load(BASE_CONDARC, Loader=yaml.FullLoader)
        channels = ["local"] + channels + ["defaults"]
        logger.info(
            "Using the following channels during build:\n  - %s",
            "\n  - ".join(channels),
        )
        condarc_options["channels"] = channels

    # dump packages at conda_root
    condarc_options["croot"] = os.path.join(
        os.environ["CONDA_ROOT"], "conda-bld"
    )

409
    recipes = load_order_file(order)
André Anjos's avatar
André Anjos committed
410
411
412
413
414
415
416
417
418
419
420

    import itertools
    from .. import bootstrap
    from ..build import base_build as _build

    # combine all versions of python with recipes
    if python:
        recipes = list(itertools.product(python, recipes))
    else:
        recipes = list(itertools.product([None], recipes))

421
    for k, (pyver, recipe) in enumerate(recipes):
André Anjos's avatar
André Anjos committed
422
423
424
425
        echo_normal("\n" + (80 * "="))
        pytext = "for python-%s " % pyver if pyver is not None else ""
        echo_normal(
            'Building "%s" %s(%d/%d)'
426
            % (recipe, pytext, k + 1, len(recipes))
André Anjos's avatar
André Anjos committed
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
        )
        echo_normal((80 * "=") + "\n")
        if not os.path.exists(os.path.join(recipe, "meta.yaml")):
            logger.info('Ignoring directory "%s" - no meta.yaml found' % recipe)
            continue

        variants_file = select_conda_build_config(
            paths=[recipe, os.curdir],
            branch=os.environ.get("CI_COMMIT_REF_NAME"),
        )
        logger.info("Conda build configuration file: %s", variants_file)

        _build(
            bootstrap=bootstrap,
            server=SERVER,
            intranet=True,
            group=group,
            recipe_dir=recipe,
            conda_build_config=variants_file,
            python_version=pyver,
            condarc_options=condarc_options,
448
        )
449
450


André Anjos's avatar
André Anjos committed
451
452
@ci.command(
    epilog="""
453
454
455
456
457
458
Examples:

  1. Tests the current package

     $ bdt ci test -vv

André Anjos's avatar
André Anjos committed
459
460
461
462
463
464
465
466
467
468
"""
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
469
470
471
472
@verbosity_option()
@bdt.raise_on_error
@click.pass_context
def test(ctx, dry_run):
473
    """Tests packages.
474

475
476
477
    This command tests packages in the CI infrastructure.  It is **not**
    meant to be used outside this context.
    """
478

André Anjos's avatar
André Anjos committed
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
    group = os.environ["CI_PROJECT_NAMESPACE"]
    if group not in ("bob", "beat"):
        # defaults back to bob - no other server setups are available as of now
        group = "bob"

    # Use custom variants and append files if available on recipe-dir
    recipe_dir = os.path.join(os.path.realpath(os.curdir), "conda")

    condarc = select_user_condarc(
        paths=[recipe_dir, os.curdir],
        branch=os.environ.get("CI_COMMIT_REF_NAME"),
    )
    if condarc is not None:
        logger.info("Condarc configuration file: %s", condarc)

    variants_file = select_conda_build_config(
        paths=[recipe_dir, os.curdir],
        branch=os.environ.get("CI_COMMIT_REF_NAME"),
    )
    logger.info("Conda build configuration file: %s", variants_file)

    append_file = select_conda_recipe_append(
        paths=[recipe_dir, os.curdir],
        branch=os.environ.get("CI_COMMIT_REF_NAME"),
    )
    logger.info("Conda build recipe-append file: %s", append_file)

    from .test import test

508
509
510
    base_path = os.path.join(os.environ["CONDA_ROOT"], "conda-bld", "*",
            os.environ["CI_PROJECT_NAME"])

André Anjos's avatar
André Anjos committed
511
512
    ctx.invoke(
        test,
André Anjos's avatar
André Anjos committed
513
514
        package=glob.glob(base_path + "*.conda") + \
                glob.glob(base_path + "*.tar.bz2"),
André Anjos's avatar
André Anjos committed
515
516
517
518
519
520
521
522
523
524
525
526
527
528
        condarc=condarc,
        config=variants_file,
        append_file=append_file,
        server=SERVER,
        group=group,
        private=(os.environ["CI_PROJECT_VISIBILITY"] != "public"),
        stable="CI_COMMIT_TAG" in os.environ,
        dry_run=dry_run,
        ci=True,
    )


@ci.command(
    epilog="""
André Anjos's avatar
André Anjos committed
529
530
531
532
533
534
Examples:

  1. Builds the current package

     $ bdt ci build -vv

André Anjos's avatar
André Anjos committed
535
536
537
538
539
540
541
542
543
544
"""
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
545
@click.option(
André Anjos's avatar
André Anjos committed
546
547
548
549
    "-r",
    "--recipe-dir",
    default=os.path.join(os.path.realpath(os.curdir), "conda"),
    help="Custom recipe folder for build. Useful for debugging.",
550
)
André Anjos's avatar
André Anjos committed
551
552
@verbosity_option()
@bdt.raise_on_error
553
@click.pass_context
554
def build(ctx, dry_run, recipe_dir):
555
    """Builds packages.
André Anjos's avatar
André Anjos committed
556

557
558
559
    This command builds packages in the CI infrastructure.  It is
    **not** meant to be used outside this context.
    """
André Anjos's avatar
André Anjos committed
560

André Anjos's avatar
André Anjos committed
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
    group = os.environ["CI_PROJECT_NAMESPACE"]
    if group not in ("bob", "beat"):
        # defaults back to bob - no other server setups are available as of now
        group = "bob"

    # Use custom variants and append files if available on recipe-dir
    condarc = select_user_condarc(
        paths=[recipe_dir, os.curdir],
        branch=os.environ.get("CI_COMMIT_REF_NAME"),
    )
    if condarc is not None:
        logger.info("Condarc configuration file: %s", condarc)

    variants_file = select_conda_build_config(
        paths=[recipe_dir, os.curdir],
        branch=os.environ.get("CI_COMMIT_REF_NAME"),
    )
    logger.info("Conda build configuration file: %s", variants_file)

    append_file = select_conda_recipe_append(
        paths=[recipe_dir, os.curdir],
        branch=os.environ.get("CI_COMMIT_REF_NAME"),
    )
    logger.info("Conda build recipe-append file: %s", append_file)

    from .build import build

    ctx.invoke(
        build,
        recipe_dir=[recipe_dir],
        python=os.environ["PYTHON_VERSION"],  # python version
        condarc=condarc,
        config=variants_file,
        no_test=False,
        append_file=append_file,
        server=SERVER,
        group=group,
        private=(os.environ["CI_PROJECT_VISIBILITY"] != "public"),
        stable="CI_COMMIT_TAG" in os.environ,
        dry_run=dry_run,
        ci=True,
    )


@ci.command(
    epilog="""
607
608
609
610
611
612
Examples:

  1. Cleans the current build (and prints what it cleans)

     $ bdt ci clean -vv

André Anjos's avatar
André Anjos committed
613
614
"""
)
615
616
617
618
@verbosity_option()
@bdt.raise_on_error
@click.pass_context
def clean(ctx):
619
    """Cleans builds.
620

621
622
623
    This command cleans builds in the CI infrastructure.  It is **not**
    meant to be used outside this context.
    """
624

André Anjos's avatar
André Anjos committed
625
626
    from ..build import git_clean_build
    from ..bootstrap import run_cmdline
627

André Anjos's avatar
André Anjos committed
628
    git_clean_build(run_cmdline, verbose=(ctx.meta["verbosity"] >= 3))
629
630


André Anjos's avatar
André Anjos committed
631
632
@ci.command(
    epilog="""
633
634
635
636
637
638
Examples:

  1. Runs the nightly builds following a list of packages in a file:

     $ bdt ci nightlies -vv order.txt

André Anjos's avatar
André Anjos committed
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
"""
)
@click.argument(
    "order",
    required=True,
    type=click.Path(file_okay=True, dir_okay=False, exists=True),
    nargs=1,
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
655
656
657
658
@verbosity_option()
@bdt.raise_on_error
@click.pass_context
def nightlies(ctx, order, dry_run):
659
    """Runs nightly builds.
660

661
    This command can run nightly builds for packages listed on a file.
662

663
    The build or each package happens in a few phases:
664

665
666
667
668
669
670
671
672
    1. Package is checked out and switched to the requested branch (master if not
       set otherwise)
    2. A build string is calculated from current dependencies.  If the package
       has already been compiled, it is downloaded from the respective conda
       channel and tested.  If the test does not pass, the package is completely
       rebuilt
    3. If the rebuild is successful, the new package is uploaded to the
       respective conda channel, and the program continues with the next package
673

674
675
676
    Dependencies are searched with priority to locally built packages.  For this
    reason, the input file **must** be provided in the right dependence order.
    """
677

André Anjos's avatar
André Anjos committed
678
679
    # loads dirnames from order file (accepts # comments and empty lines)
    packages = read_packages(order)
680

André Anjos's avatar
André Anjos committed
681
    token = os.environ["CI_JOB_TOKEN"]
682

André Anjos's avatar
André Anjos committed
683
684
685
    import git
    from .build import build
    from urllib.request import urlopen
686

André Anjos's avatar
André Anjos committed
687
688
689
    # loaded all recipes, now cycle through them implementing what is described
    # in the documentation of this function
    for n, (package, branch) in enumerate(packages):
690

André Anjos's avatar
André Anjos committed
691
692
693
694
695
        echo_normal("\n" + (80 * "="))
        echo_normal(
            "Building %s@%s (%d/%d)" % (package, branch, n + 1, len(packages))
        )
        echo_normal((80 * "=") + "\n")
696

André Anjos's avatar
André Anjos committed
697
        group, name = package.split("/", 1)
698

André Anjos's avatar
André Anjos committed
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
        clone_to = os.path.join(
            os.environ["CI_PROJECT_DIR"], "src", group, name
        )
        dirname = os.path.dirname(clone_to)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        # clone the repo, shallow version, on the specified branch
        logger.info('Cloning "%s", branch "%s" (depth=1)...', package, branch)
        git.Repo.clone_from(
            "https://gitlab-ci-token:%s@gitlab.idiap.ch/%s" % (token, package),
            clone_to,
            branch=branch,
            depth=1,
        )
714

André Anjos's avatar
André Anjos committed
715
716
717
718
719
        # determine package visibility
        private = (
            urlopen("https://gitlab.idiap.ch/%s" % package).getcode() != 200
        )
        stable = "STABLE" in os.environ
720

André Anjos's avatar
André Anjos committed
721
722
        # Use custom variants and append files if available on recipe-dir
        recipe_dir = os.path.join(clone_to, "conda")
723

André Anjos's avatar
André Anjos committed
724
725
726
727
728
729
        condarc = select_user_condarc(
            paths=[recipe_dir, os.curdir],
            branch=os.environ.get("CI_COMMIT_REF_NAME"),
        )
        if condarc is not None:
            logger.info("Condarc configuration file: %s", condarc)
730

André Anjos's avatar
André Anjos committed
731
732
733
734
735
        variants_file = select_conda_build_config(
            paths=[recipe_dir, os.curdir],
            branch=os.environ.get("CI_COMMIT_REF_NAME"),
        )
        logger.info("Conda build configuration file: %s", variants_file)
736

André Anjos's avatar
André Anjos committed
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
        append_file = select_conda_recipe_append(
            paths=[recipe_dir, os.curdir],
            branch=os.environ.get("CI_COMMIT_REF_NAME"),
        )
        logger.info("Conda build recipe-append file: %s", append_file)

        ctx.invoke(
            build,
            recipe_dir=[recipe_dir],
            python=os.environ["PYTHON_VERSION"],  # python version
            condarc=condarc,
            config=variants_file,
            no_test=False,
            append_file=append_file,
            server=SERVER,
            group=group,
            private=private,
            stable=stable,
            dry_run=dry_run,
            ci=True,
757
        )
758

André Anjos's avatar
André Anjos committed
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
        is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"

        # re-deploys a new conda package if it was rebuilt and it is the master
        # branch
        # n.b.: can only arrive here if dry_run was ``False`` (no need to check
        # again)
        if "BDT_BUILD" in os.environ and is_master:
            tarball = os.environ["BDT_BUILD"]
            del os.environ["BDT_BUILD"]
            deploy_conda_package(
                tarball,
                arch=None,
                stable=stable,
                public=(not private),
                username=os.environ["DOCUSER"],
                password=os.environ["DOCPASS"],
                overwrite=False,
                dry_run=dry_run,
            )

        # removes the documentation to avoid permissions issues with the following
        # projects being built
        local_docs = os.path.join(os.environ["CI_PROJECT_DIR"], "sphinx")
        if os.path.exists(local_docs):
            logger.debug(
                "Sphinx output was generated during test/rebuild "
                "of %s - Erasing...",
                package,
            )
            shutil.rmtree(local_docs)


@ci.command(
    epilog="""
793
794
795
796
797
798
Examples:

  1. Prepares the docs for the subsequent `bdt ci build ...`:

     $ bdt ci docs -vv requirements.txt

André Anjos's avatar
André Anjos committed
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
"""
)
@click.argument(
    "requirement",
    required=True,
    type=click.Path(file_okay=True, dir_okay=False, exists=True),
    nargs=1,
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
815
816
817
818
@verbosity_option()
@bdt.raise_on_error
@click.pass_context
def docs(ctx, requirement, dry_run):
819
    """Prepares documentation build.
820

821
822
    This command:
      \b
823

824
825
826
      1. Clones all the necessary packages necessary to build the bob/beat
         documentation
      \b
827

828
829
      2. Generates the `extra-intersphinx.txt` and `nitpick-exceptions.txt` file
      \b
830

831
832
    This command is supposed to be run **instead** of `bdt ci build...`
    """
833

André Anjos's avatar
André Anjos committed
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
    packages = read_packages(requirement)

    import git

    token = os.environ["CI_JOB_TOKEN"]

    # loaded all recipes, now cycle through them implementing what is described
    # in the documentation of this function
    extra_intersphinx = []
    nitpick = []
    doc_path = os.path.join(os.environ["CI_PROJECT_DIR"], "doc")

    for n, (package, branch) in enumerate(packages):

        group, name = package.split("/", 1)

        clone_to = os.path.join(doc_path, group, name)
        dirname = os.path.dirname(clone_to)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        # clone the repo, shallow version, on the specified branch
        if dry_run:
            logger.info(
                'Cloning "%s" [%d/%d], branch "%s" (depth=1) to %s...',
                package,
                n + 1,
                len(packages),
                branch,
                clone_to,
            )
        else:
            if os.path.exists(clone_to):
                logger.info(
                    'Repo "%s" [%d/%d], already cloned at %s; '
                    'updating branch "%s"...',
                    package,
                    n + 1,
                    len(packages),
                    clone_to,
                    branch,
                )
                git.Git(clone_to).pull("origin", branch)
            else:
                logger.info(
                    'Cloning "%s" [%d/%d], branch "%s" (depth=1) to %s...',
                    package,
                    n + 1,
                    len(packages),
                    branch,
                    clone_to,
                )
                git.Repo.clone_from(
                    "https://gitlab-ci-token:%s@gitlab.idiap.ch/%s"
                    % (token, package),
                    clone_to,
                    branch=branch,
                    depth=1,
                )

            # Copying the content from extra_intersphinx
            extra_intersphinx_path = os.path.join(
                clone_to, "doc", "extra-intersphinx.txt"
            )
            if os.path.exists(extra_intersphinx_path):
                with open(extra_intersphinx_path) as f:
                    extra_intersphinx += comment_cleanup(f.readlines())

            test_requirements_path = os.path.join(
                clone_to, "doc", "test-requirements.txt"
            )
            if os.path.exists(test_requirements_path):
                with open(test_requirements_path) as f:
                    extra_intersphinx += comment_cleanup(f.readliens())

            requirements_path = os.path.join(clone_to, "requirements.txt")
            if os.path.exists(requirements_path):
                with open(requirements_path) as f:
                    extra_intersphinx += comment_cleanup(f.readlines())

            nitpick_path = os.path.join(
                clone_to, "doc", "nitpick-exceptions.txt"
            )
            if os.path.exists(nitpick_path):
                with open(nitpick_path) as f:
                    nitpick += comment_cleanup(f.readlines())

    logger.info("Generating (extra) sphinx files...")

    # Making unique lists and removing all bob/beat references
    if not dry_run:
925

André Anjos's avatar
André Anjos committed
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
        # extra requirements for sphinx
        group = os.environ["CI_PROJECT_NAMESPACE"]
        extra_intersphinx = set(
            [
                k.strip()
                for k in extra_intersphinx
                if not k.strip().startswith((group, "gridtk"))
            ]
        )
        data = "\n".join(uniq(sorted(extra_intersphinx)))
        logger.info('Contents of "doc/extra-intersphinx.txt":\n%s', data)
        with open(os.path.join(doc_path, "extra-intersphinx.txt"), "w") as f:
            f.write(data)

        # nitpick exceptions
        data = "\n".join(uniq(sorted(nitpick)))
        logger.info('Contents of "doc/nitpick-exceptions.txt":\n%s', data)
        with open(os.path.join(doc_path, "nitpick-exceptions.txt"), "w") as f:
            f.write(data)

    logger.info("Building documentation...")
    ctx.invoke(build, dry_run=dry_run)
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984


@ci.command(
    epilog="""
Examples:

  1. Cleans-up the excess of beta packages from all conda channels via WebDAV:

     $ bdt ci -vv clean-betas --dry-run

     Notice this does not do anything.  Remove the --dry-run flag to execute


  2. Really removes (recursively), the excess of beta packages

     $ bdt ci -vv clean-betas

"""
)
@click.option(
    "-d",
    "--dry-run/--no-dry-run",
    default=False,
    help="Only goes through the actions, but does not execute them "
    "(combine with the verbosity flags - e.g. ``-vvv``) to enable "
    "printing to help you understand what will be done",
)
@verbosity_option()
@bdt.raise_on_error
def clean_betas(dry_run):
    """Cleans-up the excess of beta packages from a conda channel via WebDAV

    ATTENTION: There is no undo!  Use --dry-run to test before using.
    """

    is_master = os.environ["CI_COMMIT_REF_NAME"] == "master"
    if not is_master and dry_run == False:
985
986
        logger.warn("Forcing dry-run mode - not in master branch")
        logger.warn("... considering this is **not** a periodic run!")
987
988
989
        dry_run = True

    if dry_run:
990
991
        logger.warn("!!!! DRY RUN MODE !!!!")
        logger.warn("Nothing is being executed on server.")
992

993
994
995
996
997
998
    import re
    if os.environ["CI_PROJECT_NAMESPACE"] == "beat":
        includes = re.compile(r'^beat.*')
    else:
        includes = re.compile(r'^(bob|batl|gridtk).*')

999
    cleanup(
1000
1001
1002
            dry_run=dry_run,
            username=os.environ["DOCUSER"],
            password=os.environ["DOCPASS"],
1003
            includes=includes,
1004
            )