Commit 5914cc65 authored by André Anjos's avatar André Anjos 💬

Merge branch 'docs' into 'master'

Created the CLI command 'bdt ci docs' to replace the before_build.sh

See merge request !59
parents 7fe0424d 8e5aad1d
Pipeline #30385 passed with stages
in 5 minutes and 40 seconds
......@@ -72,3 +72,24 @@ def is_stable(package, refname, tag, repodir):
logger.info('No tag information available at build')
logger.info('Considering this to be a pre-release build')
return False
def read_packages(filename):
"""
Return a python list of tuples (repository, branch), given a file containing
one package (and branch) per line. Comments are excluded
"""
# loads dirnames from order file (accepts # comments and empty lines)
packages = []
with open(filename, 'rt') as f:
for line in f:
line = line.partition('#')[0].strip()
if line:
if ',' in line: #user specified a branch
path, branch = [k.strip() for k in line.split(',', 1)]
packages.append((path, branch))
else:
packages.append((line, 'master'))
return packages
# This YAML file contains descriptions for the CI of most of our Bob/BEAT/BATL
# packages - do **not** modify it unless you know what you're doing (and up
# to!)
# Definition of global variables (all stages)
variables:
PYTHONUNBUFFERED: "1"
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
BOOTSTRAP: "https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/bootstrap.py"
# Definition of our build pipeline order
stages:
- build
- deploy
# Build targets
.build_template: &build_job
stage: build
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci docs -vv requirements.txt
- bdt ci clean -vv
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker
image: continuumio/conda-concourse-ci
artifacts:
expire_in: 1 week
paths:
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
build_linux_36:
<<: *linux_build_job
variables:
GITLAB_CHECKOUT_STRATEGY: "https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.idiap.ch/"
PYTHON_VERSION: "3.6"
BUILD_EGG: "true"
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci build -vv
- bdt ci clean -vv
artifacts:
expire_in: 1 week
paths:
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
# Test targets (not normally used)
.test_template: &test_job
stage: test
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci test -vv
- bdt ci clean -vv
cache: &test_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.test_linux_template: &linux_test_job
<<: *test_job
tags:
- docker
image: continuumio/conda-concourse-ci
cache:
<<: *test_caches
key: "linux-cache"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- bdt ci deploy -vv
- bdt ci clean -vv
dependencies:
- build_linux_36
tags:
- docker
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
......@@ -14,6 +14,7 @@ from . import bdt
from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
WEBDAV_PATHS, BASE_CONDARC
from ..deploy import deploy_conda_package, deploy_documentation
from ..ci import read_packages
from ..log import verbosity_option, get_logger, echo_normal
logger = get_logger(__name__)
......@@ -539,16 +540,7 @@ def nightlies(ctx, order, dry_run):
"""
# loads dirnames from order file (accepts # comments and empty lines)
packages = []
with open(order, 'rt') as f:
for line in f:
line = line.partition('#')[0].strip()
if line:
if ',' in line: #user specified a branch
path, branch = [k.strip() for k in line.split(',', 1)]
packages.append((path, branch))
else:
packages.append((line, 'master'))
packages = read_packages(order)
token = os.environ['CI_JOB_TOKEN']
......@@ -636,3 +628,102 @@ def nightlies(ctx, order, dry_run):
logger.debug('Sphinx output was generated during test/rebuild ' \
'of %s - Erasing...', package)
shutil.rmtree(local_docs)
@ci.command(epilog='''
Examples:
1. Prepares the docs for the subsequent `bdt ci build ...`:
$ bdt ci docs -vv requirements.txt
''')
@click.argument('requirement', required=True, type=click.Path(file_okay=True,
dir_okay=False, exists=True), nargs=1)
@click.option('-d', '--dry-run/--no-dry-run', default=False,
help='Only goes through the actions, but does not execute them ' \
'(combine with the verbosity flags - e.g. ``-vvv``) to enable ' \
'printing to help you understand what will be done')
@verbosity_option()
@bdt.raise_on_error
@click.pass_context
def docs(ctx, requirement, dry_run):
"""Prepares documentation build
This command:
1. Clones all the necessary packages necessary to build the bob documentation
2. Generates the `extra-intersphinx.txt` and `nitpick-exceptions.txt` file
This command is supposed to be run before `bdt ci build...`
"""
packages = read_packages(requirement)
import git
token = os.environ['CI_JOB_TOKEN']
group = os.environ['CI_PROJECT_NAMESPACE']
# loaded all recipes, now cycle through them implementing what is described
# in the documentation of this function
extra_intersphinx = []
nitpick = []
doc_path = os.path.join(os.environ['CI_PROJECT_DIR'], 'doc')
for n, (package, branch) in enumerate(packages):
echo_normal('\n' + (80*'='))
echo_normal('Getting %s@%s (%d/%d)' % (package, branch, n+1,
len(packages)))
echo_normal((80*'=') + '\n')
clone_to = os.path.join(doc_path, package)
dirname = os.path.dirname(clone_to)
if not os.path.exists(dirname):
os.makedirs(dirname)
# clone the repo, shallow version, on the specified branch
if dry_run:
logger.info('Cloning "%s", branch "%s" (depth=1)...', package, branch)
else:
if os.path.exists(clone_to):
logger.info('Repo "%s", already cloned; pulling from master...', package)
git.Git(clone_to).pull("origin", branch)
else:
logger.info('Cloning "%s", branch "%s" (depth=1)...', package, branch)
git.Repo.clone_from('https://gitlab-ci-token:%s@gitlab.idiap.ch/%s' % \
(token, group+"/"+package), clone_to, branch=branch, depth=1)
# Copying the content from extra_intersphinx
extra_intersphinx_path = os.path.join(clone_to, "doc",
"extra-intersphinx.txt")
test_requirements_path = os.path.join(clone_to, "doc",
"test-requirements.txt")
requirements_path = os.path.join(clone_to, "requirements.txt")
if os.path.exists(extra_intersphinx_path):
extra_intersphinx += open(extra_intersphinx_path).readlines()
if os.path.exists(test_requirements_path):
extra_intersphinx += open(test_requirements_path).readlines()
if os.path.exists(requirements_path):
extra_intersphinx += open(requirements_path).readlines()
nitpick_path = os.path.join(clone_to, "doc", "nitpick-exceptions.txt")
if os.path.exists(nitpick_path):
nitpick += open(nitpick_path).readlines()
logger.info('Generating sphinx files...')
# Making unique lists and removing all bob references
if not dry_run:
extra_intersphinx = list(set([e for e in extra_intersphinx if group not in e ]))
nitpick = list(set([e for e in nitpick]))
# Removing projects that are part of the group
open(os.path.join(doc_path, "extra-intersphinx.txt"), "w").writelines(extra_intersphinx)
open(os.path.join(doc_path, "nitpick-exceptions.txt"), "w").writelines(nitpick)
logger.info('Building documentation...')
ctx.invoke(build, dry_run=dry_run)
......@@ -90,6 +90,7 @@ test:
- bdt ci readme --help
- bdt ci clean --help
- bdt ci nightlies --help
- bdt ci docs --help
- sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc sphinx
- if [ -n "${CI_PROJECT_DIR}" ]; then mv sphinx "${CI_PROJECT_DIR}/"; fi
......
......@@ -79,6 +79,7 @@ setup(
'readme = bob.devtools.scripts.ci:readme',
'pypi = bob.devtools.scripts.ci:pypi',
'nightlies = bob.devtools.scripts.ci:nightlies',
'docs = bob.devtools.scripts.ci:docs',
],
},
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment