Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.devtools
Commits
da9d8a52
Commit
da9d8a52
authored
Jan 08, 2019
by
André Anjos
💬
Browse files
[scritpts] Implement build support; Remove old cb-output script (incorporate into build)
parent
090ac45b
Changes
8
Hide whitespace changes
Inline
Side-by-side
bob/devtools/bootstrap.py
View file @
da9d8a52
...
...
@@ -10,25 +10,45 @@ import logging
logger
=
logging
.
getLogger
(
__name__
)
import
yaml
from
conda_build.api
import
get_or_merge_config
,
render
,
output_yaml
def
get_rendered_recipe
(
conda
,
recipe_dir
,
python
,
config
):
def
make_conda_config
(
config
,
python
,
append_file
,
condarc
):
from
conda_build.api
import
get_or_merge_config
from
conda_build.conda_interface
import
url_path
with
open
(
condarc
,
'rb'
)
as
f
:
condarc_options
=
yaml
.
load
(
f
)
retval
=
get_or_merge_config
(
None
,
variant_config_files
=
config
,
python
=
python
,
append_sections_file
=
append_file
,
**
condarc_options
)
retval
.
channel_urls
=
[]
for
url
in
condarc_options
[
'channels'
]:
# allow people to specify relative or absolute paths to local channels
# These channels still must follow conda rules - they must have the
# appropriate platform-specific subdir (e.g. win-64)
if
os
.
path
.
isdir
(
url
):
if
not
os
.
path
.
isabs
(
url
):
url
=
os
.
path
.
normpath
(
os
.
path
.
abspath
(
os
.
path
.
join
(
os
.
getcwd
(),
url
)))
url
=
url_path
(
url
)
retval
.
channel_urls
.
append
(
url
)
return
retval
def
get_rendered_metadata
(
recipe_dir
,
config
):
'''Renders the recipe and returns the interpreted YAML file'''
# equivalent command execute - in here we use the conda API
cmd
=
[
conda
,
'render'
,
'--variant-config-files'
,
config
,
'--python'
,
python
,
recipe_dir
,
]
logger
.
debug
(
'$ '
+
' '
.
join
(
cmd
))
from
conda_build.api
import
render
return
render
(
recipe_dir
,
config
=
config
)
def
get_parsed_recipe
(
metadata
):
'''Renders the recipe and returns the interpreted YAML file'''
# do the real job
config
=
get_or_merge_config
(
None
,
variant_config_files
=
config
,
python
=
python
)
metadata
=
render
(
recipe_dir
,
config
=
config
)
from
conda_build.api
import
output_yaml
output
=
output_yaml
(
metadata
[
0
][
0
])
return
yaml
.
load
(
output
)
...
...
@@ -37,9 +57,10 @@ def remove_pins(deps):
return
[
l
.
split
()[
0
]
for
l
in
deps
]
def
parse_dependencies
(
conda
,
recipe_dir
,
python
,
config
):
def
parse_dependencies
(
recipe_dir
,
config
):
recipe
=
get_rendered_recipe
(
conda
,
recipe_dir
,
python
,
config
)
metadata
=
get_rendered_metadata
(
recipe_dir
,
config
)
recipe
=
get_parsed_recipe
(
metadata
)
return
remove_pins
(
recipe
[
'requirements'
].
get
(
'build'
,
[]))
+
\
remove_pins
(
recipe
[
'requirements'
].
get
(
'host'
,
[]))
+
\
recipe
[
'requirements'
].
get
(
'run'
,
[])
+
\
...
...
bob/devtools/conda.py
View file @
da9d8a52
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Utilities for deadling with conda packages'''
import
re
import
logging
import
platform
logger
=
logging
.
getLogger
(
__name__
)
def
osname
():
"""Returns the current OS name as recognized by conda"""
r
=
'unknown'
if
platform
.
system
().
lower
()
==
'linux'
:
r
=
'linux'
elif
platform
.
system
().
lower
()
==
'darwin'
:
r
=
'osx'
else
:
raise
RuntimeError
(
'Unsupported system "%s"'
%
platform
.
system
())
if
platform
.
machine
().
lower
()
==
'x86_64'
:
r
+=
'-64'
else
:
raise
RuntimeError
(
'Unsupported machine type "%s"'
%
platform
.
machine
())
return
r
def
should_skip_build
(
metadata_tuples
):
"""Takes the output of render_recipe as input and evaluates if this
recipe's build should be skipped.
"""
return
all
(
m
[
0
].
skip
()
for
m
in
metadata_tuples
)
"""Takes the output of render_recipe as input and evaluates if this
recipe's build should be skipped.
"""
return
all
(
m
[
0
].
skip
()
for
m
in
metadata_tuples
)
def
next_build_number
(
channel_url
,
name
,
version
,
python
):
"""Calculates the next build number of a package given the channel
This function returns the next build number (integer) for a package given its
recipe, dependencies, name, version and python version. It looks on the
channel URL provided and figures out if any clash would happen and what would
be the highest build number available for that configuration.
Args:
channel_url: The URL where to look for packages clashes (normally a beta
channel)
name: The name of the package
version: The version of the package
python: The version of python as 2 digits (e.g.: "2.7" or "3.6")
Returns: The next build number with the current configuration. Zero (0) is
returned if no match is found. Also returns the URLs of the packages it
finds with matches on the name, version and python-version.
"""
from
conda.exports
import
get_index
# no dot in py_ver
py_ver
=
python
.
replace
(
'.'
,
''
)
# get the channel index
logger
.
debug
(
'Downloading channel index from %s'
,
channel_url
)
index
=
get_index
(
channel_urls
=
[
channel_url
],
prepend
=
False
)
# search if package with the same version exists
build_number
=
0
urls
=
[]
for
dist
in
index
:
if
dist
.
name
==
name
and
dist
.
version
==
version
:
match
=
re
.
match
(
'py[2-9][0-9]+'
,
dist
.
build_string
)
if
match
and
match
.
group
()
==
'py{}'
.
format
(
py_ver
):
logger
.
debug
(
"Found match at %s for %s-%s-py%s"
,
index
[
dist
].
url
,
name
,
version
,
py_ver
)
build_number
=
max
(
build_number
,
dist
.
build_number
+
1
)
urls
.
append
(
index
[
dist
].
url
)
urls
=
[
url
.
replace
(
channel_url
,
''
)
for
url
in
urls
]
return
build_number
,
urls
bob/devtools/data/recipe_append.yaml
0 → 100644
View file @
da9d8a52
build
:
script_env
:
-
DOCSERVER
bob/devtools/scripts/bootstrap.py
View file @
da9d8a52
...
...
@@ -8,16 +8,20 @@ logger = logging.getLogger(__name__)
import
pkg_resources
import
click
import
yaml
from
.
import
bdt
from
..log
import
verbosity_option
from
..bootstrap
import
parse_dependencies
,
conda_create
from
..bootstrap
import
parse_dependencies
,
conda_create
,
make_conda_config
DEFAULT_CONDARC
=
pkg_resources
.
resource_filename
(
__name__
,
os
.
path
.
join
(
'..'
,
'data'
,
'build-condarc'
))
DEFAULT_VARIANT
=
pkg_resources
.
resource_filename
(
__name__
,
os
.
path
.
join
(
'..'
,
'data'
,
'conda_build_config.yaml'
))
DEFAULT_APPEND
=
pkg_resources
.
resource_filename
(
__name__
,
os
.
path
.
join
(
'..'
,
'data'
,
'recipe_append.yaml'
))
DEFAULT_DOCSERVER
=
'http://www.idiap.ch'
@
click
.
command
(
epilog
=
'''
...
...
@@ -69,14 +73,21 @@ Examples:
@
click
.
option
(
'-m'
,
'--config'
,
'--variant-config-files'
,
show_default
=
True
,
default
=
DEFAULT_VARIANT
,
help
=
'overwrites the path leading to '
\
'variant configuration file to use'
)
@
click
.
option
(
'-a'
,
'--append-file'
,
show_default
=
True
,
default
=
DEFAULT_APPEND
,
help
=
'overwrites the path leading to '
\
'appended configuration file to use'
)
@
click
.
option
(
'-D'
,
'--docserver'
,
show_default
=
True
,
default
=
DEFAULT_DOCSERVER
,
help
=
'Server used for uploading artifacts '
\
'and other goodies'
)
@
click
.
option
(
'-d'
,
'--dry-run/--no-dry-run'
,
default
=
False
,
help
=
'Only goes through the actions, but does not execute them '
\
'(combine with the verbosity flags - e.g. ``-vvv``) to enable '
\
'printing to help you understand what will be done'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
bootstrap
(
name
,
recipe_dir
,
python
,
overwrite
,
condarc
,
config
,
dry_run
):
"""This program uses conda to build a development environment for a recipe
def
bootstrap
(
name
,
recipe_dir
,
python
,
overwrite
,
condarc
,
config
,
append_file
,
docserver
,
dry_run
):
"""Creates a development environment for a recipe
It uses the conda render API to render a recipe and install an environment
containing all build/host, run and test dependencies of a package. It does
...
...
@@ -106,9 +117,13 @@ def bootstrap(name, recipe_dir, python, overwrite, condarc, config, dry_run):
"properly?"
)
# set condarc before continuing
logger
.
debug
(
'$ export
CONDARC=%s
'
,
condarc
)
logger
.
debug
(
"[var]
CONDARC=%s
"
,
condarc
)
os
.
environ
[
'CONDARC'
]
=
condarc
deps
=
parse_dependencies
(
conda
,
recipe_dir
,
python
,
config
)
logger
.
debug
(
"[var] DOCSERVER=%s"
,
docserver
)
os
.
environ
[
'DOCSERVER'
]
=
docserver
conda_config
=
make_conda_config
(
config
,
python
,
append_file
,
condarc
)
deps
=
parse_dependencies
(
recipe_dir
,
conda_config
)
status
=
conda_create
(
conda
,
name
,
overwrite
,
condarc
,
deps
,
dry_run
)
click
.
echo
(
'Execute on your shell: "conda activate %s"'
%
name
)
bob/devtools/scripts/build.py
0 → 100644
View file @
da9d8a52
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import
os
import
sys
import
logging
logger
=
logging
.
getLogger
(
__name__
)
import
pkg_resources
import
click
from
.
import
bdt
from
..log
import
verbosity_option
from
..conda
import
next_build_number
,
osname
from
..bootstrap
import
get_rendered_metadata
,
get_parsed_recipe
from
.bootstrap
import
DEFAULT_CONDARC
,
DEFAULT_VARIANT
,
DEFAULT_APPEND
,
\
DEFAULT_DOCSERVER
@
click
.
command
(
epilog
=
'''
Examples:
1. Builds recipe from one of our build dependencies (inside bob.conda):
$ cd bob.conda
$ bdt build -vv conda/libblitz
2. Builds recipe from one of our packages, for Python 3.6 (if that is not
already the default for you):
$ bdt build --python=3.6 -vv path/to/conda/dir
3. To build multiple recipes, just pass the paths to them:
$ bdt build --python=3.6 -vv path/to/recipe-dir/1 path/to/recipe-dir/2
'''
)
@
click
.
argument
(
'recipe-dir'
,
required
=
False
,
type
=
click
.
Path
(
file_okay
=
False
,
dir_okay
=
True
,
exists
=
True
),
nargs
=-
1
)
@
click
.
option
(
'-p'
,
'--python'
,
default
=
(
'%d.%d'
%
sys
.
version_info
[:
2
]),
show_default
=
True
,
help
=
'Version of python to build the '
\
'environment for [default: %(default)s]'
)
@
click
.
option
(
'-r'
,
'--condarc'
,
default
=
DEFAULT_CONDARC
,
show_default
=
True
,
help
=
'overwrites the path leading to the condarc file to use'
,)
@
click
.
option
(
'-m'
,
'--config'
,
'--variant-config-files'
,
show_default
=
True
,
default
=
DEFAULT_VARIANT
,
help
=
'overwrites the path leading to '
\
'variant configuration file to use'
)
@
click
.
option
(
'-c'
,
'--channel'
,
show_default
=
True
,
default
=
'https://www.idiap.ch/software/bob/conda/label/beta'
,
help
=
'Channel URL where this package is meant to be uploaded to, '
\
'after a successful build - typically, this is a beta channel'
)
@
click
.
option
(
'-n'
,
'--no-test'
,
is_flag
=
True
,
help
=
'Do not test the package, only builds it'
)
@
click
.
option
(
'-a'
,
'--append-file'
,
show_default
=
True
,
default
=
DEFAULT_APPEND
,
help
=
'overwrites the path leading to '
\
'appended configuration file to use'
)
@
click
.
option
(
'-D'
,
'--docserver'
,
show_default
=
True
,
default
=
DEFAULT_DOCSERVER
,
help
=
'Server used for uploading artifacts '
\
'and other goodies'
)
@
click
.
option
(
'-d'
,
'--dry-run/--no-dry-run'
,
default
=
False
,
help
=
'Only goes through the actions, but does not execute them '
\
'(combine with the verbosity flags - e.g. ``-vvv``) to enable '
\
'printing to help you understand what will be done'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
build
(
recipe_dir
,
python
,
condarc
,
config
,
channel
,
no_test
,
append_file
,
docserver
,
dry_run
):
"""Runs conda-build with a standard configuration and environment
This command wraps the execution of conda-build so that you use the same
``condarc`` and ``conda_build_config.yaml`` file we use for our CI. It
always set ``--no-anaconda-upload``.
Note that both files are embedded within bob.devtools - you may need to
update your environment before trying this.
"""
# if we are in a dry-run mode, let's let it be known
if
dry_run
:
logger
.
warn
(
'!!!! DRY RUN MODE !!!!'
)
logger
.
warn
(
'Nothing will be really built'
)
recipe_dir
=
recipe_dir
or
[
os
.
path
.
join
(
os
.
path
.
realpath
(
'.'
),
'conda'
)]
logger
.
debug
(
"[var] CONDARC=%s"
,
condarc
)
from
..bootstrap
import
make_conda_config
conda_config
=
make_conda_config
(
config
,
python
,
append_file
,
condarc
)
logger
.
debug
(
"[var] DOCSERVER=%s"
,
docserver
)
os
.
environ
[
'DOCSERVER'
]
=
docserver
for
d
in
recipe_dir
:
if
not
os
.
path
.
exists
(
d
):
raise
RuntimeError
(
"The directory %s does not exist"
%
recipe_dir
)
version_candidate
=
os
.
path
.
join
(
d
,
'..'
,
'version.txt'
)
if
os
.
path
.
exists
(
version_candidate
):
version
=
open
(
version_candidate
).
read
().
rstrip
()
logger
.
debug
(
"[var] BOB_PACKAGE_VERSION=%s"
,
version
)
os
.
environ
[
'BOB_PACKAGE_VERSION'
]
=
version
# pre-renders the recipe - figures out package name and version
metadata
=
get_rendered_metadata
(
d
,
conda_config
)
# checks we should actually build this recipe
from
..conda
import
should_skip_build
if
should_skip_build
(
metadata
):
logger
.
warn
(
'Skipping UNSUPPORTED build of "%s" for py%s on %s'
,
d
,
python
.
replace
(
'.'
,
''
),
osname
())
return
0
# converts the metadata output into parsed yaml and continues the process
rendered_recipe
=
get_parsed_recipe
(
metadata
)
# if a channel URL was passed, set the build number
if
channel
:
build_number
,
_
=
next_build_number
(
channel
,
rendered_recipe
[
'package'
][
'name'
],
rendered_recipe
[
'package'
][
'version'
],
python
)
else
:
build_number
=
0
logger
.
debug
(
"[var] BOB_BUILD_NUMBER=%s"
,
build_number
)
os
.
environ
[
'BOB_BUILD_NUMBER'
]
=
str
(
build_number
)
# we don't execute the following command, it is just here for logging
# purposes. we directly use the conda_build API.
logger
.
info
(
'Building %s-%s-py%s (build: %d) for %s'
,
rendered_recipe
[
'package'
][
'name'
],
rendered_recipe
[
'package'
][
'version'
],
python
.
replace
(
'.'
,
''
),
build_number
,
osname
())
if
not
dry_run
:
from
conda_build.api
import
build
build
(
d
,
config
=
conda_config
,
notest
=
no_test
)
bob/devtools/scripts/cb_output.py
deleted
100644 → 0
View file @
090ac45b
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import
logging
logger
=
logging
.
getLogger
(
__name__
)
import
click
from
click.testing
import
CliRunner
import
conda_build.api
as
cb
from
.
import
bdt
from
..log
import
verbosity_option
from
..conda
import
should_skip_build
@
click
.
command
(
context_settings
=
dict
(
ignore_unknown_options
=
True
,
allow_extra_args
=
True
,
),
epilog
=
'''
\b
Examples:
$ bdt cb-output conda_recipe_dir
$ bdt cb-output ../bob.conda/conda/kaldi -m ../bob.admin/gitlab/conda_build_config.yaml --python 3.6
'''
)
@
click
.
argument
(
'recipe_path'
)
@
click
.
option
(
'-m'
,
'--variant-config-files'
,
help
=
'see conda build --help'
)
@
click
.
option
(
'--python'
,
help
=
'see conda build --help'
)
@
verbosity_option
()
@
bdt
.
raise_on_error
def
cb_output
(
recipe_path
,
variant_config_files
,
python
):
"""Outputs name(s) of package(s) that would be generated by conda build.
This command accepts extra unknown arguments so you can give it the same
arguments that you would give to conda build.
As of now, it only parses -m/--variant_config_files and --python and other
arguments are ignored.
"""
clirunner
=
CliRunner
()
with
clirunner
.
isolation
():
# render
config
=
cb
.
get_or_merge_config
(
None
,
variant_config_files
=
variant_config_files
,
python
=
python
)
metadata_tuples
=
cb
.
render
(
recipe_path
,
config
=
config
)
# check if build(s) should be skipped
if
should_skip_build
(
metadata_tuples
):
return
0
paths
=
cb
.
get_output_file_paths
(
metadata_tuples
,
config
=
config
)
click
.
echo
(
'
\n
'
.
join
(
sorted
(
paths
)))
conda/meta.yaml
View file @
da9d8a52
...
...
@@ -57,6 +57,7 @@ test:
-
bdt visibility --help
-
bdt dumpsphinx --help
-
bdt bootstrap --help
-
bdt build --help
-
sphinx-build -aEW ${PREFIX}/share/doc/{{ name }}/doc {{ project_dir }}/sphinx
about
:
...
...
setup.py
View file @
da9d8a52
...
...
@@ -39,13 +39,13 @@ setup(
'bdt = bob.devtools.scripts.bdt:main'
,
],
'bdt.cli'
:
[
'cb-output = bob.devtools.scripts.cb_output:cb_output'
,
'release = bob.devtools.scripts.release:release'
,
'changelog = bob.devtools.scripts.changelog:changelog'
,
'lasttag = bob.devtools.scripts.lasttag:lasttag'
,
'visibility = bob.devtools.scripts.visibility:visibility'
,
'dumpsphinx = bob.devtools.scripts.dumpsphinx:dumpsphinx'
,
'bootstrap = bob.devtools.scripts.bootstrap:bootstrap'
,
'build = bob.devtools.scripts.build:build'
,
],
},
classifiers
=
[
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment