Commit d486fbb4 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'devtools' into 'master'

Use bdt for building all packages

See merge request bob/bob.conda!401
parents 970211ce b6c6966b
Pipeline #26969 passed with stage
in 13 minutes and 27 seconds
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
BOB_PACKAGE_VERSION: "unknown"
PYTHONUNBUFFERED: 1
STABLE: "true"
# Definition of our build pipeline order
stages:
- build
# Build targets
.build_template: &build_job
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
script:
- scripts/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
key: "$CI_BUILD_NAME"
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
- ${CONDA_ROOT}/conda-bld/src_cache
build_linux:
<<: *build_job
tags:
- docker
image: continuumio/conda-concourse-ci
build_macosx:
<<: *build_job
tags:
- macosx
include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/base-build.yaml'
# Conda recipes for bob dependencies not available in the defaults channel
This package contains scripts and recipes for building dependencies packages of
Bob for Conda.
This package contains scripts and recipes for building dependence packages of
Bob/BEAT for Conda.
## Folder structure
There are two folders inside this repository and they are organized like this:
- `conda`: A folder to keep the recipes for packages.
- `scripts`: Contains some useful scripts.
There is a single folder called `conda`, within this repository, where you
should keep all conda recipes inside.
## Adding a new package
......@@ -21,10 +19,12 @@ If you want to create a new recipe for a package, first:
the official guide is the reference): https://conda-forge.org/docs/
3. Look at similar recipes in https://github.com/AnacondaRecipes/
Look at the .gitlab-ci.yml file here to see how packages are built and tested.
Add your recipe to the folder called `conda` and ensure your package appears in
the file `order.txt`. If your package depends on another package being built
by this repository, then ensure it is built after its dependence.
If that works, upload your changes on bob.conda on a branch and create a merge
request. Wait for the tests to pass on the MR and make sure everything
request. Wait for the tests to pass on the MR and make sure everything
completes well, by inspecting the log files. If all is good, assign the merge
request to one of the package maintainers.
......
......@@ -12,18 +12,17 @@ source:
sha256: {{ sha256 }}
build:
noarch: python
number: 0
script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv"
script: "{{ PYTHON }} -m pip install --no-deps --ignore-installed --no-cache-dir ."
entry_points:
- black = black:main
requirements:
host:
- python >=3.6
- python
- pip
run:
- python >=3.6
- python
- click >=6.5
- attrs >=18.1.0
- appdirs
......
......@@ -17,7 +17,7 @@ build:
number: 0
entry_points:
- nodeenv = nodeenv:main
script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv"
script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir"
requirements:
host:
......
......@@ -19,7 +19,7 @@ build:
- pre-commit = pre_commit.main:main
- pre-commit-validate-config = pre_commit.clientlib:validate_config_main
- pre-commit-validate-manifest = pre_commit.clientlib:validate_manifest_main
script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv"
script: "{{ PYTHON }} -m pip install --no-deps --ignore-installed --no-cache-dir -vvv ."
requirements:
host:
......
......@@ -15,7 +15,7 @@ source:
build:
number: 0
script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv"
script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-cache-dir"
requirements:
host:
......
{% set name = "python-gitlab" %}
{% set version = "1.7.0" %}
package:
name: {{ name|lower }}
version: {{ version }}
source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
sha256: 401ef8929db4dcb5b08e0a2263a0a70599fc7e5b27615f956ac26d245802d09e
build:
number: 0
script: "{{ PYTHON }} -m pip install . -vv"
entry_points:
- gitlab = gitlab.cli:main
requirements:
host:
- python
- pip
run:
- python
- requests
- six
test:
imports:
- gitlab
about:
home: https://github.com/python-gitlab/python-gitlab
license: LGPL-3.0
license_family: LGPL
license_file: COPYING
summary: 'Python wrapper for the GitLab API'
description: |
python-gitlab is a Python package providing access to the GitLab
server API. It supports the v4 API of GitLab, and provides a CLI
tool (gitlab).
doc_url: https://python-gitlab.readthedocs.io/
dev_url: https://github.com/python-gitlab/python-gitlab
extra:
recipe-maintainers:
- anjos
......@@ -11,7 +11,7 @@ source:
build:
number: 0
script: python -m pip install --no-deps --ignore-installed .
script: "{{ PYTHON }} -m pip install --no-deps --ignore-installed --no-cache-dir ."
requirements:
host:
......@@ -24,7 +24,7 @@ requirements:
test:
imports:
- sphinxcontrib
- sphinxcontrib.httpdomain
about:
home: https://github.com/sphinx-contrib/httpdomain
......
......@@ -13,13 +13,12 @@ source:
build:
number: 0
noarch: python
script: python setup.py install --single-version-externally-managed --record record.txt
script: "{{ PYTHON }} -m pip install --no-deps --ignore-installed --no-cache-dir ."
requirements:
build:
- python
- setuptools
- pip
- sphinx
run:
......
conda/vlfeat
conda/giflib
conda/libblitz
conda/libmatio
conda/nasm
conda/openh264
conda/ffmpeg
conda/libsvm
conda/openfst
conda/sox
conda/libogg
conda/speexdsp
conda/speex
conda/kaldi
conda/ipdb
conda/mne
conda/zc.buildout
conda/zc.recipe.egg
conda/mr.developer
conda/cyvlfeat
conda/dlib
conda/pyedflib
conda/menpo
conda/menpofit
conda/schema
conda/madmom
conda/docker-pycreds
conda/websocket-client
conda/docker-py
conda/oset
conda/sphinxcontrib-programoutput
conda/sphinxcontrib-httpdomain
conda/aniso8601
conda/flask-restful
conda/bob-devel
conda/beat-devel
conda/black
conda/aspy.yaml
conda/cfgv
conda/identify
conda/zipp
conda/importlib-metadata
conda/importlib-resources
conda/nodeenv
conda/pre-commit
conda/pytest-qt
#!/usr/bin/env bash
# Thu 15 Feb 15:36:30 2018 CET
basedir=`pwd`
source ${basedir}/_ci/functions.sh
# Builds all package in our CI. Packages previously built are skipped as per
# option to conda-build.
PACKAGES=()
PACKAGES+=('conda/python-gitlab')
PACKAGES+=('conda/ipdb')
PACKAGES+=('conda/mne')
PACKAGES+=('conda/vlfeat')
PACKAGES+=('conda/zc.buildout')
PACKAGES+=('conda/zc.recipe.egg')
PACKAGES+=('conda/mr.developer')
PACKAGES+=('conda/cyvlfeat')
PACKAGES+=('conda/dlib')
PACKAGES+=('conda/giflib')
PACKAGES+=('conda/libblitz')
PACKAGES+=('conda/libmatio')
PACKAGES+=('conda/nasm')
PACKAGES+=('conda/openh264')
PACKAGES+=('conda/ffmpeg')
PACKAGES+=('conda/libsvm')
PACKAGES+=('conda/openfst')
PACKAGES+=('conda/pyedflib')
PACKAGES+=('conda/sox')
PACKAGES+=('conda/libogg')
PACKAGES+=('conda/speexdsp')
PACKAGES+=('conda/speex')
PACKAGES+=('conda/kaldi')
PACKAGES+=('conda/menpo')
PACKAGES+=('conda/menpofit')
PACKAGES+=('conda/schema')
PACKAGES+=('conda/madmom')
PACKAGES+=('conda/docker-pycreds')
PACKAGES+=('conda/websocket-client')
PACKAGES+=('conda/docker-py')
PACKAGES+=('conda/oset')
PACKAGES+=('conda/sphinxcontrib-programoutput')
PACKAGES+=('conda/sphinxcontrib-httpdomain')
PACKAGES+=('conda/aniso8601')
PACKAGES+=('conda/flask-restful')
PACKAGES+=('conda/bob-devel')
PACKAGES+=('conda/beat-devel')
PACKAGES+=('conda/black')
PACKAGES+=('conda/aspy.yaml')
PACKAGES+=('conda/cfgv')
PACKAGES+=('conda/identify')
PACKAGES+=('conda/zipp')
PACKAGES+=('conda/importlib-metadata')
PACKAGES+=('conda/importlib-resources')
PACKAGES+=('conda/nodeenv')
PACKAGES+=('conda/pre-commit')
PACKAGES+=('conda/pytest-qt')
current=0
# Makes sure we activate the base environment if available
run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
run_cmd conda activate base
export_env PATH
# Set conda channels so that the public/conda is the first
set_conda_channels "public" "tag"
for p in "${PACKAGES[@]}"; do
((current++))
echo "";
log_debug " ==========================================================";
log_debug " Building ${p} (${current}/${#PACKAGES[@]})...";
log_debug " ==========================================================";
echo "";
log_info "$ ${CONDA_ROOT}/bin/python ${basedir}/scripts/check_exists.py ${SCRIPTS_DIR}/conda_build_config.yaml ${basedir}/${p} ${DOCSERVER}/${CONDA_CHANNELS[0]}"
${CONDA_ROOT}/bin/python ${basedir}/scripts/check_exists.py ${SCRIPTS_DIR}/conda_build_config.yaml ${basedir}/${p} ${DOCSERVER}/${CONDA_CHANNELS[0]}
status=$?
if [[ "${status}" == "0" ]]; then
log_info 'Skipping package build (exists on channel URL)'
else
run_cmd ${CONDA_ROOT}/bin/conda build --skip-existing --no-anaconda-upload --variant-config-files ${SCRIPTS_DIR}/conda_build_config.yaml ${basedir}/${p}
fi
if [[ "${IS_MASTER}" == "true" ]]; then
deploy_conda_packages ${CONDA_CHANNELS[0]} ""
else
log_info 'Skipping package upload (not on master branch)'
fi
done
#!/usr/bin/env python
# Mon 26 Feb 11:11:30 2018 CET
# Checks if the combination of package-name, version and build number already
# exist on the provided channel. If so, exits with 0 - otherwise, non-zero.
import os
import re
import sys
import yaml
import subprocess
from conda.exports import get_index
def check_on_channel(channel_url, name, version, build_number):
"""Checks on the given channel if a package with the specs exist"""
# get the channel index
print('Retrieving conda channel index (%s)...' % channel_url)
index = get_index(channel_urls=[channel_url], prepend=False)
# search if package with the same version/py_ver/build_number exists
print('Checking for %s-%s_*_%s...' % (name, version, build_number))
urls = []
for dist in index:
#print('checking %s-%s_%s' % (dist.name, dist.version, dist.build_string))
if dist.name == name and \
dist.version == version and \
dist.build_string.endswith('_%s' % build_number):
print('Found matching package (%s-%s_%s)' % \
(dist.name, dist.version, dist.build_string))
return True
print('No matches found among %d packages' % len(index))
return False
def get_rendered_recipe(build_config, path):
print('Rendering conda recipe (%s)...' % path)
conda=os.path.join(os.path.dirname(sys.executable), 'conda')
cmd = [conda, 'render', '--variant-config-files', build_config, path]
print('$ %s' % ' '.join(cmd))
from conda_build.api import get_or_merge_config, render, output_yaml
config = get_or_merge_config(None, variant_config_files=build_config)
metadata = render(path, config=config)
output = output_yaml(metadata[0][0])
return yaml.load(output)
def main():
parsed = get_rendered_recipe(sys.argv[1], sys.argv[2])
if parsed is not None:
if check_on_channel(sys.argv[3], parsed['package']['name'],
parsed['package']['version'], parsed['build']['number']):
sys.exit(0)
else:
print('Recipe %s is not enabled for the current architecture' % sys.argv[2])
sys.exit(0)
sys.exit(1)
if __name__ == '__main__':
if len(sys.argv) != 4:
print('usage: %s build-config recipe-dir channel' % sys.argv[0])
sys.exit(1)
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment