Initial conda-ci port

.gitlab-ci.yml

+# This build file uses template features from YAML so it is generic enough for
+# any Bob project. Don't modify it unless you know what you're doing.
+
+# Definition of global variables (all stages)
+variables:
+  CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+
+
+# Definition of our build pipeline order
 stages:
   - build
+  - docker
+  - deploy
+  - pypi
 
-variables:
-  PREFIX: /opt/beat.env.web/usr
 
-build:
+# Build targets
+.build_template: &build_job
   stage: build
   before_script:
-  - ${PREFIX}/bin/python --version
-  - docker info
+    - mkdir _ci
+    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+    - chmod 755 _ci/install.sh
+    - ./_ci/install.sh _ci master #installs ci support scripts
+    - ./_ci/before_build.sh
+  script:
+    - ./_ci/build.sh
+  after_script:
+    - ./_ci/after_build.sh
+  cache: &build_caches
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
+.build_linux_template: &linux_build_job
+  <<: *build_job
+  tags:
+    - docker
+  image: continuumio/conda-concourse-ci
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+  cache:
+    <<: *build_caches
+    key: "linux-cache"
+
+
+.build_macosx_template: &macosx_build_job
+  <<: *build_job
+  tags:
+    - macosx
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
+  cache:
+    <<: *build_caches
+    key: "macosx-cache"
+
+
+# Docker host based testing (must be run inside dind or docker-enabled host)
+.docker_test_linux_template: &linux_docker_job
+  stage: docker
+  before_script:
+    # safe keep artifacts as before_build.sh will erase those...
+    - mv ${CONDA_ROOT}/conda-bld .
+    - ./_ci/install.sh _ci master #updates ci support scripts
+    - ./_ci/before_build.sh
+    - mv conda-bld ${CONDA_ROOT}
+    - ./scripts/before_test.sh
+  script:
+    - export BEAT_DOCKER_TESTS=true
+    - BOB_TEST_ONLY=true ./_ci/build.sh
+  after_script:
+    - ./_ci/after_build.sh
+
+
+build_linux_27:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "2.7"
+
+
+build_linux_36:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "3.6"
+    BUILD_EGG: "true"
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - dist/*.zip
+      - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+
+
+build_macosx_27:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "2.7"
+
+
+build_macosx_36:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "3.6"
+
+
+# Docker host based testing
+docker_linux_27:
+  <<: *linux_docker_job
+  variables:
+    PYTHON_VERSION: "2.7"
+  dependencies:
+    - build_linux_27
+  tags:
+    - docker-build
+
+
+docker_linux_36:
+  <<: *linux_docker_job
+  variables:
+    PYTHON_VERSION: "3.6"
+  dependencies:
+    - build_linux_36
+  tags:
+    - docker-build
+
+
+# Deploy targets
+.deploy_template: &deploy_job
+  stage: deploy
+  before_script:
+    - ./_ci/install.sh _ci master #updates ci support scripts
+  script:
+    - ./_ci/deploy.sh
+  dependencies:
+    - build_linux_27
+    - build_linux_36
+    - build_macosx_27
+    - build_macosx_36
+  tags:
+    - deployer
+
+
+deploy_beta:
+  <<: *deploy_job
+  environment: beta
+  only:
+    - 1.4.x
+
+
+deploy_stable:
+  <<: *deploy_job
+  environment: stable
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+
+pypi:
+  stage: pypi
+  environment: pypi
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+  before_script:
+    - ./_ci/install.sh _ci master #updates ci support scripts
   script:
-  - git clean -ffdx
-  - ${PREFIX}/bin/python bootstrap-buildout.py
-  - ./bin/buildout
-  - ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ${PREFIX}/bin/nosetests -sv ${CI_PROJECT_NAME}
-  - ./bin/python ${PREFIX}/bin/coverage report
-  - ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
-  - ./bin/python ${PREFIX}/bin/sphinx-build doc html
+    - ./_ci/pypi.sh
+  dependencies:
+    - build_linux_36
   tags:
-  - docker-build
+    - deployer

MANIFEST.in

-include LICENSE.AGPL README.rst buildout.cfg bootstrap-buildout.py
-recursive-include doc conf.py *.rst *.png *.svg *.ico *.pdf
+include LICENSE.AGPL README.rst version.txt requirements.txt
+include buildout.cfg develop.cfg
+recursive-include scripts *.sh
+recursive-include doc conf.py *.rst *.png *.ico *.pdf
+recursive-include beat *.checksum *.index *.json

bootstrap-buildout.py

-##############################################################################
-#
-# Copyright (c) 2006 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Bootstrap a buildout-based project
-
-Simply run this script in a directory containing a buildout.cfg.
-The script accepts buildout command-line options, so you can
-use the -c option to specify an alternate configuration file.
-"""
-
-import os
-import shutil
-import sys
-import tempfile
-
-from optparse import OptionParser
-
-tmpeggs = tempfile.mkdtemp()
-
-usage = '''\
-[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
-
-Bootstraps a buildout-based project.
-
-Simply run this script in a directory containing a buildout.cfg, using the
-Python that you want bin/buildout to use.
-
-Note that by using --find-links to point to local resources, you can keep
-this script from going over the network.
-'''
-
-parser = OptionParser(usage=usage)
-parser.add_option("-v", "--version", help="use a specific zc.buildout version")
-
-parser.add_option("-t", "--accept-buildout-test-releases",
-                  dest='accept_buildout_test_releases',
-                  action="store_true", default=False,
-                  help=("Normally, if you do not specify a --version, the "
-                        "bootstrap script and buildout gets the newest "
-                        "*final* versions of zc.buildout and its recipes and "
-                        "extensions for you.  If you use this flag, "
-                        "bootstrap and buildout will get the newest releases "
-                        "even if they are alphas or betas."))
-parser.add_option("-c", "--config-file",
-                  help=("Specify the path to the buildout configuration "
-                        "file to be used."))
-parser.add_option("-f", "--find-links",
-                  help=("Specify a URL to search for buildout releases"))
-parser.add_option("--allow-site-packages",
-                  action="store_true", default=False,
-                  help=("Let bootstrap.py use existing site packages"))
-parser.add_option("--setuptools-version",
-                  help="use a specific setuptools version")
-
-
-options, args = parser.parse_args()
-
-######################################################################
-# load/install setuptools
-
-try:
-    if options.allow_site_packages:
-        import setuptools
-        import pkg_resources
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
-
-ez = {}
-exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
-
-if not options.allow_site_packages:
-    # ez_setup imports site, which adds site packages
-    # this will remove them from the path to ensure that incompatible versions
-    # of setuptools are not in the path
-    import site
-    # inside a virtualenv, there is no 'getsitepackages'.
-    # We can't remove these reliably
-    if hasattr(site, 'getsitepackages'):
-        for sitepackage_path in site.getsitepackages():
-            sys.path[:] = [x for x in sys.path if sitepackage_path not in x]
-
-setup_args = dict(to_dir=tmpeggs, download_delay=0)
-
-if options.setuptools_version is not None:
-    setup_args['version'] = options.setuptools_version
-
-ez['use_setuptools'](**setup_args)
-import setuptools
-import pkg_resources
-
-# This does not (always?) update the default working set.  We will
-# do it.
-for path in sys.path:
-    if path not in pkg_resources.working_set.entries:
-        pkg_resources.working_set.add_entry(path)
-
-######################################################################
-# Install buildout
-
-ws = pkg_resources.working_set
-
-cmd = [sys.executable, '-c',
-       'from setuptools.command.easy_install import main; main()',
-       '-mZqNxd', tmpeggs]
-
-find_links = os.environ.get(
-    'bootstrap-testing-find-links',
-    options.find_links or
-    ('http://downloads.buildout.org/'
-     if options.accept_buildout_test_releases else None)
-)
-if find_links:
-    cmd.extend(['-f', find_links])
-
-setuptools_path = ws.find(
-    pkg_resources.Requirement.parse('setuptools')).location
-
-requirement = 'zc.buildout'
-version = options.version
-if version is None and not options.accept_buildout_test_releases:
-    # Figure out the most recent final version of zc.buildout.
-    import setuptools.package_index
-    _final_parts = '*final-', '*final'
-
-    def _final_version(parsed_version):
-        try:
-            return not parsed_version.is_prerelease
-        except AttributeError:
-            # Older setuptools
-            for part in parsed_version:
-                if (part[:1] == '*') and (part not in _final_parts):
-                    return False
-            return True
-
-    index = setuptools.package_index.PackageIndex(
-        search_path=[setuptools_path])
-    if find_links:
-        index.add_find_links((find_links,))
-    req = pkg_resources.Requirement.parse(requirement)
-    if index.obtain(req) is not None:
-        best = []
-        bestv = None
-        for dist in index[req.project_name]:
-            distv = dist.parsed_version
-            if _final_version(distv):
-                if bestv is None or distv > bestv:
-                    best = [dist]
-                    bestv = distv
-                elif distv == bestv:
-                    best.append(dist)
-        if best:
-            best.sort()
-            version = best[-1].version
-if version:
-    requirement = '=='.join((requirement, version))
-cmd.append(requirement)
-
-import subprocess
-if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
-    raise Exception(
-        "Failed to execute command:\n%s" % repr(cmd)[1:-1])
-
-######################################################################
-# Import and run buildout
-
-ws.add_entry(tmpeggs)
-ws.require(requirement)
-import zc.buildout.buildout
-
-if not [a for a in args if '=' not in a]:
-    args.append('bootstrap')
-
-# if -c was provided, we push it back into args for buildout' main function
-if options.config_file is not None:
-    args[0:0] = ['-c', options.config_file]
-
-zc.buildout.buildout.main(args)
-shutil.rmtree(tmpeggs)

buildout.cfg

-; vim: set fileencoding=utf-8 :
-; Andre Anjos <andre.anjos@idiap.ch>
-; Tue 02 Jun 2015 13:05:59 CEST
-
 [buildout]
 parts = scripts
-extensions = mr.developer
-auto-checkout = *
+eggs = beat.cmdline
 develop = .
 newest = false
-eggs = beat.cmdline
-       beat.backend.python
-       beat.core
-       ipdb
-
-[sources]
-beat.core = git https://gitlab.idiap.ch/beat/beat.core branch=1.6.x
-beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python branch=1.5.x
 
 [scripts]
 recipe = bob.buildout:scripts

conda/meta.yaml

+{% set name = 'beat.cmdline' %}
+{% set project_dir = environ.get('RECIPE_DIR') + '/..' %}
+
+package:
+  name: {{ name }}
+  version: {{ environ.get('BOB_PACKAGE_VERSION', '0.0.1') }}
+
+build:
+  entry_points:
+    - beat = beat.cmdline.scripts.beat:main
+  number: {{ environ.get('BOB_BUILD_NUMBER', 0) }}
+  run_exports:
+    - {{ pin_subpackage(name) }}
+  script:
+    - cd {{ project_dir }}
+    {% if environ.get('BUILD_EGG') %}
+    - python setup.py sdist --formats=zip
+    {% endif %}
+    - python setup.py install --single-version-externally-managed --record record.txt
+
+requirements:
+  host:
+    - python {{ python }}
+    - setuptools {{ setuptools }}
+  run:
+    - python
+    - setuptools
+    - beat.backend.python
+    - beat.core
+
+test:
+  requires:
+    - bob-devel {{ bob_devel }}.*
+    - beat-devel {{ beat_devel }}.*
+    - bob.extension
+    - nose
+    - coverage
+    - sphinx
+    - sphinx_rtd_theme
+    - sphinxcontrib-programoutput
+
+  imports:
+    - {{ name }}
+
+  commands:
+    - worker --help
+    {% if environ.get('BEAT_DOCKER_TESTS', False)  %}
+    - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
+    {% else %}
+    - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }} --exclude=".*test_docker.*"
+    {% endif %}
+    - sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
+    - sphinx-build -aEb doctest {{ project_dir }}/doc sphinx
+    - conda inspect linkages -p $PREFIX {{ name }}  # [not win]
+    - conda inspect objects -p $PREFIX {{ name }}  # [osx]
+
+about:
+  home: https://www.idiap.ch/software/beat/
+  license: AGPLv3
+  summary: Command-line client for the BEAT platform
+  license_family: AGPL

develop.cfg

+; vim: set fileencoding=utf-8 :
+; Andre Anjos <andre.anjos@idiap.ch>
+; Tue 02 Jun 2015 13:05:59 CEST
+
+[buildout]
+parts = scripts
+extensions = mr.developer
+auto-checkout = *
+develop = .
+newest = false
+eggs = beat.cmdline
+       beat.backend.python
+       beat.core
+       ipdb
+
+[sources]
+beat.core = git https://gitlab.idiap.ch/beat/beat.core branch=1.6.x
+beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python branch=1.5.x
+
+[scripts]
+recipe = bob.buildout:scripts

doc/conf.py

@@ -6,7 +6,7 @@
 # Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
 # Contact: beat.support@idiap.ch                                              #
 #                                                                             #
-# This file is part of the beat.web module of the BEAT platform.              #
+# This file is part of the beat.core module of the BEAT platform.             #
 #                                                                             #
 # Commercial License Usage                                                    #
 # Licensees holding valid commercial BEAT licenses may use this file in       #
@@ -25,16 +25,11 @@
 #                                                                             #
 ###############################################################################
 
-
 import os
 import sys
 import glob
 import pkg_resources
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
@@ -46,30 +41,50 @@ needs_sphinx = '1.3'
 extensions = [
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
-    'sphinx.ext.imgmath',
     'sphinx.ext.ifconfig',
     'sphinx.ext.autodoc',
     'sphinx.ext.autosummary',
     'sphinx.ext.doctest',
-    'sphinx.ext.intersphinx',
     'sphinx.ext.graphviz',
+    'sphinx.ext.intersphinx',
     'sphinx.ext.napoleon',
     'sphinx.ext.viewcode',
+    'sphinx.ext.mathjax',
     'sphinxcontrib.programoutput',
-]
+    #'matplotlib.sphinxext.plot_directive',
+    ]
+
+# Be picky about warnings
+nitpicky = True
+
+# Ignores stuff we can't easily resolve on other project's sphinx manuals
+nitpick_ignore = []
+
+# Allows the user to override warnings from a separate file
+if os.path.exists('nitpick-exceptions.txt'):
+    for line in open('nitpick-exceptions.txt'):
+        if line.strip() == "" or line.startswith("#"):
+            continue
+        dtype, target = line.split(None, 1)
+        target = target.strip()
+        try: # python 2.x
+            target = unicode(target)
+        except NameError:
+            pass
+        nitpick_ignore.append((dtype, target))
 
 # Always includes todos
 todo_include_todos = True
 
-# Create numbers on figures with captions
-numfig = True
-
 # Generates auto-summary automatically
 autosummary_generate = True
 
+# Create numbers on figures with captions
+numfig = True
+
 # If we are on OSX, the 'dvipng' path maybe different
 dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
-if os.path.exists(dvipng_osx): imgmath_dvipng = dvipng_osx
+if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -112,10 +127,7 @@ release = distribution.version
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = [
-    'api/modules.rst',
-  'api/beat.rst',
-]
+exclude_patterns = ['links.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None
@@ -139,7 +151,7 @@ pygments_style = 'sphinx'
 
 # Some variables which are useful for generated material
 project_variable = project.replace('.', '_')
-short_description = u'Biometrics Evaluation and Testing Platform (Core Modules)'
+short_description = u'Command-line client for the BEAT platform'
 owner = [u'Idiap Research Institute']
 
 
@@ -167,12 +179,12 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = 'img/beat.svg'
+html_logo = 'img/logo.png'
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = 'img/beat.ico'
+html_favicon = 'img/favicon.ico'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -224,83 +236,63 @@ html_favicon = 'img/beat.ico'
 htmlhelp_basename = project_variable + u'_doc'
 
 
-# -- Options for LaTeX output --------------------------------------------------
-
-# The paper size ('letter' or 'a4').
-latex_paper_size = 'a4'
-
-# The font size ('10pt', '11pt' or '12pt').
-latex_font_size = '10pt'
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
-latex_documents = [(
-    'index',
-    project_variable + '.tex',
-    short_description,
-    owner[0],
-    'manual',
-)]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-latex_logo = ''
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
+# -- Post configuration --------------------------------------------------------
 
 # Included after all input documents
 rst_epilog = """
 .. |project| replace:: BEAT
-.. |url| replace:: https://www.beat-eu.org/platform/
 .. |version| replace:: %s
 .. |current-year| date:: %%Y
 """ % (version,)
 
-# -- Options for manual page output --------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [(
-    'index',
-    project_variable,
-    short_description,
-    owner,
-    1,
-)]
-
 # Default processing flags for sphinx
 autoclass_content = 'class'
 autodoc_member_order = 'bysource'
 autodoc_default_flags = [
-    'members',
+  'members',
   'undoc-members',
   'show-inheritance',
-]
-
-python_version = '.'.join(str(k) for k in sys.version_info[0:2])
-python_link = 'http://docs.python.org/' + python_version + '/'
-numpy_link = 'http://docs.scipy.org/doc/numpy/'
-intersphinx_mapping = dict(
-    python=(python_link, None),
-    numpy=(numpy_link, None),
-)
+  ]
+
+# For inter-documentation mapping:
+doc_server = [
+    'http://www.idiap.ch/software/beat/docs/beat/%(name)s/%(version)s/',
+    'http://www.idiap.ch/software/beat/docs/beat/%(name)s/master/',
+    'http://www.idiap.ch/software/beat/docs/beat/%(name)s/1.6.x/',
+    'http://www.idiap.ch/software/beat/docs/beat/%(name)s/1.5.x/',
+    'http://www.idiap.ch/software/bob/docs/beat/%(name)s/%(version)s/',
+    'http://www.idiap.ch/software/bob/docs/beat/%(name)s/master/',
+    'http://www.idiap.ch/software/bob/docs/beat/%(name)s/1.6.x/',
+    'http://www.idiap.ch/software/bob/docs/beat/%(name)s/1.5.x/',
+    ]
+doc_server = '|'.join(doc_server)
+from bob.extension.utils import link_documentation, load_requirements
+sphinx_requirements = "extra-intersphinx.txt"
+if os.path.exists(sphinx_requirements):
+  intersphinx_mapping = link_documentation(
+      additional_packages=['python','numpy'] + \
+          load_requirements(sphinx_requirements),
+      server=doc_server,
+      )
+else:
+  intersphinx_mapping = link_documentation(server=doc_server)
+
+# We want to remove all private (i.e. _. or __.__) members
+# that are not in the list of accepted functions
+accepted_private_functions = ['__array__']
+
+def member_function_test(app, what, name, obj, skip, options):
+  # test if we have a private function
+  if len(name) > 1 and name[0] == '_':
+    # test if this private function should be allowed
+    if name not in accepted_private_functions:
+      # omit privat functions that are not in the list of accepted private functions
+      return skip
+    else:
+      # test if the method is documented
+      if not hasattr(obj, '__doc__') or not obj.__doc__: