Support for new conda-based CI/CD pipelines

.gitignore

@@ -78,3 +78,34 @@ user_conf.json
 
 # pycache
 __pycache__
+
+*~
+*.swp
+*.pyc
+*.so
+bin
+eggs
+parts
+.installed.cfg
+.mr.developer.cfg
+*.egg-info
+develop-eggs
+sphinx
+doc/api
+src
+dist
+.nfs*
+.gdb_history
+build
+*.egg
+opsnr.stt
+.coverage
+.DS_Store
+html/
+record.txt
+_ci/
+miniconda.sh
+miniconda/
+miniconda.cached/
+conda/recipe_append.yaml
+conda-bld/

.gitlab-ci.yml

+# This build file uses template features from YAML so it is generic enough for
+# any Bob project. Don't modify it unless you know what you're doing.
+
+# Definition of global variables (all stages)
+variables:
+  CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
+
+
+# Definition of our build pipeline order
+stages:
+  - build
+  - docker
+  - deploy
+  - pypi
+
+
+# Build targets
+.build_template: &build_job
+  stage: build
+  before_script:
+    - mkdir _ci
+    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+    - chmod 755 _ci/install.sh
+    - ./_ci/install.sh _ci master #installs ci support scripts
+    - ./_ci/before_build.sh
+  script:
+    - ./_ci/build.sh
+  after_script:
+    - ./_ci/after_build.sh
+  cache: &build_caches
+    paths:
+      - miniconda.sh
+      - ${CONDA_ROOT}/pkgs/*.tar.bz2
+      - ${CONDA_ROOT}/pkgs/urls.txt
+
+
+.build_linux_template: &linux_build_job
+  <<: *build_job
+  tags:
+    - docker
+  image: continuumio/conda-concourse-ci
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+  cache:
+    <<: *build_caches
+    key: "linux-cache"
+
+
+.build_macosx_template: &macosx_build_job
+  <<: *build_job
+  tags:
+    - macosx
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
+  cache:
+    <<: *build_caches
+    key: "macosx-cache"
+
+
+# Docker host based testing (must be run inside dind or docker-enabled host)
+.docker_test_linux_template: &linux_docker_job
+  stage: docker
+  before_script:
+    # safe keep artifacts as before_build.sh will erase those...
+    - mv ${CONDA_ROOT}/conda-bld .
+    - ./_ci/install.sh _ci master #updates ci support scripts
+    - ./_ci/before_build.sh
+    - mv conda-bld ${CONDA_ROOT}
+    - ./scripts/before_test.sh
+  script:
+    - export BEAT_DOCKER_TESTS=true
+    - BOB_TEST_ONLY=true ./_ci/build.sh
+  after_script:
+    - ./_ci/after_build.sh
+
+
+build_linux_27:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "2.7"
+
+
+build_linux_36:
+  <<: *linux_build_job
+  variables:
+    PYTHON_VERSION: "3.6"
+    BUILD_EGG: "true"
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - _ci/
+      - dist/*.zip
+      - sphinx
+      - ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
+
+
+build_macosx_27:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "2.7"
+
+
+build_macosx_36:
+  <<: *macosx_build_job
+  variables:
+    PYTHON_VERSION: "3.6"
+
+
+# Docker host based testing
+docker_linux_27:
+  <<: *linux_docker_job
+  variables:
+    PYTHON_VERSION: "2.7"
+  dependencies:
+    - build_linux_27
+  tags:
+    - docker-build
+
+
+docker_linux_36:
+  <<: *linux_docker_job
+  variables:
+    PYTHON_VERSION: "3.6"
+  dependencies:
+    - build_linux_36
+  tags:
+    - docker-build
+
+
+# Deploy targets
+.deploy_template: &deploy_job
+  stage: deploy
+  before_script:
+    - ./_ci/install.sh _ci master #updates ci support scripts
+  script:
+    - ./_ci/deploy.sh
+  dependencies:
+    - build_linux_27
+    - build_linux_36
+    - build_macosx_27
+    - build_macosx_36
+  tags:
+    - deployer
+
+
+deploy_beta:
+  <<: *deploy_job
+  environment: beta
+  only:
+    - master
+
+
+deploy_stable:
+  <<: *deploy_job
+  environment: stable
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+
+
+pypi:
+  stage: pypi
+  environment: pypi
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+  before_script:
+    - ./_ci/install.sh _ci master #updates ci support scripts
+  script:
+    - ./_ci/pypi.sh
+  dependencies:
+    - build_linux_36
+  tags:
+    - deployer

MANIFEST.in

+include LICENSE.AGPL README.rst version.txt requirements.txt buildout.cfg
+recursive-include scripts *.sh
+recursive-include doc conf.py *.rst *.png *.ico
+recursive-include beat/editor/templates *.jinja2
+recursive-include beat/editor/js *

README.rst

+.. vim: set fileencoding=utf-8 :
+
+.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/          ..
+.. Contact: beat.support@idiap.ch                                             ..
+..                                                                            ..
+.. This file is part of the beat.editor module of the BEAT platform.            ..
+..                                                                            ..
+.. Commercial License Usage                                                   ..
+.. Licensees holding valid commercial BEAT licenses may use this file in      ..
+.. accordance with the terms contained in a written agreement between you     ..
+.. and Idiap. For further information contact tto@idiap.ch                    ..
+..                                                                            ..
+.. Alternatively, this file may be used under the terms of the GNU Affero     ..
+.. Public License version 3 as published by the Free Software and appearing   ..
+.. in the file LICENSE.AGPL included in the packaging of this file.           ..
+.. The BEAT platform is distributed in the hope that it will be useful, but   ..
+.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
+.. or FITNESS FOR A PARTICULAR PURPOSE.                                       ..
+..                                                                            ..
+.. You should have received a copy of the GNU Affero Public License along     ..
+.. with the BEAT platform. If not, see http://www.gnu.org/licenses/.          ..
+
+.. image:: https://img.shields.io/badge/docs-stable-yellow.svg
+   :target: https://www.idiap.ch/software/beat/docs/beat/beat.editor/stable/index.html
+.. image:: https://img.shields.io/badge/docs-latest-orange.svg
+   :target: https://www.idiap.ch/software/beat/docs/beat/beat.editor/master/index.html
+.. image:: https://gitlab.idiap.ch/beat/beat.editor/badges/master/build.svg
+   :target: https://gitlab.idiap.ch/beat/beat.editor/commits/master
+.. image:: https://gitlab.idiap.ch/beat/beat.editor/badges/master/coverage.svg
+   :target: https://gitlab.idiap.ch/beat/beat.editor/commits/master
+.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
+   :target: https://gitlab.idiap.ch/beat/beat.editor
+.. image:: https://img.shields.io/pypi/v/beat.editor.svg
+   :target: https://pypi.python.org/pypi/beat.editor
+
+
+===============================
+ Local editor for BEAT objects
+===============================
+
+This package part of BEAT_, an open-source evaluation platform for data science
+algorithms and workflows. It contains the source code for a local editor for
+BEAT objects.
+
+
+Installation
+------------
+
+Complete BEAT's `installation`_ instructions. Then, to install this package,
+run::
+
+  $ conda install beat.editor
+
+
+Contact
+-------
+
+For questions or reporting issues to this software package, contact our
+development `mailing list`_.
+
+
+.. Place your references here:
+.. _beat: https://www.idiap.ch/software/beat
+.. _installation: https://www.idiap.ch/software/beat/install
+.. _mailing list: https://www.idiap.ch/software/beat/discuss

beat/__init__.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.editor module of the BEAT platform.           #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+# see https://docs.python.org/3/library/pkgutil.html
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)

server.py → beat/editor/scripts/server.py

@@ -16,11 +16,8 @@ import simplejson
 from flask import Flask, request
 from flask_restful import Resource, Api
 from flask_cors import CORS
-import templates
 
-app = Flask(__name__)
-api = Api(app)
-CORS(app)
+from .. import templates
 
 def get_user_conf():
     """Reads & returns the user configuration in a dict"""
@@ -444,14 +441,19 @@ def gen_beat_endpoint(entity):
     return BeatEndpoint
 
 
-api.add_resource(Home, '/')
-api.add_resource(Settings, '/settings')
-api.add_resource(Layout, '/layout')
-api.add_resource(Templates, '/templates')
-api.add_resource(Environments, '/environments')
-for entity in BeatEntity:
-    val = entity.value
-    api.add_resource(gen_beat_endpoint(entity), '/' + val)
+def main():
 
-if __name__ == '__main__':
-    app.run(debug=True)
+  app = Flask(__name__)
+  api = Api(app)
+  CORS(app)
+
+  api.add_resource(Home, '/')
+  api.add_resource(Settings, '/settings')
+  api.add_resource(Layout, '/layout')
+  api.add_resource(Templates, '/templates')
+  api.add_resource(Environments, '/environments')
+  for entity in BeatEntity:
+      val = entity.value
+      api.add_resource(gen_beat_endpoint(entity), '/' + val)
+
+  app.run(debug=True)

buildout.cfg

+[buildout]
+parts = scripts
+eggs = beat.editor
+develop = .
+newest = false
+
+[scripts]
+recipe = bob.buildout:scripts

conda/meta.yaml

+{% set name = 'beat.editor' %}
+{% set project_dir = environ.get('RECIPE_DIR') + '/..' %}
+
+package:
+  name: {{ name }}
+  version: {{ environ.get('BOB_PACKAGE_VERSION', '0.0.1') }}
+
+build:
+  entry_points:
+    - beatedit = beat.editor.scripts.server:main
+  number: {{ environ.get('BOB_BUILD_NUMBER', 0) }}
+  run_exports:
+    - {{ pin_subpackage(name) }}
+  script:
+    - cd {{ project_dir }}
+    {% if environ.get('BUILD_EGG') %}
+    - python setup.py sdist --formats=zip
+    {% endif %}
+    - python setup.py install --single-version-externally-managed --record record.txt
+
+requirements:
+  host:
+    - python {{ python }}
+    - setuptools {{ setuptools }}
+    - nodejs {{ nodejs }}
+  run:
+    - python
+    - setuptools
+    - simplejson
+    - jinja2
+    - flask
+    - flask-cors
+    - flask-restful
+    - enum34  # [py<34]
+
+test:
+  source_files:
+    - js
+
+  requires:
+    - bob-devel {{ bob_devel }}.*
+    - beat-devel {{ beat_devel }}.*
+    - bob.extension
+    - nose
+    - coverage
+    - sphinx
+    - sphinx_rtd_theme
+    - nodejs
+
+  imports:
+    - {{ name }}
+
+  commands:
+    - beatedit --help
+    - nosetests --with-coverage --cover-package={{ name }} -sv {{ name }}
+    - sphinx-build -aEW {{ project_dir }}/doc {{ project_dir }}/sphinx
+    - sphinx-build -aEb doctest {{ project_dir }}/doc sphinx
+    - conda inspect linkages -p $PREFIX {{ name }}  # [not win]
+    - conda inspect objects -p $PREFIX {{ name }}  # [osx]
+
+about:
+  home: https://www.idiap.ch/software/beat/
+  license: AGPLv3
+  summary: Local editor for BEAT objects
+  license_family: AGPL

doc/api.rst

+
+=====
+ API
+=====
+
+This section includes information for using the Python API of
+``beat.editor``.
+
+.. automodule:: beat.editor
+

doc/conf.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+
+import os
+import sys
+import glob
+import pkg_resources
+
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+needs_sphinx = '1.3'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.doctest',
+    'sphinx.ext.graphviz',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.mathjax',
+    #'matplotlib.sphinxext.plot_directive'
+    ]
+
+# Be picky about warnings
+nitpicky = True
+
+# Ignores stuff we can't easily resolve on other project's sphinx manuals
+nitpick_ignore = []
+
+# Allows the user to override warnings from a separate file
+if os.path.exists('nitpick-exceptions.txt'):
+    for line in open('nitpick-exceptions.txt'):
+        if line.strip() == "" or line.startswith("#"):
+            continue
+        dtype, target = line.split(None, 1)
+        target = target.strip()
+        try: # python 2.x
+            target = unicode(target)
+        except NameError:
+            pass
+        nitpick_ignore.append((dtype, target))
+
+# Always includes todos
+todo_include_todos = True
+
+# Generates auto-summary automatically
+autosummary_generate = True
+
+# Create numbers on figures with captions
+numfig = True
+
+# If we are on OSX, the 'dvipng' path maybe different
+dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
+if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'beat.backend.python'
+import time
+copyright = u'%s, Idiap Research Institute' % time.strftime('%Y')
+
+# Grab the setup entry
+distribution = pkg_resources.require(project)[0]
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = distribution.version
+# The full version, including alpha/beta/rc tags.
+release = distribution.version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['links.rst']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# Some variables which are useful for generated material
+project_variable = project.replace('.', '_')
+short_description = u'Biometrics Evaluation and Testing Platform (Python backend)'
+owner = [u'Idiap Research Institute']
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = project_variable
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+html_logo = 'img/logo.png'
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+html_favicon = 'img/favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = project_variable + u'_doc'
+
+
+# -- Post configuration --------------------------------------------------------
+
+# Included after all input documents
+rst_epilog = """
+.. |project| replace:: BEAT
+.. |version| replace:: %s
+.. |current-year| date:: %%Y
+""" % (version,)
+
+# Default processing flags for sphinx
+autoclass_content = 'class'
+autodoc_member_order = 'bysource'
+autodoc_default_flags = [
+  'members',
+  'undoc-members',
+  'show-inheritance',
+  ]
+
+# For inter-documentation mapping:
+from bob.extension.utils import link_documentation, load_requirements
+sphinx_requirements = "extra-intersphinx.txt"
+if os.path.exists(sphinx_requirements):
+  intersphinx_mapping = link_documentation(
+      additional_packages=['python','numpy'] + \
+          load_requirements(sphinx_requirements)
+          )
+else:
+  intersphinx_mapping = link_documentation()
+
+# Adds simplejson, pyzmq links
+intersphinx_mapping['http://simplejson.readthedocs.io/en/stable/'] = None
+intersphinx_mapping['http://pyzmq.readthedocs.io/en/stable/'] = None
+
+# We want to remove all private (i.e. _. or __.__) members
+# that are not in the list of accepted functions
+accepted_private_functions = ['__array__']
+
+def member_function_test(app, what, name, obj, skip, options):
+  # test if we have a private function
+  if len(name) > 1 and name[0] == '_':
+    # test if this private function should be allowed
+    if name not in accepted_private_functions:
+      # omit privat functions that are not in the list of accepted private functions
+      return skip
+    else:
+      # test if the method is documented
+      if not hasattr(obj, '__doc__') or not obj.__doc__:
+        return skip
+  return False
+
+def setup(app):
+  app.connect('autodoc-skip-member', member_function_test)