Merge branch 'update-ci' into 'master'

Update CI and documentation linking

See merge request !272

beat/web/settings/test.py

@@ -53,6 +53,7 @@ LOGGING['loggers']['beat.core']['handlers'] = ['discard']
 LOGGING['loggers']['beat.web']['handlers'] = ['discard']
 LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard']
 
+BASE_DIR = os.path.dirname(os.path.abspath(__name__))
 PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix'))
 ALGORITHMS_ROOT  = os.path.join(PREFIX, 'algorithms')
 PLOTTERS_ROOT    = os.path.join(PREFIX, 'plotters')

buildout.cfg

@@ -3,7 +3,7 @@ extends = common.cfg
 develop = .
 
 [sources]
-beat.examples = git git@gitlab.idiap.ch:beat/beat.examples egg=false
+beat.examples = git https://gitlab.idiap.ch/beat/beat.examples egg=false
 
 [node]
 npms += protractor

common.cfg

 [buildout]
-parts = scripts node bower help django
+parts = scripts node bower django
 extensions = mr.developer
 auto-checkout = *
 newest = false
@@ -51,10 +51,3 @@ packages = jquery#~1.11.3
            angular-ui-codemirror
 executable = ${buildout:bin-directory}/bower --config.interactive=false
 downloads = static
-
-[help]
-base-path = .
-recipe = collective.recipe.cmd
-cmds = ${buildout:bin-directory}/sphinx-build ${help:base-path}/doc/user ${help:base-path}/beat/web/static/guide
-on_install = true
-on_update = true

dev.yml

@@ -6,8 +6,8 @@ channels:
 dependencies:
   # packages you may pin are here
   - python=3.6
-  - bob-devel=2018.12.29
-  - beat-devel=2018.12.29
+  - bob-devel=2019.02.11
+  - beat-devel=2019.02.11
 
   # requirements.txt, they are indirectly pinned through the above
   - beat.core

doc/admin/conf.py

@@ -50,12 +50,6 @@ extensions = [
     'sphinxcontrib.programoutput',
     ]
 
-import sphinx
-if sphinx.__version__ >= "1.4.1":
-    extensions.append('sphinx.ext.imgmath')
-else:
-    extensions.append('sphinx.ext.pngmath')
-
 # Always includes todos
 todo_include_todos = True
 
@@ -63,7 +57,7 @@ todo_include_todos = True
 numfig = True
 
 # If we are on OSX, the 'dvipng' path maybe different
-dvipng_osx = '/opt/local/libexec/texlive/binaries/dvipng'
+dvipng_osx = '/Library/TeX/texbin/dvipng'
 if os.path.exists(dvipng_osx): pngmath_dvipng = dvipng_osx
 
 # Add any paths that contain templates here, relative to this directory.
@@ -293,35 +287,51 @@ texinfo_documents = [
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
 
-def smaller_than(v1, v2):
-  """Compares scipy/numpy version numbers"""
-
-  c1 = v1.split('.')
-  c2 = v2.split('.')[:len(c1)] #clip to the compared version
-  for i, k in enumerate(c2):
-    n1 = c1[i]
-    n2 = c2[i]
-    try:
-      n1 = int(n1)
-      n2 = int(n2)
-    except ValueError:
-      n1 = str(n1)
-      n2 = str(n2)
-    if n1 > n2: return False
-  return True
-
-# Some name mangling to find the correct sphinx manuals for some packages
-numpy_version = __import__('numpy').version.version
-if smaller_than(numpy_version, '1.5.z'):
-    numpy_version = '.'.join(numpy_version.split('.')[:-1]) + '.x'
-else:
-    numpy_version = '.'.join(numpy_version.split('.')[:-1]) + '.0'
-numpy_manual = 'http://docs.scipy.org/doc/numpy-%s/' % numpy_version
+# Default processing flags for sphinx
+autoclass_content = 'class'
+autodoc_member_order = 'bysource'
+autodoc_default_flags = [
+  'members',
+  'undoc-members',
+  'show-inheritance',
+  ]
+
 
+if not 'BOB_DOCUMENTATION_SERVER' in os.environ:
+  # notice we need to overwrite this for BEAT projects - defaults from Bob are
+  # not OK
+  os.environ['BOB_DOCUMENTATION_SERVER'] = "https://www.idiap.ch/software/beat/docs/beat/%(name)s/%(version)s/|https://www.idiap.ch/software/beat/docs/beat/%(name)s/master/"
 
 # For inter-documentation mapping:
-intersphinx_mapping = {
-    'http://docs.python.org/%d.%d/' % sys.version_info[:2]: None,
-    numpy_manual: None,
-    'http://matplotlib.sourceforge.net/': None,
-    }
+from bob.extension.utils import link_documentation, load_requirements
+sphinx_requirements = "extra-intersphinx.txt"
+if os.path.exists(sphinx_requirements):
+  intersphinx_mapping = link_documentation(
+      additional_packages=['python','numpy'] + \
+          load_requirements(sphinx_requirements))
+else:
+  intersphinx_mapping = link_documentation()
+
+# Adds simplejson, pyzmq links
+intersphinx_mapping['http://simplejson.readthedocs.io/en/stable/'] = None
+
+# We want to remove all private (i.e. _. or __.__) members
+# that are not in the list of accepted functions
+accepted_private_functions = ['__array__']
+
+def member_function_test(app, what, name, obj, skip, options):
+  # test if we have a private function
+  if len(name) > 1 and name[0] == '_':
+    # test if this private function should be allowed
+    if name not in accepted_private_functions:
+      # omit private functions that are not in the list of accepted private
+      # functions
+      return skip
+    else:
+      # test if the method is documented
+      if not hasattr(obj, '__doc__') or not obj.__doc__:
+        return skip
+  return False
+
+def setup(app):
+  app.connect('autodoc-skip-member', member_function_test)

scripts/after_build.sh

-#!/usr/bin/env bash
-
-source _ci/functions.sh
-
-BEAT_TEST_PREFIX=`cat $TEST_PREFIX_PATH_FILE`
-export_env BEAT_TEST_PREFIX
-
-log_info "Deleting BEAT test prefix $BEAT_TEST_PREFIX"
-run_cmd rm -rf ${BEAT_TEST_PREFIX} ${TEST_PREFIX_PATH_FILE}

scripts/before_build.sh

-#!/usr/bin/env bash
-
-source _ci/functions.sh
-
-# Makes sure we activate the base environment if available
-run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
-run_cmd conda activate base
-export_env PATH
-
-# installs the base conda environment
-run_cmd conda env create -f dev.yml
-conda activate beat-web
-
-# runs buildout
-run_cmd buildout
-
-# pull docker images
-thisdir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-run_cmd ${thisdir}/docker_pull.sh master

scripts/build.sh

-#!/usr/bin/env bash
-
-source _ci/functions.sh
-
-# Makes sure we activate the base environment if available
-run_cmd source ${CONDA_ROOT}/etc/profile.d/conda.sh
-run_cmd conda activate beat-web
-export_env PATH
-
-COVERAGE_FILE=.coverage.django
-export_env COVERAGE_FILE
-
-run_cmd ./bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
-
-BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.ci
-COVERAGE_FILE=.coverage.cmdline
-NOSE_WITH_COVERAGE=1
-NOSE_COVER_PACKAGE=beat.web
-
-export_env BEAT_CMDLINE_TEST_PLATFORM
-export_env COVERAGE_FILE
-export_env NOSE_WITH_COVERAGE
-export_env NOSE_COVER_PACKAGE
-
-run_cmd ./bin/coverage run --source=beat.cmdline ./bin/nosetests -sv beat.cmdline
-
-unset COVERAGE_FILE
-
-run_cmd ./bin/coverage combine .coverage.django .coverage.cmdline
-run_cmd ./bin/coverage report
-run_cmd ./bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
-run_cmd ./bin/sphinx-build doc/api sphinx/api
-run_cmd ./bin/sphinx-build doc/admin sphinx/admin
-run_cmd ./bin/sphinx-build doc/user sphinx/user
-
-# Builds a zip file that can be uploaded to PyPI
-python setup.py sdist --formats=zip