Commit 72a0e44b authored by Amir Mohammadi's avatar Amir Mohammadi
Browse files

[Automatic] update links and the ci mostly

parent 7c8164d1
Pipeline #4523 failed with stage
in 4 minutes and 28 seconds
# This build file is defined in two parts: 1) a generic set of instructions you
# probably **don't** need to change and 2) a part you may have to tune to your
# project. It heavily uses template features from YAML to help you in only
# changing a minimal part of it and avoid code duplication to a maximum while
# still providing a nice pipeline display on your package.
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# 1) Generic instructions (only change if you know what you're doing)
# -------------------------------------------------------------------
# Definition of our build pipeline
stages:
- build
- test
- docs
- wheels
- deploy
# Global variables
variables:
CONDA_PREFIX: env
# ---------
# Templates
# ---------
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
......@@ -27,104 +22,98 @@ variables:
stage: build
before_script:
- git clean -ffdx
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/7/raw | tr -d '\r' > bootstrap-conda.sh
- chmod 755 ./bootstrap-conda.sh
- ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
variables: &build_variables
BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/"
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh
script:
- ./bin/buildout
- if [ -x ./bin/bob_dbmanage.py ]; then ./bin/bob_dbmanage.py all download --force; fi
- ./bin/sphinx-build doc sphinx
- ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG}
- ./_ci/build.sh
after_script:
- rm -rf ${CONDA_PREFIX}
- ./_ci/after_build.sh
artifacts:
expire_in: 1 day
expire_in: 1 week
paths:
- bootstrap-conda.sh
- _ci/
- dist/
- sphinx/
# Template for building on a Linux machine
.build_linux_template: &linux_build_job
<<: *build_job
variables: &linux_build_variables
<<: *build_variables
CONDA_FOLDER: "/local/conda"
CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
# Template for building on a Mac OSX machine
.build_mac_template: &macosx_build_job
<<: *build_job
variables: &macosx_build_variables
<<: *build_variables
CONDA_FOLDER: "/opt/conda"
MACOSX_DEPLOYMENT_TARGET: "10.9"
CFLAGS: "-pthread -coverage"
CXXFLAGS: "-pthread -coverage"
LDFLAGS: "-lpthread"
# Template for the test stage - re-install from uploaded wheels
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
- source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX}
- pip install --use-wheel --no-index --pre dist/*.whl
- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh
script:
- cd ${CONDA_PREFIX}
- python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())"
- coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME}
- coverage report
- sphinx-build -b doctest ../doc ../sphinx
- ./_ci/test.sh
after_script:
- rm -rf ${CONDA_PREFIX}
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against one combination of python 2.x and 3.x if it is a python
# only package, otherwise, needs to run in both pythons to all supported
# architectures (Linux and Mac OSX 64-bit)
# Needs to run against one supported architecture, platform and python version
.wheels_template: &wheels_job
stage: wheels
environment: intranet
only:
- master
- tags
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/8/raw | tr -d '\r' > upload-wheel.sh
- chmod 755 upload-wheel.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./upload-wheel.sh
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
environment: intranet
only:
- master
before_script:
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/9/raw | tr -d '\r' > upload-sphinx.sh
- chmod 755 upload-sphinx.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_docs.sh
script:
- ./_ci/docs.sh
after_script:
- ./_ci/after_docs.sh
# Template for the deployment stage - re-installs from uploaded wheels
# Needs to run on a single architecture only
# Will deploy your package to PyPI and other required services
# Only runs for tags
.deploy_template: &deploy_job
stage: deploy
environment: internet
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_deploy.sh
script:
- ./upload-sphinx.sh
- ./_ci/deploy.sh
after_script:
- ./_ci/after_deploy.sh
# 2) Package specific instructions (you may tune this if needed)
# --------------------------------------------------------------
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_linux_27:
<<: *linux_build_job
<<: *build_job
variables: &linux_27_build_variables
<<: *linux_build_variables
PYTHON_VER: "2.7"
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
......@@ -139,6 +128,15 @@ test_linux_27:
wheels_linux_27:
<<: *wheels_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
deploy_linux_27:
<<: *deploy_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
......@@ -147,10 +145,9 @@ wheels_linux_27:
# Linux + Python 3.4: Builds and tests
build_linux_34:
<<: *linux_build_job
<<: *build_job
variables: &linux_34_build_variables
<<: *linux_build_variables
PYTHON_VER: "3.4"
PYTHON_VERSION: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-linux
......@@ -164,12 +161,11 @@ test_linux_34:
- conda-linux
# Linux + Python 3.5: Builds, tests, uploads wheel
# Linux + Python 3.5: Builds, tests and uploads wheel
build_linux_35:
<<: *linux_build_job
<<: *build_job
variables: &linux_35_build_variables
<<: *linux_build_variables
PYTHON_VER: "3.5"
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
......@@ -184,6 +180,7 @@ test_linux_35:
wheels_linux_35:
<<: *wheels_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
......@@ -191,6 +188,7 @@ wheels_linux_35:
docs_linux_35:
<<: *docs_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
......@@ -199,10 +197,9 @@ docs_linux_35:
# Mac OSX + Python 2.7: Builds and tests
build_macosx_27:
<<: *macosx_build_job
<<: *build_job
variables: &macosx_27_build_variables
<<: *macosx_build_variables
PYTHON_VER: "2.7"
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
......@@ -218,10 +215,9 @@ test_macosx_27:
# Mac OSX + Python 3.4: Builds and tests
build_macosx_34:
<<: *macosx_build_job
<<: *build_job
variables: &macosx_34_build_variables
<<: *macosx_build_variables
PYTHON_VER: "3.4"
PYTHON_VERSION: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-macosx
......@@ -237,10 +233,9 @@ test_macosx_34:
# Mac OSX + Python 3.5: Builds and tests
build_macosx_35:
<<: *macosx_build_job
<<: *build_job
variables: &macosx_35_build_variables
<<: *macosx_build_variables
PYTHON_VER: "3.5"
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
......@@ -251,4 +246,4 @@ test_macosx_35:
dependencies:
- build_macosx_35
tags:
- conda-macosx
\ No newline at end of file
- conda-macosx
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Pavel Korshunov <pavel.korshunov@idiap.ch>
# Mon 12 Oct 14:43:22 CEST 2015
"""
Replay attack database implementation of bob.db.verification.utils.Database interface.
It is an extension of an SQL-based database interface, which directly talks to Replay database, for
verification experiments (good to use in bob.bio.base framework).
"""
from .query import Database as ReplayDatabase
from .models import File as ReplayFile, Client, Protocol
from bob.db.verification.utils import File as BaseFile
from bob.db.verification.utils import Database as BaseDatabase
class File(BaseFile, ReplayFile):
def __init__(self, f):
"""
Initializes this File object with an File equivalent from the underlying SQl-based interface for
Replay database.
"""
BaseFile.__init__(self, client_id=f.client_id, path=f.path, file_id=f.id)
self.__f = f
# def load(self, directory=None, extension='.hdf5'):
# return self.__f.load(directory=directory, extension=extension)
# load.__doc__ = ReplayFile.load.__doc__
# def get_client_id(self):
# """
# Get ID of the client that this file belongs to.
# """
# return self.__f.client_id
class Database(BaseDatabase):
"""
Implements verification API for querying Replay database.
"""
__doc__ = __doc__
def __init__(self, original_directory=None, original_extension=None):
# call base class constructors to open a session to the database
BaseDatabase.__init__(self, original_directory=original_directory, original_extension=original_extension)
self.__db = ReplayDatabase()
self.group_mapping = dict(zip(('train', 'devel', 'test'), ('world', 'dev', 'eval')))
self.reverse_group_mapping = dict(zip(('world', 'dev', 'eval'), ('train', 'devel', 'test')))
def convert_group_names_bio(self, group_names):
"""
In the SQL Database group names are ('train', 'devel', 'test')
But bob.bio.base expects the names to be: ('world', 'dev', 'eval')
"""
if group_names is None:
return None
if isinstance(group_names, str):
return self.group_mapping.get(group_names)
return [self.group_mapping[g] for g in group_names]
def convert_group_names_sql(self, group_names):
if group_names is None:
return None
if isinstance(group_names, str):
return self.reverse_group_mapping.get(group_names)
return [self.reverse_group_mapping[g] for g in group_names]
def protocols(self):
"""Returns all registered protocols"""
return self.__db.protocols() + [Protocol('licit')]
def protocol_names(self):
"""Returns all registered protocol names"""
return [p.name for p in self.protocols()]
def clients(self, groups=None, protocol=None, **kwargs):
"""Returns a list of Clients for the specific query by the user.
If no parameters are specified - return all clients.
Keyword Parameters:
protocol
Protocol is ignored in this context, since its choice has no influence on clients.
groups
The groups to which the subjects attached to the models belong ('dev', 'eval', 'world')
Returns: A list containing the ids of all models belonging to the given group.
"""
# if protocol == '.':
# protocol = None
# protocol = self.check_parameters_for_validity(protocol, "protocol", self.protocol_names(), None)
groups = self.check_parameters_for_validity(groups, "group", self.groups(), self.groups())
groups = self.convert_group_names_sql(groups)
retval = []
if groups:
q = self.__db.session.query(Client).filter(Client.set.in_(groups))
q = q.order_by(Client.id)
retval = list(q)
return retval
def groups(self, protocol=None):
return self.convert_group_names_bio(self.__db.groups())
def model_ids(self, groups=None, protocol=None, **kwargs):
return [client.id for client in self.clients(groups=groups, protocol=protocol, **kwargs)]
def annotations(self, file):
"""Will return the bounding box annotation of all frames of the video."""
# fn = 10 # 10th frame number
annots = file.bbx(directory=self.original_directory)
# bob uses the (y, x) format
annotations = dict()
for i in range(annots.shape[0]):
topleft = (annots[i][2], annots[i][1])
bottomright = (annots[i][2] + annots[i][4], annots[i][1] + annots[i][3])
annotations[str(i)] = {'topleft': topleft, 'bottomright': bottomright}
return annotations
def objects(self, groups=None, protocol=None, purposes=None, model_ids=None, **kwargs):
"""This function returns lists of File objects, which fulfill the given restrictions.
Keyword parameters:
groups : str or [str]
The groups of which the clients should be returned.
Usually, groups are one or more elements of ('world', 'dev', 'eval')
protocol
The protocol for which the clients should be retrieved.
The protocol is dependent on your database.
If you do not have protocols defined, just ignore this field.
purposes : str or [str]
The purposes for which File objects should be retrieved.
Usually, purposes are one of ('enroll', 'probe').
model_ids : [various type]
The model ids for which the File objects should be retrieved.
What defines a 'model id' is dependent on the database.
In cases, where there is only one model per client, model ids and client ids are identical.
In cases, where there is one model per file, model ids and file ids are identical.
But, there might also be other cases.
"""
if protocol == '.':
protocol = None
valid_protocols = [x + '-licit' for x in self.protocol_names()]
valid_protocols += [x + '-spoof' for x in self.protocol_names()]
protocol = self.check_parameter_for_validity(protocol, "protocol", valid_protocols, 'grandtest')
groups = self.check_parameters_for_validity(groups, "group", self.groups(), self.groups())
purposes = self.check_parameters_for_validity(purposes, "purpose", ('enroll', 'probe'), ('enroll', 'probe'))
purposes = list(purposes)
groups = self.convert_group_names_sql(groups)
# protocol licit is not defined in the low level API
# so do a hack here.
if '-licit' in protocol:
# for licit we return the grandtest protocol
protocol = protocol.replace('-licit', '')
# The low-level API has only "attack", "real", "enroll" and "probe"
# should translate to "real" or "attack" depending on the protocol.
# enroll does not to change.
if 'probe' in purposes:
purposes.remove('probe')
purposes.append('real')
if len(purposes) == 1:
# making the model_ids to None will return all clients which make
# the impostor data also available.
model_ids = None
elif model_ids:
raise NotImplementedError(
'Currently returning both enroll and probe for specific '
'client(s) in the licit protocol is not supported. '
'Please specify one purpose only.')
elif '-spoof' in protocol:
protocol = protocol.replace('-spoof', '')
# you need to replace probe with attack and real for the spoof protocols.
# I am adding the real here also to create positives scores also.
if 'probe' in purposes:
purposes.remove('probe')
purposes.append('attack')
purposes.append('real')
else:
raise ValueError('Valid protocols are: ' + ' '.join(valid_protocols))
# now, query the actual Replay database
objects = self.__db.objects(groups=groups, protocol=protocol, cls=purposes, clients=model_ids, **kwargs)
# make sure to return verification.utils representation of a file, not the database one
# also make sure you replace client ids with spoof/metatdata1/metadata2/...
retval = []
for f in objects:
if f.is_real():
retval.append(File(f))
else:
temp = File(f)
temp.client_id = 'attack'
retval.append(temp)
return retval
......@@ -30,9 +30,29 @@ extensions = [
import sphinx
if sphinx.__version__ >= "1.4.1":
extensions.append('sphinx.ext.imgmath')
imgmath_image_format = 'svg'
else:
extensions.append('sphinx.ext.pngmath')
# Be picky about warnings
nitpicky = True
# Ignores stuff we can't easily resolve on other project's sphinx manuals
nitpick_ignore = []
# Allows the user to override warnings from a separate file
if os.path.exists('nitpick-exceptions.txt'):
for line in open('nitpick-exceptions.txt'):
if line.strip() == "" or line.startswith("#"):
continue
dtype, target = line.split(None, 1)
target = target.strip()
try: # python 2.x
target = unicode(target)
except NameError:
pass
nitpick_ignore.append((dtype, target))
# Always includes todos
todo_include_todos = True
......@@ -216,8 +236,13 @@ autodoc_default_flags = [
]
# For inter-documentation mapping:
from bob.extension.utils import link_documentation
intersphinx_mapping = link_documentation()
from bob.extension.utils import link_documentation, load_requirements
sphinx_requirements = "extra-intersphinx.txt"
if os.path.exists(sphinx_requirements):
intersphinx_mapping = link_documentation(additional_packages=load_requirements(sphinx_requirements))
else:
intersphinx_mapping = link_documentation()
# We want to remove all private (i.e. _. or __.__) members
# that are not in the list of accepted functions
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment