Commit 79d615ac authored by Amir MOHAMMADI's avatar Amir MOHAMMADI
Browse files

Merge branch 'clean-up' into 'master'

Clean up



See merge request !2
parents 82d32d1f 1fe6ed87
Pipeline #4604 passed with stages
in 26 minutes and 58 seconds
# This build file is defined in two parts: 1) a generic set of instructions you
# probably **don't** need to change and 2) a part you may have to tune to your
# project. It heavily uses template features from YAML to help you in only
# changing a minimal part of it and avoid code duplication to a maximum while
# still providing a nice pipeline display on your package.
# This build file heavily uses template features from YAML so it is generic
# enough for any Bob project. Don't modify it unless you know what you're
# doing.
# 1) Generic instructions (only change if you know what you're doing)
# -------------------------------------------------------------------
# Definition of our build pipeline
stages:
- build
- test
- docs
- wheels
- deploy
# Global variables
variables:
CONDA_PREFIX: env
# ---------
# Templates
# ---------
# Template for the build stage
# Needs to run on all supported architectures, platforms and python versions
......@@ -27,104 +22,98 @@ variables:
stage: build
before_script:
- git clean -ffdx
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/7/raw | tr -d '\r' > bootstrap-conda.sh
- chmod 755 ./bootstrap-conda.sh
- ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
variables: &build_variables
BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/"
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_build.sh
script:
- ./bin/buildout
- if [ -x ./bin/bob_dbmanage.py ]; then ./bin/bob_dbmanage.py all download --force; fi
- ./bin/sphinx-build doc sphinx
- ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG}
- ./_ci/build.sh
after_script:
- rm -rf ${CONDA_PREFIX}
- ./_ci/after_build.sh
artifacts:
expire_in: 1 day
expire_in: 1 week
paths:
- bootstrap-conda.sh
- _ci/
- dist/
- sphinx/
# Template for building on a Linux machine
.build_linux_template: &linux_build_job
<<: *build_job
variables: &linux_build_variables
<<: *build_variables
CONDA_FOLDER: "/local/conda"
CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
# Template for building on a Mac OSX machine
.build_mac_template: &macosx_build_job
<<: *build_job
variables: &macosx_build_variables
<<: *build_variables
CONDA_FOLDER: "/opt/conda"
MACOSX_DEPLOYMENT_TARGET: "10.9"
CFLAGS: "-pthread -coverage"
CXXFLAGS: "-pthread -coverage"
LDFLAGS: "-lpthread"
# Template for the test stage - re-install from uploaded wheels
# Template for the test stage - re-installs from uploaded wheels
# Needs to run on all supported architectures, platforms and python versions
.test_template: &test_job
stage: test
before_script:
- ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
- source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX}
- pip install --use-wheel --no-index --pre dist/*.whl
- ./_ci/install.sh _ci #updates
- ./_ci/before_test.sh
script:
- cd ${CONDA_PREFIX}
- python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())"
- coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME}
- coverage report
- sphinx-build -b doctest ../doc ../sphinx
- ./_ci/test.sh
after_script:
- rm -rf ${CONDA_PREFIX}
- ./_ci/after_test.sh
# Template for the wheel uploading stage
# Needs to run against one combination of python 2.x and 3.x if it is a python
# only package, otherwise, needs to run in both pythons to all supported
# architectures (Linux and Mac OSX 64-bit)
# Needs to run against one supported architecture, platform and python version
.wheels_template: &wheels_job
stage: wheels
environment: intranet
only:
- master
- tags
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
before_script:
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/8/raw | tr -d '\r' > upload-wheel.sh
- chmod 755 upload-wheel.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_wheels.sh
script:
- ./upload-wheel.sh
- ./_ci/wheels.sh
after_script:
- ./_ci/after_wheels.sh
# Template for (latest) documentation upload stage
# Only one real job needs to do this
.docs_template: &docs_job
stage: docs
environment: intranet
only:
- master
before_script:
- curl --silent https://gitlab.idiap.ch/bob/bob/snippets/9/raw | tr -d '\r' > upload-sphinx.sh
- chmod 755 upload-sphinx.sh
- ./_ci/install.sh _ci #updates
- ./_ci/before_docs.sh
script:
- ./upload-sphinx.sh
- ./_ci/docs.sh
after_script:
- ./_ci/after_docs.sh
# 2) Package specific instructions (you may tune this if needed)
# --------------------------------------------------------------
# Template for the deployment stage - re-installs from uploaded wheels
# Needs to run on a single architecture only
# Will deploy your package to PyPI and other required services
# Only runs for tags
.deploy_template: &deploy_job
stage: deploy
environment: internet
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci #updates
- ./_ci/before_deploy.sh
script:
- ./_ci/deploy.sh
after_script:
- ./_ci/after_deploy.sh
# -------------
# Build Targets
# -------------
# Linux + Python 2.7: Builds, tests, uploads wheel
# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
build_linux_27:
<<: *linux_build_job
<<: *build_job
variables: &linux_27_build_variables
<<: *linux_build_variables
PYTHON_VER: "2.7"
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-linux
......@@ -139,6 +128,15 @@ test_linux_27:
wheels_linux_27:
<<: *wheels_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
- conda-linux
deploy_linux_27:
<<: *deploy_job
variables: *linux_27_build_variables
dependencies:
- build_linux_27
tags:
......@@ -147,10 +145,9 @@ wheels_linux_27:
# Linux + Python 3.4: Builds and tests
build_linux_34:
<<: *linux_build_job
<<: *build_job
variables: &linux_34_build_variables
<<: *linux_build_variables
PYTHON_VER: "3.4"
PYTHON_VERSION: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-linux
......@@ -164,12 +161,11 @@ test_linux_34:
- conda-linux
# Linux + Python 3.5: Builds, tests, uploads wheel
# Linux + Python 3.5: Builds, tests and uploads wheel
build_linux_35:
<<: *linux_build_job
<<: *build_job
variables: &linux_35_build_variables
<<: *linux_build_variables
PYTHON_VER: "3.5"
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-linux
......@@ -184,6 +180,7 @@ test_linux_35:
wheels_linux_35:
<<: *wheels_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
......@@ -191,6 +188,7 @@ wheels_linux_35:
docs_linux_35:
<<: *docs_job
variables: *linux_35_build_variables
dependencies:
- build_linux_35
tags:
......@@ -198,57 +196,54 @@ docs_linux_35:
# Mac OSX + Python 2.7: Builds and tests
#build_macosx_27:
# <<: *macosx_build_job
# variables: &macosx_27_build_variables
# <<: *macosx_build_variables
# PYTHON_VER: "2.7"
# WHEEL_TAG: "py27"
# tags:
# - conda-macosx
#test_macosx_27:
# <<: *test_job
# variables: *macosx_27_build_variables
# dependencies:
# - build_macosx_27
# tags:
# - conda-macosx
build_macosx_27:
<<: *build_job
variables: &macosx_27_build_variables
PYTHON_VERSION: "2.7"
WHEEL_TAG: "py27"
tags:
- conda-macosx
test_macosx_27:
<<: *test_job
variables: *macosx_27_build_variables
dependencies:
- build_macosx_27
tags:
- conda-macosx
# Mac OSX + Python 3.4: Builds and tests
#build_macosx_34:
# <<: *macosx_build_job
# variables: &macosx_34_build_variables
# <<: *macosx_build_variables
# PYTHON_VER: "3.4"
# WHEEL_TAG: "py3"
# tags:
# - conda-macosx
#test_macosx_34:
# <<: *test_job
# variables: *macosx_34_build_variables
# dependencies:
# - build_macosx_34
# tags:
# - conda-macosx
build_macosx_34:
<<: *build_job
variables: &macosx_34_build_variables
PYTHON_VERSION: "3.4"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_34:
<<: *test_job
variables: *macosx_34_build_variables
dependencies:
- build_macosx_34
tags:
- conda-macosx
# Mac OSX + Python 3.5: Builds and tests
#build_macosx_35:
# <<: *macosx_build_job
# variables: &macosx_35_build_variables
# <<: *macosx_build_variables
# PYTHON_VER: "3.5"
# WHEEL_TAG: "py3"
# tags:
# - conda-macosx
#test_macosx_35:
# <<: *test_job
# variables: *macosx_35_build_variables
# dependencies:
# - build_macosx_35
# tags:
# - conda-macosx
\ No newline at end of file
build_macosx_35:
<<: *build_job
variables: &macosx_35_build_variables
PYTHON_VERSION: "3.5"
WHEEL_TAG: "py3"
tags:
- conda-macosx
test_macosx_35:
<<: *test_job
variables: *macosx_35_build_variables
dependencies:
- build_macosx_35
tags:
- conda-macosx
......@@ -21,7 +21,7 @@
This package is part of the signal-processing and machine learning toolbox
Bob_. This package provides a Bob interface for the Replay-Mobile database for
face antispoofing experiments for mobile devices.
face presentation attack (antispoofing) experiments for mobile devices.
Installation
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.dos.anjos@gmail.com>
# Wed 18 May 09:28:44 2011
"""The Replay-Mobile Database accessors for Bob
"""
......@@ -9,6 +7,7 @@
from .query import Database
from .models import Client, File, Protocol, RealAccess, Attack
def get_config():
"""Returns a string containing the configuration information.
"""
......
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Sushil Bhattacharjee <sushil.bhattacharjee@idiap.ch>
# Tue 20 Mar 19:20:22 2012 CET
"""Checks for installed files.
"""
......@@ -12,6 +10,7 @@ import sys
# Driver API
# ==========
def checkfiles(args):
"""Checks existence files based on your criteria"""
......@@ -19,13 +18,13 @@ def checkfiles(args):
db = Database()
r = db.objects(
protocol=args.protocol,
support=args.support,
protocol=args.protocol,
support=args.support,
groups=args.group,
cls=args.cls,
light=args.light,
clients=args.client,
)
)
# go through all files, check if they are available on the filesystem
good = []
......@@ -33,7 +32,7 @@ def checkfiles(args):
for f in r:
if os.path.exists(f.make_path(args.directory, args.extension)):
good.append(f)
else:
else:
bad.append(f)
# report
......@@ -45,11 +44,12 @@ def checkfiles(args):
if bad:
for f in bad:
output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
output.write('%d files (out of %d) were not found at "%s"\n' % \
(len(bad), len(r), args.directory))
output.write('%d files (out of %d) were not found at "%s"\n' %
(len(bad), len(r), args.directory))
return 0
def add_command(subparsers):
"""Add specific subcommands that the action "checkfiles" can use"""
......@@ -62,7 +62,7 @@ def add_command(subparsers):
db = Database()
if not db.is_valid():
protocols = ('waiting','for','database','creation')
protocols = ('waiting', 'for', 'database', 'creation')
clients = tuple()
else:
protocols = [k.name for k in db.protocols()]
......@@ -77,6 +77,6 @@ def add_command(subparsers):
parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the check to those files shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
parser.add_argument('--self-test', dest="selftest", default=False,
action='store_true', help=SUPPRESS)
action='store_true', help=SUPPRESS)
parser.set_defaults(func=checkfiles) #action
parser.set_defaults(func=checkfiles) # action
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Sushil Bhattacharjee <sushil.bhattacharjee@idiap.ch>
# Fri Nov 13 11:11:11 2015 +0200
#
# Copyright (C) 2015 Idiap Research Institute, Martigny, Switzerland
"""This script creates the Replay-Attack database in a single pass.
"""
import os
import fnmatch
from .models import *
def add_clients(session, protodir, verbose):
"""Add clients to the replay attack database."""
for client in open(os.path.join(protodir, 'clients.txt'), 'rt'):
s = client.strip().split(' ', 2)
if not s: continue #empty line
if not s:
continue # empty line
id = int(s[0])
set = s[1]
if verbose: print("Adding client %d on '%s' set..." % (id, set))
if verbose:
print("Adding client %d on '%s' set..." % (id, set))
session.add(Client(id, set))
def add_real_lists(session, protodir, verbose):
"""Adds all RCD filelists"""
......@@ -34,16 +33,17 @@ def add_real_lists(session, protodir, verbose):
"""Parses the RCD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[0].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
propuse =v[2]
client_id = int(v[0].replace('client', ''))
path = os.path.splitext(f)[0] # keep only the filename stem
propuse = v[2]
device = v[3]
light = v[4]
return [client_id, path, light,device], [propuse,device]
return [client_id, path, light, device], [propuse, device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
if not s:
continue # emtpy line
filefields, realfields = parse_real_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
......@@ -51,7 +51,6 @@ def add_real_lists(session, protodir, verbose):
realfields.insert(0, file)
session.add(RealAccess(*realfields))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-train.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-devel.txt'))
add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-test.txt'))
......@@ -71,21 +70,22 @@ def add_attack_lists(session, protodir, verbose):
"""Parses the RAD filename and break it in the relevant chunks."""
v = os.path.splitext(os.path.basename(f))[0].split('_')
client_id = int(v[1].replace('client',''))
path = os.path.splitext(f)[0] #keep only the filename stem
client_id = int(v[1].replace('client', ''))
path = os.path.splitext(f)[0] # keep only the filename stem
light = v[7]
#attack_support = f.split('/')[-2]
# attack_support = f.split('/')[-2]
attack_support = v[4] #fixed, hand
attack_device = v[3] #print, mattescreen
sample_type = v[6] #photo or video
sample_device = v[5] #tablet or mobile
attack_support = v[4] # fixed, hand
attack_device = v[3] # print, mattescreen
sample_type = v[6] # photo or video
sample_device = v[5] # tablet or mobile
return [client_id, path, light ,sample_device], [attack_support, attack_device, sample_type, sample_device]
return [client_id, path, light, sample_device], [attack_support, attack_device, sample_type, sample_device]
for fname in open(filename, 'rt'):
s = fname.strip()
if not s: continue #emtpy line
if not s:
continue # emtpy line
filefields, attackfields = parse_attack_filename(s)
filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
file = File(*filefields)
......@@ -93,42 +93,43 @@ def add_attack_lists(session, protodir, verbose):
attackfields.insert(0, file)
session.add(Attack(*attackfields))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
def define_protocols(session, protodir, verbose):
"""Defines all available protocols"""
#figures out which protocols to use
# figures out which protocols to use
valid = {}
# Three grandtest protocols
#files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]
# Three grandtest protocols
# files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]
files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train' ,'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']
files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train', 'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']
for fname in files_protocol:
s = fname.split('-', 5)
folder = fname.split('/', 1)[0]
consider = True
files = {}
for grp in ('train', 'devel', 'test'):
# check attack file
if len(s)<6:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder,s[1],s[2],s[3], grp))
else :
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder,s[1],s[2],s[3], grp))
if len(s) < 6:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder, s[1], s[2], s[3], grp))
else:
attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder, s[1], s[2], s[3], grp))
if not os.path.exists(attack):
if verbose:
print("Not considering protocol %s as attack list '%s' was not found" % (s[1], attack))
consider = False
# check real file
real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder, s[3], grp))
if not os.path.exists(real):
alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder, s[3], grp))
if not os.path.exists(alt_real):
if verbose:
print("Not considering protocol %s as real list '%s' or '%s' were not found" % (s[1], real, alt_real))
......@@ -136,26 +137,30 @@ def define_protocols(session, protodir, verbose):
else:
real = alt_real
if consider: files[grp] = (attack, real)
if consider:
files[grp] = (attack, real)
if consider: valid[s[1]] = files
if consider:
valid[s[1]] = files
for protocol, groups in valid.items():
if verbose: print("Creating protocol '%s'..." % protocol)
if verbose:
print("Creating protocol '%s'..." % protocol)
# create protocol on the protocol table
obj = Protocol(name=protocol)
for grp, flist in groups.items():