Merge branch 'clean-up' into 'master'

Clean up See merge request !2

Merge branch 'clean-up' into 'master'
Clean up See merge request !2
79d615ac · Amir MOHAMMADI · 82d32d1f · 1fe6ed87 · 79d615ac · 79d615ac
Commit 79d615ac authored Oct 05, 2016 by Amir MOHAMMADI
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
-# This build file is defined in two parts: 1) a generic set of instructions you
-# probably **don't** need to change and 2) a part you may have to tune to your
-# project. It heavily uses template features from YAML to help you in only
-# changing a minimal part of it and avoid code duplication to a maximum while
-# still providing a nice pipeline display on your package.
+# This build file heavily uses template features from YAML so it is generic
+# enough for any Bob project. Don't modify it unless you know what you're
+# doing.


-# 1) Generic instructions (only change if you know what you're doing)
-# -------------------------------------------------------------------
-
 # Definition of our build pipeline
 stages:
  - build
  - test
  - docs
  - wheels
+  - deploy


-# Global variables
-variables:
-  CONDA_PREFIX: env
-
+# ---------
+# Templates
+# ---------

 # Template for the build stage
 # Needs to run on all supported architectures, platforms and python versions
@@ -27,104 +22,98 @@ variables:
  stage: build
  before_script:
    - git clean -ffdx
-    - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/7/raw | tr -d '\r' > bootstrap-conda.sh
-    - chmod 755 ./bootstrap-conda.sh
-    - ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
-  variables: &build_variables
-    BOB_DOCUMENTATION_SERVER: "http://www.idiap.ch/software/bob/docs/latest/bob/%s/master/"
+    - mkdir _ci
+    - curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
+    - chmod 755 _ci/install.sh
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_build.sh
  script:
-    - ./bin/buildout
-    - if [ -x ./bin/bob_dbmanage.py ]; then ./bin/bob_dbmanage.py all download --force; fi
-    - ./bin/sphinx-build doc sphinx
-    - ./bin/python setup.py bdist_wheel --python-tag ${WHEEL_TAG}
+    - ./_ci/build.sh
  after_script:
-    - rm -rf ${CONDA_PREFIX}
+    - ./_ci/after_build.sh
  artifacts:
-    expire_in: 1 day
+    expire_in: 1 week
    paths:
-      - bootstrap-conda.sh
+      - _ci/
      - dist/
      - sphinx/


-# Template for building on a Linux machine
-.build_linux_template: &linux_build_job
-  <<: *build_job
-  variables: &linux_build_variables
-    <<: *build_variables
-    CONDA_FOLDER: "/local/conda"
-    CFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
-    CXXFLAGS: "-D_GLIBCXX_USE_CXX11_ABI=0 -coverage"
-
-
-# Template for building on a Mac OSX machine
-.build_mac_template: &macosx_build_job
-  <<: *build_job
-  variables: &macosx_build_variables
-    <<: *build_variables
-    CONDA_FOLDER: "/opt/conda"
-    MACOSX_DEPLOYMENT_TARGET: "10.9"
-    CFLAGS: "-pthread -coverage"
-    CXXFLAGS: "-pthread -coverage"
-    LDFLAGS: "-lpthread"
-
-
-# Template for the test stage - re-install from uploaded wheels
+# Template for the test stage - re-installs from uploaded wheels
 # Needs to run on all supported architectures, platforms and python versions
 .test_template: &test_job
  stage: test
  before_script:
-    - ./bootstrap-conda.sh ${CONDA_FOLDER} ${PYTHON_VER} ${CONDA_PREFIX}
-    - source ${CONDA_FOLDER}/bin/activate ${CONDA_PREFIX}
-    - pip install --use-wheel --no-index --pre dist/*.whl
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_test.sh
  script:
-    - cd ${CONDA_PREFIX}
-    - python -c "from ${CI_PROJECT_NAME} import get_config; print(get_config())"
-    - coverage run --source=${CI_PROJECT_NAME} ./bin/nosetests -sv ${CI_PROJECT_NAME}
-    - coverage report
-    - sphinx-build -b doctest ../doc ../sphinx
+    - ./_ci/test.sh
  after_script:
-    - rm -rf ${CONDA_PREFIX}
+    - ./_ci/after_test.sh


 # Template for the wheel uploading stage
-# Needs to run against one combination of python 2.x and 3.x if it is a python
-# only package, otherwise, needs to run in both pythons to all supported
-# architectures (Linux and Mac OSX 64-bit)
+# Needs to run against one supported architecture, platform and python version
 .wheels_template: &wheels_job
  stage: wheels
+  environment: intranet
  only:
    - master
-    - tags
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
  before_script:
-    - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/8/raw | tr -d '\r' > upload-wheel.sh
-    - chmod 755 upload-wheel.sh
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_wheels.sh
  script:
-    - ./upload-wheel.sh
+    - ./_ci/wheels.sh
+  after_script:
+    - ./_ci/after_wheels.sh


 # Template for (latest) documentation upload stage
 # Only one real job needs to do this
 .docs_template: &docs_job
  stage: docs
+  environment: intranet
  only:
    - master
  before_script:
-    - curl --silent https://gitlab.idiap.ch/bob/bob/snippets/9/raw | tr -d '\r' > upload-sphinx.sh
-    - chmod 755 upload-sphinx.sh
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_docs.sh
  script:
-    - ./upload-sphinx.sh
+    - ./_ci/docs.sh
+  after_script:
+    - ./_ci/after_docs.sh


-# 2) Package specific instructions (you may tune this if needed)
-# --------------------------------------------------------------
+# Template for the deployment stage - re-installs from uploaded wheels
+# Needs to run on a single architecture only
+# Will deploy your package to PyPI and other required services
+# Only runs for tags
+.deploy_template: &deploy_job
+  stage: deploy
+  environment: internet
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - branches
+  before_script:
+    - ./_ci/install.sh _ci #updates
+    - ./_ci/before_deploy.sh
+  script:
+    - ./_ci/deploy.sh
+  after_script:
+    - ./_ci/after_deploy.sh
+
+
+# -------------
+# Build Targets
+# -------------

-# Linux + Python 2.7: Builds, tests, uploads wheel
+# Linux + Python 2.7: Builds, tests, uploads wheel and deploys (if needed)
 build_linux_27:
-  <<: *linux_build_job
+  <<: *build_job
  variables: &linux_27_build_variables
-    <<: *linux_build_variables
-    PYTHON_VER: "2.7"
+    PYTHON_VERSION: "2.7"
    WHEEL_TAG: "py27"
  tags:
    - conda-linux
@@ -139,6 +128,15 @@ test_linux_27:

 wheels_linux_27:
  <<: *wheels_job
+  variables: *linux_27_build_variables
+  dependencies:
+    - build_linux_27
+  tags:
+    - conda-linux
+
+deploy_linux_27:
+  <<: *deploy_job
+  variables: *linux_27_build_variables
  dependencies:
    - build_linux_27
  tags:
@@ -147,10 +145,9 @@ wheels_linux_27:

 # Linux + Python 3.4: Builds and tests
 build_linux_34:
-  <<: *linux_build_job
+  <<: *build_job
  variables: &linux_34_build_variables
-    <<: *linux_build_variables
-    PYTHON_VER: "3.4"
+    PYTHON_VERSION: "3.4"
    WHEEL_TAG: "py3"
  tags:
    - conda-linux
@@ -164,12 +161,11 @@ test_linux_34:
    - conda-linux


-# Linux + Python 3.5: Builds, tests, uploads wheel
+# Linux + Python 3.5: Builds, tests and uploads wheel
 build_linux_35:
-  <<: *linux_build_job
+  <<: *build_job
  variables: &linux_35_build_variables
-    <<: *linux_build_variables
-    PYTHON_VER: "3.5"
+    PYTHON_VERSION: "3.5"
    WHEEL_TAG: "py3"
  tags:
    - conda-linux
@@ -184,6 +180,7 @@ test_linux_35:

 wheels_linux_35:
  <<: *wheels_job
+  variables: *linux_35_build_variables
  dependencies:
    - build_linux_35
  tags:
@@ -191,6 +188,7 @@ wheels_linux_35:

 docs_linux_35:
  <<: *docs_job
+  variables: *linux_35_build_variables
  dependencies:
    - build_linux_35
  tags:
@@ -198,57 +196,54 @@ docs_linux_35:


 # Mac OSX + Python 2.7: Builds and tests
-#build_macosx_27:
-#  <<: *macosx_build_job
-#  variables: &macosx_27_build_variables
-#    <<: *macosx_build_variables
-#    PYTHON_VER: "2.7"
-#    WHEEL_TAG: "py27"
-#  tags:
-#    - conda-macosx
-
-#test_macosx_27:
-#  <<: *test_job
-#  variables: *macosx_27_build_variables
-#  dependencies:
-#    - build_macosx_27
-#  tags:
-#    - conda-macosx
+build_macosx_27:
+  <<: *build_job
+  variables: &macosx_27_build_variables
+    PYTHON_VERSION: "2.7"
+    WHEEL_TAG: "py27"
+  tags:
+    - conda-macosx
+
+test_macosx_27:
+  <<: *test_job
+  variables: *macosx_27_build_variables
+  dependencies:
+    - build_macosx_27
+  tags:
+    - conda-macosx


 # Mac OSX + Python 3.4: Builds and tests
-#build_macosx_34:
-#  <<: *macosx_build_job
-#  variables: &macosx_34_build_variables
-#    <<: *macosx_build_variables
-#    PYTHON_VER: "3.4"
-#    WHEEL_TAG: "py3"
-#  tags:
-#    - conda-macosx
-
-#test_macosx_34:
-#  <<: *test_job
-#  variables: *macosx_34_build_variables
-#  dependencies:
-#    - build_macosx_34
-#  tags:
-#    - conda-macosx
+build_macosx_34:
+  <<: *build_job
+  variables: &macosx_34_build_variables
+    PYTHON_VERSION: "3.4"
+    WHEEL_TAG: "py3"
+  tags:
+    - conda-macosx
+
+test_macosx_34:
+  <<: *test_job
+  variables: *macosx_34_build_variables
+  dependencies:
+    - build_macosx_34
+  tags:
+    - conda-macosx


 # Mac OSX + Python 3.5: Builds and tests
-#build_macosx_35:
-#  <<: *macosx_build_job
-#  variables: &macosx_35_build_variables
-#    <<: *macosx_build_variables
-#    PYTHON_VER: "3.5"
-#    WHEEL_TAG: "py3"
-#  tags:
-#    - conda-macosx
-
-#test_macosx_35:
-#  <<: *test_job
-#  variables: *macosx_35_build_variables
-#  dependencies:
-#    - build_macosx_35
-#  tags:
-#    - conda-macosx
\ No newline at end of file
+build_macosx_35:
+  <<: *build_job
+  variables: &macosx_35_build_variables
+    PYTHON_VERSION: "3.5"
+    WHEEL_TAG: "py3"
+  tags:
+    - conda-macosx
+
+test_macosx_35:
+  <<: *test_job
+  variables: *macosx_35_build_variables
+  dependencies:
+    - build_macosx_35
+  tags:
+    - conda-macosx
--- a/README.rst
+++ b/README.rst
@@ -21,7 +21,7 @@

 This package is part of the signal-processing and machine learning toolbox
 Bob_. This package provides a Bob interface for the Replay-Mobile database for
-face antispoofing experiments for mobile devices.
+face presentation attack (antispoofing) experiments for mobile devices.


 Installation

--- a/bob/db/replaymobile/__init__.py
+++ b/bob/db/replaymobile/__init__.py
 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
-# Andre Anjos <andre.dos.anjos@gmail.com>
-# Wed 18 May 09:28:44 2011

 """The Replay-Mobile Database accessors for Bob
 """
@@ -9,6 +7,7 @@
 from .query import Database
 from .models import Client, File, Protocol, RealAccess, Attack

+
 def get_config():
  """Returns a string containing the configuration information.
  """

--- a/bob/db/replaymobile/checkfiles.py
+++ b/bob/db/replaymobile/checkfiles.py
 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
-# Sushil Bhattacharjee <sushil.bhattacharjee@idiap.ch>
-# Tue 20 Mar 19:20:22 2012 CET

 """Checks for installed files.
 """
@@ -12,6 +10,7 @@ import sys
 # Driver API
 # ==========

+
 def checkfiles(args):
  """Checks existence files based on your criteria"""

@@ -19,13 +18,13 @@ def checkfiles(args):
  db = Database()

  r = db.objects(
-      protocol=args.protocol, 
-      support=args.support, 
+      protocol=args.protocol,
+      support=args.support,
      groups=args.group,
      cls=args.cls,
      light=args.light,
      clients=args.client,
-      )
+  )

  # go through all files, check if they are available on the filesystem
  good = []
@@ -33,7 +32,7 @@ def checkfiles(args):
  for f in r:
    if os.path.exists(f.make_path(args.directory, args.extension)):
      good.append(f)
-    else: 
+    else:
      bad.append(f)

  # report
@@ -45,11 +44,12 @@ def checkfiles(args):
  if bad:
    for f in bad:
      output.write('Cannot find file "%s"\n' % (f.make_path(args.directory, args.extension),))
-    output.write('%d files (out of %d) were not found at "%s"\n' % \
-        (len(bad), len(r), args.directory))
+    output.write('%d files (out of %d) were not found at "%s"\n' %
+                 (len(bad), len(r), args.directory))

  return 0

+
 def add_command(subparsers):
  """Add specific subcommands that the action "checkfiles" can use"""

@@ -62,7 +62,7 @@ def add_command(subparsers):
  db = Database()

  if not db.is_valid():
-    protocols = ('waiting','for','database','creation')
+    protocols = ('waiting', 'for', 'database', 'creation')
    clients = tuple()
  else:
    protocols = [k.name for k in db.protocols()]
@@ -77,6 +77,6 @@ def add_command(subparsers):
  parser.add_argument('-l', '--light', dest="light", default='', help="if given, this value will limit the check to those files shot under a given lighting. (defaults to '%(default)s')", choices=db.lights())
  parser.add_argument('-C', '--client', dest="client", default=None, type=int, help="if given, limits the dump to a particular client (defaults to '%(default)s')", choices=clients)
  parser.add_argument('--self-test', dest="selftest", default=False,
-      action='store_true', help=SUPPRESS)
+                      action='store_true', help=SUPPRESS)

-  parser.set_defaults(func=checkfiles) #action
+  parser.set_defaults(func=checkfiles)  # action
--- a/bob/db/replaymobile/create.py
+++ b/bob/db/replaymobile/create.py
 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
-# Sushil Bhattacharjee <sushil.bhattacharjee@idiap.ch>
-# Fri Nov 13 11:11:11 2015 +0200
-#
-# Copyright (C) 2015 Idiap Research Institute, Martigny, Switzerland

 """This script creates the Replay-Attack database in a single pass.
 """

 import os
-import fnmatch

 from .models import *

+
 def add_clients(session, protodir, verbose):
  """Add clients to the replay attack database."""

  for client in open(os.path.join(protodir, 'clients.txt'), 'rt'):
    s = client.strip().split(' ', 2)
-    if not s: continue #empty line
+    if not s:
+      continue  # empty line
    id = int(s[0])
    set = s[1]
-    if verbose: print("Adding client %d on '%s' set..." % (id, set))
+    if verbose:
+      print("Adding client %d on '%s' set..." % (id, set))
    session.add(Client(id, set))

+
 def add_real_lists(session, protodir, verbose):
  """Adds all RCD filelists"""

@@ -34,16 +33,17 @@ def add_real_lists(session, protodir, verbose):
      """Parses the RCD filename and break it in the relevant chunks."""

      v = os.path.splitext(os.path.basename(f))[0].split('_')
-      client_id = int(v[0].replace('client',''))
-      path = os.path.splitext(f)[0] #keep only the filename stem
-      propuse =v[2]
+      client_id = int(v[0].replace('client', ''))
+      path = os.path.splitext(f)[0]  # keep only the filename stem
+      propuse = v[2]
      device = v[3]
      light = v[4]
-      return [client_id, path, light,device], [propuse,device]
+      return [client_id, path, light, device], [propuse, device]

    for fname in open(filename, 'rt'):
      s = fname.strip()
-      if not s: continue #emtpy line
+      if not s:
+        continue  # emtpy line
      filefields, realfields = parse_real_filename(s)
      filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
      file = File(*filefields)
@@ -51,7 +51,6 @@ def add_real_lists(session, protodir, verbose):
      realfields.insert(0, file)
      session.add(RealAccess(*realfields))

-
  add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-train.txt'))
  add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-devel.txt'))
  add_real_list(session, os.path.join(protodir, 'alldevices/real-alldevices-test.txt'))
@@ -71,21 +70,22 @@ def add_attack_lists(session, protodir, verbose):
      """Parses the RAD filename and break it in the relevant chunks."""

      v = os.path.splitext(os.path.basename(f))[0].split('_')
-      client_id = int(v[1].replace('client',''))
-      path = os.path.splitext(f)[0] #keep only the filename stem
+      client_id = int(v[1].replace('client', ''))
+      path = os.path.splitext(f)[0]  # keep only the filename stem
      light = v[7]
-      #attack_support = f.split('/')[-2]
+      # attack_support = f.split('/')[-2]

-      attack_support = v[4] #fixed, hand
-      attack_device = v[3] #print, mattescreen
-      sample_type = v[6] #photo or video
-      sample_device = v[5] #tablet or mobile
+      attack_support = v[4]  # fixed, hand
+      attack_device = v[3]  # print, mattescreen
+      sample_type = v[6]  # photo or video
+      sample_device = v[5]  # tablet or mobile

-      return [client_id, path, light ,sample_device], [attack_support, attack_device, sample_type, sample_device]
+      return [client_id, path, light, sample_device], [attack_support, attack_device, sample_type, sample_device]

    for fname in open(filename, 'rt'):
      s = fname.strip()
-      if not s: continue #emtpy line
+      if not s:
+        continue  # emtpy line
      filefields, attackfields = parse_attack_filename(s)
      filefields[0] = session.query(Client).filter(Client.id == filefields[0]).one()
      file = File(*filefields)
@@ -93,42 +93,43 @@ def add_attack_lists(session, protodir, verbose):
      attackfields.insert(0, file)
      session.add(Attack(*attackfields))

-  add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
-  add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
-  add_attack_list(session,os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
+  add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-train.txt'))
+  add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-devel.txt'))
+  add_attack_list(session, os.path.join(protodir, 'alldevices/attack-grandtest-allsupports-alldevices-test.txt'))
+

 def define_protocols(session, protodir, verbose):
  """Defines all available protocols"""

-  #figures out which protocols to use
+  # figures out which protocols to use
  valid = {}
-  # Three grandtest protocols 
-  #files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]
+  # Three grandtest protocols
+  # files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'mobile/attack-mobilegrandtest-allsupports-mobile-train' ,'tablet/attack-tabletgrandtest-allsupports-tablet-train' ]

-  files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train' ,'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']
+  files_protocol = ['alldevices/attack-grandtest-allsupports-alldevices-train.txt', 'alldevices/attack-print-allsupports-alldevices-train', 'alldevices/attack-mattescreen-fixed-alldevices-alltype-train']

  for fname in files_protocol:
    s = fname.split('-', 5)
    folder = fname.split('/', 1)[0]
    consider = True
    files = {}
-    
+
    for grp in ('train', 'devel', 'test'):

      # check attack file
-      if len(s)<6:
-      	attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder,s[1],s[2],s[3], grp))
-      else :
-        attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder,s[1],s[2],s[3], grp))
+      if len(s) < 6:
+        attack = os.path.join(protodir, '%s/attack-%s-%s-%s-%s.txt' % (folder, s[1], s[2], s[3], grp))
+      else:
+        attack = os.path.join(protodir, '%s/attack-%s-%s-%s-alltype-%s.txt' % (folder, s[1], s[2], s[3], grp))
      if not os.path.exists(attack):
        if verbose:
          print("Not considering protocol %s as attack list '%s' was not found" % (s[1], attack))
        consider = False

      # check real file
-      real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
+      real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder, s[3], grp))
      if not os.path.exists(real):
-       alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder,s[3],grp))
+       alt_real = os.path.join(protodir, '%s/real-%s-%s.txt' % (folder, s[3], grp))
       if not os.path.exists(alt_real):
         if verbose:
           print("Not considering protocol %s as real list '%s' or '%s' were not found" % (s[1], real, alt_real))
@@ -136,26 +137,30 @@ def define_protocols(session, protodir, verbose):
       else:
         real = alt_real

-      if consider: files[grp] = (attack, real)
+      if consider:
+        files[grp] = (attack, real)

-    if consider: valid[s[1]] = files
+    if consider:
+      valid[s[1]] = files

  for protocol, groups in valid.items():
-    if verbose: print("Creating protocol '%s'..." % protocol)
+    if verbose:
+      print("Creating protocol '%s'..." % protocol)

    # create protocol on the protocol table
    obj = Protocol(name=protocol)

    for grp, flist in groups.items():