Major refactoring to add new protocols and to match new data organization, as in Idiap DDP

MANIFEST.in

-include README.rst
+include README.rst boostrap.py buildout.cfg
 recursive-include docs *.py *.rst
 recursive-include xbob *.sql3

README.rst

@@ -24,7 +24,7 @@ There are a few ways to achieve this:
 
 The package is available in two different distribution formats:
 
-1. You can download it from `PyPI <http://pypi.python.org/pypi>`_, or
+1. You can download it from `PyPI <http://pypi.python.org/pypi/xbob.db.mobio>`_, or
 
 2. You can download it in its source form from `its git repository
    <https://github.com/bioidiap/xbob.db.mobio>`_. When you download the
@@ -72,3 +72,38 @@ lines::
   [sources]
   xbob.db.mobio = git https://github.com/bioidiap/xbob.db.mobio.git
   ...
+
+MOBIO protocols
+===============
+
+There were initially two protocols defined on the Phase 2 of the database,
+which were called 'female' and 'male'. Later on, the number of protocols
+has increased, considering the additional data recorded using laptops, which
+has led to 8 protocols.
+
+The two initial protocols 'female' and 'male' now correspond to the protocols
+called 'mobile0-female' and 'mobile0-male', respectively. The training, 
+development and evaluation sets are indeed identical.
+
+However, if you want to use the same ZT score normalization files as in this 
+publication::
+
+  @article{McCool_IET_BMT_2013,
+    title = {Session variability modelling for face authentication},
+    author = {McCool, Chris and Wallace, Roy and McLaren, Mitchell and El Shafey, Laurent and Marcel, S{\'{e}}bastien},
+    month = sep,
+    journal = {IET Biometrics},
+    volume = {2},
+    number = {3},
+    year = {2013},
+    pages = {117-129},
+    issn = {2047-4938},
+    doi = {10.1049/iet-bmt.2012.0059},
+  }
+
+You have to specify optional arguments::
+
+  1. `speech_type = 'p'` when calling the `tobjects()` method
+
+  2. `speech_type = ['p','r','l','f']` when calling the `zobjects()` method
+

setup.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # Laurent El Shafey <laurent.el-shafey@idiap.ch>
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from setuptools import setup, find_packages
 
@@ -9,9 +23,9 @@ from setuptools import setup, find_packages
 setup(
 
     name='xbob.db.mobio',
-    version='1.0.5a0',
+    version='1.1.0',
     description='MOBIO Database Access API for Bob',
-    url='http://github.com/bioidiap/xbob.db.mobio',
+    url='https://pypi.python.org/pypi/xbob.db.mobio',
     license='GPLv3',
     author='Laurent El Shafey',
     author_email='laurent.el-shafey@idiap.ch',
@@ -24,7 +38,7 @@ setup(
 
     install_requires=[
       'setuptools',
-      'six',  # py2/3 compatibility library
+      'six',  # py2/3 compatibility library 
       'bob',  # base signal proc./machine learning library
       'xbob.db.verification.utils>=0.1.4' # defines a set of utilities for face verification databases like this one.
     ],
@@ -44,6 +58,10 @@ setup(
       'bob.test': [
         'mobio = xbob.db.mobio.test:MobioDatabaseTest',
         ],
+      # scripts 
+      'console_scripts': [
+        'generate_filelist = xbob.db.mobio.generate_filelist:main',
+        ],
       },
 
     classifiers = [

xbob/db/mobio/__init__.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # Laurent El Shafey <laurent.el-shafey@idiap.ch>
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 """The Mobio database
 """

xbob/db/mobio/driver.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # Laurent El Shafey <laurent.el-shafey@idiap.ch>
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 """Commands the MOBIO database can respond to.
 """
@@ -139,7 +153,7 @@ class Interface(BaseInterface):
     parser = subparsers.add_parser('dumplist', help=dumplist.__doc__)
     parser.add_argument('-d', '--directory', help="if given, this path will be prepended to every entry returned.")
     parser.add_argument('-e', '--extension', help="if given, this extension will be appended to every entry returned.")
-    parser.add_argument('-p', '--protocol', help="if given, limits the check to a particular subset of the data that corresponds to the given protocol.", choices=db.protocol_names() if db.is_valid() else ())
+    parser.add_argument('-p', '--protocol', help="if given, limits the check to a particular subset of the data that corresponds to the given protocol.", choices=list(db.protocol_names()).extend(['male', 'female']) if db.is_valid() else ())
     parser.add_argument('-u', '--purpose', help="if given, this value will limit the output files to those designed for the given purposes.", choices=db.purposes() if db.is_valid() else ())
     parser.add_argument('-C', '--client', type=int, help="if given, limits the dump to a particular client.", choices=db.model_ids() if db.is_valid() else ())
     parser.add_argument('-g', '--group', help="if given, this value will limit the output files to those belonging to a particular protocolar group.", choices=db.groups() if db.is_valid() else ())

xbob/db/mobio/generate_filelist.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# Elie Khoury <Elie.Khoury@idiap.ch>
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import sys, os
+import argparse
+from .query import Database
+
+
+def sort_by_ids(files):
+    """Returns a sorted version of the given list of File's (or other structures that define an 'id' data member).
+    The files will be sorted according to their id, and duplicate entries will be removed."""
+    sorted_files = sorted(files, cmp=lambda x,y: cmp(x.id, y.id))
+    return [f for i,f in enumerate(sorted_files) if not i or sorted_files[i-1].id != f.id]  
+    
+
+def sort_by_pathes(files):
+    """Returns a sorted version of the given list of File's (or other structures that define an 'id' data member).
+    The files will be sorted according to their id, and duplicate entries will be removed."""
+    sorted_files = sorted(files, cmp=lambda x,y: cmp(x.path, y.path))
+    return [f for i,f in enumerate(sorted_files) if not i or sorted_files[i-1].path != f.path]  
+
+def ensure_dir(dirname):
+  """ Creates the directory dirname if it does not already exist,
+      taking into account concurrent 'creation' on the grid.
+      An exception is thrown if a file (rather than a directory) already 
+      exists. """
+  try:
+    # Tries to create the directory
+    os.makedirs(dirname)
+  except OSError:
+    # Check that the directory exists
+    if os.path.isdir(dirname): pass
+    else: raise
+    
+def main():
+  """Executes the main function"""
+
+  parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+  parser.add_argument('-o', '--output-dir', metavar='DIR', type=str, dest='output_dir', default='./protocols/', help='Output directory (defaults to "%(default)s")')
+
+  parser.add_argument('-p', '--protocol-name', type=str, dest='protocol_name', default='mobile0-male', help=' Protocol name (defaults to "%(default)s")')
+
+  parser.add_argument('-g', '--gender-dependent', action='store_true', dest='gender_dependent', default=False, help='Use gender dependent Training data (defaults to "%(default)s")')
+
+  parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help="Increase some verbosity")
+
+  args = parser.parse_args()
+  
+
+  ########################
+  # Loading Hiperparameters
+  #########################
+  m_output_dir = args.output_dir 
+  m_protocol_name = args.protocol_name
+  m_gender_dependent = args.gender_dependent
+  
+  # verify that the protocol name exists
+  db = Database()
+  if m_protocol_name not in db.protocol_names():
+    raise ValueError("The given protocol name '%s' does not exist."%m_protocol_name)
+  
+  gender_value = 'female'
+  if m_protocol_name.find(gender_value) == -1:
+    gender_value = 'male'
+  print('gender: %s' % gender_value)
+  
+ 
+
+  ### List for world group
+  if (m_gender_dependent):
+    world_dir = m_output_dir +'/' + m_protocol_name + '_GD/' + 'norm'
+    #world_dir = m_output_dir +'/' + m_protocol_name + '_GD/' + 'world'
+    dev_dir = m_output_dir +'/' + m_protocol_name + '_GD/' + 'dev'
+    eval_dir = m_output_dir +'/' + m_protocol_name + '_GD/' + 'eval'
+  else:
+    world_dir = m_output_dir +'/' + m_protocol_name + '/' + 'norm'
+    #world_dir = m_output_dir +'/' + m_protocol_name + '/' + 'world'
+    dev_dir = m_output_dir +'/' + m_protocol_name + '/' + 'dev'
+    eval_dir = m_output_dir +'/' + m_protocol_name + '/' + 'eval'
+
+  # ensure directories
+  ensure_dir(world_dir)
+  ensure_dir(dev_dir)
+  ensure_dir(eval_dir)
+  
+  ### List for world group (norm/train_world.lst)
+  if (m_gender_dependent):
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="world", gender = gender_value))
+  else:
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="world"))
+  
+  files =  sort_by_pathes(files)
+  known = set()
+  file_list = open(world_dir+'/train_world.lst', 'w')
+  
+  for file in files:
+    if file.path not in known and not known.add(file.path):
+      file_list.write(file.path + ' ' + (str(file.client_id)).zfill(3) + '\n') 
+  file_list.close()  
+  
+  
+  ### List of DEV.clients (dev/for_models.lst) group
+  if (m_gender_dependent):
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="dev", purposes='enrol', gender = gender_value))
+  else:
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="dev", purposes='enrol'))
+  
+  files = sort_by_pathes(files)
+  known = set()
+  file_list = open(dev_dir+'/for_models.lst', 'w')
+  
+  for file in files:
+    if file.path not in known and not known.add(file.path):
+      
+      file_list.write(file.path + ' ' + (str(file.client_id)).zfill(3) + ' ' + (str(file.client_id)).zfill(3) + '\n') 
+  file_list.close() 
+  
+  ### List of DEV.trials (dev/for_scores.lst) group
+  if (m_gender_dependent):
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="dev", purposes='probe', gender = gender_value))
+  else:
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="dev", purposes='probe'))
+  
+  files = sort_by_pathes(files)
+  
+  clients =sort_by_ids(db.clients(protocol=m_protocol_name, groups="dev")) 
+  
+  known = set()
+  file_list = open(dev_dir+'/for_scores.lst', 'w')
+  #file_list = open(dev_dir+'/trials.lst', 'w')
+  
+  for file in files:
+    if file.path not in known and not known.add(file.path):
+      for c in clients:
+        file_list.write(file.path + ' ' + (str(c.id)).zfill(3) + ' ' + (str(c.id)).zfill(3) + ' ' + (str(file.client_id)).zfill(3) +'\n') 
+  file_list.close() 
+  
+
+  ### List of EVAL.clients (eval/for_models.lst) group
+  if (m_gender_dependent):
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="eval", purposes='enrol', gender = gender_value))
+  else:
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="eval", purposes='enrol'))
+  
+  files = sort_by_pathes(files)
+  known = set()
+  file_list = open(eval_dir+'/for_models.lst', 'w')
+  
+  for file in files:
+    if file.path not in known and not known.add(file.path):
+      
+      file_list.write(file.path + ' ' + (str(file.client_id)).zfill(3) + ' ' + (str(file.client_id)).zfill(3) + '\n') 
+  file_list.close() 
+    
+  ### List of EVAL.trials (eval/for_scores.lst) group
+  if (m_gender_dependent):
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="eval", purposes='probe', gender = gender_value))
+  else:
+    files = sort_by_ids(db.objects(protocol=m_protocol_name, groups="eval", purposes='probe'))
+  
+  files = sort_by_pathes(files)
+  clients =sort_by_ids(db.clients(protocol=m_protocol_name, groups="eval"))
+  known = set()
+  file_list = open(eval_dir+'/for_scores.lst', 'w')
+  #file_list = open(eval_dir+'/trials.lst', 'w')
+  
+  for file in files:
+    if file.path not in known and not known.add(file.path):
+      for c in clients:
+        file_list.write(file.path + ' ' + (str(c.id)).zfill(3) + ' ' + (str(c.id)).zfill(3) + ' ' + (str(file.client_id)).zfill(3) +'\n') 
+  file_list.close() 
+  
+  
+        
+if __name__ == "__main__":
+  main()  
+

xbob/db/mobio/models.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
 # Laurent El Shafey <laurent.el-shafey@idiap.ch>
+#
+# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 """Table models and functionality for the Mobio database.
 """
@@ -84,20 +98,27 @@ class TModel(Base):
 
   __tablename__ = 'tmodel'
 
-  id = Column(String(9), primary_key=True) # Overhead in size is negligible
+  # Unique identifier for this TModel object
+  id = Column(Integer, primary_key=True)
+  # Model id (only unique for a given protocol)
+  mid = Column(String(9))
   client_id = Column(Integer, ForeignKey('client.id')) # for SQL
+  protocol_id = Column(Integer, ForeignKey('protocol.id')) # for SQL
 
   # for Python: A direct link to the client
   client = relationship("Client", backref=backref("tmodels", order_by=id))
+  # for Python: A direct link to the protocol
+  protocol = relationship("Protocol", backref=backref("tmodels", order_by=id))
   # for Python: A direct link to the files
   files = relationship("File", secondary=tmodel_file_association, backref=backref("tmodels", order_by=id))
 
-  def __init__(self, mid, client_id):
-    self.id = mid
+  def __init__(self, mid, client_id, protocol_id):
+    self.mid = mid
     self.client_id = client_id
+    self.protocol_id = protocol_id
 
   def __repr__(self):
-    return "TModel('%s')" % self.id
+    return "TModel('%s', '%s')" % (self.mid, self.protocol_id)
 
 class File(Base, xbob.db.verification.utils.File):
   """Generic file container"""
@@ -150,12 +171,15 @@ class Protocol(Base):
   id = Column(Integer, primary_key=True)
   # Name of the protocol associated with this object
   name = Column(String(20), unique=True)
+  gender_choices = ('female','male')
+  gender = Column(Enum(*gender_choices))
 
-  def __init__(self, name):
+  def __init__(self, name, gender):
     self.name = name
+    self.gender = gender
 
   def __repr__(self):
-    return "Protocol('%s')" % (self.name,)
+    return "Protocol('%s','%s')" % (self.name, self.gender)
 
 class ProtocolPurpose(Base):
   """MOBIO protocol purposes"""