Move metadata into separate tarball

.gitignore

@@ -18,3 +18,4 @@ build
 *.egg
 src/
 db.sql3
+bob/db/3dfv/data/

MANIFEST.in

 include README.rst bootstrap-buildout.py buildout.cfg develop.cfg LICENSE version.txt requirements.txt
 recursive-include doc *.py *.rst
-recursive-include bob *.sql3 *.txt.gz
+recursive-include bob *.csv *.sql3 *.txt

bob/db/3dfv/create.py

@@ -7,7 +7,6 @@
 import os
 import re
 import csv
-import gzip
 import pkg_resources
 
 from .models import *
@@ -69,8 +68,8 @@ def add_files(db_session, verbose):
   dir_re = re.compile(r'^(?P<id>\d{3})$')
 
   fname = pkg_resources.resource_filename(__name__, os.path.join('data',
-    'files.txt.gz'))
-  with gzip.open(fname, 'rb') as flist:
+    'files.txt'))
+  with open(fname, 'rb') as flist:
     for f in flist:
       info = try_get_metadata(f.strip().decode())
 
@@ -125,8 +124,8 @@ def add_protocols(session, verbose):
       print("Created %s" % protocol)
 
     # training data
-    train_filename = os.path.join(protocol_dir, name, 'train.txt.gz')
-    with gzip.open(train_filename, 'rb') as f:
+    train_filename = os.path.join(protocol_dir, name, 'train.txt')
+    with open(train_filename, 'rb') as f:
       for filename in f:
         file_ = retrieve_file(session, filename.strip().decode())
         protocol.training_set.append(file_)
@@ -134,8 +133,8 @@ def add_protocols(session, verbose):
           print("Added %s to %s" % (file_, protocol))
 
     # enrollment data
-    models_filename = os.path.join(protocol_dir, name, 'dev-models.txt.gz')
-    with gzip.open(models_filename, 'rb') as f:
+    models_filename = os.path.join(protocol_dir, name, 'dev-models.txt')
+    with open(models_filename, 'rb') as f:
       for row in f:
         filename, model_ref = row.split()
         file_ = retrieve_file(session, filename.decode())
@@ -149,8 +148,8 @@ def add_protocols(session, verbose):
           print("Added %s to %s" % (file_, model))
 
     # probing data
-    probes_filename = os.path.join(protocol_dir, name, 'dev-probes.txt.gz')
-    with gzip.open(probes_filename, 'rb') as f:
+    probes_filename = os.path.join(protocol_dir, name, 'dev-probes.txt')
+    with open(probes_filename, 'rb') as f:
       for filename in f:
         file_ = retrieve_file(session, filename.decode())
         probe = Probe('dev', protocol, file_)

bob/db/3dfv/data/metadata.csv

-Id,Age,Gender,Skin Color,Occupation
-1,48,f,2,x
-2,24,f,3,x
-3,54,f,2,x
-4,49,f,3,x
-5,48,m,3,x
-6,18,m,3,x
-7,36,m,1,x
-8,69,f,2,x
-9,69,m,2,x
-10,26,m,x,x
-11,26,m,3,2
-12,45,m,3,x
-13,24,m,2,x
-14,24,f,3,x
-15,72,m,1,x
-16,25,f,2,x
-17,65,f,x,x
-18,50,m,x,x
-19,64,f,1,x
-21,40,m,4,1
-22,58,f,1,1
-23,39,m,2,1
-24,68,m,2,2
-25,72,f,2,9
-26,68,f,2,8
-27,49,f,5,3
-28,27,m,3,2
-29,25,m,3,0
-30,72,f,2,1
-31,73,f,2,2
-32,53,f,2,2
-33,59,m,2,4
-34,60,f,2,3
-35,57,f,2,3
-36,48,m,3,7
-37,57,f,2,9
-38,27,m,2,2
-41,29,f,2,x
-42,28,f,4,x
-43,35,m,2,1
-44,26,f,2,1
-46,34,f,2,9
-47,30,f,1,2
-48,68,m,3,2
-49,65,f,3,2
-50,27,m,3,7
-51,28,f,2,7
-53,26,f,2,2
-54,57,f,2,2
-55,56,m,2,8
-56,57,m,2,2
-57,54,f,3,4
-58,52,f,3,9
-59,47,f,3,3
-60,47,m,3,2
-61,27,f,2,3
-62,29,m,3,3
-63,42,f,2,3
-64,84,m,3,8
-65,47,f,3,9
-66,36,f,3,4
-67,35,m,3,2
-68,25,f,5,2
-69,25,f,3,2
-70,25,f,2,3
-71,56,f,2,3
-72,57,f,2,3
-73,63,m,2,3
-74,56,f,2,2
-75,57,f,2,3
-76,57,f,2,3
-77,44,f,2,4
-78,52,f,2,9
-79,63,m,2,3
-80,65,f,2,4
-81,36,m,2,2
-82,29,f,2,2
-83,29,m,2,2
-84,23,m,2,2
-85,24,f,4,4
-86,39,f,4,1
-87,31,f,2,2
-88,36,m,6,4
-89,31,f,2,3
-90,28,m,3,2
-91,52,m,2,2
-92,30,m,2,8
-93,35,m,2,2
-94,28,m,2,8
-101,28,m,5,2
-102,36,m,3,2
-103,27,m,2,2
-104,31,m,4,2
-105,37,m,3,2
-106,43,m,3,2
-107,53,m,3,3
-108,27,f,2,2
-109,24,m,3,2
-110,32,f,3,1
-111,25,f,3,2
-112,25,f,2,2
-113,33,m,2,2
-114,40,m,2,2
-115,25,m,2,2
-116,29,f,2,1
-117,23,f,2,1
-118,27,m,2,1
-119,30,m,5,2
-120,27,m,4,2
-121,31,m,4,2
-122,45,f,3,1
-123,60,f,3,3
-124,26,f,4,2
-125,49,m,3,3
-126,28,m,3,2
-127,63,m,2,3
-128,33,m,3,2
-129,21,f,2,2
-130,29,m,4,2
-131,58,m,3,2
-132,39,m,2,2
-133,31,m,5,2
-134,27,m,2,2
-135,20,m,3,4
-136,55,f,3,1
-137,50,m,2,2
-138,35,m,2,2
-139,54,m,3,2
-140,32,m,5,2
-141,23,f,5,9
-142,25,f,2,9
-143,26,m,3,2
-144,26,f,2,1
-145,36,f,3,2
-146,63,m,2,1
-147,50,m,2,2
-148,29,f,4,2
-149,28,f,3,2
-150,33,f,3,2
-152,67,f,2,4
-153,67,m,3,7

bob/db/3dfv/driver.py

@@ -6,6 +6,7 @@
 
 import os
 import sys
+import pkg_resources
 from bob.db.base.driver import Interface as BaseInterface
 
 
@@ -67,22 +68,27 @@ def checkfiles(args):
 
 class Interface(BaseInterface):
 
+
   def name(self):
     return '3dfv'
 
+
   def version(self):
     import pkg_resources  # part of setuptools
     return pkg_resources.require('bob.db.%s' % self.name())[0].version
 
+
   def files(self):
+    basedir = pkg_resources.resource_filename(__name__, '')
+    filelist = os.path.join(basedir, 'files.txt')
+    return [os.path.join(basedir, k.strip()) for k in \
+        open(filelist, 'rt').readlines() if k.strip()]
 
-    from pkg_resources import resource_filename
-    raw_files = ('db.sql3',)
-    return [resource_filename(__name__, k) for k in raw_files]
 
   def type(self):
     return 'sqlite'
 
+
   def add_commands(self, parser):
 
     from . import __doc__ as docs

bob/db/3dfv/files.txt

+db.sql3
+data/files.txt
+data/metadata.csv
+data/protocols/central/train.txt
+data/protocols/central/dev-probes.txt
+data/protocols/central/dev-models.txt