Re-organized metadata and procedure to upload/download it

MANIFEST.in

 include README.rst bootstrap-buildout.py buildout.cfg develop.cfg LICENSE version.txt requirements.txt
-recursive-include bob/db/hci_tagging *.csv *.txt *.hdf5
+recursive-include bob/db/hci_tagging *.csv *.txt *.hdf5 *.face

bob/db/hci_tagging/__init__.py

@@ -46,8 +46,10 @@ class Database(object):
 
     """
 
+    proto_basedir = os.path.join('data', 'protocols')
+
     if protocol in ('cvpr14',):
-      d = resource_filename(__name__, os.path.join('protocols/cvpr14', 'li_samples_cvpr14.txt'))
+      d = resource_filename(__name__, os.path.join(proto_basedir, 'cvpr14', 'li_samples_cvpr14.txt'))
       with open(d, 'rt') as f: sessions = f.read().split()
       return [File(**k) for k in self.metadata if k['basedir'] in sessions]
 
@@ -58,15 +60,15 @@ class Database(object):
       else:
         files = []
         if 'train' in subset:
-          d = resource_filename(__name__, os.path.join('protocols/all', 'train.txt'))
+          d = resource_filename(__name__, os.path.join(proto_basedir, 'all', 'train.txt'))
           with open(d, 'rt') as f: sessions = f.read().split()
           files += [File(**k) for k in self.metadata if k['basedir'] in sessions]
         if 'dev' in subset:
-          d = resource_filename(__name__, os.path.join('protocols/all', 'dev.txt'))
+          d = resource_filename(__name__, os.path.join(proto_basedir, 'all', 'dev.txt'))
           with open(d, 'rt') as f: sessions = f.read().split()
           files += [File(**k) for k in self.metadata if k['basedir'] in sessions]
         if 'test' in subset:
-          d = resource_filename(__name__, os.path.join('protocols/all', 'test.txt'))
+          d = resource_filename(__name__, os.path.join(proto_basedir, 'all', 'test.txt'))
           with open(d, 'rt') as f: sessions = f.read().split()
           files += [File(**k) for k in self.metadata if k['basedir'] in sessions]
 

bob/db/hci_tagging/driver.py

@@ -46,6 +46,12 @@ def dumplist(args):
 def create_meta(args):
   """Runs the face detection, heart-rate estimation, save outputs at package"""
 
+  if not args.force:
+    raise RuntimeError("This method will re-write the internal HDF5 files, " \
+        "which contain vital metadata used for generating results." \
+        " Make sure this is what you want to do reading the API for this " \
+        "package first (special attention to the method " \
+        ":py:meth:`File.run_face_detector`).")
 
   from . import Database
   db = Database()
@@ -57,15 +63,15 @@ def create_meta(args):
     objects = objects[:args.limit]
 
   if args.grid_count:
-    print len(objects)
+    print(len(objects))
     sys.exit(0)
 
   # if we are on a grid environment, just find what I have to process.
   if os.environ.has_key('SGE_TASK_ID'):
     pos = int(os.environ['SGE_TASK_ID']) - 1
     if pos >= len(objects):
-      raise RuntimeError, "Grid request for job %d on a setup with %d jobs" % \
-          (pos, len(objects))
+      raise RuntimeError("Grid request for job %d on a setup with %d jobs" % \
+          (pos, len(objects)))
     objects = [objects[pos]]
 
   if args.selftest:
@@ -76,10 +82,10 @@ def create_meta(args):
   for obj in objects:
     output = obj.make_path(basedir, '.hdf5')
     if os.path.exists(output) and not args.force:
-      print "Skipping `%s' (meta file exists)" % obj.make_path()
+      print("Skipping `%s' (meta file exists)" % obj.make_path())
       continue
     try:
-      print "Creating meta data for `%s'..." % obj.make_path()
+      print("Creating meta data for `%s'..." % obj.make_path())
       bb = obj.run_face_detector(args.directory, max_frames=1)[0]
       hr = obj.estimate_heartrate_in_bpm(args.directory)
       if bb and hr:
@@ -97,12 +103,12 @@ def create_meta(args):
         h5.set_attribute('units', 'beats-per-minute', 'heartrate')
         h5.close()
       else:
-        print "Skipping `%s': Missing Bounding box and/or Heart-rate" % (obj.stem,)
-        print " -> Bounding box: %s" % bb
-        print " -> Heart-rate  : %s" % hr
+        print("Skipping `%s': Missing Bounding box and/or Heart-rate" % (obj.stem,))
+        print(" -> Bounding box: %s" % bb)
+        print(" -> Heart-rate  : %s" % hr)
 
     except IOError as e:
-      print "Skipping `%s': %s" % (obj.stem, str(e))
+      print("Skipping `%s': %s" % (obj.stem, str(e)))
       continue
 
     finally:
@@ -128,35 +134,35 @@ def debug(args):
     objects = objects[:args.limit]
 
   if args.grid_count:
-    print len(objects)
+    print(len(objects))
     sys.exit(0)
 
   # if we are on a grid environment, just find what I have to process.
   if os.environ.has_key('SGE_TASK_ID'):
     pos = int(os.environ['SGE_TASK_ID']) - 1
     if pos >= len(objects):
-      raise RuntimeError, "Grid request for job %d on a setup with %d jobs" % \
-          (pos, len(objects))
+      raise RuntimeError("Grid request for job %d on a setup with %d jobs" % \
+          (pos, len(objects)))
     objects = [objects[pos]]
 
   basedir = 'debug'
 
   for obj in objects:
-    print "Creating debug data for `%s'..." % obj.make_path()
+    print("Creating debug data for `%s'..." % obj.make_path())
     try:
 
       detections = obj.run_face_detector(args.directory)
       # save annotated video file
       output = obj.make_path(args.output_directory, '.avi')
-      print "Annotating video `%s'" % output
+      print("Annotating video `%s'" % output)
       utils.annotate_video(obj.load_video(args.directory), detections, output)
 
-      print "Annotating heart-rate `%s'" % output
+      print("Annotating heart-rate `%s'" % output)
       output = obj.make_path(args.output_directory, '.pdf')
       utils.explain_heartrate(obj, args.directory, output)
 
     except IOError as e:
-      print "Skipping `%s': %s" % (obj.stem, str(e))
+      print("Skipping `%s': %s" % (obj.stem, str(e)))
       continue
 
     finally:
@@ -222,7 +228,9 @@ def upload(arguments):
   # compress
   import tarfile
   f = tarfile.open(target_file, 'w:bz2')
-  for n,p in zip(names, paths): f.add(p, n)
+  for k,(n,p) in enumerate(zip(names, paths)):
+    print("+ [%d/%d] %s" % (k+1, len(names), n))
+    f.add(p, n)
   f.close()
 
   # set permissions for sane Idiap storage

bob/db/hci_tagging/models.py

 #!/usr/bin/env python
 # vim: set fileencoding=utf-8 :
-# Andre Anjos <andre.anjos@idiap.ch>
-# Wed 30 Sep 2015 12:13:47 CEST
 
 import os
 import pkg_resources
@@ -90,6 +88,10 @@ class File(bob.db.base.File):
     """
 
     path = os.path.join(directory, self.basedir, self.video_stem + '.avi')
+
+    if not os.path.exists(path):
+      raise IOError("Video file `%s' is not available - have you downloaded the database raw files from the original site?" % (path,))
+
     return bob.io.base.load(path)
 
 
@@ -109,6 +111,10 @@ class File(bob.db.base.File):
     """
 
     path = os.path.join(directory, self.basedir, self.video_stem + '.avi')
+
+    if not os.path.exists(path):
+      raise IOError("Video file `%s' is not available - have you downloaded the database raw files from the original site?" % (path,))
+
     return bob.io.video.reader(path)
 
 
@@ -119,9 +125,9 @@ class File(bob.db.base.File):
 
        This method is deprecated and serves only as a development basis to
        clean-up the :py:meth:`load_face_detection`, which for now relies on
-       HDF5 files shipped with this package. Technically, the output of this
-       method and the detected faces shipped should be the same as of today,
-       13 december 2016.
+       ``.face`` files shipped with this package. Technically, the output of
+       this method and the detected faces shipped should be the same as of
+       today, 13 december 2016.
 
 
     Parameters:
@@ -167,22 +173,22 @@ class File(bob.db.base.File):
 
     """
 
-    data_dir = pkg_resources.resource_filename(__name__, 'data')
-    path = self.make_path(data_dir, '.hdf5')
-
-    if os.path.exists(path):
-      f = bob.io.base.HDF5File(path)
-      f.cd('face_detector')
-
-      detections = {}
-      for k,(y,x,h,w) in enumerate(zip(f.get('topleft_y'), f.get('topleft_x'),
-          f.get('height'), f.get('width'))):
-        detections[k] = bob.ip.facedetect.BoundingBox(topleft=(y,x),
-            size=(h,w))
+    basedir = os.path.join('data', 'bbox')
+    data_dir = pkg_resources.resource_filename(__name__, basedir)
+    path = self.make_path(data_dir, '.face')
 
-      return detections
+    if not os.path.exists(path):
+      raise IOError("Face bounding-box file `%s' is not available - have you ran the metadata generation step or `./bin/bob_dbmanage.py hci_tagging download'?" % (path,))
 
-    return None
+    retval = {}
+    with open(path, 'rt') as f:
+      for row in f:
+        if not row.strip(): continue
+        p = row.split()
+        # .face file: <frame> <x> <y> <width> <height>
+        # BoundingBox ctor: top left (y, x), size (height, width)
+        retval[int(p[0])] = bob.ip.facedetect.BoundingBox((float(p[2]), float(p[1])), (float(p[4]), float(p[3])))
+    return retval
 
 
   def estimate_heartrate_in_bpm(self, directory):
@@ -207,22 +213,21 @@ class File(bob.db.base.File):
 
 
   def load_heart_rate_in_bpm(self):
-    """Loads the heart-rate from locally stored files if they exist, fails
-    gracefully otherwise, returning `None`"""
+    """Loads heart-rate from locally stored files, raises if it isn't there"""
 
     data_dir = pkg_resources.resource_filename(__name__, 'data')
     path = self.make_path(data_dir, '.hdf5')
 
-    if os.path.exists(path):
-      f = bob.io.base.HDF5File(path)
-      return f.get('heartrate')
+    if not os.path.exists(path):
+      raise IOError("Metadata file `%s' is not available - have you ran the metadata generation step or `./bin/bob_dbmanage.py hci_tagging download'?" % (path,))
 
-    return None
+    f = bob.io.base.HDF5File(path)
+    return f.get('heartrate')
 
 
   def load_drmf_keypoints(self):
     """Loads the 66-keypoints coming from the Discriminative Response Map
-    Fitting (DRMF) landmark detector.
+    Fitting (DRMF) landmark detector. Raises if metadata file isn't there.
 
     Reference: http://ibug.doc.ic.ac.uk/resources/drmf-matlab-code-cvpr-2013/.
 
@@ -239,11 +244,11 @@ class File(bob.db.base.File):
     data_dir = pkg_resources.resource_filename(__name__, 'data')
     path = self.make_path(data_dir, '.hdf5')
 
-    if os.path.exists(path):
-      f = bob.io.base.HDF5File(path)
-      return f.get('drmf_landmarks66')
+    if not os.path.exists(path):
+      raise IOError("Metadata file `%s' is not available - have you ran the metadata generation step or `./bin/bob_dbmanage.py hci_tagging download'?" % (path,))
 
-    return None
+    f = bob.io.base.HDF5File(path)
+    return f.get('drmf_landmarks66')
 
 
   def save(self, data, directory=None, extension='.hdf5'):

bob/db/hci_tagging/protocols/all/dev.txt

-Sessions/2331
-Sessions/1546
-Sessions/2273
-Sessions/2215
-Sessions/1476
-Sessions/2927
-Sessions/2304
-Sessions/1719
-Sessions/2212
-Sessions/2526
-Sessions/665
-Sessions/1811
-Sessions/715
-Sessions/2542
-Sessions/1735
-Sessions/3422
-Sessions/2623
-Sessions/2661
-Sessions/3461
-Sessions/3426
-Sessions/3401
-Sessions/3447
-Sessions/2555
-Sessions/2932
-Sessions/1694
-Sessions/2620
-Sessions/2577
-Sessions/2915
-Sessions/2971
-Sessions/3389
-Sessions/2885
-Sessions/1795
-Sessions/3446
-Sessions/1474
-Sessions/666
-Sessions/269
-Sessions/3507
-Sessions/381
-Sessions/3472
-Sessions/3490
-Sessions/3404
-Sessions/370
-Sessions/3464
-Sessions/2710
-Sessions/2900
-Sessions/3413
-Sessions/2322
-Sessions/1705
-Sessions/2508
-Sessions/2500
-Sessions/2223
-Sessions/2557
-Sessions/2296
-Sessions/2239
-Sessions/3400
-Sessions/2877
-Sessions/1745
-Sessions/3495
-Sessions/2630
-Sessions/2217
-Sessions/2232
-Sessions/1696
-Sessions/2929
-Sessions/1541
-Sessions/1510
-Sessions/2255
-Sessions/3407
-Sessions/1817
-Sessions/3497
-Sessions/2652
-Sessions/672
-Sessions/2943
-Sessions/741
-Sessions/2643
-Sessions/308
-Sessions/3405
-Sessions/1698
-Sessions/678
-Sessions/767
-Sessions/3488
-Sessions/744
-Sessions/2524
-Sessions/1801
-Sessions/373
-Sessions/1473
-Sessions/2327
-Sessions/2607
-Sessions/2516
-Sessions/2260
-Sessions/1481
-Sessions/3387
-Sessions/686
-Sessions/3425
-Sessions/700
-Sessions/2274
-Sessions/2572
-Sessions/1764
-Sessions/2913
-Sessions/323
-Sessions/3496
-Sessions/758
-Sessions/1797
-Sessions/2648
-Sessions/2963
-Sessions/1755
-Sessions/3441
-Sessions/2579
-Sessions/306
-Sessions/2525
-Sessions/2224
-Sessions/2926
-Sessions/729
-Sessions/1715
-Sessions/1722
-Sessions/2872
-Sessions/3477
-Sessions/1518
-Sessions/1767
-Sessions/683
-Sessions/2564
-Sessions/3437
-Sessions/1548
-Sessions/3409
-Sessions/2921
-Sessions/2471
-Sessions/2323
-Sessions/774
-Sessions/2576
-Sessions/367
-Sessions/309
-Sessions/685
-Sessions/2241
-Sessions/329
-Sessions/3486
-Sessions/2983
-Sessions/713
-Sessions/2267
-Sessions/2514
-Sessions/2868
-Sessions/1787
-Sessions/2229
-Sessions/733
-Sessions/2226
-Sessions/2681
-Sessions/1768
-Sessions/354
-Sessions/761
-Sessions/2916
-Sessions/3424
-Sessions/2603
-Sessions/2912
-Sessions/3454
-Sessions/2625
-Sessions/2946
-Sessions/2871
-Sessions/1699
-Sessions/2242
-Sessions/2891
-Sessions/2722
-Sessions/2582
-Sessions/2256
-Sessions/1781
-Sessions/1765
-Sessions/3428
-Sessions/2332
-Sessions/3393
-Sessions/2549
-Sessions/2496
-Sessions/766
-Sessions/727
-Sessions/290
-Sessions/316
-Sessions/2684
-Sessions/765
-Sessions/2980
-Sessions/3423
-Sessions/705
-Sessions/775
-Sessions/2262
-Sessions/2636
-Sessions/294
-Sessions/2952
-Sessions/2711
-Sessions/753
-Sessions/2218
-Sessions/2567
-Sessions/768
-Sessions/2532
-Sessions/2686
-Sessions/2278
-Sessions/1508
-Sessions/1488
-Sessions/2254
-Sessions/3438
-Sessions/2505
-Sessions/2632
-Sessions/2597
-Sessions/3386
-Sessions/1549
-Sessions/3443
-Sessions/736
-Sessions/2919
-Sessions/1502
-Sessions/1552
-Sessions/2494
-Sessions/2513
-Sessions/2477
-Sessions/3398
-Sessions/655
-Sessions/695
-Sessions/2935
-Sessions/3430
-Sessions/3452
-Sessions/2729
-Sessions/2486
-Sessions/2244
-Sessions/708
-Sessions/764
-Sessions/2981
-Sessions/2970
-Sessions/334
-Sessions/1535
-Sessions/1498
-Sessions/771
-Sessions/2658
-Sessions/690
-Sessions/2214
-Sessions/3403
-Sessions/2482
-Sessions/2288
-Sessions/3494
-Sessions/3462
-Sessions/2895
-Sessions/724
-Sessions/2485
-Sessions/2923
-Sessions/2593
-Sessions/1702
-Sessions/2967
-Sessions/2893
-Sessions/706
-Sessions/716
-Sessions/3394
-Sessions/1714
-Sessions/317
-Sessions/320
-Sessions/2672
-Sessions/1707
-Sessions/657
-Sessions/1736
-Sessions/336
-Sessions/2707
-Sessions/2474
-Sessions/2640
-Sessions/1520
-Sessions/748
-Sessions/697
-Sessions/291
-Sessions/772
-Sessions/756
-Sessions/1782
-Sessions/1775
-Sessions/277
-Sessions/313
-Sessions/3471
-Sessions/275
-Sessions/746
-Sessions/3508
-Sessions/2928
-Sessions/2314
-Sessions/734
-Sessions/2938
-Sessions/274
-Sessions/340
-Sessions/351
-Sessions/2329
-Sessions/2892
-Sessions/2595
-Sessions/2889
-Sessions/3460
-Sessions/377
-Sessions/2924
-Sessions/2951
-Sessions/2675
-Sessions/696
-Sessions/344
-Sessions/3475
-Sessions/2556
-Sessions/2610
-Sessions/282
-Sessions/755
-Sessions/372
-Sessions/688
-Sessions/704
-Sessions/279
-Sessions/1800
-Sessions/670
-Sessions/1743
-Sessions/283
-Sessions/1747
-Sessions/3470
-Sessions/2627
-Sessions/662
-Sessions/2674
-Sessions/2504
-Sessions/335
-Sessions/1806
-Sessions/358
-Sessions/779
-Sessions/2642
-Sessions/1809
-Sessions/273
-Sessions/2324
-Sessions/2300
-Sessions/1740
-Sessions/3434
-Sessions/2493
-Sessions/2501
-Sessions/363
-Sessions/2979
-Sessions/2491
-Sessions/338
-Sessions/1799
-Sessions/3417
-Sessions/1802
-Sessions/677
-Sessions/2476
-Sessions/2230
-Sessions/3505
-Sessions/2272
-Sessions/1763
-Sessions/2873
-Sessions/2950
-Sessions/2608
-Sessions/2691
-Sessions/2309
-Sessions/3385
-Sessions/357
-Sessions/3501
-Sessions/2568
-Sessions/1556
-Sessions/2228
-Sessions/747
-Sessions/368
-Sessions/1706
-Sessions/2679
-Sessions/2718
-Sessions/2248
-Sessions/2937
-Sessions/2968
-Sessions/328