Commit 71770894 authored by André Anjos's avatar André Anjos 💬
Browse files

Re-organized metadata and procedure to upload/download it

parent 44e0aad5
include README.rst bootstrap-buildout.py buildout.cfg develop.cfg LICENSE version.txt requirements.txt
recursive-include bob/db/hci_tagging *.csv *.txt *.hdf5
recursive-include bob/db/hci_tagging *.csv *.txt *.hdf5 *.face
......@@ -46,8 +46,10 @@ class Database(object):
"""
proto_basedir = os.path.join('data', 'protocols')
if protocol in ('cvpr14',):
d = resource_filename(__name__, os.path.join('protocols/cvpr14', 'li_samples_cvpr14.txt'))
d = resource_filename(__name__, os.path.join(proto_basedir, 'cvpr14', 'li_samples_cvpr14.txt'))
with open(d, 'rt') as f: sessions = f.read().split()
return [File(**k) for k in self.metadata if k['basedir'] in sessions]
......@@ -58,15 +60,15 @@ class Database(object):
else:
files = []
if 'train' in subset:
d = resource_filename(__name__, os.path.join('protocols/all', 'train.txt'))
d = resource_filename(__name__, os.path.join(proto_basedir, 'all', 'train.txt'))
with open(d, 'rt') as f: sessions = f.read().split()
files += [File(**k) for k in self.metadata if k['basedir'] in sessions]
if 'dev' in subset:
d = resource_filename(__name__, os.path.join('protocols/all', 'dev.txt'))
d = resource_filename(__name__, os.path.join(proto_basedir, 'all', 'dev.txt'))
with open(d, 'rt') as f: sessions = f.read().split()
files += [File(**k) for k in self.metadata if k['basedir'] in sessions]
if 'test' in subset:
d = resource_filename(__name__, os.path.join('protocols/all', 'test.txt'))
d = resource_filename(__name__, os.path.join(proto_basedir, 'all', 'test.txt'))
with open(d, 'rt') as f: sessions = f.read().split()
files += [File(**k) for k in self.metadata if k['basedir'] in sessions]
......
......@@ -46,6 +46,12 @@ def dumplist(args):
def create_meta(args):
"""Runs the face detection, heart-rate estimation, save outputs at package"""
if not args.force:
raise RuntimeError("This method will re-write the internal HDF5 files, " \
"which contain vital metadata used for generating results." \
" Make sure this is what you want to do reading the API for this " \
"package first (special attention to the method " \
":py:meth:`File.run_face_detector`).")
from . import Database
db = Database()
......@@ -57,15 +63,15 @@ def create_meta(args):
objects = objects[:args.limit]
if args.grid_count:
print len(objects)
print(len(objects))
sys.exit(0)
# if we are on a grid environment, just find what I have to process.
if os.environ.has_key('SGE_TASK_ID'):
pos = int(os.environ['SGE_TASK_ID']) - 1
if pos >= len(objects):
raise RuntimeError, "Grid request for job %d on a setup with %d jobs" % \
(pos, len(objects))
raise RuntimeError("Grid request for job %d on a setup with %d jobs" % \
(pos, len(objects)))
objects = [objects[pos]]
if args.selftest:
......@@ -76,10 +82,10 @@ def create_meta(args):
for obj in objects:
output = obj.make_path(basedir, '.hdf5')
if os.path.exists(output) and not args.force:
print "Skipping `%s' (meta file exists)" % obj.make_path()
print("Skipping `%s' (meta file exists)" % obj.make_path())
continue
try:
print "Creating meta data for `%s'..." % obj.make_path()
print("Creating meta data for `%s'..." % obj.make_path())
bb = obj.run_face_detector(args.directory, max_frames=1)[0]
hr = obj.estimate_heartrate_in_bpm(args.directory)
if bb and hr:
......@@ -97,12 +103,12 @@ def create_meta(args):
h5.set_attribute('units', 'beats-per-minute', 'heartrate')
h5.close()
else:
print "Skipping `%s': Missing Bounding box and/or Heart-rate" % (obj.stem,)
print " -> Bounding box: %s" % bb
print " -> Heart-rate : %s" % hr
print("Skipping `%s': Missing Bounding box and/or Heart-rate" % (obj.stem,))
print(" -> Bounding box: %s" % bb)
print(" -> Heart-rate : %s" % hr)
except IOError as e:
print "Skipping `%s': %s" % (obj.stem, str(e))
print("Skipping `%s': %s" % (obj.stem, str(e)))
continue
finally:
......@@ -128,35 +134,35 @@ def debug(args):
objects = objects[:args.limit]
if args.grid_count:
print len(objects)
print(len(objects))
sys.exit(0)
# if we are on a grid environment, just find what I have to process.
if os.environ.has_key('SGE_TASK_ID'):
pos = int(os.environ['SGE_TASK_ID']) - 1
if pos >= len(objects):
raise RuntimeError, "Grid request for job %d on a setup with %d jobs" % \
(pos, len(objects))
raise RuntimeError("Grid request for job %d on a setup with %d jobs" % \
(pos, len(objects)))
objects = [objects[pos]]
basedir = 'debug'
for obj in objects:
print "Creating debug data for `%s'..." % obj.make_path()
print("Creating debug data for `%s'..." % obj.make_path())
try:
detections = obj.run_face_detector(args.directory)
# save annotated video file
output = obj.make_path(args.output_directory, '.avi')
print "Annotating video `%s'" % output
print("Annotating video `%s'" % output)
utils.annotate_video(obj.load_video(args.directory), detections, output)
print "Annotating heart-rate `%s'" % output
print("Annotating heart-rate `%s'" % output)
output = obj.make_path(args.output_directory, '.pdf')
utils.explain_heartrate(obj, args.directory, output)
except IOError as e:
print "Skipping `%s': %s" % (obj.stem, str(e))
print("Skipping `%s': %s" % (obj.stem, str(e)))
continue
finally:
......@@ -222,7 +228,9 @@ def upload(arguments):
# compress
import tarfile
f = tarfile.open(target_file, 'w:bz2')
for n,p in zip(names, paths): f.add(p, n)
for k,(n,p) in enumerate(zip(names, paths)):
print("+ [%d/%d] %s" % (k+1, len(names), n))
f.add(p, n)
f.close()
# set permissions for sane Idiap storage
......
This diff is collapsed.
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# Andre Anjos <andre.anjos@idiap.ch>
# Wed 30 Sep 2015 12:13:47 CEST
import os
import pkg_resources
......@@ -90,6 +88,10 @@ class File(bob.db.base.File):
"""
path = os.path.join(directory, self.basedir, self.video_stem + '.avi')
if not os.path.exists(path):
raise IOError("Video file `%s' is not available - have you downloaded the database raw files from the original site?" % (path,))
return bob.io.base.load(path)
......@@ -109,6 +111,10 @@ class File(bob.db.base.File):
"""
path = os.path.join(directory, self.basedir, self.video_stem + '.avi')
if not os.path.exists(path):
raise IOError("Video file `%s' is not available - have you downloaded the database raw files from the original site?" % (path,))
return bob.io.video.reader(path)
......@@ -119,9 +125,9 @@ class File(bob.db.base.File):
This method is deprecated and serves only as a development basis to
clean-up the :py:meth:`load_face_detection`, which for now relies on
HDF5 files shipped with this package. Technically, the output of this
method and the detected faces shipped should be the same as of today,
13 december 2016.
``.face`` files shipped with this package. Technically, the output of
this method and the detected faces shipped should be the same as of
today, 13 december 2016.
Parameters:
......@@ -167,22 +173,22 @@ class File(bob.db.base.File):
"""
data_dir = pkg_resources.resource_filename(__name__, 'data')
path = self.make_path(data_dir, '.hdf5')
if os.path.exists(path):
f = bob.io.base.HDF5File(path)
f.cd('face_detector')
detections = {}
for k,(y,x,h,w) in enumerate(zip(f.get('topleft_y'), f.get('topleft_x'),
f.get('height'), f.get('width'))):
detections[k] = bob.ip.facedetect.BoundingBox(topleft=(y,x),
size=(h,w))
basedir = os.path.join('data', 'bbox')
data_dir = pkg_resources.resource_filename(__name__, basedir)
path = self.make_path(data_dir, '.face')
return detections
if not os.path.exists(path):
raise IOError("Face bounding-box file `%s' is not available - have you ran the metadata generation step or `./bin/bob_dbmanage.py hci_tagging download'?" % (path,))
return None
retval = {}
with open(path, 'rt') as f:
for row in f:
if not row.strip(): continue
p = row.split()
# .face file: <frame> <x> <y> <width> <height>
# BoundingBox ctor: top left (y, x), size (height, width)
retval[int(p[0])] = bob.ip.facedetect.BoundingBox((float(p[2]), float(p[1])), (float(p[4]), float(p[3])))
return retval
def estimate_heartrate_in_bpm(self, directory):
......@@ -207,22 +213,21 @@ class File(bob.db.base.File):
def load_heart_rate_in_bpm(self):
"""Loads the heart-rate from locally stored files if they exist, fails
gracefully otherwise, returning `None`"""
"""Loads heart-rate from locally stored files, raises if it isn't there"""
data_dir = pkg_resources.resource_filename(__name__, 'data')
path = self.make_path(data_dir, '.hdf5')
if os.path.exists(path):
f = bob.io.base.HDF5File(path)
return f.get('heartrate')
if not os.path.exists(path):
raise IOError("Metadata file `%s' is not available - have you ran the metadata generation step or `./bin/bob_dbmanage.py hci_tagging download'?" % (path,))
return None
f = bob.io.base.HDF5File(path)
return f.get('heartrate')
def load_drmf_keypoints(self):
"""Loads the 66-keypoints coming from the Discriminative Response Map
Fitting (DRMF) landmark detector.
Fitting (DRMF) landmark detector. Raises if metadata file isn't there.
Reference: http://ibug.doc.ic.ac.uk/resources/drmf-matlab-code-cvpr-2013/.
......@@ -239,11 +244,11 @@ class File(bob.db.base.File):
data_dir = pkg_resources.resource_filename(__name__, 'data')
path = self.make_path(data_dir, '.hdf5')
if os.path.exists(path):
f = bob.io.base.HDF5File(path)
return f.get('drmf_landmarks66')
if not os.path.exists(path):
raise IOError("Metadata file `%s' is not available - have you ran the metadata generation step or `./bin/bob_dbmanage.py hci_tagging download'?" % (path,))
return None
f = bob.io.base.HDF5File(path)
return f.get('drmf_landmarks66')
def save(self, data, directory=None, extension='.hdf5'):
......
Sessions/2331
Sessions/1546
Sessions/2273
Sessions/2215
Sessions/1476
Sessions/2927
Sessions/2304
Sessions/1719
Sessions/2212
Sessions/2526
Sessions/665
Sessions/1811
Sessions/715
Sessions/2542
Sessions/1735
Sessions/3422
Sessions/2623
Sessions/2661
Sessions/3461
Sessions/3426
Sessions/3401
Sessions/3447
Sessions/2555
Sessions/2932
Sessions/1694
Sessions/2620
Sessions/2577
Sessions/2915
Sessions/2971
Sessions/3389
Sessions/2885
Sessions/1795
Sessions/3446
Sessions/1474
Sessions/666
Sessions/269
Sessions/3507
Sessions/381
Sessions/3472
Sessions/3490
Sessions/3404
Sessions/370
Sessions/3464
Sessions/2710
Sessions/2900
Sessions/3413
Sessions/2322
Sessions/1705
Sessions/2508
Sessions/2500
Sessions/2223
Sessions/2557
Sessions/2296
Sessions/2239
Sessions/3400
Sessions/2877
Sessions/1745
Sessions/3495
Sessions/2630
Sessions/2217
Sessions/2232
Sessions/1696
Sessions/2929
Sessions/1541
Sessions/1510
Sessions/2255
Sessions/3407
Sessions/1817
Sessions/3497
Sessions/2652
Sessions/672
Sessions/2943
Sessions/741
Sessions/2643
Sessions/308
Sessions/3405
Sessions/1698
Sessions/678
Sessions/767
Sessions/3488
Sessions/744
Sessions/2524
Sessions/1801
Sessions/373
Sessions/1473
Sessions/2327
Sessions/2607
Sessions/2516
Sessions/2260
Sessions/1481
Sessions/3387
Sessions/686
Sessions/3425
Sessions/700
Sessions/2274
Sessions/2572
Sessions/1764
Sessions/2913
Sessions/323
Sessions/3496
Sessions/758
Sessions/1797
Sessions/2648
Sessions/2963
Sessions/1755
Sessions/3441
Sessions/2579
Sessions/306
Sessions/2525
Sessions/2224
Sessions/2926
Sessions/729
Sessions/1715
Sessions/1722
Sessions/2872
Sessions/3477
Sessions/1518
Sessions/1767
Sessions/683
Sessions/2564
Sessions/3437
Sessions/1548
Sessions/3409
Sessions/2921
Sessions/2471
Sessions/2323
Sessions/774
Sessions/2576
Sessions/367
Sessions/309
Sessions/685
Sessions/2241
Sessions/329
Sessions/3486
Sessions/2983
Sessions/713
Sessions/2267
Sessions/2514
Sessions/2868
Sessions/1787
Sessions/2229
Sessions/733
Sessions/2226
Sessions/2681
Sessions/1768
Sessions/354
Sessions/761
Sessions/2916
Sessions/3424
Sessions/2603
Sessions/2912
Sessions/3454
Sessions/2625
Sessions/2946
Sessions/2871
Sessions/1699
Sessions/2242
Sessions/2891
Sessions/2722
Sessions/2582
Sessions/2256
Sessions/1781
Sessions/1765
Sessions/3428
Sessions/2332
Sessions/3393
Sessions/2549
Sessions/2496
Sessions/766
Sessions/727
Sessions/290
Sessions/316
Sessions/2684
Sessions/765
Sessions/2980
Sessions/3423
Sessions/705
Sessions/775
Sessions/2262
Sessions/2636
Sessions/294
Sessions/2952
Sessions/2711
Sessions/753
Sessions/2218
Sessions/2567
Sessions/768
Sessions/2532
Sessions/2686
Sessions/2278
Sessions/1508
Sessions/1488
Sessions/2254
Sessions/3438
Sessions/2505
Sessions/2632
Sessions/2597
Sessions/3386
Sessions/1549
Sessions/3443
Sessions/736
Sessions/2919
Sessions/1502
Sessions/1552
Sessions/2494
Sessions/2513
Sessions/2477
Sessions/3398
Sessions/655
Sessions/695
Sessions/2935
Sessions/3430
Sessions/3452
Sessions/2729
Sessions/2486
Sessions/2244
Sessions/708
Sessions/764
Sessions/2981
Sessions/2970
Sessions/334
Sessions/1535
Sessions/1498
Sessions/771
Sessions/2658
Sessions/690
Sessions/2214
Sessions/3403
Sessions/2482
Sessions/2288
Sessions/3494
Sessions/3462
Sessions/2895
Sessions/724
Sessions/2485
Sessions/2923
Sessions/2593
Sessions/1702
Sessions/2967
Sessions/2893
Sessions/706
Sessions/716
Sessions/3394
Sessions/1714
Sessions/317
Sessions/320
Sessions/2672
Sessions/1707
Sessions/657
Sessions/1736
Sessions/336
Sessions/2707
Sessions/2474
Sessions/2640
Sessions/1520
Sessions/748
Sessions/697
Sessions/291
Sessions/772
Sessions/756
Sessions/1782
Sessions/1775
Sessions/277
Sessions/313
Sessions/3471
Sessions/275
Sessions/746
Sessions/3508
Sessions/2928
Sessions/2314
Sessions/734
Sessions/2938
Sessions/274
Sessions/340
Sessions/351
Sessions/2329
Sessions/2892
Sessions/2595
Sessions/2889
Sessions/3460
Sessions/377
Sessions/2924
Sessions/2951
Sessions/2675
Sessions/696
Sessions/344
Sessions/3475
Sessions/2556
Sessions/2610
Sessions/282
Sessions/755
Sessions/372
Sessions/688
Sessions/704
Sessions/279
Sessions/1800
Sessions/670
Sessions/1743
Sessions/283
Sessions/1747
Sessions/3470
Sessions/2627
Sessions/662
Sessions/2674
Sessions/2504
Sessions/335
Sessions/1806
Sessions/358
Sessions/779
Sessions/2642
Sessions/1809
Sessions/273
Sessions/2324
Sessions/2300
Sessions/1740
Sessions/3434
Sessions/2493
Sessions/2501
Sessions/363
Sessions/2979
Sessions/2491
Sessions/338
Sessions/1799
Sessions/3417
Sessions/1802
Sessions/677
Sessions/2476
Sessions/2230
Sessions/3505
Sessions/2272
Sessions/1763
Sessions/2873
Sessions/2950
Sessions/2608
Sessions/2691
Sessions/2309
Sessions/3385
Sessions/357
Sessions/3501
Sessions/2568
Sessions/1556
Sessions/2228
Sessions/747
Sessions/368
Sessions/1706
Sessions/2679
Sessions/2718
Sessions/2248
Sessions/2937
Sessions/2968
Sessions/328