diff --git a/README.rst b/README.rst index 17923360b1d4b2f4b323ee318ec11ab779c453d6..0b20abf90901481f69597ac63ee03b334d240cca 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,7 @@ It includes examples with three different complexities: * An example building an UBM/GMM model on top of DCT blocks. The face verification experiments are executed on the `AT&T`_ database, which is a toy database that is perfectly useful for this example, but **not** to publish scientific papers. +Anyways, the functionality inside this package will work with any other face verification database of Bob_. Installation diff --git a/bob/example/faceverify/__init__.py b/bob/example/faceverify/__init__.py index 7fd08b7b53bef2b57cc3729801185b8473af3cf3..511b76c203d5b7cfca2acc2e289b212652cdd80f 100644 --- a/bob/example/faceverify/__init__.py +++ b/bob/example/faceverify/__init__.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from .utils import load_images + def get_config(): """Returns a string containing the configuration information. """ diff --git a/bob/example/faceverify/data/dct_feature.hdf5 b/bob/example/faceverify/data/dct_feature.hdf5 index 2e43ff74cc80ca16b7165a8df3e1f2e9149fedd8..794c9e62def9341183a53c5f571237f5614d98d0 100644 Binary files a/bob/example/faceverify/data/dct_feature.hdf5 and b/bob/example/faceverify/data/dct_feature.hdf5 differ diff --git a/bob/example/faceverify/data/gabor_model.hdf5 b/bob/example/faceverify/data/gabor_model.hdf5 index 3244215773b159c03e769604cb106a3ca96dd64e..afda99bb96e249e4dd1ab2dab3e4d4d631e522ae 100644 Binary files a/bob/example/faceverify/data/gabor_model.hdf5 and b/bob/example/faceverify/data/gabor_model.hdf5 differ diff --git a/bob/example/faceverify/data/gabor_probe.hdf5 b/bob/example/faceverify/data/gabor_probe.hdf5 index a6be2757df123f9c5f2724673da566fbf2464fb0..4d1b87881507b17e7b7cb3e209760e00abc2ff29 100644 Binary files a/bob/example/faceverify/data/gabor_probe.hdf5 and b/bob/example/faceverify/data/gabor_probe.hdf5 differ diff --git a/bob/example/faceverify/data/pca_model.hdf5 b/bob/example/faceverify/data/pca_model.hdf5 index 245a5b6507b79f5508b60e3f6b0c92253198a7f8..1bfcca4a24aa3dc851ea6174c1eb53802149d69f 100644 Binary files a/bob/example/faceverify/data/pca_model.hdf5 and b/bob/example/faceverify/data/pca_model.hdf5 differ diff --git a/bob/example/faceverify/data/pca_probe.hdf5 b/bob/example/faceverify/data/pca_probe.hdf5 index 6509e8bd55c5c8c9ac9402a1074dacdcba5b200f..525d3544b5c4b9e3435bed17dd08611266637f3c 100644 Binary files a/bob/example/faceverify/data/pca_probe.hdf5 and b/bob/example/faceverify/data/pca_probe.hdf5 differ diff --git a/bob/example/faceverify/data/pca_projector.hdf5 b/bob/example/faceverify/data/pca_projector.hdf5 index e4dfd6dd78cff450f05c0f09a80600c52cb25adc..9f2ac87626b72815f2775a16f7f7e2cd4adbbf7b 100644 Binary files a/bob/example/faceverify/data/pca_projector.hdf5 and b/bob/example/faceverify/data/pca_projector.hdf5 differ diff --git a/bob/example/faceverify/dct_ubm.py b/bob/example/faceverify/dct_ubm.py index e18628857d146b24456146cd7280b5cbbca0408f..53a2dbe7ea598a72d2ab03412d42dbfe13ef50c4 100644 --- a/bob/example/faceverify/dct_ubm.py +++ b/bob/example/faceverify/dct_ubm.py @@ -37,10 +37,10 @@ matplotlib.rc('font', family='serif') matplotlib.rc('lines', linewidth = 4) from matplotlib import pyplot -from .utils import atnt_database_directory +from .utils import atnt_database_directory, load_images -# setup the logging system +# setup the logging system (used by C++ functionality in Bob) import logging formatter = logging.Formatter("%(name)s@%(asctime)s -- %(levelname)s: %(message)s") logger = logging.getLogger("bob") @@ -49,18 +49,6 @@ for handler in logger.handlers: logger.setLevel(logging.INFO) -def load_images(db, group = None, purpose = None, client_id = None, database_directory = None, image_extension = '.pgm'): - """Reads the images for the given group and the given client id from the given database""" - # get the file names from the database - files = db.objects(groups = group, purposes = purpose, model_ids = client_id) - # iterate through the list of file names - images = {} - for k in files: - # load image and linearize it into a vector - images[k.id] = bob.io.base.load(k.make_path(database_directory, image_extension)).astype(numpy.float64) - return images - - # Parameters of the DCT extraction DCT_BLOCK_SIZE = 12 DCT_BLOCK_OVERLAP = 11 @@ -84,7 +72,7 @@ def train(training_features, number_of_gaussians = NUMBER_OF_GAUSSIANS): """Trains the UBM/GMM module with the given set of training DCT features""" # create array set used for training - training_set = numpy.vstack([v for v in training_features.values()]) + training_set = numpy.vstack(training_features) input_size = training_set.shape[1] # create the KMeans and UBM machine @@ -115,7 +103,7 @@ def train(training_features, number_of_gaussians = NUMBER_OF_GAUSSIANS): def enroll(model_features, ubm, gmm_trainer): """Enrolls the GMM model for the given model features (which should stem from the same identity)""" # create array set used for enrolling - enroll_set = numpy.vstack(model_features.values()) + enroll_set = numpy.vstack(model_features) # create a GMM from the UBM gmm = bob.learn.misc.GMMMachine(ubm) @@ -142,81 +130,109 @@ def stats(probe_feature, ubm): def main(): """This function will perform an a DCT block extraction and a UBM/GMM modeling test on the AT&T database""" - # use the bob.db interface to retrieve information about the Database - atnt_db = bob.db.atnt.Database() - # Check the existence of the AT&T database and download it if not # Also check if the AT&T database directory is overwritten by the command line image_directory = atnt_database_directory(sys.argv[1] if len(sys.argv) > 1 else None) + # use the bob.db interface to retrieve information about the Database + db = bob.db.atnt.Database(original_directory = image_directory) + + # The protocol does not exist for AT&T, but to be able to exchange the database, we define it here. + protocol = None + + # The group is 'dev', throughout + group = 'dev' + + # The images of the AT&T database are already cropped, so we don't need to specify a face cropper. + face_cropper = None + # For other databases you might want to use: +# face_cropper = bob.ip.base.FaceEyesNorm(crop_size = (80,64), right_eye = (16,15), left_eye = (16,48)) + + # After image cropping, we apply an image preprocessor + preprocessor = bob.ip.base.TanTriggs() ##################################################################### ### UBM Training # load all training images - training_images = load_images(atnt_db, group = 'world', database_directory = image_directory) + training_files = db.training_files(protocol = protocol) + print("Loading %d training images" % len(training_files)) + training_images = load_images(db, training_files, face_cropper, preprocessor) - print("Extracting training features") - training_features = {} - for key, image in training_images.items(): - training_features[key] = extract_feature(image) + print("Extracting %d training features" % len(training_files)) + training_features = [extract_feature(image) for image in training_images] print("Training UBM model") ubm = train(training_features) ##################################################################### ### GMM model enrollment - print("Enrolling GMM models") gmm_trainer = bob.learn.misc.MAP_GMMTrainer() gmm_trainer.max_iterations = 1 gmm_trainer.set_prior_gmm(ubm) + # enroll a GMM model for each model identity (i.e., each client) - model_ids = [client.id for client in atnt_db.clients(groups = 'dev')] + model_ids = db.model_ids(groups = group) + print("Enrolling %d GMM models" % len(model_ids)) + # generate models for each model ID models = {} for model_id in model_ids: - # load images for the current model id - model_images = load_images(atnt_db, group = 'dev', purpose = 'enroll', client_id = model_id, database_directory = image_directory) - models_for_current_id = {} - # extract model features - for key, image in model_images.items(): - models_for_current_id[key] = extract_feature(image) - # enroll model for the current identity from these features - model = enroll(models_for_current_id, ubm, gmm_trainer) - models[model_id] = model + # load enroll images for the current model ID + enroll_images = load_images(db, db.enroll_files(protocol = protocol, groups = group, model_id = model_id), face_cropper, preprocessor) + # extract features + enroll_features = [extract_feature(enroll_image) for enroll_image in enroll_images] + models[model_id] = enroll(enroll_features, ubm, gmm_trainer) + ##################################################################### ### probe stats - - print("Computing probe statistics") - probe_images = load_images(atnt_db, group = 'dev', purpose = 'probe', database_directory = image_directory) + probe_files = db.probe_files(protocol = protocol, groups = group) + print("Computing %d probe statistics" % len(probe_files)) + probe_images = load_images(db, probe_files, face_cropper, preprocessor) + # extract probe features and store them by probe ID (which is the File.id) probes = {} - for key, image in probe_images.items(): - # extract probe features - probe_feature = extract_feature(image) - # compute GMM statistics - probes[key] = stats(probe_feature, ubm) + for i in range(len(probe_files)): + # provide status information + print("\rProbe", i+1, "of", len(probe_files), end='') + sys.stdout.flush() + + probe_id = probe_files[i].id + probe_feature = extract_feature(probe_images[i]) + probes[probe_id] = stats(probe_feature, ubm) ##################################################################### - ### compute scores, we here choose a simple Euclidean distance measure + ### compute scores, using the linear_scoring function positive_scores = [] negative_scores = [] - print("Computing scores") + print("\nComputing scores") distance_function = bob.learn.misc.linear_scoring # iterate through models and probes and compute scores - for model_id, model_gmm in models.items(): - for probe_key, probe_stats in probes.items(): + model_count = 1 + for model_id, model in models.items(): + # provide status information + print("\rModel", model_count, "of", len(models), end='') + sys.stdout.flush() + model_count += 1 + + # the client ID that is attached to the model + model_client_id = db.get_client_id_from_model_id(model_id) + # get the probe files, which should be compared with this model + model_probe_files = db.probe_files(protocol = protocol, groups = group, model_id = model_id) + for probe_file in model_probe_files: + # get the according probe statistics using the File.id of the probe file + probe_stats = probes[probe_file.id] # compute score - score = distance_function([model_gmm], ubm, [probe_stats])[0,0] + score = distance_function([model], ubm, [probe_stats])[0,0] # check if this is a positive score - if model_id == atnt_db.get_client_id_from_file_id(probe_key): + if model_client_id == probe_file.client_id: positive_scores.append(score) else: negative_scores.append(score) - print("Evaluation") + print("\nEvaluation") # convert list of scores to numpy arrays positives = numpy.array(positive_scores) negatives = numpy.array(negative_scores) diff --git a/bob/example/faceverify/eigenface.py b/bob/example/faceverify/eigenface.py index 6e61b82a4ddc17a9dfa3a155aa02b700603b4028..68cbbc062bfc6cefa67e8f1c4a57bebac6953962 100644 --- a/bob/example/faceverify/eigenface.py +++ b/bob/example/faceverify/eigenface.py @@ -23,6 +23,7 @@ from __future__ import print_function import bob.db.atnt import bob.io.base import bob.io.image +import bob.ip.base import bob.learn.linear import bob.measure @@ -36,19 +37,7 @@ matplotlib.rc('font', family='serif') matplotlib.rc('lines', linewidth = 4) from matplotlib import pyplot -from .utils import atnt_database_directory - - -def load_images(db, group = None, purpose = None, database_directory = None, image_extension = '.pgm'): - """Reads the images for the given group and the given purpose from the given database""" - # get the file names from the database - files = db.objects(groups = group, purposes = purpose) - # iterate through the list of file names - images = {} - for k in files: - # load image - images[k.id] = bob.io.base.load(k.make_path(database_directory, image_extension)).astype(numpy.float64) - return images +from .utils import atnt_database_directory, load_images # The number of eigenfaces that should be kept @@ -61,7 +50,7 @@ def train(training_images): # create array set used for training # iterate through the training examples and linearize the images - training_set = numpy.vstack([image.flatten() for image in training_images.values()]) + training_set = numpy.vstack([image.flatten() for image in training_images]) # training the PCA returns a machine that can be used for projection pca_machine, eigen_values = pca_trainer.train(training_set) @@ -83,19 +72,32 @@ DISTANCE_FUNCTION = scipy.spatial.distance.euclidean def main(): """This function will perform an eigenface test on the AT&T database""" - # use the bob.db interface to retrieve information about the Database - atnt_db = bob.db.atnt.Database() - # Check the existence of the AT&T database and download it if not # Also check if the AT&T database directory is overwritten by the command line image_directory = atnt_database_directory(sys.argv[1] if len(sys.argv) > 1 else None) + # use the bob.db interface to retrieve information about the Database + db = bob.db.atnt.Database(original_directory = image_directory) + + # The protocol does not exist for AT&T, but to be able to exchange the database, we define it here. + protocol = None + + # The group is 'dev', throughout + group = 'dev' + + # The images of the AT&T database are already cropped, so we don't need to specify a face cropper. + face_cropper = None + # For other databases you might want to use: +# face_cropper = bob.ip.base.FaceEyesNorm(crop_size = (80,64), right_eye = (16,15), left_eye = (16,48)) + ##################################################################### ### Training # load all training images - training_images = load_images(atnt_db, group = 'world', database_directory = image_directory) + training_files = db.training_files(protocol = protocol) + print("Loading %d training images" % len(training_files)) + training_images = load_images(db, training_files, face_cropper) print("Training PCA machine") pca_machine = train(training_images) @@ -103,28 +105,24 @@ def main(): ##################################################################### ### extract eigenface features of model and probe images - # load model and probe images - model_images = load_images(atnt_db, group = 'dev', purpose = 'enroll', database_directory = image_directory) - probe_images = load_images(atnt_db, group = 'dev', purpose = 'probe', database_directory = image_directory) - - print("Extracting models") - model_features = {} - for key, image in model_images.items(): - model_features[key] = extract_feature(image, pca_machine) - - # enroll models from 5 features by simply storing all features - model_ids = [client.id for client in atnt_db.clients(groups = 'dev')] - models = dict((model_id, []) for model_id in model_ids) # note: py26 compat. - # iterate over model features - for file_id, image in model_features.items(): - model_id = atnt_db.get_client_id_from_file_id(file_id) - # "enroll" model by collecting all model features of this client - models[model_id].append(model_features[file_id]) - - print("Extracting probes") - probe_features = {} - for key, image in probe_images.items(): - probe_features[key] = extract_feature(image, pca_machine) + model_ids = db.model_ids(groups = group) + print("Extracting %d models" % len(model_ids)) + # generate models for each model ID + models = {} + for model_id in model_ids: + # load enroll images for the current model ID + enroll_images = load_images(db, db.enroll_files(protocol = protocol, groups = group, model_id = model_id), face_cropper) + # extract features for all enroll images and store all of them + models[model_id] = [extract_feature(enroll_image, pca_machine) for enroll_image in enroll_images] + + probe_files = db.probe_files(protocol = protocol, groups = group) + print("Extracting %d probes" % len(probe_files)) + probe_images = load_images(db, probe_files, face_cropper) + # extract probe features and store them by probe ID (which is the File.id) + probes = {} + for i in range(len(probe_files)): + probe_id = probe_files[i].id + probes[probe_id] = extract_feature(probe_images[i], pca_machine) ##################################################################### @@ -136,7 +134,13 @@ def main(): # iterate through models and probes and compute scores for model_id, model in models.items(): - for probe_key, probe_feature in probe_features.items(): + # the client ID that is attached to the model + model_client_id = db.get_client_id_from_model_id(model_id) + # get the probe files, which should be compared with this model + model_probe_files = db.probe_files(protocol = protocol, groups = group, model_id = model_id) + for probe_file in model_probe_files: + # get the according probe feature using the File.id of the probe file + probe_feature = probes[probe_file.id] # compute scores for all model features scores = [- DISTANCE_FUNCTION(model_feature, probe_feature) for model_feature in model] # the final score is the average distance @@ -144,7 +148,7 @@ def main(): score = numpy.sum(scores) # check if this is a positive score - if model_id == atnt_db.get_client_id_from_file_id(probe_key): + if model_client_id == probe_file.client_id: positive_scores.append(score) else: negative_scores.append(score) diff --git a/bob/example/faceverify/gabor_graph.py b/bob/example/faceverify/gabor_graph.py index cffa74e1b501d5e4e38d25ce19eeca0fd487d638..18c0357a59584e5f57c2a0bee3feba2c27bc4120 100644 --- a/bob/example/faceverify/gabor_graph.py +++ b/bob/example/faceverify/gabor_graph.py @@ -37,24 +37,7 @@ matplotlib.rc('font', family='serif') matplotlib.rc('lines', linewidth = 4) from matplotlib import pyplot -from .utils import atnt_database_directory - - -# To preprocess the AT&T images, we use the TanTriggs algorithm -preprocessor = bob.ip.base.TanTriggs() - -def load_images(db, group = None, purpose = None, database_directory = None, image_extension = '.pgm'): - """Reads the images for the given group and the given purpose from the given database""" - # get the file names from the database - files = db.objects(groups = group, purposes = purpose) - # iterate through the list of file names - images = {} - for k in files: - # load image and linearize it into a vector - images[k.id] = bob.io.base.load(k.make_path(database_directory, image_extension)).astype(numpy.float64) - # preprocess the images - images[k.id] = preprocessor(images[k.id]) - return images +from .utils import atnt_database_directory, load_images # define Gabor wavelet transform class globally since it is reused for all images @@ -62,14 +45,14 @@ gabor_wavelet_transform = bob.ip.gabor.Transform(k_max = 0.25 * math.pi) # pre-allocate Gabor wavelet transform image in the desired size trafo_image = numpy.ndarray((gabor_wavelet_transform.number_of_wavelets, 112, 92), numpy.complex128) -def extract_feature(image, graph): +def extract_feature(image, extractor): """Extracts the Gabor graphs from the given image""" # perform Gabor wavelet transform on the image gabor_wavelet_transform.transform(image, trafo_image) # extract the Gabor graphs from the feature image - gabor_graph = graph.extract(trafo_image) + gabor_graph = extractor.extract(trafo_image) # return the extracted graph return gabor_graph @@ -81,48 +64,62 @@ SIMILARITY_FUNCTION = bob.ip.gabor.Similarity('PhaseDiffPlusCanberra', gabor_wav def main(): """This function will perform Gabor graph comparison test on the AT&T database.""" - # use the bob.db interface to retrieve information about the Database - atnt_db = bob.db.atnt.Database() - # Check the existence of the AT&T database and download it if not # Also check if the AT&T database directory is overwritten by the command line image_directory = atnt_database_directory(sys.argv[1] if len(sys.argv) > 1 else None) + # use the bob.db interface to retrieve information about the Database + db = bob.db.atnt.Database(original_directory = image_directory) + + # The protocol does not exist for AT&T, but to be able to exchange the database, we define it here. + protocol = None + + # The group is 'dev', throughout + group = 'dev' + + # The images of the AT&T database are already cropped, so we don't need to specify a face cropper. + face_cropper = None + # For other databases you might want to use: +# face_cropper = bob.ip.base.FaceEyesNorm(crop_size = (80,64), right_eye = (16,15), left_eye = (16,48)) + + # After image cropping, we apply an image preprocessor + preprocessor = bob.ip.base.TanTriggs() + + # The image resolution of the (cropped) images, which might change with the database + image_resolution = (112, 92) ##################################################################### ### Training # for Gabor graphs, no training is required. - print("Creating Gabor graph machine") - # create a machine that will produce tight Gabor graphs with inter-node distance (4,4) - graph_machine = bob.ip.gabor.Graph(first=(8,6), last=(104,86), step=(4,4)) + # create a machine that will produce Gabor graphs with inter-node distance (4,4) + graph_extractor = bob.ip.gabor.Graph(first=(8,6), last=(image_resolution[0]-8, image_resolution[1]-6), step=(4,4)) ##################################################################### ### extract Gabor graph features for all model and probe images - # load all model and probe images - model_images = load_images(atnt_db, group = 'dev', purpose = 'enroll', database_directory = image_directory) - probe_images = load_images(atnt_db, group = 'dev', purpose = 'probe', database_directory = image_directory) - - print("Extracting models") - model_features = {} - for key, image in model_images.items(): - model_features[key] = extract_feature(image, graph_machine) - - # enroll models from 5 features by simply storing all features - model_ids = [client.id for client in atnt_db.clients(groups = 'dev')] - models = dict((model_id, []) for model_id in model_ids) # note: py26 compat. - # iterate over model features - for key, image in model_features.items(): - model_id = atnt_db.get_client_id_from_file_id(key) - # "enroll" model by collecting all model features of this client - models[model_id].append(model_features[key]) - - print("Extracting probes") - probe_features = {} - for key, image in probe_images.items(): - probe_features[key] = extract_feature(image, graph_machine) + ##################################################################### + ### extract eigenface features of model and probe images + + model_ids = db.model_ids(groups = group) + print("Extracting %d models" % len(model_ids)) + # generate models for each model ID + models = {} + for model_id in model_ids: + # load enroll images for the current model ID + enroll_images = load_images(db, db.enroll_files(protocol = protocol, groups = group, model_id = model_id), face_cropper, preprocessor) + # extract features for all enroll images and store all of them + models[model_id] = [extract_feature(enroll_image, graph_extractor) for enroll_image in enroll_images] + + probe_files = db.probe_files(protocol = protocol, groups = group) + print("Extracting %d probes" % len(probe_files)) + probe_images = load_images(db, probe_files, face_cropper, preprocessor) + # extract probe features and store them by probe ID (which is the File.id) + probes = {} + for i in range(len(probe_files)): + probe_id = probe_files[i].id + probes[probe_id] = extract_feature(probe_images[i], graph_extractor) ##################################################################### ### compute scores, we here choose a simple Euclidean distance measure @@ -134,21 +131,32 @@ def main(): # iterate through models and probes and compute scores model_count = 1 for model_id, model in models.items(): + # provide status information print("\rModel", model_count, "of", len(models), end='') sys.stdout.flush() model_count += 1 - for probe_key, probe_feature in probe_features.items(): - # compute local scores for each model gabor jet and each probe jet - scores = numpy.ndarray((len(model), len(probe_feature)), dtype = numpy.float) - for model_feature_index in range(len(model)): - for gabor_jet_index in range(len(probe_feature)): - scores[model_feature_index, gabor_jet_index] = SIMILARITY_FUNCTION(model[model_feature_index][gabor_jet_index], probe_feature[gabor_jet_index]) + # the client ID that is attached to the model + model_client_id = db.get_client_id_from_model_id(model_id) + # get the probe files, which should be compared with this model + model_probe_files = db.probe_files(protocol = protocol, groups = group, model_id = model_id) + for probe_file in model_probe_files: + # get the according probe feature using the File.id of the probe file + probe_feature = probes[probe_file.id] + # compute local scores for each model gabor jet and each probe jet + score = 0. + for gabor_jet_index in range(len(probe_feature)): + scores = [] + # compute the similarity to all model jets + for model_feature_index in range(len(model)): + scores.append(SIMILARITY_FUNCTION(model[model_feature_index][gabor_jet_index], probe_feature[gabor_jet_index])) + # .. and take the most similar one + score += max(scores) # the final score is computed as the average over all positions, taking the most similar model jet - score = numpy.average(numpy.max(scores, axis = 0)) + score /= len(probe_feature) # check if this is a positive score - if model_id == atnt_db.get_client_id_from_file_id(probe_key): + if model_client_id == probe_file.client_id: positive_scores.append(score) else: negative_scores.append(score) diff --git a/bob/example/faceverify/tests.py b/bob/example/faceverify/tests.py index 199dd0c6e1d68bc00ee193bbd1c05084c28f8c57..db076285e9f37fd1c3211e2d7ff0f226d1d3bd44 100644 --- a/bob/example/faceverify/tests.py +++ b/bob/example/faceverify/tests.py @@ -84,9 +84,9 @@ class FaceVerifyExampleTest(unittest.TestCase): raise SkipTest("Skipping the tests since importing from bob.example.faceverify.eigenface raised exception '%s'"%e) # open database - atnt_db = bob.db.atnt.Database() + atnt_db = bob.db.atnt.Database(self.m_database_dir) # test if all training images are loaded - images = load_images(atnt_db, group = 'world', database_directory = self.m_database_dir) + images = load_images(atnt_db, atnt_db.training_files()) self.assertEqual(len(images), 200) # test that the training works (for speed reasons, we limit the number of training files) @@ -114,7 +114,7 @@ class FaceVerifyExampleTest(unittest.TestCase): # compute score score = DISTANCE_FUNCTION(model, probe) - self.assertAlmostEqual(score, 3498.308154114) + self.assertAlmostEqual(score, 2433.76349382) def test02_gabor_graph(self): @@ -125,9 +125,9 @@ class FaceVerifyExampleTest(unittest.TestCase): raise SkipTest("Skipping the tests since importing from bob.example.faceverify.gabor_graph raised exception '%s'"%e) # open database - atnt_db = bob.db.atnt.Database() + atnt_db = bob.db.atnt.Database(self.m_database_dir) # test if all training images are loaded - images = load_images(atnt_db, group = 'world', database_directory = self.m_database_dir) + images = load_images(atnt_db, atnt_db.training_files(), preprocessor = bob.ip.base.TanTriggs()) self.assertEqual(len(images), 200) # extract features; for test purposes we wil use smaller features with inter-node-distance 8 @@ -150,7 +150,7 @@ class FaceVerifyExampleTest(unittest.TestCase): # compute score score = numpy.mean([SIMILARITY_FUNCTION.similarity(model[i], probe[i]) for i in range(len(model))]) - self.assertAlmostEqual(score, 0.414937662799) + self.assertAlmostEqual(score, 0.4727632139) def test03_dct_ubm(self): @@ -161,10 +161,9 @@ class FaceVerifyExampleTest(unittest.TestCase): raise SkipTest("Skipping the tests since importing from bob.example.faceverify.dct_ubm raised exception '%s'"%e) # open database - atnt_db = bob.db.atnt.Database() + atnt_db = bob.db.atnt.Database(self.m_database_dir) # test if all training images are loaded - images = load_images(atnt_db, group = 'world', database_directory = self.m_database_dir) - keys = sorted(images.keys()) + images = load_images(atnt_db, atnt_db.training_files()) self.assertEqual(len(images), 200) # test that the original DCT extraction works @@ -176,32 +175,24 @@ class FaceVerifyExampleTest(unittest.TestCase): self.assertTrue(numpy.allclose(feature_ref, dct_feature)) # extract features for several images -# features = {i : extract_feature(images[i]) for i in keys[:13]} - features = {} - for i in keys[:13]: features[i] = extract_feature(images[i]) + features = [extract_feature(images[i]) for i in range(13)] # train the UBM with several features, and a limited number of Gaussians -# ubm = train({i : features[i] for i in keys[:10]}) - trainset = {} - for i in keys[:10]: trainset[i] = features[i] + trainset = features[:10] ubm = train(trainset, number_of_gaussians = 2) if regenerate_references: ubm.save(bob.io.base.HDF5File(self.resource('dct_ubm.hdf5'), 'w')) - # load PCA reference and check that it is still similar + # load GMM reference and check that it is still similar ubm_ref = bob.learn.misc.GMMMachine(bob.io.base.HDF5File(self.resource('dct_ubm.hdf5'))) self.assertTrue(ubm_ref.is_similar_to(ubm)) - # enroll a model with two features enroller = bob.learn.misc.MAP_GMMTrainer() enroller.max_iterations = 1 enroller.set_prior_gmm(ubm) -# model = enroll({i : features[i] for i in keys[10:12]}, ubm, enroller) - enrollset = {} - for i in keys[10:12]: enrollset[i] = features[i] - model = enroll(enrollset, ubm, enroller) + model = enroll(features[10:12], ubm, enroller) if regenerate_references: model.save(bob.io.base.HDF5File(self.resource('dct_model.hdf5'), 'w')) @@ -209,7 +200,7 @@ class FaceVerifyExampleTest(unittest.TestCase): self.assertTrue(model_ref.is_similar_to(model)) # compute probe statistics - probe = stats(features[keys[12]], ubm) + probe = stats(features[12], ubm) if regenerate_references: probe.save(bob.io.base.HDF5File(self.resource('dct_probe.hdf5'), 'w')) diff --git a/bob/example/faceverify/utils.py b/bob/example/faceverify/utils.py index ada3d5a90f0bc233b74e037c63d78fcd9618b118..4f490ec2ec787fa95e99a6e51363d47e30cc1032 100644 --- a/bob/example/faceverify/utils.py +++ b/bob/example/faceverify/utils.py @@ -18,9 +18,12 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os +import bob.io.base +import bob.ip.color +import numpy def atnt_database_directory(atnt_user_directory = None): - """Checks, where the AT&T database is located and downloads it on need.""" + #Checks, where the AT&T database is located and downloads it on need if atnt_user_directory is not None: # a user directory is specified atnt_default_directory = atnt_user_directory @@ -71,3 +74,46 @@ def atnt_database_directory(atnt_user_directory = None): return atnt_default_directory +def load_images(database, files, face_cropper = None, preprocessor = None): + """Loads the original images for the given list of File objects. + + Keyword Parameters: + + database : :py:class:`bob.db.verification.utils.Database` or derived from it + The database interface to query. + + files : [:py:class:`bob.db.verification.utils.File`] or derived from it + A list of ``File`` objects for which the images should be loaded. + + face_cropper : :py:class:`bob.ip.base.FaceEyesNorm` or ``None`` + If a face cropper is given, the face will be cropped using the eye annotations of the database. + + preprocessor : ``function``, ``object`` or ``None`` + If a preprocessor is given, it will be applied to the (cropped) faces. + """ + file_names = database.original_file_names(files) + # iterate through the list of file names and load images + images = [bob.io.base.load(file_name) for file_name in file_names] + # convert color to gray images if necessary + gray_images = [] + for image in images: + gray_image = bob.ip.color.rgb_to_gray(image).astype(numpy.float64) if len(image.shape) == 3 else image + gray_images.append(gray_image.astype(numpy.float64)) + + if face_cropper is not None: + cropped_images = [] + for i in range(len(files)): + # get the annotations for the files + annotations = database.annotations(files[i]) + assert 'reye' in annotations and 'leye' in annotations + cropped_images.append(face_cropper(gray_images[i], right_eye = annotations['reye'], left_eye = annotations['leye'])) + else: + cropped_images = gray_images + + if preprocessor is not None: + preprocessed_images = [preprocessor(i) for i in cropped_images] + else: + preprocessed_images = cropped_images + + return preprocessed_images + diff --git a/buildout.cfg b/buildout.cfg index 2719550ebd5290ea0f27ed31bbcd3bf0d4fa78b1..dc613b1f63bdff4c75137234fba9ff8ab017dd2d 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -20,6 +20,7 @@ develop = src/bob.extension src/bob.learn.linear src/bob.learn.misc src/bob.ip.base + src/bob.ip.color src/bob.ip.gabor src/bob.db.base src/bob.db.verification.utils @@ -44,6 +45,7 @@ bob.learn.activation = git https://github.com/bioidiap/bob.learn.activation bob.learn.linear = git https://github.com/bioidiap/bob.learn.linear bob.learn.misc = git https://github.com/bioidiap/bob.learn.misc bob.ip.base = git https://github.com/bioidiap/bob.ip.base +bob.ip.color = git https://github.com/bioidiap/bob.ip.color bob.ip.gabor = git https://github.com/bioidiap/bob.ip.gabor bob.db.base = git https://github.com/bioidiap/bob.db.base bob.db.verification.utils = git https://github.com/bioidiap/bob.db.verification.utils @@ -52,3 +54,4 @@ bob.db.atnt = git https://github.com/bioidiap/bob.db.atnt [scripts] recipe = bob.buildout:scripts +dependent-scripts = true diff --git a/doc/conf.py b/doc/conf.py index 8d7099a937d73f3e462d857219ecd90136d10ae5..0a5864e223b5504502a0113e012d2ee239595589 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -249,11 +249,11 @@ man_pages = [ # Default processing flags for sphinx autoclass_content = 'both' autodoc_member_order = 'bysource' -autodoc_default_flags = ['members', 'undoc-members', 'inherited-members', 'show-inheritance'] +autodoc_default_flags = ['members', 'inherited-members', 'show-inheritance'] # For inter-documentation mapping: from bob.extension.utils import link_documentation -intersphinx_mapping = link_documentation(['python', 'numpy', 'scipy', 'bob.io.base', 'bob.measure', 'bob.ip.base', 'bob.ip.gabor', 'bob.learn.linear', 'bob.learn.misc', 'bob.db.atnt', 'bob.db.verification.utils']) +intersphinx_mapping = link_documentation(['python', 'numpy', 'scipy', 'bob.io.base', 'bob.measure', 'bob.ip.base', 'bob.ip.gabor', 'bob.learn.linear', 'bob.learn.misc', 'bob.db.atnt', 'bob.db.banca', 'bob.db.verification.utils']) def setup(app): diff --git a/doc/dct_ubm.png b/doc/dct_ubm.png index 0d597b674136ae91687da448de1c2584d44fb5f9..1e75a091ce3840900d4ee5fa409e399302dd86f2 100644 Binary files a/doc/dct_ubm.png and b/doc/dct_ubm.png differ diff --git a/doc/examples.rst b/doc/examples.rst index d4eed9b3b70e81240ed2cffdeb7c9168f0f42c27..ca9e8868eba2d9a3f95ea32b31acb226c2b5c663 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -15,21 +15,26 @@ import bob.io.base import bob.io.image import bob.ip.base + import bob.ip.color import bob.ip.gabor import bob.learn.linear import bob.measure - training_features = [(numpy.random.rand(400) * 255) for i in range(20)] - probe_features = [(numpy.random.rand(400) * 255) for i in range(5)] - model_features = {1: [(numpy.random.rand(400) * 255) for i in range(5)]} - models = [[(numpy.random.rand(400) * 255) for i in range(5)]] - training_image = numpy.random.rand(20,20) * 255 - model_image = numpy.random.rand(20,20) * 255 - probe_image = numpy.random.rand(20,20) * 255 + training_images = [(numpy.random.rand(400) * 255) for i in range(20)] + probe_images = [(numpy.random.rand(400) * 255) for i in range(200)] + enroll_images = [(numpy.random.rand(400) * 255) for i in range(5)] + + image = (numpy.random.rand(3,100,100) * 255).astype(numpy.uint8) + gray_image = image[0] positives = numpy.random.rand(100) negatives = numpy.random.rand(500) - 0.5 + def load_images(x): return images + def extract_feature(x): return x + def enroll(x): return x + def compare(x,y): return numpy.sum(x-y) + To run the examples, just call the scripts from within the ``bin`` directory, e.g.: .. code-block:: sh @@ -62,126 +67,223 @@ While the eigenface experiment should be finished in a couple of seconds, the Ga The example code that is presented here differ slightly from the code in the source files. Here, only the concepts of the functions should be clarified, while the source files contain code that is better arranged and computes faster. - In this small example, we use the :ref:`bob.db.atnt <bob.db.atnt>` database. This database is compatible with all other :py:class:`bob.db.verification.utils.Database`\s, so any other :ref:`verification_databases` can be used in a direct replacement of :ref:`bob.db.atnt <bob.db.atnt>`. -The Eigenface Example -~~~~~~~~~~~~~~~~~~~~~ -The eigenface example follows the work-flow that is presented in the original paper *Eigenfaces for Recognition* [TP91]_ by Turk and Pentland. -First, it creates an :py:class:`bob.db.atnt.Database` object to query the :ref:`bob.db.atnt <bob.db.atnt>` database: + +Overall Structure of an Experiment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All three examples have the same basic structure, which shows the simplicity of implementing any face recognition algorithm with the provided database interface. +First, the :py:class:`bob.db.atnt.Database` database interface is created, which will be be used to assure that we follow the default evaluation protocol of the database: .. doctest:: - >>> atnt_db = bob.db.atnt.Database() + >>> db = bob.db.atnt.Database(original_directory = "...") -For training the projection matrix, the training images (which are in gray-scale) are be read using the generic :py:func:`bob.io.base.load` function, which returns the data as a :py:class:`numpy.ndarray`: +In total, there are four steps for each algorithm: -.. doctest:: +1. Training: - >>> training_image_files = atnt_db.objects(groups = 'world') - >>> for training_file in training_image_files: - ... # load image - ... training_image = bob.io.base.load(training_file.make_path(...)) #doctest:+SKIP - ... # linearize pixels - ... training_feature = training_image.flatten() + Images from the training set are loaded (using the :py:func:`bob.example.faceverify.utils.load_images` function discussed below). + Features from the training set are extracted and used to train the model. -Since the face images in the AT&T database are already cropped, they can simply be linearized (converted into one long vector) and put into a 2D array with one sample in each row, and no :py:class:`bob.ip.base.FaceEyesNorm` needs to be applied to align the faces using the :py:meth:`bob.db.verification.utils.Database.annotations` of the eyes: + .. doctest:: -.. doctest:: + >>> training_files = db.training_files() + >>> training_images = load_images(db, training_files) # doctest: +SKIP + >>> training_features = [extract_feature(image) for image in training_images] - >>> training_set = numpy.vstack(training_features) +2. Model Enrollment: -which is then used to train a :py:class:`bob.learn.linear.Machine` using a :py:class:`bob.learn.linear.PCATrainer`: + Models are enrolled using features from several images for each model. + The enroll_files are queried for each model separately: -.. doctest:: + .. doctest:: + :hide: - >>> pca_trainer = bob.learn.linear.PCATrainer() - >>> pca_machine, eigen_values = pca_trainer.train(training_set) + >>> def load_images(x,y): return enroll_images + + .. doctest:: + :options: +NORMALIZE_WHITESPACE -For some distance functions, the eigenvalues are needed, but in our example we just ignore them. + >>> model_ids = db.model_ids(groups = 'dev') + >>> models = {} + >>> for model_id in model_ids: + ... enroll_files = db.enroll_files(groups = 'dev', model_id = model_id) + ... enroll_images = load_images(db, enroll_files) + ... enroll_features = [extract_feature(enroll_image) for enroll_image in enroll_images] + ... models[model_id] = enroll(enroll_features) -After training, the model and probe images are loaded, linearized, and projected into the eigenspace using the trained ``pca_machine``: +3. Probe Feature Extraction: -.. doctest:: + Probe features are extracted for all probe features. + They are stored in a dictionary, using the ``File.id`` of the probe ``File`` object as keys: - >>> model_image_files = atnt_db.objects(groups = 'dev', purposes = 'enroll') - >>> for model_file in model_image_files: - ... # load image - ... model_image = bob.io.base.load(model_file.make_path(...)) #doctest:+SKIP - ... # project to PCA subspace - ... model_feature = pca_machine.forward(model_image.flatten()) + .. doctest:: - >>> probe_image_files = atnt_db.objects(groups = 'dev', purposes = 'probe') - >>> for probe_file in probe_image_files: - ... # load image - ... probe_image = bob.io.base.load(probe_file.make_path(...)) #doctest:+SKIP - ... # project to PCA subspace - ... model_feature = pca_machine.forward(probe_image.flatten()) + >>> probe_files = db.probe_files(groups = 'dev') + >>> probe_images = load_images(db, probe_files) # doctest: +SKIP + >>> probes = {} + >>> for i in range(len(probe_files)): + ... probe_id = probe_files[i].id + ... probes[probe_id] = extract_feature(probe_images[i]) -To follow the evaluation protocol, we *enroll* a client model for each client, simply by collecting all model feature vectors: +4. Comparison of Model and Probe: -.. doctest:: + Now, scores are computed by comparing the created models with the extracted probe features. + Some of the protocols of our databases (e.g., the :ref:`bob.db.banca <bob.db.banca>` database) require that each model is compared only to a model-specific sub-set of probes. + Hence, for each model we have to query the database again. + Finally, we need to know if the pair was a positive pair (i.e., when model and probe came from the same client), or negative (i.e., model and probe came from different clients): + + .. doctest:: + + >>> positives = [] + >>> negatives = [] + >>> for model_id, model in models.items(): + ... model_client_id = db.get_client_id_from_model_id(model_id) + ... model_probe_files = db.probe_files(groups = 'dev', model_id = model_id) + ... for probe_file in model_probe_files: + ... score = compare(model, probes[probe_file.id]) + ... if model_client_id == probe_file.client_id: + ... positives.append(score) + ... else: + ... negatives.append(score) + +5. Evaluation of Scores: + + Finally, the two sets of scores are used to evaluate the face recognition system, using functionality of the :ref:`bob.measure <bob.measure>` package. + Here, we compute the FAR and the FRR at the EER threshold and plot the ROC curve: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE, +ELLIPSIS + + >>> threshold = bob.measure.eer_threshold(negatives, positives) + >>> FAR, FRR = bob.measure.farfrr(negatives, positives, threshold) + >>> bob.measure.plot.roc(negatives, positives, CAR=True) + [...] + + .. note:: + Here we plot the ROC curves with logarithmic FAR axis --- to highlight the interesting part of the curve, i.e., where the FAR values are small. + + +Loading Data +~~~~~~~~~~~~ + +To load the image data, I have implemented the generic :py:func:`bob.example.faceverify.utils.load_images` function: + + .. automodule:: bob.example.faceverify.utils + +This function is generic in the sense that: + +1. it can read gray level and color images: + + .. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> image = bob.io.base.load(filename) # doctest: +SKIP + >>> if len(image.shape) == 3: + ... gray_image = bob.ip.color.rgb_to_gray(image) + +2. when eyes annotations are available, it can apply a face cropper :py:class:`bob.ip.base.FaceEyesNorm` using the given eye positions, e.g.: + + .. doctest:: + + >>> face_cropper = bob.ip.base.FaceEyesNorm(crop_size = (112,92), right_eye = (20,31), left_eye = (20,62)) + >>> cropped_image = face_cropper(gray_image, right_eye = (30,40), left_eye = (32, 60)) + +3. a preprocessor can be applied to the images, which is either a function or a class like :py:class:`bob.ip.base.TanTriggs` that has the ``()`` operator (aka the ``__call__`` function) overloaded: - >>> model_ids = [client.id for client in atnt_db.clients(groups = 'dev')] - >>> for model_image_id in model_features: - ... # query the database for the model id of the current file id - ... model_id = atnt_db.get_client_id_from_file_id(model_image_id) - ... # append feature for the current model id - ... models[model_id].append(model_features[model_feature_id]) #doctest:+SKIP + .. doctest:: + >>> preprocessor = bob.ip.base.TanTriggs() + >>> preprocessed_image = preprocessor(cropped_image) -To compute the verification result, each model feature is compared to each probe feature by computing the :py:func:`scipy.spatial.distance.euclidean` distance: +Finally, this function applies some or all of these steps on a list of images. + + +The Experiments +~~~~~~~~~~~~~~~ + +For the three implemented face recognition experiments, we have chosen different ways to preprocess the images, to train the system, to extract the features, to enroll the models, to extract the probes and to compare models and probes. + +The Eigenface Example +--------------------- + +The eigenface example follows the work-flow that is presented in the original paper *Eigenfaces for Recognition* [TP91]_ by Turk and Pentland. +To train the eigenspace, the images from the training set are linearized (i.e., the pixels of an image are re-arranged to form a single vector): .. doctest:: - >>> for model in models: - ... for probe_feature in probe_features: - ... for model_feature in model: - ... score = scipy.spatial.distance.euclidean(model_feature, probe_feature) + >>> training_set = numpy.vstack([image.flatten() for image in training_images]) + +Afterwards, a :py:class:`bob.learn.linear.PCATrainer` is used to compute the PCA subspace: + +.. doctest:: -Finally, all scores of one model and one probe are averaged to get the final score for this pair. + >>> pca_trainer = bob.learn.linear.PCATrainer() + >>> pca_machine, eigen_values = pca_trainer.train(training_set) -The results are divided into a list of positive scores (model and probe are from the same identity) and a a list of negative scores (identities of model and probe differ). -Using these lists, the ROC curve is plotted using functionality from :ref:`bob.measure <bob.measure>`: +For some distance functions, the eigenvalues are needed, but in our example we just ignore them. +Finally, the number of kept eigenfaces is limited to 5: .. doctest:: - >>> bob.measure.plot.roc(negatives, positives) #doctest:+ELLIPSIS - [...] + >>> pca_machine.resize(pca_machine.shape[0], 5) -.. image:: eigenface.png - :scale: 100 % -and the performance is computed: +After training, the model and probe images are loaded, linearized, and projected into the eigenspace using the trained ``pca_machine``. +A possible ``extract_feature (image)`` function could look like: + +.. code-block:: py + + >>> pca_machine.forward(image.flatten()) + +The enrollment of the client is done by collecting all model feature vectors. +The ``enroll(features)`` function is quite simple: .. doctest:: - >>> threshold = bob.measure.eer_threshold(negatives, positives) - >>> FAR, FRR = bob.measure.farfrr(negatives, positives, threshold) + >>> def enroll(features): + ... return features -The expected result is: FAR 9.15% and FRR 9% at threshold -9276.2 +To compute the verification result, each model feature is compared to each probe feature by computing the :py:func:`scipy.spatial.distance.euclidean` distance. +Finally, all scores of one model and one probe are averaged to get the final score for this pair, which would translate into a ``compare(model, probe)`` function like: -.. note:: +.. doctest:: - Here we plot the ROC curves with logarithmic FAR axis --- to highlight the interesting part of the curve, i.e., where the FAR values are small. + >>> score = 0. + >>> for model_feature in model: + ... score += scipy.spatial.distance.euclidean(model_feature, probe) # doctest: +SKIP + >>> score /= len(model) -.. note:: +The final ROC curve of this experiment is: - Computing eigenfaces with such a low amount of training data is usually not an excellent idea. - Hence, the performance in this example is relatively poor. +.. image:: eigenface.png + :scale: 100 % +and the expected FAR and FRR performance is: FAR 9.15% and FRR 9% at threshold -9276.2 -Gabor jet comparisons -~~~~~~~~~~~~~~~~~~~~~ +.. note:: + Computing eigenfaces with such a low amount of training data is usually not an excellent idea. + Hence, the performance in this example is relatively poor. + + +Gabor jet Comparison +-------------------- A better face verification example uses Gabor jet features [WFKM97]_ . In this example we do not define a face graph, but instead we use the :py:class:`bob.ip.gabor.Jet`\s at several grid positions in the image. -To do that, we define: +In opposition to the Eigenface example above, here we preprocess the images with :py:class:`bob.ip.base.TanTriggs`. + +The Gabor graph extraction does not require a training stage. +Therefore, in opposition to the eigenface example, the training images are not used in this example. + +Instead, to extract Gabor grid graph features, it is sufficient to use: .. doctest:: - >>> graph = bob.ip.gabor.Graph((8,6), (104,86), (4,4)) + >>> extractor = bob.ip.gabor.Graph((8,6), (104,86), (4,4)) that will create Gabor graphs with node positions from (8,6) to (104,86) with step size (4,4). @@ -190,24 +292,13 @@ that will create Gabor graphs with node positions from (8,6) to (104,86) with st The resolution of the images in the AT&T database is 92x112. Of course, there are ways to automatically get the size of the images, but for brevity we hard-coded the resolution of the images. -.. note:: - - The Gabor graph extraction does not require a training stage. - Therefore, in opposition to the eigenface example, the training images are not used in this example. - -Now, the Gabor graph features can be extracted from the model and probe images. -Here is the code for the model graphs only: +Now, the Gabor graph features can be extracted from the model and probe images: .. doctest:: - >>> gabor_wavelet_transform = bob.ip.gabor.Transform() - >>> model_image_files = atnt_db.objects(groups = 'dev', purposes = 'enroll') - >>> for model_file in model_image_files: - ... # load image - ... model_image = bob.io.base.load(model_file.make_path(...)) #doctest:+SKIP - ... # create the Gabor transformed image - ... trafo_image = gabor_wavelet_transform.transform(model_image) - ... model_feature = graph.extract(model_image) + >>> gabor_wavelet_transform = bob.ip.gabor.Transform() + >>> trafo_image = gabor_wavelet_transform.transform(preprocessed_image) + >>> model_graph = extractor.extract(trafo_image) For model enrollment, as above we simply collect all enrollment features. To compare the Gabor graphs, several methods can be applied. @@ -218,18 +309,17 @@ Again, many choices for the Gabor jet comparison exist, here we take a novel Gab >>> similarity_function = bob.ip.gabor.Similarity("PhaseDiffPlusCanberra", gabor_wavelet_transform) Since we have several local features, we can exploit this fact. -For each local position, we compute the similarity between the probe feature at this position and all model features and take the maximum value: +In the ``compare(model, probe)`` function, we compute the similarity between the probe feature at this position and all model features and take the maximum value for each grid position: .. code-block:: python - >>> for model_id in model_ids: - ... for probe_feature in probe_features: - ... for model_feature in models[model_id]: - ... for node_index in range(len(probe_feature)): - ... scores[...] = similarity_function.similarity(model_feature[node_index], probe_feature[node_index]) - ... score = numpy.average(numpy.max(scores, axis = 0)) + >>> for gabor_jet_index in range(len(probe_feature)): + ... scores = [] + ... for model_feature_index in range(len(model)): + ... scores.append(SIMILARITY_FUNCTION(model[model_feature_index][gabor_jet_index], probe_feature[gabor_jet_index])) + ... score += max(scores) + >>> score /= len(probe_feature) -The evaluation is identical to the evaluation in the eigenface example. Since this method is better for suited for small image databases, the resulting verification rates are better. The expected ROC curve is: @@ -242,25 +332,21 @@ while the expected verification result is: FAR 3% and FRR 3% at distance thresho The UBM/GMM modeling of DCT Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The last example shows a quite complicated, but successful algorithm. +Again, images are preprocessed with :py:class:`bob.ip.base.TanTriggs`. The first step is the feature extraction of the training image features and the collection of them in a 2D array. In this experiment we will use *Discrete Cosine Transform* (DCT) block features using :py:class:`bob.ip.base.DCTFeatures` [MM09]_: .. doctest:: - >>> dct_extractor = bob.ip.base.DCTFeatures(45, (12, 12), (11, 11)) - >>> training_image_files = atnt_db.objects(groups = 'world') - >>> for training_file in training_image_files: - ... # load training image - ... training_image = bob.io.base.load(training_file.make_path(...)) #doctest:+SKIP - ... # extract DCT block features - ... training_features = dct_extractor(training_image) + >>> dct_extractor = bob.ip.base.DCTFeatures(45, (12, 12), (11, 11)) + >>> features = dct_extractor(preprocessed_image) Hence, from every image, several DCT block features are extracted independently. All these features are mixed together to build the training set: .. code-block:: python - >>> training_set = numpy.vstack(training_features_list) + >>> training_set = numpy.vstack(training_features) With these training features, a *universal background model* (UBM) is computed [RQD00]_. It is a *Gaussian Mixture Model* (GMM) that holds information about the overall distribution of DCT features in facial images. @@ -268,52 +354,37 @@ The UBM model is trained using a :py:class:`bob.learn.misc.KMeansTrainer` to est .. code-block:: python - >>> kmeans_machine = bob.learn.misc.KMeansMachine(...) - >>> kmeans_trainer = bob.learn.misc.KMeansTrainer() - >>> kmeans_trainer.train(kmeans, training_set) + >>> kmeans_machine = bob.learn.misc.KMeansMachine(...) + >>> kmeans_trainer = bob.learn.misc.KMeansTrainer() + >>> kmeans_trainer.train(kmeans, training_set) Afterward, the UBM is initialized with the results of the k-means training: .. code-block:: python - >>> ubm = bob.learn.misc.GMMMachine(...) - >>> ubm.means = kmeans_machine.means - >>> [variances, weights] = kmeans_machine.get_variances_and_weights_for_each_cluster(training_set) - >>> ubm.variances = variances - >>> ubm.weights = weights + >>> ubm = bob.learn.misc.GMMMachine(...) + >>> ubm.means = kmeans_machine.means + >>> [variances, weights] = kmeans_machine.get_variances_and_weights_for_each_cluster(training_set) + >>> ubm.variances = variances + >>> ubm.weights = weights and a :py:class:`bob.learn.misc.ML_GMMTrainer` is used to compute the actual UBM model: .. code-block:: python - >>> trainer = bob.learn.misc.ML_GMMTrainer() - >>> trainer.train(ubm, training_set) + >>> trainer = bob.learn.misc.ML_GMMTrainer() + >>> trainer.train(ubm, training_set) After UBM training, the next step is the model enrollment. Here, a separate GMM model is generated by shifting the UBM towards the mean of the model features [MM09]_. -For this purpose, we need to get the model images sorted by identity: - -.. code-block:: python - - >>> model_ids = atnt_db.client_ids(groups = 'dev') - -Now, we load the images for each identity, extract the DCT features and enroll a model for each identity. For that purpose, a :py:class:`bob.learn.misc.MAP_GMMTrainer` is used: .. code-block:: python - >>> gmm_trainer = bob.learn.misc.MAP_GMMTrainer() - >>> # ... initialize GMM trainer ... - >>> for model_id in model_ids: - ... model_files = db.objects(groups = 'dev', purposes = 'enroll', client_ids = model_id) - ... model_feature_set_list = [] - ... for model_file in model_files: - ... # ... load model image ... - ... model_dct_blocks = dct_extractor(model_image) - ... model_feature_set_list.append(model_dct_blocks) - ... model_feature_set = numpy.vstack(model_feature_set_list) - ... model_gmm = bob.learn.misc.GMMMachine(ubm) - ... gmm_trainer.train(model_gmm, model_feature_set) + >>> gmm_trainer = bob.learn.misc.MAP_GMMTrainer() + >>> enroll_set = numpy.vstack(enroll_features) + >>> model_gmm = bob.learn.misc.GMMMachine(ubm) + >>> gmm_trainer.train(model_gmm, model_feature_set) Also the probe image need some processing. @@ -322,21 +393,15 @@ Afterward, the :py:class:`bob.learn.misc.GMMStats` statistics for each probe fil .. code-block:: python - >>> probe_image_files = atnt_db.objects(groups = 'dev', purposes = 'probe', ...) - >>> for probe_file in probe_image_files: - ... # ... load probe image ... - ... probe_dct_blocks = dct_extractor(probe_image_blocks) - ... probe_gmm_stats = bob.learn.misc.GMMStats() - ... gmm_stats.init() - ... ubm.acc_statistics(probe_dct_blocks, probe_gmm_stats) + >>> probe_set = numpy.vstack([probe_feature]) + >>> gmm_stats = bob.learn.misc.GMMStats() + >>> ubm.acc_statistics(probe_dct_blocks, gmm_stats) Finally, the scores for the probe files are computed using the :py:func:`bob.learn.misc.linear_scoring` function: .. code-block:: python - >>> for model_gmm in models: - ... for probe_gmm_stats in probes: - ... score = bob.learn.misc.linear_scoring([model_gmm], ubm, [probe_gmm_stats])[0,0] + >>> score = bob.learn.misc.linear_scoring([model], ubm, [probe_stats])[0,0] Again, the evaluation of the scores is identical to the previous examples. The expected ROC curve is: @@ -344,7 +409,7 @@ The expected ROC curve is: .. image:: dct_ubm.png :scale: 100 % -The expected result is: FAR 5% and FRR 5% at distance threshold 7640.95 +The expected result is: FAR 3.15% and FRR 3% at distance threshold 2301.58 .. [TP91] Matthew Turk and Alex Pentland. Eigenfaces for recognition. Journal of Cognitive Neuroscience, 3(1):71-86, 1991. diff --git a/doc/index.rst b/doc/index.rst index 6b7a1f8fbef81478e06138e70b506bf8ca8ba915..55e7e31239f0256d5b908701dab3fce82ed79431 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,6 +16,7 @@ It includes examples with three different complexities: * An example building an UBM/GMM model on top of DCT blocks. The face verification experiments are executed using the protocols for the `AT&T`_ database implemented in :ref:`bob.db.atnt <bob.db.atnt>`. +However, the code is set up such that it will work with any other of the :ref:`verification_databases` as well. .. warning:: The `AT&T`_ database is a toy database that is perfectly useful for this example, but **not** to publish scientific papers. diff --git a/setup.py b/setup.py index 82756c58aca041a567210316a99d22cfcfc878b4..8d0a5a0217de4085724a4b001e8ec6149e22d9da 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ setup( "setuptools", "bob.io.image", # image IO "bob.ip.base", # tan-triggs, dct-blocks + "bob.ip.color", # color image conversion "bob.ip.gabor", # gabor graph "bob.learn.linear", # eigenfaces "bob.learn.misc", # ubm-gmm