From 04c20aa22cca5b150a48478512dd067d73d5bca4 Mon Sep 17 00:00:00 2001 From: Guillaume HEUSCH <guillaume.heusch@idiap.ch> Date: Wed, 3 Oct 2018 11:20:10 +0200 Subject: [PATCH] manage to somehow create a table with Clients ... --- bob/db/fargo/create.py | 16 +++++++++-- bob/db/fargo/driver.py | 13 ++++++--- bob/db/fargo/models.py | 25 ++++++++++------- bob/db/fargo/query.py | 62 ++++++++++++++++++++++++++++++++++-------- setup.py | 12 -------- 5 files changed, 88 insertions(+), 40 deletions(-) diff --git a/bob/db/fargo/create.py b/bob/db/fargo/create.py index f5a6130..7f1f7cc 100644 --- a/bob/db/fargo/create.py +++ b/bob/db/fargo/create.py @@ -4,6 +4,9 @@ import os from .models import * +from bob.db.base.driver import Interface as BaseInterface + + import bob.core logger = bob.core.log.setup('bob.db.fargo') @@ -24,8 +27,14 @@ def add_clients(session, imagesdir, verbose=True): if verbose: logger.info("Adding clients ...") for d in os.listdir(imagesdir): - print(d) - + client_id = int(d) + if client_id <= 25: + group = 'train' + elif client_id <= 50: + group = 'dev' + else: + group = 'eval' + session.add(Client(client_id, group)) def create_tables(args): """Creates all necessary tables (only to be used at the first time)""" @@ -43,6 +52,7 @@ def create(args): from bob.db.base.utils import session_try_nolock + print(args) dbfile = args.files[0] if args.recreate: @@ -74,7 +84,7 @@ def add_command(subparsers): parser.add_argument('-v', '--verbose', action='count', default=0, help="Do SQL operations in a verbose way") parser.add_argument('-i', '--imagesdir', action='store', - default='/idiap/temp/heusch/bob.project.fargo/images/' + default='/idiap/temp/heusch/bob.project.fargo/images/', metavar='DIR', help="Change the path to the extracted images of the FARGO database (defaults to %(default)s)") diff --git a/bob/db/fargo/driver.py b/bob/db/fargo/driver.py index cab9b99..46ed1b9 100644 --- a/bob/db/fargo/driver.py +++ b/bob/db/fargo/driver.py @@ -71,19 +71,24 @@ class Interface(BaseInterface): return pkg_resources.require('bob.db.%s' % self.name())[0].version def files(self): - return () + from pkg_resources import resource_filename + raw_files = ('db.sql3',) + return [resource_filename(__name__, k) for k in raw_files] def type(self): - return 'text' + return 'sqlite' def add_commands(self, parser): from . import __doc__ as docs - subparsers = self.setup_parser(parser, - "FARGO database", docs) + subparsers = self.setup_parser(parser, "FARGO database", docs) import argparse + + # example: get the "create" action from a submodule + from .create import add_command as create_command + create_command(subparsers) # the "dumplist" action parser = subparsers.add_parser('dumplist', help=dumplist.__doc__) diff --git a/bob/db/fargo/models.py b/bob/db/fargo/models.py index 920c795..e63ccb6 100644 --- a/bob/db/fargo/models.py +++ b/bob/db/fargo/models.py @@ -2,13 +2,17 @@ # vim: set fileencoding=utf-8 : import os + from sqlalchemy import Table, Column, Integer, String, ForeignKey from bob.db.base.sqlalchemy_migration import Enum, relationship from sqlalchemy.orm import backref from sqlalchemy.ext.declarative import declarative_base + +import bob.db.base import numpy import bob.io.base import bob.io.image + import bob.core logger = bob.core.log.setup('bob.db.fargo') @@ -21,7 +25,7 @@ class Client(Base): __tablename__ = 'client' - set_choices = ('train', 'devel', 'test') + set_choices = ('train', 'dev', 'eval') """Possible groups to which clients may belong to""" id = Column(Integer, primary_key=True) @@ -38,7 +42,7 @@ class Client(Base): return "Client('%s', '%s')" % (self.id, self.set) -class File(Base): +class File(Base, bob.db.base.File): """Generic file container""" __tablename__ = 'file' @@ -59,6 +63,10 @@ class File(Base): modality_choices = ('rgb', 'nir', 'depth') modality = Column(Enum(*modality_choices)) + # recordings + recording_choices = ('0', '1') + recording = Column(Enum(*recording_choices)) + # shot (i.e. images extracted from the video sequence) # TODO: How to handle that with varying number of poses ? - Guillaume HEUSCH, 02-10-2018 shot_choices = tuple(['{:0>2d}'.format(s) for s in range(10)]) @@ -74,17 +82,16 @@ class File(Base): # path of this file in the database path = Column(String(100), unique=True) - # for Python - client = relationship(Client, backref=backref('files', order_by=id)) - """A direct link to the client object that this file belongs to""" - def __init__(self, client, path, light, device): - self.client = client - self.path = path + def __init__(self, id_file, client_id, path): + bob.db.base.File.__init__(self, path=path) + self.id = id_file + self.client_id = client_id self.light = light self.device = device self.pose = pose self.modality = modality + self.recording = recording self.shot = shot def __repr__(self): @@ -105,11 +112,9 @@ class File(Base): Returns a string containing the newly generated file path. """ - if not directory: directory = '' if not extension: extension = '' return str(os.path.join(directory, self.path + extension)) - diff --git a/bob/db/fargo/query.py b/bob/db/fargo/query.py index 28cf170..1ddcb23 100644 --- a/bob/db/fargo/query.py +++ b/bob/db/fargo/query.py @@ -1,18 +1,58 @@ #!/usr/bin/env python # encoding: utf-8 -import bob.bio.base -from bob.bio.face.database import FaceBioFile +import os +from bob.db.base import utils +from .models import * -class Database(bob.bio.base.database.FileListBioDatabase): - """Wrapper class for the FARGO database for face verification - """ +#from .driver import Interface +#INFO = Interface() +#SQLITE_FILE = INFO.files()[0] - def __init__(self, original_directory=None, original_extension=None, **kwargs): +import bob.db.base + +class Database(bob.db.base.SQLiteDatabase): + + def __init__(self, + original_directory=None, + original_extension=None, + annotation_directory=None, + annotation_extension=None): + # call base class constructor - from pkg_resources import resource_filename - folder = resource_filename(__name__, 'lists') - super(Database, self).__init__(folder, 'fargo', bio_file_class=FaceBioFile, - original_directory=original_directory, - original_extension=original_extension, **kwargs) + # File -> from models.py + super(Database, self).__init__(SQLITE_FILE, File, original_directory, original_extension) + + self.annotation_directory = annotation_directory + self.annotation_extension = annotation_extension + + def objects(self, protocol='mc-rgb', groups=None, modality='rgb'): + """ Return a set of file + + Parameters + ---------- + protocol: py:obj:str + One of the protocols + groups: py:ojg:str or py:obj:tuple of py:obj:str + One of the groups ('world', 'dev', 'eval') or several of them + + Returns + ------- + py:obj:list + A list of File + + Raises + ------ + + """ + + protocol = self.check_parameters_for_validity(protocol, "protocol", self.protocols()) + groups = self.check_parameters_for_validity(groups, "group", self.groups()) + + retval = [] + # training set: all images for the client id 1 to 25, controlled conditions, all device, all recordings + if 'train' in groups: + q = self.query(Client).join(File) + q = q.filter(Client.set.in_(groups)) + retval += list(q) diff --git a/setup.py b/setup.py index e19ea2d..bb4c10c 100644 --- a/setup.py +++ b/setup.py @@ -55,18 +55,6 @@ setup( 'fargo = bob.db.fargo.driver:Interface', ], - 'bob.bio.database' : [ - 'fargo_public_MC_RGB = bob.db.fargo.config:database_public_MC_RGB', - 'fargo_public_UD_RGB = bob.db.fargo.config:database_public_UD_RGB', - 'fargo_public_UO_RGB = bob.db.fargo.config:database_public_UO_RGB', - 'fargo_public_MC_NIR = bob.db.fargo.config:database_public_MC_NIR', - 'fargo_public_UD_NIR = bob.db.fargo.config:database_public_UD_NIR', - 'fargo_public_UO_NIR = bob.db.fargo.config:database_public_UO_NIR', - 'fargo_public_MC_depth = bob.db.fargo.config:database_public_MC_depth', - 'fargo_public_UD_depth = bob.db.fargo.config:database_public_UD_depth', - 'fargo_public_UO_depth = bob.db.fargo.config:database_public_UO_depth', - ], - }, classifiers=[ -- GitLab