Merge branch 'cross_validation' into 'master'

Cross validation

See merge request !22

bob/learn/pytorch/architectures/FASNet.py

@@ -8,7 +8,7 @@ class FASNet(nn.Module):
 	"""PyTorch Reimplementation of Lucena, Oeslle, et al. "Transfer learning using 
 	convolutional neural networks for face anti-spoofing."
 	International Conference Image Analysis and Recognition. Springer, Cham, 2017.
-	eferenced from keras implementation: https://github.com/OeslleLucena/FASNet
+	Referenced from keras implementation: https://github.com/OeslleLucena/FASNet
 
 	Attributes:
 		pretrained: bool

bob/learn/pytorch/architectures/MCCNN.py

@@ -43,7 +43,7 @@ class MCCNN(nn.Module):
   	The path to download the pretrained LightCNN model from.
   
   """
-  def __init__(self, block=resblock, layers=[1, 2, 3, 4], num_channels=4, verbosity_level=2):
+  def __init__(self, block=resblock, layers=[1, 2, 3, 4], num_channels=4, verbosity_level=2, use_sigmoid=True):
     """ Init function
 
     Parameters
@@ -51,6 +51,9 @@ class MCCNN(nn.Module):
 
     num_channels: int
       The number of channels present in the input
+    use_sigmoid: bool
+      Whether to use sigmoid in eval phase. If set to `False` do not use
+      sigmoid in eval phase. Training phase is not affected.
     verbosity_level: int
       Verbosity level.
     
@@ -58,6 +61,7 @@ class MCCNN(nn.Module):
     super(MCCNN, self).__init__()
 
     self.num_channels=num_channels
+    self.use_sigmoid=use_sigmoid
 
     self.lcnn_layers=['conv1','block1','group1','block2', 'group2','block3','group3','block4','group4','fc']
 
@@ -211,7 +215,7 @@ class MCCNN(nn.Module):
 
     output = self.linear2fc(output)
 
-    if self.training:
+    if  self.training or self.use_sigmoid:
 
       output=nn.Sigmoid()(output)
 

bob/learn/pytorch/config/mccnn/wmca_mccnn.py → bob/learn/pytorch/config/mccnn/wmca_mccnn_cv.py

@@ -7,7 +7,7 @@ from bob.learn.pytorch.datasets import DataFolder
 
 from bob.pad.face.database import BatlPadDatabase
 
-from bob.learn.pytorch.datasets import ChannelSelect
+from bob.learn.pytorch.datasets import ChannelSelect, RandomHorizontalFlipImage
 
 
 #==============================================================================
@@ -36,17 +36,38 @@ from bob.learn.pytorch.datasets import ChannelSelect
 
 data_folder_train='/idiap/temp/ageorge/WMCA/preprocessed/'
 
+output_base_path='/idiap/temp/ageorge/Pytorch_WMCA/MCCNNv1/' 
+
+unseen_protocols=['','-LOO_fakehead','-LOO_flexiblemask','-LOO_glasses','-LOO_papermask','-LOO_prints','-LOO_replay','-LOO_rigidmask']
+
+PROTOCOL_INDEX=0 
+####################################################################
+
 frames=50
 
 extension='.h5'
 
 train_groups=['train'] # only 'train' group is used for training the network
 
-protocols="grandtest-color*depth*infrared*thermal-{}".format(frames) # makeup is excluded anyway here
+val_groups=['dev']
+
+do_crossvalidation=True
+#=======================
+
+if do_crossvalidation:
+	phases=['train','val']
+else:
+	phases=['train']
+
+groups={"train":['train'],"val":['dev']}
+
+
+
+protocols="grandtest-color-50"+unseen_protocols[PROTOCOL_INDEX] # makeup is excluded anyway here
 
 exlude_attacks_list=["makeup"]
 
-bob_hldi_instance_train = BatlPadDatabase(
+bob_hldi_instance = BatlPadDatabase(
     protocol=protocols,
     original_directory=data_folder_train,
     original_extension=extension,
@@ -58,50 +79,71 @@ bob_hldi_instance_train = BatlPadDatabase(
 #==============================================================================
 # Initialize the torch dataset, subselect channels from the pretrained files if needed.
 
-SELECTED_CHANNELS = [0,1,2,3] # selects  color, depth, infrared and thermal
+SELECTED_CHANNELS = [0,1,2,3] 
+####################################################################
 
-img_transform_train = transforms.Compose([ChannelSelect(selected_channels = SELECTED_CHANNELS),transforms.ToPILImage(),transforms.RandomHorizontalFlip(),transforms.ToTensor()])# Add p=0.5 later
 
-dataset = DataFolder(data_folder=data_folder_train,
-					 transform=img_transform_train,
-					 extension='.hdf5',
-					 bob_hldi_instance=bob_hldi_instance_train,
-					 groups=train_groups,
-					 protocol=protocols,
-					 purposes=['real', 'attack'],
-					 allow_missing_files=True)
+img_transform={}
 
-#==============================================================================
-# Load the architecture
+img_transform['train'] = transforms.Compose([ChannelSelect(selected_channels = SELECTED_CHANNELS),RandomHorizontalFlipImage(p=0.5),transforms.ToTensor()])
 
-NUM_CHANNELS = 4
+img_transform['val'] = transforms.Compose([ChannelSelect(selected_channels = SELECTED_CHANNELS),transforms.ToTensor()])
+
+dataset={}
+
+for phase in phases:
+
+	dataset[phase] = DataFolder(data_folder=data_folder_train,
+						 transform=img_transform[phase],
+						 extension='.hdf5',
+						 bob_hldi_instance=bob_hldi_instance,
+						 groups=groups[phase],
+						 protocol=protocols,
+						 purposes=['real', 'attack'],
+						 allow_missing_files=True)
 
-assert(len(SELECTED_CHANNELS)==NUM_CHANNELS)
 
-network=MCCNN(num_channels = NUM_CHANNELS)
 
 #==============================================================================
 # Specify other training parameters
 
-batch_size = 64
-num_workers = 4
+NUM_CHANNELS = len(SELECTED_CHANNELS)
+
+ADAPTED_LAYERS = 'conv1-block1-group1-ffc'
+####################################################################
+ADAPT_REF_CHANNEL = False
+####################################################################
+
+
+
+batch_size = 32
+num_workers = 0
 epochs=25
 learning_rate=0.0001
 seed = 3
-output_dir = 'training_mccn'
 use_gpu = False
-adapted_layers = 'conv1-group1-block1-ffc'
-adapt_reference_channel = False
+adapted_layers = ADAPTED_LAYERS
+adapt_reference_channel = ADAPT_REF_CHANNEL
 verbose = 2
+UID = "_".join([str(i) for i in SELECTED_CHANNELS])+"_"+str(ADAPT_REF_CHANNEL)+"_"+ADAPTED_LAYERS+"_"+str(NUM_CHANNELS)+"_"+protocols
+training_logs= output_base_path+UID+'/train_log_dir/'
+output_dir = output_base_path+UID
 
 
+#==============================================================================
+# Load the architecture
+
+
+assert(len(SELECTED_CHANNELS)==NUM_CHANNELS)
+
+network=MCCNN(num_channels = NUM_CHANNELS)
 #==============================================================================
 """
 Note: Running in GPU
 
 jman submit --queue gpu \
---name mccnn \
---log-dir /idiap/user/ageorge/WORK/COMMON_ENV_PAD_BATL_DB/training_mccn/logs/ \
+--name mccnnv2 \
+--log-dir /idiap/temp/ageorge/Pytorch_WMCA/MCCNNv2/logs/ \
 --environment="PYTHONUNBUFFERED=1" -- \
 ./bin/train_mccnn.py \
 /idiap/user/ageorge/WORK/COMMON_ENV_PAD_BATL_DB/src/bob.learn.pytorch/bob/learn/pytorch/config/mccnn/wmca_mccnn.py --use-gpu -vvv

bob/learn/pytorch/extractor/image/FASNet.py

+import numpy as np 
+import torch
+from torch.autograd import Variable
+
+import torchvision.transforms as transforms
+
+from bob.learn.pytorch.architectures import FASNet
+from bob.bio.base.extractor import Extractor
+
+import logging
+logger = logging.getLogger("bob.learn.pytorch")
+
+class FASNetExtractor(Extractor):
+  """ The class implementing the FASNet score computation.
+
+  Attributes
+  ----------
+  network: :py:class:`torch.nn.Module`
+      The network architecture
+  transforms: :py:mod:`torchvision.transforms`
+      The transform from numpy.array to torch.Tensor
+
+  """
+  
+  def __init__(self, transforms = transforms.Compose([transforms.ToPILImage(),transforms.Resize(224, interpolation=2),transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                 std=[0.229, 0.224, 0.225])]), model_file=None):
+    """ Init method
+
+    Parameters
+    ----------
+    num_channels_used: int
+      The number of channels to be used by the network. This could be 
+      different from the number of channels present in the input image. For instance, 
+      when used together with 'ChannelSelect' transform. The value of `num_channels_used`
+      should be the number of channels eventually used by the network (i.e., output of transform).
+    model_file: str
+      The path of the trained PAD network to load
+    transforms: :py:mod:`torchvision.transforms` 
+      tranform to be applied on the image
+    
+    """
+
+    Extractor.__init__(self, skip_extractor_training=True)
+    
+    # model
+    self.transforms = transforms 
+    self.network = FASNet()
+
+    
+    #self.network=self.network.to(device)
+
+    if model_file is None:
+      # do nothing (used mainly for unit testing) 
+      logger.debug("No pretrained file provided")
+      pass
+    else:
+
+
+      # With the new training
+      logger.debug('Starting to load the pretrained PAD model')
+      cp = torch.load(model_file)
+      if 'state_dict' in cp:
+        self.network.load_state_dict(cp['state_dict'])
+
+      logger.debug('Loaded the pretrained PAD model')    
+ 
+    self.network.eval()
+
+  def __call__(self, image):
+    """ Extract features from an image
+
+    Parameters
+    ----------
+    image : 3D :py:class:`numpy.ndarray` (floats)
+      The multi-channel image to extract the score from. Its size must be num_channelsx128x128;
+      Note: the value of `num_channels` is the number of channels as obtained from the preprocessed
+      data. The actual number of channels used may vary, for instance
+      if `ChannelSelect` transform is used, the number of channels used would change.
+      
+    Returns
+    -------
+    output : float
+      The extracted feature is a scalar values ~1 for bonafide and ~0 for PAs
+    
+    """
+   
+    input_image = np.rollaxis(np.rollaxis(image, 2),2) # changes to 128x128xnum_channels
+    input_image = self.transforms(input_image)
+    input_image = input_image.unsqueeze(0)
+    
+    output = self.network.forward(Variable(input_image))
+    output = output.data.numpy().flatten()
+
+    # output is a scalar score
+
+    return output
\ No newline at end of file

bob/learn/pytorch/extractor/image/__init__.py

@@ -3,6 +3,7 @@ from .LightCNN29 import LightCNN29Extractor
 from .LightCNN29v2 import LightCNN29v2Extractor
 from .MCCNN import MCCNNExtractor
 from .MCCNNv2 import MCCNNv2Extractor
+from .FASNet import FASNetExtractor
 
 __all__ = [_ for _ in dir() if not _.startswith('_')]
 

bob/learn/pytorch/scripts/train_fasnet.py

+#!/usr/bin/env python
+# encoding: utf-8
+
+
+""" Train a FASNet for face PAD
+
+Usage:
+  %(prog)s <configuration> 
+            [--model=<string>] [--batch-size=<int>] [--num-workers=<int>][--epochs=<int>] 
+            [--learning-rate=<float>][--do-crossvalidation][--seed=<int>] 
+            [--output-dir=<path>] [--use-gpu] [--verbose ...]
+
+Arguments:
+  <configuration>  A configuration file, defining the dataset and the network
+
+Options:
+  -h, --help                            Shows this help message and exits
+      --model=<string>                  Filename of the model to load (if any). 
+      --batch-size=<int>                Batch size [default: 64]
+      --num-workers=<int>               Number subprocesses to use for data loading [default: 0]
+      --epochs=<int>                    Number of training epochs [default: 20]
+      --learning-rate=<float>           Learning rate [default: 0.01]
+      --do-crossvalidation              Whether to perform cross validation [default: False]
+  -S, --seed=<int>                      The random seed [default: 3] 
+  -o, --output-dir=<path>               Dir to save stuff [default: training]
+  -g, --use-gpu                         Use the GPU
+  -v, --verbose                         Increase the verbosity (may appear multiple times).
+
+Note that arguments provided directly by command-line will override the ones in the configuration file.
+
+Example:
+
+  To run the training process 
+
+    $ %(prog)s config.py 
+
+See '%(prog)s --help' for more information.
+
+"""
+
+import os, sys
+import pkg_resources
+
+import torch
+import numpy
+from docopt import docopt
+
+import bob.core
+logger = bob.core.log.setup("bob.learn.pytorch")
+
+from bob.extension.config import load
+from bob.learn.pytorch.trainers import FASNetTrainer
+from bob.learn.pytorch.utils import get_parameter
+
+version = pkg_resources.require('bob.learn.pytorch')[0].version
+
+def main(user_input=None):
+  
+  # Parse the command-line arguments
+  if user_input is not None:
+      arguments = user_input
+  else:
+      arguments = sys.argv[1:]
+
+  prog = os.path.basename(sys.argv[0])
+  completions = dict(prog=prog, version=version,)
+  args = docopt(__doc__ % completions,argv=arguments,version='Train a FASNet (%s)' % version,)
+
+  # load configuration file
+  configuration = load([os.path.join(args['<configuration>'])])
+  
+  # get the pre-trained model file, if any
+  model = args['--model']
+  if hasattr(configuration, 'model'):
+    model = configuration.model
+  
+  # get various parameters, either from config file or command-line 
+  batch_size = get_parameter(args, configuration, 'batch_size', 64)
+  num_workers = get_parameter(args, configuration, 'num_workers', 0)
+  epochs = get_parameter(args, configuration, 'epochs', 20)
+  learning_rate = get_parameter(args, configuration, 'learning_rate', 0.01)
+  seed = get_parameter(args, configuration, 'seed', 3)
+  output_dir = get_parameter(args, configuration, 'output_dir', 'training')
+  use_gpu = get_parameter(args, configuration, 'use_gpu', False)
+  verbosity_level = get_parameter(args, configuration, 'verbose', 0)
+  do_crossvalidation  = get_parameter(args, configuration, 'do_crossvalidation', False)
+  
+  bob.core.log.set_verbosity_level(logger, verbosity_level)
+  bob.io.base.create_directories_safe(output_dir)
+
+  # print parameters
+  logger.debug("Model file = {}".format(model))
+  logger.debug("Batch size = {}".format(batch_size))
+  logger.debug("Num workers = {}".format(num_workers))
+  logger.debug("Epochs = {}".format(epochs))
+  logger.debug("Learning rate = {}".format(learning_rate))
+  logger.debug("Seed = {}".format(seed))
+  logger.debug("Output directory = {}".format(output_dir))
+  logger.debug("Use GPU = {}".format(use_gpu))
+  logger.debug("Perform cross validation = {}".format(do_crossvalidation))
+
+  # process on the arguments / options
+  torch.manual_seed(seed)
+  if use_gpu:
+    torch.cuda.manual_seed_all(seed)
+  if torch.cuda.is_available() and not use_gpu:
+    logger.warn("You have a CUDA device, so you should probably run with --use-gpu")
+
+  # get data
+  if hasattr(configuration, 'dataset'):
+    
+    dataloader={}
+
+    if not do_crossvalidation:
+
+      logger.info("There are {} training samples".format(len(configuration.dataset['train'])))
+
+      dataloader['train'] = torch.utils.data.DataLoader(configuration.dataset['train'], batch_size=batch_size, num_workers=num_workers, shuffle=True)
+
+    else:
+
+      dataloader['train'] = torch.utils.data.DataLoader(configuration.dataset['train'], batch_size=batch_size, num_workers=num_workers, shuffle=True)
+      dataloader['val'] = torch.utils.data.DataLoader(configuration.dataset['val'], batch_size=batch_size, num_workers=num_workers, shuffle=True)
+
+      logger.info("There are {} training samples".format(len(configuration.dataset['train'])))
+      logger.info("There are {} validation samples".format(len(configuration.dataset['val'])))
+    
+  else:
+    logger.error("Please provide a dataset in your configuration file !")
+    sys.exit()
+  
+  # train the network
+  if hasattr(configuration, 'network'):
+    trainer = FASNetTrainer(configuration.network, batch_size=batch_size, use_gpu=use_gpu, verbosity_level=verbosity_level,tf_logdir=output_dir+'/tf_logs',do_crossvalidation=do_crossvalidation)
+    trainer.train(dataloader, n_epochs=epochs, learning_rate=learning_rate, output_dir=output_dir, model=model)
+  else:
+    logger.error("Please provide a network in your configuration file !")
+    sys.exit()

bob/learn/pytorch/scripts/train_mccnn.py

@@ -7,7 +7,7 @@
 Usage:
   %(prog)s <configuration> 
             [--model=<string>] [--batch-size=<int>] [--num-workers=<int>][--epochs=<int>] 
-            [--learning-rate=<float>] [--adapted-layers=<string>] [--adapt-reference-channel] [--seed=<int>] 
+            [--learning-rate=<float>] [--adapted-layers=<string>] [--adapt-reference-channel] [--do-crossvalidation][--seed=<int>] 
             [--output-dir=<path>] [--use-gpu] [--verbose ...]
 
 Arguments:
@@ -22,6 +22,7 @@ Options:
       --learning-rate=<float>           Learning rate [default: 0.01]
       --adapted-layers=<string>         Layers to adapt in the training [default: conv1-block1-group1-ffc]
       --adapt-reference-channel         Flag deciding whether to adapt default channel as well [default: False]
+      --do-crossvalidation              Whether to perform cross validation [default: False]
   -S, --seed=<int>                      The random seed [default: 3] 
   -o, --output-dir=<path>               Dir to save stuff [default: training]
   -g, --use-gpu                         Use the GPU
@@ -86,6 +87,7 @@ def main(user_input=None):
   verbosity_level = get_parameter(args, configuration, 'verbose', 0)
   adapted_layers = get_parameter(args, configuration, 'adapted_layers', 'conv1-block1-group1-ffc')
   adapt_reference_channel = get_parameter(args, configuration, 'adapt_reference_channel', False)
+  do_crossvalidation  = get_parameter(args, configuration, 'do_crossvalidation', False)
   
   bob.core.log.set_verbosity_level(logger, verbosity_level)
   bob.io.base.create_directories_safe(output_dir)
@@ -101,6 +103,7 @@ def main(user_input=None):
   logger.debug("Use GPU = {}".format(use_gpu))
   logger.debug("Adapted layers = {}".format(adapted_layers))
   logger.debug("Adapt reference channel = {}".format(adapt_reference_channel))
+  logger.debug("Perform cross validation = {}".format(do_crossvalidation))
 
   # process on the arguments / options
   torch.manual_seed(seed)
@@ -111,15 +114,30 @@ def main(user_input=None):
 
   # get data
   if hasattr(configuration, 'dataset'):
-    dataloader = torch.utils.data.DataLoader(configuration.dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True)
-    logger.info("There are {} training samples".format(len(configuration.dataset)))
+    
+    dataloader={}
+
+    if not do_crossvalidation:
+
+      logger.info("There are {} training samples".format(len(configuration.dataset['train'])))
+
+      dataloader['train'] = torch.utils.data.DataLoader(configuration.dataset['train'], batch_size=batch_size, num_workers=num_workers, shuffle=True)
+
+    else:
+
+      dataloader['train'] = torch.utils.data.DataLoader(configuration.dataset['train'], batch_size=batch_size, num_workers=num_workers, shuffle=True)
+      dataloader['val'] = torch.utils.data.DataLoader(configuration.dataset['val'], batch_size=batch_size, num_workers=num_workers, shuffle=True)
+
+      logger.info("There are {} training samples".format(len(configuration.dataset['train'])))
+      logger.info("There are {} validation samples".format(len(configuration.dataset['val'])))
+    
   else:
     logger.error("Please provide a dataset in your configuration file !")
     sys.exit()
   
   # train the network
   if hasattr(configuration, 'network'):
-    trainer = MCCNNTrainer(configuration.network, batch_size=batch_size, use_gpu=use_gpu, adapted_layers=adapted_layers, adapt_reference_channel=adapt_reference_channel, verbosity_level=verbosity_level,tf_logdir=output_dir+'/tf_logs')
+    trainer = MCCNNTrainer(configuration.network, batch_size=batch_size, use_gpu=use_gpu, adapted_layers=adapted_layers, adapt_reference_channel=adapt_reference_channel, verbosity_level=verbosity_level,tf_logdir=output_dir+'/tf_logs',do_crossvalidation=do_crossvalidation)
     trainer.train(dataloader, n_epochs=epochs, learning_rate=learning_rate, output_dir=output_dir, model=model)
   else:
     logger.error("Please provide a network in your configuration file !")

bob/learn/pytorch/test/test.py

@@ -220,21 +220,95 @@ class DummyDataSetMCCNN(Dataset):
     sample = data, label
     return sample 
 
+
 def test_MCCNNtrainer():
 
   from ..architectures import MCCNN
   net = MCCNN(num_channels=4)
 
-  dataloader = torch.utils.data.DataLoader(DummyDataSetMCCNN(), batch_size=32, shuffle=True)
-  
+  dataloader={}
+  dataloader['train'] = torch.utils.data.DataLoader(DummyDataSetMCCNN(), batch_size=32, shuffle=True)
+
   from ..trainers import MCCNNTrainer
-  trainer = MCCNNTrainer(net, verbosity_level=3)
+  trainer = MCCNNTrainer(net, verbosity_level=3, do_crossvalidation=False)
   trainer.train(dataloader, n_epochs=1, output_dir='.')
 
   import os
   assert os.path.isfile('model_1_0.pth')
 
   os.remove('model_1_0.pth')   
+
+
+def test_MCCNNtrainer_cv():
+
+  from ..architectures import MCCNN
+  net = MCCNN(num_channels=4)
+
+  dataloader={}
+  dataloader['train'] = torch.utils.data.DataLoader(DummyDataSetMCCNN(), batch_size=32, shuffle=True)
+  dataloader['val'] = torch.utils.data.DataLoader(DummyDataSetMCCNN(), batch_size=32, shuffle=True)
+
+  from ..trainers import MCCNNTrainer
+  trainer = MCCNNTrainer(net, verbosity_level=3, do_crossvalidation=True)
+  trainer.train(dataloader, n_epochs=1, output_dir='.')
+
+  import os
+  assert os.path.isfile('model_1_0.pth')
+  assert os.path.isfile('model_100_0.pth') # the best model
+
+  os.remove('model_1_0.pth')  
+  os.remove('model_100_0.pth')   
+
+
+class DummyDataSetFASNet(Dataset):
+  def __init__(self):
+    pass
+  def __len__(self):
+    return 100
+  def __getitem__(self, idx):
+    data =  numpy.random.rand(3, 224,224).astype("float32")
+    label = numpy.random.randint(2)
+    sample = data, label
+    return sample 
+
+def test_FASNettrainer():
+
+  from ..architectures import FASNet
+  net = FASNet()
+
+  dataloader={}
+  dataloader['train'] = torch.utils.data.DataLoader(DummyDataSetFASNet(), batch_size=32, shuffle=True)
+
+  from ..trainers import FASNetTrainer
+  trainer = FASNetTrainer(net, verbosity_level=3,do_crossvalidation=False)
+  trainer.train(dataloader, n_epochs=1, output_dir='.')
+
+  import os
+  assert os.path.isfile('model_1_0.pth')
+
+  os.remove('model_1_0.pth') 
+
+
+def test_FASNettrainer_cv():
+
+  from ..architectures import FASNet
+  net = FASNet()
+
+  dataloader={}
+  dataloader['train'] = torch.utils.data.DataLoader(DummyDataSetFASNet(), batch_size=32, shuffle=True)
+  dataloader['val'] = torch.utils.data.DataLoader(DummyDataSetFASNet(), batch_size=32, shuffle=True)
+
+  from ..trainers import FASNetTrainer
+  trainer = FASNetTrainer(net, verbosity_level=3,do_crossvalidation=True)
+  trainer.train(dataloader, n_epochs=1, output_dir='.')
+
+  import os
+  assert os.path.isfile('model_1_0.pth')
+  assert os.path.isfile('model_100_0.pth')
+
+  os.remove('model_1_0.pth')
+  os.remove('model_100_0.pth')
+
 class DummyDataSetGAN(Dataset):
   def __init__(self):
     pass
@@ -356,6 +430,14 @@ def test_extractors():