diff --git a/bob/ip/pytorch_extractor/MLPAlgorithm.py b/bob/ip/pytorch_extractor/MLPAlgorithm.py
index 3c2daa6032978544fc52bf864c4f3167b4a7936f..571aaa617a9dbf291c83e418a5fc2127f78d58bc 100644
--- a/bob/ip/pytorch_extractor/MLPAlgorithm.py
+++ b/bob/ip/pytorch_extractor/MLPAlgorithm.py
@@ -23,6 +23,8 @@ from bob.pad.base.utils import convert_list_of_frame_cont_to_array
import bob.io.base
+import torch
+
# =============================================================================
# Main body :
@@ -34,36 +36,15 @@ class MLPAlgorithm(Algorithm):
Attributes
-----------
- config_file: str
- Relative name of the config file.
- The path should be relative to ``config_group``,
- for example: "test_data/mlp_algo_test_config.py".
-
- This file **must** contain at least the following definitions:
-
- Function namely ``transform``, which takes numpy.ndarray as input,
- and returns a transformed Tensor. The dimensionality of the output
- tensor must match the format expected by the MLP.
- A ``Network`` class, defining your network architecture. Note, if your
- class is named differently, import it as ``Network``, for example:
- ``from bob.learn.pytorch.architectures import MyNetwork as Network``
-
- Optional: ``network_kwargs`` to be used for ``Network`` initialization.
- For example, if you don't want to use a sigmoid in the output of the
- MLP, set the kwargs accodingly. Note: in current algorithm the
- ``forward()`` method of the ``Network`` is used for feature extraction.
-
- config_group: str
- Group/package name containing the configuration file. Usually all
- configs should be stored in this folder/place.
- For example: "bob.ip.pytorch_extractor".
- Both ``config_file`` and ``config_group`` are used to access the
- configuration module.
+ network : object
+ An instance of an MLP Network to be used for score computation.
+ Note: in current algorith the ``forward()`` method of the Network
+ is used for score computation. For example, if you don't want to use
+ a sigmoid in the output of an MLP, set the kwargs accodingly.
model_file : str
- A paths to the model file to be used for network initialization.
- The network structure is defined in the config file.
+ A paths to the model file to be used for ``network`` initialization.
url : str
URL to download the pretrained models from.
@@ -91,16 +72,14 @@ class MLPAlgorithm(Algorithm):
"""
def __init__(self,
- config_file,
- config_group,
+ network,
model_file = None,
url = None,
archive_extension = '.tar.gz',
frame_level_scores_flag = True,
mean_std_norm_flag = True):
- super(MLPAlgorithm, self).__init__(config_file = config_file,
- config_group = config_group,
+ super(MLPAlgorithm, self).__init__(network = network,
model_file = model_file,
url = url,
archive_extension = archive_extension,
@@ -109,8 +88,8 @@ class MLPAlgorithm(Algorithm):
performs_projection=True,
requires_projector_training=True)
- self.config_file = config_file
- self.config_group = config_group
+ self.transform = lambda x : torch.Tensor(x).unsqueeze(0)
+ self.network = network
self.model_file = model_file
self.url = url
self.archive_extension = archive_extension
@@ -221,7 +200,7 @@ class MLPAlgorithm(Algorithm):
----------
feature : FrameContainer or :py:class:`numpy.ndarray`
Two types of inputs are accepted.
- A Frame Container conteining the features of an individual frmaes,
+ A Frame Container containing the features of an individual frmaes,
see ``bob.bio.video.utils.FrameContainer``.
Or a ND feature array of the size (n_samples x n_features).
@@ -250,15 +229,11 @@ class MLPAlgorithm(Algorithm):
self.features_mean,
self.features_std)
- # kwargs for the transform_and_net_forward function:
- function_kwargs = {}
- function_kwargs["config_file"] = self.config_file
- function_kwargs["config_group"] = self.config_group
- function_kwargs["model_file"] = self.model_file
- function_kwargs["color_input_flag"] = False
-
scores = transform_and_net_forward(feature = feature,
- **function_kwargs)
+ transform = self.transform,
+ network = self.network,
+ model_file = self.model_file,
+ color_input_flag = False)
return scores
diff --git a/bob/ip/pytorch_extractor/MultiNetPatchExtractor.py b/bob/ip/pytorch_extractor/MultiNetPatchExtractor.py
index dc4469213c67ad82b55621a709b4abb014a0b651..19fb520ccfcc126ba8c1500a72f95ec6bd216194 100644
--- a/bob/ip/pytorch_extractor/MultiNetPatchExtractor.py
+++ b/bob/ip/pytorch_extractor/MultiNetPatchExtractor.py
@@ -1,8 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
-Created on Thu Oct 11 15:32:02 2018
-
@author: Olegs Nikisins
"""
@@ -13,7 +11,6 @@ from bob.bio.base.extractor import Extractor
import numpy as np
from bob.ip.pytorch_extractor.utils import reshape_flat_patches
-from bob.ip.pytorch_extractor.utils import combinations
from bob.ip.pytorch_extractor.utils import transform_and_net_forward
from bob.ip.pytorch_extractor.utils import load_pretrained_model
@@ -37,35 +34,21 @@ class MultiNetPatchExtractor(Extractor, object):
Attributes
-----------
- config_file: str
- Relative name of the config file.
- The path should be relative to ``config_group``,
- for example: "test_data/multi_net_patch_extractor_test_config.py".
-
- This file **must** contain at least the following definitions:
+ transform : object
Function namely ``transform``, which is a Compose transformation of
torchvision package, to be applied to the input samples.
- A ``Network`` class, defining your network architecture. Note, if your
- class is named differently, import it as ``Network``, for example:
- ``from bob.learn.pytorch.architectures import MyNetwork as Network``
-
- Optional: ``network_kwargs`` to be used for ``Network`` initialization.
- For example, if you want to use the latent embeddings of the autoencoder
- class, set the kwargs accodingly. Note: in current extractor the
- ``forward()`` method of the ``Network`` is used for feature extraction.
-
- config_group: str
- Group/package name containing the configuration file. Usually all
- configs should be stored in this folder/place.
- For example: "bob.ip.pytorch_extractor".
- Both ``config_file`` and ``config_group`` are used to access the
- configuration module.
+ network : object
+ An instance of the Network to be used for feature extraction.
+ Note: in current extractor the ``forward()`` method of the Network
+ is used for feature extraction. For example, if you want to use the
+ latent embeddings of the autoencoder class, initialize the network
+ accordingly.
model_file : [str]
- A list of paths to the model files to be used for network initialization.
- The network structure is defined in the config file.
+ A list of paths to the model files to be used for ``network``
+ initialization.
patches_num : [int]
A list of inices specifying which patches will be selected for
@@ -95,7 +78,7 @@ class MultiNetPatchExtractor(Extractor, object):
``urls``. The downloaded models **will be placed to the locations**
specified in ``model_file`` list.
- For example, the pretrained model for the autoencoder pre-trained on
+ For example, a model for an autoencoder pre-trained on
RGB faces of the size (3(channels) x 128 x 128) and fine-tuned
on the BW-NIR-D data can be found here:
["https://www.idiap.ch/software/bob/data/bob/bob.ip.pytorch_extractor/master/"
@@ -110,8 +93,9 @@ class MultiNetPatchExtractor(Extractor, object):
"""
# =========================================================================
- def __init__(self, config_file,
- config_group,
+ def __init__(self,
+ transform,
+ network,
model_file,
patches_num,
patch_reshape_parameters = None,
@@ -122,8 +106,8 @@ class MultiNetPatchExtractor(Extractor, object):
Init method.
"""
- super(MultiNetPatchExtractor, self).__init__(config_file = config_file,
- config_group = config_group,
+ super(MultiNetPatchExtractor, self).__init__(transform = transform,
+ network = network,
model_file = model_file,
patches_num = patches_num,
patch_reshape_parameters = patch_reshape_parameters,
@@ -131,8 +115,8 @@ class MultiNetPatchExtractor(Extractor, object):
urls = urls,
archive_extension = archive_extension)
- self.config_file = config_file
- self.config_group = config_group
+ self.transform = transform
+ self.network = network
self.model_file = model_file
self.patches_num = patches_num
self.patch_reshape_parameters = patch_reshape_parameters
@@ -151,8 +135,6 @@ class MultiNetPatchExtractor(Extractor, object):
patches : 2D :py:class:`numpy.ndarray`
An array containing flattened patches. The dimensions are:
``num_patches x len_of_flat_patch``
- Optional: the last column of the array can also be a binary mask.
- This case is also handled.
Returns
--------
@@ -160,19 +142,6 @@ class MultiNetPatchExtractor(Extractor, object):
Feature vector.
"""
- # kwargs for the transform_and_net_forward function:
- function_kwargs = {}
- function_kwargs["config_file"] = self.config_file
- function_kwargs["config_group"] = self.config_group
- function_kwargs["model_file"] = self.model_file
- function_kwargs["color_input_flag"] = self.color_input_flag
-
- # convert all values in the dictionary to the list if not a list already:
- function_kwargs = {k:[v] if not isinstance(v, list) else v for (k,v) in function_kwargs.items()}
-
- # compute all possible key-value combinations:
- function_kwargs = combinations(function_kwargs) # function_kwargs is now a list with kwargs
-
# select patches specified by indices:
patches_selected = [patches[idx] for idx in self.patches_num] # convert to list to make it iterable
@@ -181,11 +150,11 @@ class MultiNetPatchExtractor(Extractor, object):
features_all_patches = []
- # make sure the model_file and urls are not None:
+ # make sure the model_file and urls are not None, but lists:
if self.model_file is None:
- self.model_file = [self.model_file]
+ self.model_file = [None] * len(self.patches_num)
if self.urls is None:
- self.urls = [self.urls]
+ self.urls = [None] * len(self.patches_num)
for idx, patch in enumerate(patches_3d):
@@ -194,16 +163,24 @@ class MultiNetPatchExtractor(Extractor, object):
url = self.urls[self.patches_num[idx]],
archive_extension = self.archive_extension)
- if len(function_kwargs) == 1: # patches are passed through the same network:
+ if len(self.model_file) == 1: # patches are passed through the same network:
- features = transform_and_net_forward(feature = patch, **function_kwargs[0])
+ features = transform_and_net_forward(feature = patch,
+ transform = self.transform,
+ network = self.network,
+ model_file = self.model_file[0],
+ color_input_flag = self.color_input_flag)
else: # patches are passed through different networks:
- features = transform_and_net_forward(feature = patch, **function_kwargs[self.patches_num[idx]])
+ features = transform_and_net_forward(feature = patch,
+ transform = self.transform,
+ network = self.network,
+ model_file = self.model_file[idx],
+ color_input_flag = self.color_input_flag)
-# print ("The model we use for patch {} is:".format(str(idx)))
-# print (function_kwargs[self.patches_num(idx)]["model_file"])
+# print ("The model we use for patch {} is:".format(str(idx)))
+# print (self.model_file[idx])
features_all_patches.append(features)
@@ -211,4 +188,3 @@ class MultiNetPatchExtractor(Extractor, object):
return features
-
diff --git a/bob/ip/pytorch_extractor/test.py b/bob/ip/pytorch_extractor/test.py
index a856a20bfde9b253e66d89aa8368b92abb01347a..04fec22c7fa5b7b520a74dd4ca602169dc2aca81 100644
--- a/bob/ip/pytorch_extractor/test.py
+++ b/bob/ip/pytorch_extractor/test.py
@@ -49,6 +49,8 @@ def test_multi_net_patch_extractor():
"""
from bob.ip.pytorch_extractor import MultiNetPatchExtractor
+ from bob.learn.pytorch.architectures import ConvAutoencoder
+ from torchvision import transforms
# =========================================================================
# prepare the test data:
@@ -62,9 +64,13 @@ def test_multi_net_patch_extractor():
# =========================================================================
# test the extractor:
- CONFIG_FILE = "test_data/multi_net_patch_extractor_test_config.py" # config containing an instance of Composed Transform and a Network class to be used in feature extractor
- CONFIG_GROUP = "bob.ip.pytorch_extractor"
- # use specific/unique model for each patch. Models pre-trained on CelebA and fine-tuned (3 layers) on BATL:
+ # transform to be applied to input patches:
+ TRANSFORM = transforms.Compose([transforms.ToTensor(),
+ transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+ ])
+
+ # use latent embeddings in the feature extractor:
+ NETWORK = ConvAutoencoder(return_latent_embedding = True)
MODEL_FILE = None
@@ -72,16 +78,16 @@ def test_multi_net_patch_extractor():
PATCH_RESHAPE_PARAMETERS = [3, 128, 128] # reshape vectorized patches to this dimensions before passing to the Network
COLOR_INPUT_FLAG = True
- image_extractor = MultiNetPatchExtractor(config_file = CONFIG_FILE,
- config_group = CONFIG_GROUP,
- model_file = MODEL_FILE,
- patches_num = PATCHES_NUM,
- patch_reshape_parameters = PATCH_RESHAPE_PARAMETERS,
- color_input_flag = COLOR_INPUT_FLAG,
- urls = None)
+ extractor = MultiNetPatchExtractor(transform = TRANSFORM,
+ network = NETWORK,
+ model_file = MODEL_FILE,
+ patches_num = PATCHES_NUM,
+ patch_reshape_parameters = PATCH_RESHAPE_PARAMETERS,
+ color_input_flag = COLOR_INPUT_FLAG,
+ urls = None)
# pass through encoder only, compute latent vector:
- latent_vector = image_extractor(patch_flat)
+ latent_vector = extractor(patch_flat)
# test:
assert latent_vector.shape == (1296,)
@@ -101,6 +107,8 @@ def test_mlp_algorithm():
"""
from bob.ip.pytorch_extractor import MLPAlgorithm
+ from bob.learn.pytorch.architectures import TwoLayerMLP
+
# =========================================================================
# prepare the test data / feature vector:
@@ -110,16 +118,19 @@ def test_mlp_algorithm():
# =========================================================================
# test the extractor:
- CONFIG_FILE = "test_data/mlp_algo_test_config.py" # config containing an instance of Composed Transform and a Network class to be used in feature extractor
- CONFIG_GROUP = "bob.ip.pytorch_extractor"
+ # don't use the sigmoid in the output:
+ NETWORK = TwoLayerMLP(in_features = 1296,
+ n_hidden_relu = 10,
+ apply_sigmoid = False)
+
MODEL_FILE = None
- algorithm = MLPAlgorithm(config_file = CONFIG_FILE,
- config_group = CONFIG_GROUP,
+ algorithm = MLPAlgorithm(network = NETWORK,
model_file = MODEL_FILE,
url = None,
archive_extension = '.tar.gz',
- frame_level_scores_flag = True)
+ frame_level_scores_flag = True,
+ mean_std_norm_flag = True)
# pass through encoder only, compute latent vector:
score = algorithm.project(features)
diff --git a/bob/ip/pytorch_extractor/test_data/__init__.py b/bob/ip/pytorch_extractor/test_data/__init__.py
deleted file mode 100755
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/bob/ip/pytorch_extractor/test_data/mlp_algo_test_config.py b/bob/ip/pytorch_extractor/test_data/mlp_algo_test_config.py
deleted file mode 100644
index aad962df80ef766defa311b05619b517d8ae7682..0000000000000000000000000000000000000000
--- a/bob/ip/pytorch_extractor/test_data/mlp_algo_test_config.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-"""
-@author: Olegs Nikisins
-"""
-#==============================================================================
-# Import here:
-
-import torch
-
-
-#==============================================================================
-# Define parameters here:
-
-"""
-Transformations to be applied to the input 1D numpy arrays (feature vectors).
-Only conversion to Tensor and unsqueezing is needed to match the input of
-TwoLayerMLP network
-"""
-
-def transform(x):
- """
- Convert input to Tensor and unsqueeze to match the input of
- TwoLayerMLP network.
-
- Arguments
- ---------
- x : numpy array
- 1D numpy array / feature vector.
-
- Return
- ------
- x_transform : Tensor
- Torch tensor, transformed ``x`` to be used as MLP input.
- """
-
- return torch.Tensor(x).unsqueeze(0)
-
-
-"""
-Define the network to be trained as a class, named ``Network``.
-Note: Do not change the name of the below class, always import as ``Network``.
-"""
-
-from bob.learn.pytorch.architectures import TwoLayerMLP as Network
-
-
-"""
-kwargs to be used for ``Network`` initialization. The name must be ``network_kwargs``.
-"""
-network_kwargs = {}
-network_kwargs['in_features'] = 1296
-network_kwargs['n_hidden_relu'] = 10
-network_kwargs['apply_sigmoid'] = False # don't use sigmoid to make the scores more even
-
-
diff --git a/bob/ip/pytorch_extractor/test_data/multi_net_patch_extractor_test_config.py b/bob/ip/pytorch_extractor/test_data/multi_net_patch_extractor_test_config.py
deleted file mode 100644
index 6e0b095d793e54bc787a83512e5134aa1f32124e..0000000000000000000000000000000000000000
--- a/bob/ip/pytorch_extractor/test_data/multi_net_patch_extractor_test_config.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-"""
-@author: Olegs Nikisins
-"""
-#==============================================================================
-# Import here:
-
-from torchvision import transforms
-
-
-#==============================================================================
-# Define parameters here:
-
-"""
-Transformations to be applied sequentially to the input PIL image.
-Note: the variable name ``transform`` must be the same in all configuration files.
-"""
-
-transform = transforms.Compose([transforms.ToTensor(),
- transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
- ])
-
-
-"""
-Define the network to be trained as a class, named ``Network``.
-Note: Do not change the name of the below class, always import as ``Network``.
-"""
-
-from bob.learn.pytorch.architectures import ConvAutoencoder as Network
-
-
-"""
-kwargs to be used for ``Network`` initialization. The name must be ``network_kwargs``.
-"""
-network_kwargs = {}
-network_kwargs['return_latent_embedding'] = True
diff --git a/bob/ip/pytorch_extractor/utils.py b/bob/ip/pytorch_extractor/utils.py
index 78ff2affe538843bd03ed1305e9be5e2922231b0..cda62c47ff80f098d1307c853736341c71171f1a 100644
--- a/bob/ip/pytorch_extractor/utils.py
+++ b/bob/ip/pytorch_extractor/utils.py
@@ -1,8 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
-Created on Fri Jan 25 14:29:55 2019
-
@author: Olegs Nikisins
"""
@@ -10,8 +8,6 @@ Created on Fri Jan 25 14:29:55 2019
# Import what is needed here:
import os
-import importlib
-
from torchvision import transforms
import numpy as np
@@ -143,52 +139,25 @@ def apply_transform_as_pil(img_array, transform):
return pil_img
-def _init_the_network(config_module):
- """
- Initialize the network, given imported configuration module.
-
- Parameters
- ----------
- config_module : object
- Module containing Network configuration parameters.
-
- Returns
- -------
- model : object
- An instance of the Network class.
- """
-
- if "network_kwargs" in dir(config_module):
- network_kwargs = config_module.network_kwargs
- model = config_module.Network(**network_kwargs)
-
- else:
- model = config_module.Network()
-
- return model
-
-
-def _transform(feature, config_module, color_input_flag):
+# =============================================================================
+def _transform(feature, color_input_flag, transform = None):
"""
- Apply transformation defined in the configuration module to the input data.
+ Apply transformation to the input data.
Currently two types of the transformation are supported:
First, Compose transformation of torchvision package.
Second, custom transformation functions, to be applied to each feature
- vector. For example, mean-std normalization.
-
- In both cases define transform() method in the ``config_module``.
+ vector. For example, mean-std normalization, or simple conversion of
+ input numpy vector to PyTorch tensors.
Parameters
----------
+
feature : :py:class:`numpy.ndarray`
ND feature array of the size (N_samples x dim1 x dim2 x ...).
- config_module : object
- Module containing Network configuration parameters.
-
color_input_flag : bool
If set to ``True``, the input is considered to be a color image of the
size ``(3, H, W)``. The tensor to be passed through the net will be
@@ -196,17 +165,21 @@ def _transform(feature, config_module, color_input_flag):
If set to ``False``, the input is considered to be a set of BW images
of the size ``(n_samples, H, W)``. The tensor to be passed through
the net will be of the size ``(n_samples, 1, H, W)``.
+
+ transform : object
+ Function namely ``transform`` to be applied to the input samples.
+ Default: None .
"""
- if "transform" in dir(config_module):
+ if transform is not None:
- if isinstance(config_module.transform, transforms.Compose):
+ if isinstance(transform, transforms.Compose):
- feature = apply_transform_as_pil(feature, config_module.transform)
+ feature = apply_transform_as_pil(feature, transform)
else:
- feature = np.stack([config_module.transform(item) for item in feature])
+ feature = np.stack([transform(item) for item in feature])
if color_input_flag:
# convert feature array to Tensor of size (1, 3, H, W)
@@ -221,20 +194,16 @@ def _transform(feature, config_module, color_input_flag):
# =============================================================================
def transform_and_net_forward(feature,
- config_file,
- config_group,
+ transform,
+ network,
model_file,
color_input_flag = False):
"""
This function performs the following steps:
- 1. Import config module.
-
- 2. Applies transformation defined in the config file to the input data.
+ 1. Applies transformation defined in the ``transform`` argument.
- 3. Initializes the Network class with optional kwargs.
-
- 4. Pass the transformed data through ``forward()`` method of the Network
+ 2. Pass the transformed data through ``forward()`` method of the Network
class.
Parameters
@@ -243,17 +212,15 @@ def transform_and_net_forward(feature,
feature : :py:class:`numpy.ndarray`
ND feature array of the size (N_samples x dim1 x dim2 x ...).
- config_file: str
- Relative name of the config file defining the network, training data,
- and training parameters. The path should be relative to
- ``config_group``, for example: "autoencoder/netN.py".
+ transform : object
+ Function namely ``transform`` to be applied to the input samples.
- config_group : str
- Group/package name containing the configuration file. Usually all
- configs should be stored in this folder/place.
- For example: "bob.pad.face.config.pytorch".
- Both ``config_file`` and ``config_group`` are used to access the
- configuration module.
+ network : object
+ An instance of the Network to be used for feature extraction.
+ Note: in current extractor the ``forward()`` method of the Network
+ is used for feature extraction. For example, if you want to use the
+ latent embeddings of the autoencoder class, initialize the network
+ accordingly.
model_file : str
A path to the model file to be used for network initialization.
@@ -274,38 +241,25 @@ def transform_and_net_forward(feature,
Output of the network per input sample/frame.
"""
- # =========================================================================
- # Create relative module name given path:
- relative_mod_name = '.' + os.path.splitext(config_file)[0].replace(os.path.sep, '.')
-
- # import configuration module:
- config_module = importlib.import_module(relative_mod_name, config_group)
-
-
# =========================================================================
# transform the input image or feature vector:
- feature_tensor = _transform(feature, config_module, color_input_flag)
-
+ feature_tensor = _transform(feature, color_input_flag, transform)
# =========================================================================
- # Initialize the model
- local_model = _init_the_network(config_module)
-
# Load the pre-trained model into the network
if model_file is not None:
model_state = torch.load(model_file, map_location=lambda storage,loc:storage)
# Initialize the state of the model:
- local_model.load_state_dict(model_state)
+ network.load_state_dict(model_state)
# Model is used for evaluation only:
- local_model.train(False)
-
+ network.train(False)
# =========================================================================
# Pass the transformed feature vector through the network:
- output_tensor = local_model.forward(Variable(feature_tensor))
+ output_tensor = network.forward(Variable(feature_tensor))
net_output = output_tensor.data.numpy().squeeze()
@@ -314,6 +268,7 @@ def transform_and_net_forward(feature,
return net_output.astype(np.float)
+# =============================================================================
def load_pretrained_model(model_path, url, archive_extension = '.tar.gz'):
"""
Loads the model from the given ``url``, if the model specified in the
@@ -322,6 +277,7 @@ def load_pretrained_model(model_path, url, archive_extension = '.tar.gz'):
Arguments
---------
+
model_path : str
Absolute file name pointing to the model.