Commit 16c6ff5d authored by Tiago Pereira's avatar Tiago Pereira Committed by Amir MOHAMMADI
Browse files

The User Guide section was rewritten. Approached the issues #22, #21 and #20

[sphinx] Preparing some plots

Documenting a little linear scoring

Changed the default initialization to RANDOM_NO_DUPLICATE

Continuing the documentation

Continuing the documentation

Addded bob.db.iris as a test requirement

Documenting

Ignoring .DS_Store files
parent 0c54d5dd
......@@ -15,5 +15,6 @@ dist
.nfs*
.gdb_history
build
.DS_Store
*.egg
src/
......@@ -15,7 +15,8 @@
static auto GMMMachine_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".GMMMachine",
"This class implements a multivariate diagonal Gaussian distribution.",
"This class implements the statistical model for multivariate diagonal mixture Gaussian distribution (GMM)."
"A GMM is defined as :math:`\\sum_{c=0}^{C} \\omega_c \\mathcal{N}(x | \\mu_c, \\sigma_c)`, where :math:`C` is the number of Gaussian components :math:`\\mu_c`, :math:`\\sigma_c` and :math:`\\omega_c` are respectively the the mean, variance and the weight of each gaussian component :math:`c`.",
"See Section 2.3.9 of Bishop, \"Pattern recognition and machine learning\", 2006"
).add_constructor(
bob::extension::FunctionDoc(
......@@ -744,7 +745,7 @@ static PyObject* PyBobLearnEMGMMMachine_loglikelihood_(PyBobLearnEMGMMMachineObj
/*** acc_statistics ***/
static auto acc_statistics = bob::extension::FunctionDoc(
"acc_statistics",
"Accumulate the GMM statistics for this sample(s). Inputs are checked.",
"Accumulate the GMM statistics (:py:class:`bob.learn.em.GMMStats)` for this sample(s). Inputs are checked.",
"",
true
)
......@@ -780,7 +781,7 @@ static PyObject* PyBobLearnEMGMMMachine_accStatistics(PyBobLearnEMGMMMachineObje
/*** acc_statistics_ ***/
static auto acc_statistics_ = bob::extension::FunctionDoc(
"acc_statistics_",
"Accumulate the GMM statistics for this sample(s). Inputs are NOT checked.",
"Accumulate the GMM statistics (:py:class:`bob.learn.em.GMMStats)` for this sample(s). Inputs are NOT checked.",
"",
true
)
......@@ -853,7 +854,7 @@ static PyObject* PyBobLearnEMGMMMachine_setVarianceThresholds_method(PyBobLearnE
/*** get_gaussian ***/
static auto get_gaussian = bob::extension::FunctionDoc(
"get_gaussian",
"Get the specified Gaussian component.",
"Get the specified Gaussian (:py:class:`bob.learn.em.Gaussian`) component.",
".. note:: An exception is thrown if i is out of range.",
true
)
......
......@@ -40,7 +40,7 @@ class KMeansTrainer
/**
* @brief Constructor
*/
KMeansTrainer(InitializationMethod=RANDOM);
KMeansTrainer(InitializationMethod=RANDOM_NO_DUPLICATE);
/**
* @brief Virtualize destructor
......
......@@ -184,7 +184,7 @@ static auto supervector_length = bob::extension::VariableDoc(
"int",
"Returns the supervector length.",
"NGaussians x NInputs: Number of Gaussian components by the feature dimensionality"
"WARNING An exception is thrown if no Universal Background Model has been set yet."
"An exception is thrown if no Universal Background Model has been set yet."
""
);
PyObject* PyBobLearnEMISVBase_getSupervectorLength(PyBobLearnEMISVBaseObject* self, void*) {
......
......@@ -88,7 +88,7 @@ int list_as_vector(PyObject* list, std::vector<blitz::Array<double,N> >& vec)
static auto ISVTrainer_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".ISVTrainer",
"ISVTrainer"
"References: [Vogt2008,McCool2013]",
"Train Intersession varibility modeling :ref:`ISV <isv>`.",
""
).add_constructor(
bob::extension::FunctionDoc(
......
......@@ -15,8 +15,8 @@
static auto IVectorMachine_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".IVectorMachine",
"An IVectorMachine consists of a Total Variability subspace :math:`T` and allows the extraction of IVector"
"References: [Dehak2010]_",
"Statistical model for the Total Variability training (:ref:`iVectors <ivector>`)"
"",
""
).add_constructor(
bob::extension::FunctionDoc(
......@@ -189,7 +189,7 @@ static auto supervector_length = bob::extension::VariableDoc(
"Returns the supervector length.",
"NGaussians x NInputs: Number of Gaussian components by the feature dimensionality"
"@warning An exception is thrown if no Universal Background Model has been set yet."
"An exception is thrown if no Universal Background Model has been set yet."
""
);
PyObject* PyBobLearnEMIVectorMachine_getSupervectorLength(PyBobLearnEMIVectorMachineObject* self, void*) {
......
......@@ -36,9 +36,9 @@ static int extract_GMMStats_1d(PyObject *list,
static auto IVectorTrainer_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".IVectorTrainer",
"IVectorTrainer"
"An IVectorTrainer to learn a Total Variability subspace :math:`$T$`"
" (and eventually a covariance matrix :math:`$\\Sigma$`).",
" References: [Dehak2010]"
"Trains the Total Variability subspace :math:`$T$` to generate :ref:`iVectors <ivector>`."
"",
""
).add_constructor(
bob::extension::FunctionDoc(
"__init__",
......
......@@ -15,8 +15,8 @@
static auto JFABase_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".JFABase",
"A JFABase instance can be seen as a container for :math:`U`, :math:`V` and :math:`D` when performing Joint Factor Analysis (JFA).\n\n"
"References: [Vogt2008]_ [McCool2013]_",
"Container for :math:`U`, :math:`V` and :math:`D` when performing Joint Factor Analysis (:ref:`JFA <jfa>`).\n\n"
"",
""
).add_constructor(
bob::extension::FunctionDoc(
......@@ -192,7 +192,7 @@ static auto supervector_length = bob::extension::VariableDoc(
"Returns the supervector length.",
"NGaussians x NInputs: Number of Gaussian components by the feature dimensionality"
"@warning An exception is thrown if no Universal Background Model has been set yet."
"An exception is thrown if no Universal Background Model has been set yet."
""
);
PyObject* PyBobLearnEMJFABase_getSupervectorLength(PyBobLearnEMJFABaseObject* self, void*) {
......
......@@ -87,8 +87,8 @@ int list_as_vector(PyObject* list, std::vector<blitz::Array<double,N> >& vec)
static auto JFATrainer_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".JFATrainer",
"JFATrainer"
"References: [Vogt2008,McCool2013]",
"Trains a Joint Factor Analysis (:ref:`JFA <jfa>`) on top of GMMs"
"",
""
).add_constructor(
bob::extension::FunctionDoc(
......
......@@ -15,7 +15,7 @@
static auto KMeansMachine_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".KMeansMachine",
"This class implements a k-means classifier.\n"
"Statistical model for the :ref:`k-means <kmeans>` .\n"
"See Section 9.1 of Bishop, \"Pattern recognition and machine learning\", 2006"
).add_constructor(
bob::extension::FunctionDoc(
......
......@@ -42,8 +42,8 @@ static inline const std::string& IM2string(bob::learn::em::KMeansTrainer::Initia
static auto KMeansTrainer_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".KMeansTrainer",
"Trains a KMeans machine."
"This class implements the expectation-maximization algorithm for a k-means machine."
"Trains a KMeans clustering :ref:`k-means <kmeans>.`"
"This class implements the expectation-maximization algorithm for a k-means."
"See Section 9.1 of Bishop, \"Pattern recognition and machine learning\", 2006"
"It uses a random initialization of the means followed by the expectation-maximization algorithm"
).add_constructor(
......
......@@ -71,6 +71,7 @@ static inline bool f(PyObject* o){return o != 0 && PyObject_IsTrue(o) > 0;}
/*** linear_scoring ***/
bob::extension::FunctionDoc linear_scoring1 = bob::extension::FunctionDoc(
"linear_scoring",
"The :ref:`Linear scoring <linearscoring>` is an approximation to the log-likelihood ratio that was shown to be as accurate and up to two orders of magnitude more efficient to compute [Glembek2009]_."
"",
0,
true
......
......@@ -17,7 +17,7 @@ static inline bool f(PyObject* o){return o != 0 && PyObject_IsTrue(o) > 0;} /*
static auto MAP_GMMTrainer_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".MAP_GMMTrainer",
"This class implements the maximum a posteriori M-step of the expectation-maximization algorithm for a GMM Machine. The prior parameters are encoded in the form of a GMM (e.g. a universal background model). The EM algorithm thus performs GMM adaptation."
"This class implements the maximum a posteriori (:ref:`MAP <map>`) M-step of the expectation-maximization algorithm for a GMM Machine. The prior parameters are encoded in the form of a GMM (e.g. a universal background model). The EM algorithm thus performs GMM adaptation."
).add_constructor(
bob::extension::FunctionDoc(
"__init__",
......
......@@ -17,7 +17,7 @@ static inline bool f(PyObject* o){return o != 0 && PyObject_IsTrue(o) > 0;} /*
static auto ML_GMMTrainer_doc = bob::extension::ClassDoc(
BOB_EXT_MODULE_PREFIX ".ML_GMMTrainer",
"This class implements the maximum likelihood M-step of the expectation-maximisation algorithm for a GMM Machine."
"This class implements the maximum likelihood M-step (:ref:`MLE <mle>`) of the expectation-maximisation algorithm for a GMM Machine."
).add_constructor(
bob::extension::FunctionDoc(
"__init__",
......
......@@ -12,7 +12,7 @@
/*** zt_norm ***/
bob::extension::FunctionDoc zt_norm = bob::extension::FunctionDoc(
"ztnorm",
"Normalise raw scores with ZT-Norm."
"Normalise raw scores with :ref:`ZT-Norm <ztnorm>`."
"Assume that znorm and tnorm have no common subject id.",
0,
true
......@@ -72,7 +72,7 @@ PyObject* PyBobLearnEM_ztNorm(PyObject*, PyObject* args, PyObject* kwargs) {
/*** t_norm ***/
bob::extension::FunctionDoc t_norm = bob::extension::FunctionDoc(
"tnorm",
"Normalise raw scores with T-Norm",
"Normalise raw scores with :ref:`T-Norm <tnorm>`",
0,
true
)
......@@ -109,7 +109,7 @@ PyObject* PyBobLearnEM_tNorm(PyObject*, PyObject* args, PyObject* kwargs) {
/*** z_norm ***/
bob::extension::FunctionDoc z_norm = bob::extension::FunctionDoc(
"znorm",
"Normalise raw scores with Z-Norm",
"Normalise raw scores with :ref:`Z-Norm <znorm>`",
0,
true
)
......
......@@ -25,6 +25,7 @@ extensions = [
'sphinx.ext.intersphinx',
'sphinx.ext.napoleon',
'sphinx.ext.viewcode',
'matplotlib.sphinxext.plot_directive'
]
import sphinx
......
This diff is collapsed.
......@@ -10,12 +10,16 @@
Expectation Maximization Machine Learning Tools
================================================
The EM algorithm is an iterative method that estimates parameters for statistical models, where the model depends on unobserved latent variables. The EM iteration alternates between performing an expectation (E) step, which creates a function for the expectation of the log-likelihood evaluated using the current estimate for the parameters, and a maximization (M) step, which computes parameters maximizing the expected log-likelihood found on the E step. These parameter-estimates are then used to determine the distribution of the latent variables in the next E step [WikiEM]_.
The EM algorithm is an iterative method that estimates parameters for statistical models, where the model depends on unobserved latent variables.
The EM iteration alternates between performing an expectation (E) step, which creates a function for the expectation of the log-likelihood
evaluated using the current estimate for the parameters, and a maximization (M) step,
which computes parameters maximizing the expected log-likelihood found on the E step.
These parameter-estimates are then used to determine the distribution of the latent variables in the next E step [WikiEM]_.
The package includes the machine definition per se and a selection of different trainers for specialized purposes:
- K-Means
- Maximum Likelihood (ML)
- Maximum a Posteriori (MAP)
- K-Means
- Inter Session Variability Modelling (ISV)
- Joint Factor Analysis (JFA)
- Total Variability Modeling (iVectors)
......@@ -47,8 +51,7 @@ References
.. [Roweis1998] Roweis, Sam. "EM algorithms for PCA and SPCA." Advances in neural information processing systems (1998): 626-632.
.. [WikiEM] `Expectation Maximization <http://en.wikipedia.org/wiki/Expectation%E2%80%93maximization_algorithm>`_
.. [Glembek2009] Glembek, Ondrej, et al. "Comparison of scoring methods used in speaker recognition with joint factor analysis." Acoustics, Speech and Signal Processing, 2009. ICASSP 2009. IEEE International Conference on. IEEE, 2009.
Indices and tables
------------------
......
import bob.db.iris
import numpy
numpy.random.seed(2) # FIXING A SEED
import bob.learn.linear
import bob.learn.em
import matplotlib.pyplot as plt
def train_ubm(features, n_gaussians):
input_size = features.shape[1]
kmeans_machine = bob.learn.em.KMeansMachine(int(n_gaussians), input_size)
ubm = bob.learn.em.GMMMachine(int(n_gaussians), input_size)
# The K-means clustering is firstly used to used to estimate the initial means, the final variances and the final weights for each gaussian component
kmeans_trainer = bob.learn.em.KMeansTrainer('RANDOM_NO_DUPLICATE')
bob.learn.em.train(kmeans_trainer, kmeans_machine, features)
# Getting the means, weights and the variances for each cluster. This is a very good estimator for the ML
(variances, weights) = kmeans_machine.get_variances_and_weights_for_each_cluster(features)
means = kmeans_machine.means
# initialize the UBM with the output of kmeans
ubm.means = means
ubm.variances = variances
ubm.weights = weights
# Creating the ML Trainer. We will adapt only the means
trainer = bob.learn.em.ML_GMMTrainer(update_means=True, update_variances=False, update_weights=False)
bob.learn.em.train(trainer, ubm, features)
return ubm
def isv_train(features, ubm):
"""
Features com lista de listas [ [data_point_1_user_1,data_point_2_user_1], [data_point_1_user_2,data_point_2_user_2] ]
"""
stats = []
for user in features:
user_stats = []
for f in user:
s = bob.learn.em.GMMStats(ubm.shape[0], ubm.shape[1])
ubm.acc_statistics(f, s)
user_stats.append(s)
stats.append(user_stats)
relevance_factor = 4
subspace_dimension_of_u = 1
isvbase = bob.learn.em.ISVBase(ubm, subspace_dimension_of_u)
trainer = bob.learn.em.ISVTrainer(relevance_factor)
# trainer.rng = bob.core.random.mt19937(int(self.init_seed))
bob.learn.em.train(trainer, isvbase, stats, max_iterations=50)
return isvbase
### GENERATING DATA
data_per_class = bob.db.iris.data()
setosa = numpy.column_stack((data_per_class['setosa'][:, 0], data_per_class['setosa'][:, 3]))
versicolor = numpy.column_stack((data_per_class['versicolor'][:, 0], data_per_class['versicolor'][:, 3]))
virginica = numpy.column_stack((data_per_class['virginica'][:, 0], data_per_class['virginica'][:, 3]))
data = numpy.vstack((setosa, versicolor, virginica))
# TRAINING THE PRIOR
ubm = train_ubm(data, 3)
isvbase = isv_train([setosa, versicolor, virginica], ubm)
# Variability direction
u0 = isvbase.u[0:2, 0] / numpy.linalg.norm(isvbase.u[0:2, 0])
u1 = isvbase.u[2:4, 0] / numpy.linalg.norm(isvbase.u[2:4, 0])
u2 = isvbase.u[4:6, 0] / numpy.linalg.norm(isvbase.u[4:6, 0])
figure, ax = plt.subplots()
plt.scatter(setosa[:, 0], setosa[:, 1], c="darkcyan", label="setosa")
plt.scatter(versicolor[:, 0], versicolor[:, 1], c="goldenrod", label="versicolor")
plt.scatter(virginica[:, 0], virginica[:, 1], c="dimgrey", label="virginica")
plt.scatter(ubm.means[:, 0], ubm.means[:, 1], c="blue", marker="x", label="centroids - mle")
#plt.scatter(ubm.means[:, 0], ubm.means[:, 1], c="blue", marker=".", label="within class varibility", s=0.01)
ax.arrow(ubm.means[0, 0], ubm.means[0, 1], u0[0], u0[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[1, 0], ubm.means[1, 1], u1[0], u1[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[2, 0], ubm.means[2, 1], u2[0], u2[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
plt.text(ubm.means[0, 0] + u0[0], ubm.means[0, 1] + u0[1] - 0.1, r'$\mathbf{U}_1$', fontsize=15)
plt.text(ubm.means[1, 0] + u1[0], ubm.means[1, 1] + u1[1] - 0.1, r'$\mathbf{U}_2$', fontsize=15)
plt.text(ubm.means[2, 0] + u2[0], ubm.means[2, 1] + u2[1] - 0.1, r'$\mathbf{U}_3$', fontsize=15)
ax.set_xticklabels("" for item in ax.get_xticklabels())
ax.set_yticklabels("" for item in ax.get_yticklabels())
# plt.grid(True)
plt.xlabel('Sepal length')
plt.ylabel('Petal width')
plt.legend()
plt.show()
import bob.db.iris
import numpy
numpy.random.seed(2) # FIXING A SEED
import bob.learn.linear
import bob.learn.em
import matplotlib.pyplot as plt
def train_ubm(features, n_gaussians):
input_size = features.shape[1]
kmeans_machine = bob.learn.em.KMeansMachine(int(n_gaussians), input_size)
ubm = bob.learn.em.GMMMachine(int(n_gaussians), input_size)
# The K-means clustering is firstly used to used to estimate the initial means, the final variances and the final weights for each gaussian component
kmeans_trainer = bob.learn.em.KMeansTrainer('RANDOM_NO_DUPLICATE')
bob.learn.em.train(kmeans_trainer, kmeans_machine, features)
# Getting the means, weights and the variances for each cluster. This is a very good estimator for the ML
(variances, weights) = kmeans_machine.get_variances_and_weights_for_each_cluster(features)
means = kmeans_machine.means
# initialize the UBM with the output of kmeans
ubm.means = means
ubm.variances = variances
ubm.weights = weights
# Creating the ML Trainer. We will adapt only the means
trainer = bob.learn.em.ML_GMMTrainer(update_means=True, update_variances=False, update_weights=False)
bob.learn.em.train(trainer, ubm, features)
return ubm
def jfa_train(features, ubm):
"""
Features com lista de listas [ [data_point_1_user_1,data_point_2_user_1], [data_point_1_user_2,data_point_2_user_2] ]
"""
stats = []
for user in features:
user_stats = []
for f in user:
s = bob.learn.em.GMMStats(ubm.shape[0], ubm.shape[1])
ubm.acc_statistics(f, s)
user_stats.append(s)
stats.append(user_stats)
subspace_dimension_of_u = 1
subspace_dimension_of_v = 1
jfa_base = bob.learn.em.JFABase(ubm, subspace_dimension_of_u, subspace_dimension_of_v)
trainer = bob.learn.em.JFATrainer()
# trainer.rng = bob.core.random.mt19937(int(self.init_seed))
bob.learn.em.train_jfa(trainer, jfa_base, stats, max_iterations=50)
return jfa_base
### GENERATING DATA
data_per_class = bob.db.iris.data()
setosa = numpy.column_stack((data_per_class['setosa'][:, 0], data_per_class['setosa'][:, 3]))
versicolor = numpy.column_stack((data_per_class['versicolor'][:, 0], data_per_class['versicolor'][:, 3]))
virginica = numpy.column_stack((data_per_class['virginica'][:, 0], data_per_class['virginica'][:, 3]))
data = numpy.vstack((setosa, versicolor, virginica))
# TRAINING THE PRIOR
ubm = train_ubm(data, 3)
jfa_base = jfa_train([setosa, versicolor, virginica], ubm)
# Variability direction U
u0 = jfa_base.u[0:2, 0] / numpy.linalg.norm(jfa_base.u[0:2, 0])
u1 = jfa_base.u[2:4, 0] / numpy.linalg.norm(jfa_base.u[2:4, 0])
u2 = jfa_base.u[4:6, 0] / numpy.linalg.norm(jfa_base.u[4:6, 0])
# Variability direction V
v0 = jfa_base.v[0:2, 0] / numpy.linalg.norm(jfa_base.v[0:2, 0])
v1 = jfa_base.v[2:4, 0] / numpy.linalg.norm(jfa_base.v[2:4, 0])
v2 = jfa_base.v[4:6, 0] / numpy.linalg.norm(jfa_base.v[4:6, 0])
figure, ax = plt.subplots()
plt.scatter(setosa[:, 0], setosa[:, 1], c="darkcyan", label="setosa")
plt.scatter(versicolor[:, 0], versicolor[:, 1], c="goldenrod", label="versicolor")
plt.scatter(virginica[:, 0], virginica[:, 1], c="dimgrey", label="virginica")
plt.scatter(ubm.means[:, 0], ubm.means[:, 1], c="blue", marker="x", label="centroids - mle")
#plt.scatter(ubm.means[:, 0], ubm.means[:, 1], c="blue", marker=".", label="within class varibility", s=0.01)
# U
ax.arrow(ubm.means[0, 0], ubm.means[0, 1], u0[0], u0[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[1, 0], ubm.means[1, 1], u1[0], u1[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[2, 0], ubm.means[2, 1], u2[0], u2[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
plt.text(ubm.means[0, 0] + u0[0], ubm.means[0, 1] + u0[1] - 0.1, r'$\mathbf{U}_1$', fontsize=15)
plt.text(ubm.means[1, 0] + u1[0], ubm.means[1, 1] + u1[1] - 0.1, r'$\mathbf{U}_2$', fontsize=15)
plt.text(ubm.means[2, 0] + u2[0], ubm.means[2, 1] + u2[1] - 0.1, r'$\mathbf{U}_3$', fontsize=15)
# V
ax.arrow(ubm.means[0, 0], ubm.means[0, 1], v0[0], v0[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[1, 0], ubm.means[1, 1], v1[0], v1[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
ax.arrow(ubm.means[2, 0], ubm.means[2, 1], v2[0], v2[1], fc="k", ec="k", head_width=0.05, head_length=0.1)
plt.text(ubm.means[0, 0] + v0[0], ubm.means[0, 1] + v0[1] - 0.1, r'$\mathbf{V}_1$', fontsize=15)
plt.text(ubm.means[1, 0] + v1[0], ubm.means[1, 1] + v1[1] - 0.1, r'$\mathbf{V}_2$', fontsize=15)
plt.text(ubm.means[2, 0] + v2[0], ubm.means[2, 1] + v2[1] - 0.1, r'$\mathbf{V}_3$', fontsize=15)
ax.set_xticklabels("" for item in ax.get_xticklabels())
ax.set_yticklabels("" for item in ax.get_yticklabels())
# plt.grid(True)
plt.xlabel('Sepal length')
plt.ylabel('Petal width')
plt.legend(loc=2)
plt.ylim([-1, 3.5])
#plt.show()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment