Commit 106df8b5 authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Merge branch 'fixes-scipy1.8' into 'master'

Fixes scipy 1.8.0

Closes #12

See merge request !19
parents db375654 2a448e14
Pipeline #58669 passed with stages
in 22 minutes and 7 seconds
......@@ -2,115 +2,130 @@ from ._library import BoostedMachine
import numpy
import scipy.optimize
import logging
logger = logging.getLogger('bob.learn.boosting')
logger = logging.getLogger("bob.learn.boosting")
class Boosting:
""" The class to boost the features from a set of training samples.
"""The class to boost the features from a set of training samples.
It iteratively adds new weak models to assemble a strong classifier.
In each round of iteration a weak machine is learned by optimizing a differentiable function.
It iteratively adds new weak models to assemble a strong classifier.
In each round of iteration a weak machine is learned by optimizing a differentiable function.
**Constructor Documentation**
**Constructor Documentation**
Keyword parameters
Keyword parameters
weak_trainer : :py:class:`bob.learn.boosting.LUTTrainer` or :py:class:`bob.learn.boosting.StumpTrainer`
The class to train weak machines.
weak_trainer : :py:class:`bob.learn.boosting.LUTTrainer` or :py:class:`bob.learn.boosting.StumpTrainer`
The class to train weak machines.
loss_function : a class derived from :py:class:`bob.learn.boosting.LossFunction`
The function to define the weights for the weak machines.
loss_function : a class derived from :py:class:`bob.learn.boosting.LossFunction`
The function to define the weights for the weak machines.
"""
"""
def __init__(self, weak_trainer, loss_function):
self.m_trainer = weak_trainer
self.m_loss_function = loss_function
def __init__(self, weak_trainer, loss_function):
self.m_trainer = weak_trainer
self.m_loss_function = loss_function
def get_loss_function(self):
"""Returns the loss function this trainer will use."""
return self.m_loss_function
def train(
self,
training_features,
training_targets,
number_of_rounds=20,
boosted_machine=None,
):
"""The function to train a boosting machine.
def get_loss_function(self):
"""Returns the loss function this trainer will use."""
return self.m_loss_function
The function boosts the training features and returns a strong classifier as a weighted combination of weak classifiers.
Keyword parameters:
def train(self, training_features, training_targets, number_of_rounds = 20, boosted_machine = None):
"""The function to train a boosting machine.
training_features : uint16 <#samples, #features> or float <#samples, #features>)
Features extracted from the training samples.
The function boosts the training features and returns a strong classifier as a weighted combination of weak classifiers.
training_targets : float <#samples, #outputs>
The values that the boosted classifier should reach for the given samples.
Keyword parameters:
number_of_rounds : int
The number of rounds of boosting, i.e., the number of weak classifiers to select.
training_features : uint16 <#samples, #features> or float <#samples, #features>)
Features extracted from the training samples.
boosted_machine :py:class:`bob.learn.boosting.BoostedMachine` or None
The machine to add the weak machines to. If not given, a new machine is created.
training_targets : float <#samples, #outputs>
The values that the boosted classifier should reach for the given samples.
Returns : :py:class:`bob.learn.boosting.BoostedMachine`
The boosted machine that is combination of the weak classifiers.
"""
number_of_rounds : int
The number of rounds of boosting, i.e., the number of weak classifiers to select.
# Initializations
if len(training_targets.shape) == 1:
training_targets = training_targets[:, numpy.newaxis]
boosted_machine :py:class:`bob.learn.boosting.BoostedMachine` or None
The machine to add the weak machines to. If not given, a new machine is created.
number_of_samples = training_features.shape[0]
number_of_outputs = training_targets.shape[1]
Returns : :py:class:`bob.learn.boosting.BoostedMachine`
The boosted machine that is combination of the weak classifiers.
"""
strong_predicted_scores = numpy.zeros((number_of_samples, number_of_outputs))
weak_predicted_scores = numpy.ndarray((number_of_samples, number_of_outputs))
# Initializations
if(len(training_targets.shape) == 1):
training_targets = training_targets[:,numpy.newaxis]
number_of_samples = training_features.shape[0]
number_of_outputs = training_targets.shape[1]
strong_predicted_scores = numpy.zeros((number_of_samples, number_of_outputs))
weak_predicted_scores = numpy.ndarray((number_of_samples, number_of_outputs))
if boosted_machine is not None:
boosted_machine(training_features, strong_predicted_scores)
else:
boosted_machine = BoostedMachine()
# Start boosting iterations for num_rnds rounds
logger.info("Starting %d rounds of boosting" % number_of_rounds)
for round in range(number_of_rounds):
logger.debug("Starting round %d" % (round+1))
# Compute the gradient of the loss function, l'(y,f(x)) using loss_class
loss_gradient = self.m_loss_function.loss_gradient(training_targets, strong_predicted_scores)
# Select the best weak machine for current round of boosting
weak_machine = self.m_trainer.train(training_features, loss_gradient)
# Compute the classification scores of the samples based only on the current round weak classifier (g_r)
weak_machine(training_features, weak_predicted_scores)
# Perform L-BFGS minimization and compute the scale (alpha_r) for current weak machine
alpha, _, flags = scipy.optimize.fmin_l_bfgs_b(
func = self.m_loss_function.loss_sum,
x0 = numpy.zeros(number_of_outputs),
fprime = self.m_loss_function.loss_gradient_sum,
args = (training_targets, strong_predicted_scores, weak_predicted_scores),
# disp = 1
)
# check output of L-BFGS
if flags['warnflag'] != 0:
msg = "too many function evaluations or too many iterations" if flags['warnflag'] == 1 else flags['task']
if (alpha == numpy.zeros(number_of_outputs)).all():
logger.warn("L-BFGS returned zero weights with error '%d': %s" % (flags['warnflag'], msg))
return boosted_machine
if boosted_machine is not None:
boosted_machine(training_features, strong_predicted_scores)
else:
logger.warn("L-BFGS returned warning '%d': %s" % (flags['warnflag'], msg))
# Update the prediction score after adding the score from the current weak classifier f(x) = f(x) + alpha_r*g_r
strong_predicted_scores += alpha * weak_predicted_scores
# Add the current weak machine into the boosting machine
boosted_machine.add_weak_machine(weak_machine, alpha)
logger.info("Finished round %d / %d" % (round+1, number_of_rounds))
return boosted_machine
boosted_machine = BoostedMachine()
# Start boosting iterations for num_rnds rounds
logger.info("Starting %d rounds of boosting" % number_of_rounds)
for round in range(number_of_rounds):
logger.debug("Starting round %d" % (round + 1))
# Compute the gradient of the loss function, l'(y,f(x)) using loss_class
loss_gradient = self.m_loss_function.loss_gradient(
training_targets, strong_predicted_scores
)
# Select the best weak machine for current round of boosting
weak_machine = self.m_trainer.train(training_features, loss_gradient)
# Compute the classification scores of the samples based only on the current round weak classifier (g_r)
weak_machine(training_features, weak_predicted_scores)
# Perform L-BFGS minimization and compute the scale (alpha_r) for current weak machine
alpha, _, flags = scipy.optimize.fmin_l_bfgs_b(
func=self.m_loss_function.loss_sum,
x0=numpy.zeros(number_of_outputs),
fprime=self.m_loss_function.loss_gradient_sum,
args=(training_targets, strong_predicted_scores, weak_predicted_scores),
# disp = 1
)
# check output of L-BFGS
if flags["warnflag"] != 0:
msg = (
"too many function evaluations or too many iterations"
if flags["warnflag"] == 1
else flags["task"]
)
if (alpha == numpy.zeros(number_of_outputs)).all():
logger.warn(
"L-BFGS returned zero weights with error '%d': %s"
% (flags["warnflag"], msg)
)
return boosted_machine
else:
logger.warn(
"L-BFGS returned warning '%d': %s" % (flags["warnflag"], msg)
)
# Update the prediction score after adding the score from the current weak classifier f(x) = f(x) + alpha_r*g_r
strong_predicted_scores += alpha * weak_predicted_scores
# Add the current weak machine into the boosting machine
boosted_machine.add_weak_machine(weak_machine, alpha)
logger.info("Finished round %d / %d" % (round + 1, number_of_rounds))
return boosted_machine
......@@ -26,7 +26,9 @@ class LossFunction(object):
Depending on the intended task, one of the two output variants should be chosen.
For classification tasks, please use the former way (#samples, #outputs), while for regression tasks, use the latter (#samples, 1).
"""
raise NotImplementedError("This is a pure abstract function. Please implement that in your derived class.")
raise NotImplementedError(
"This is a pure abstract function. Please implement that in your derived class."
)
def loss_gradient(self, targets, scores):
"""This function is to compute the gradient of the loss for the given targets and scores.
......@@ -40,7 +42,9 @@ class LossFunction(object):
Returns
loss (float <#samples, #outputs>): The gradient of the loss based on the given scores and targets.
"""
raise NotImplementedError("This is a pure abstract function. Please implement that in your derived class.")
raise NotImplementedError(
"This is a pure abstract function. Please implement that in your derived class."
)
def loss_sum(self, alpha, targets, previous_scores, current_scores):
"""The function computes the sum of the loss which is used to find the optimized values of alpha (x).
......@@ -68,7 +72,7 @@ class LossFunction(object):
losses = self.loss(targets, scores)
# compute the sum of the loss
return numpy.sum(losses, 0)
return numpy.mean(numpy.sum(losses, 0))
def loss_gradient_sum(self, alpha, targets, previous_scores, current_scores):
"""The function computes the gradient as the sum of the derivatives per sample which is used to find the optimized values of alpha.
......
......@@ -5,170 +5,195 @@ import bob
import bob.learn.boosting.utils
class TestBoosting(unittest.TestCase):
"""Class to test the LUT trainer """
def _data(self, digits = [3, 0], count = 20):
self.database = bob.learn.boosting.utils.MNIST()
# get the data
inputs, targets = [], []
for digit in digits:
input, target = self.database.data(labels = digit)
inputs.append(input[:count])
targets.append(target[:count])
return numpy.vstack(inputs), numpy.hstack(targets)
def _align_uni(self, targets):
# align target data to be used in a uni-variate classification
aligned = numpy.ones(targets.shape)
aligned[targets != targets[0]] = -1
return aligned
def _align_multi(self, targets, digits):
aligned = - numpy.ones((targets.shape[0], len(digits)))
for i, d in enumerate(digits):
aligned[targets==d, i] = 1
return aligned
def test01_stump_boosting(self):
# get test input data
inputs, targets = self._data()
aligned = self._align_uni(targets)
# for stump trainers, the exponential loss function is preferred
loss_function = bob.learn.boosting.ExponentialLoss()
weak_trainer = bob.learn.boosting.StumpTrainer()
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
machine = booster.train(inputs.astype(numpy.float64), aligned, number_of_rounds=1)
# check the result
weight = 1.83178082
self.assertEqual(machine.weights.shape, (1,1))
self.assertTrue(numpy.allclose(machine.weights, -weight))
self.assertEqual(len(machine.weak_machines), 1)
self.assertEqual(machine.indices, [483])
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.StumpMachine))
self.assertEqual(weak.threshold, 15.5)
self.assertEqual(weak.polarity, 1.)
# check first training image
single = machine(inputs[0].astype(numpy.uint16))
self.assertAlmostEqual(single, weight)
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 39 (out of 40) labels are correctly classified by a single feature position
self.assertTrue(numpy.allclose(labels * scores, weight))
self.assertEqual(numpy.count_nonzero(labels == aligned), 39)
def test02_lut_boosting(self):
# get test input data
inputs, targets = self._data()
aligned = self._align_uni(targets)
# for stump trainers, the logit loss function is preferred
loss_function = bob.learn.boosting.LogitLoss()
weak_trainer = bob.learn.boosting.LUTTrainer(256)
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
weight = 15.46452387
machine = booster.train(inputs.astype(numpy.uint16), aligned, number_of_rounds=1)
self.assertEqual(machine.weights.shape, (1,1))
self.assertTrue(numpy.allclose(machine.weights, -weight))
self.assertEqual(len(machine.weak_machines), 1)
self.assertEqual(machine.indices, [379])
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.LUTMachine))
self.assertEqual(weak.lut.shape, (256,1))
# check first training image
single = machine(inputs[0].astype(numpy.uint16))
self.assertAlmostEqual(single, weight)
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 40 (out of 40) labels are correctly classified by a single feature position
self.assertTrue(numpy.allclose(labels * scores, weight))
self.assertEqual(numpy.count_nonzero(labels == aligned), 40)
def test03_multi_shared(self):
# get test input data
digits = [1, 4, 7, 9]
inputs, targets = self._data(digits)
aligned = self._align_multi(targets, digits)
# for stump trainers, the logit loss function is preferred
loss_function = bob.learn.boosting.LogitLoss()
weak_trainer = bob.learn.boosting.LUTTrainer(256, len(digits), "shared")
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
weights = numpy.array([2.5123104, 2.19725677, 2.34455412, 1.94584326])
machine = booster.train(inputs.astype(numpy.uint16), aligned, number_of_rounds=1)
self.assertEqual(machine.weights.shape, (1,len(digits)))
self.assertTrue(numpy.allclose(machine.weights, -weights))
self.assertEqual(len(machine.weak_machines), 1)
self.assertEqual(machine.indices, [437])
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.LUTMachine))
self.assertEqual(weak.lut.shape, (256,4))
# check first training image
score = numpy.ndarray(4)
machine(inputs[0].astype(numpy.uint16), score)
self.assertTrue(numpy.allclose(score, weights * numpy.array([1., -1., -1., -1.])))
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 286 (out of 360) labels are correctly classified by a single feature position
self.assertTrue(all([numpy.allclose(numpy.abs(scores[i]), weights) for i in range(labels.shape[0])]))
self.assertEqual(numpy.count_nonzero(labels == aligned), 286)
def test04_multi_independent(self):
# get test input data
digits = [1, 4, 7, 9]
inputs, targets = self._data(digits)
aligned = self._align_multi(targets, digits)
# for stump trainers, the logit loss function is preferred
loss_function = bob.learn.boosting.LogitLoss()
weak_trainer = bob.learn.boosting.LUTTrainer(256, len(digits), "independent")
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
weights = numpy.array([2.94443872, 2.70805517, 2.34454354, 2.94443872])
machine = booster.train(inputs.astype(numpy.uint16), aligned, number_of_rounds=1)
self.assertEqual(machine.weights.shape, (1,len(digits)))
self.assertTrue(numpy.allclose(machine.weights, -weights))
self.assertEqual(len(machine.weak_machines), 1)
self.assertTrue(all(machine.indices == [215, 236, 264, 349]))
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.LUTMachine))
self.assertEqual(weak.lut.shape, (256,4))
# check first training image
score = numpy.ndarray(4)
machine(inputs[0].astype(numpy.uint16), score)
self.assertTrue(numpy.allclose(score, weights * numpy.array([1., -1., -1., -1.])))
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 294 (out of 360) labels are correctly classified by a single feature position
self.assertTrue(all([numpy.allclose(numpy.abs(scores[i]), weights) for i in range(labels.shape[0])]))
self.assertEqual(numpy.count_nonzero(labels == aligned), 294)
"""Class to test the LUT trainer"""
def _data(self, digits=[3, 0], count=20):
self.database = bob.learn.boosting.utils.MNIST()
# get the data
inputs, targets = [], []
for digit in digits:
input, target = self.database.data(labels=digit)
inputs.append(input[:count])
targets.append(target[:count])
return numpy.vstack(inputs), numpy.hstack(targets)
def _align_uni(self, targets):
# align target data to be used in a uni-variate classification
aligned = numpy.ones(targets.shape)
aligned[targets != targets[0]] = -1
return aligned
def _align_multi(self, targets, digits):
aligned = -numpy.ones((targets.shape[0], len(digits)))
for i, d in enumerate(digits):
aligned[targets == d, i] = 1
return aligned
def test01_stump_boosting(self):
# get test input data
inputs, targets = self._data()
aligned = self._align_uni(targets)
# for stump trainers, the exponential loss function is preferred
loss_function = bob.learn.boosting.ExponentialLoss()
weak_trainer = bob.learn.boosting.StumpTrainer()
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
machine = booster.train(
inputs.astype(numpy.float64), aligned, number_of_rounds=1
)
# check the result
weight = 1.83178082
self.assertEqual(machine.weights.shape, (1, 1))
self.assertTrue(numpy.allclose(machine.weights, -weight))
self.assertEqual(len(machine.weak_machines), 1)
self.assertEqual(machine.indices, [483])
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.StumpMachine))
self.assertEqual(weak.threshold, 15.5)
self.assertEqual(weak.polarity, 1.0)
# check first training image
single = machine(inputs[0].astype(numpy.uint16))
self.assertAlmostEqual(single, weight)
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 39 (out of 40) labels are correctly classified by a single feature position
self.assertTrue(numpy.allclose(labels * scores, weight))
self.assertEqual(numpy.count_nonzero(labels == aligned), 39)
def test02_lut_boosting(self):
# get test input data
inputs, targets = self._data()
aligned = self._align_uni(targets)
# for stump trainers, the logit loss function is preferred
loss_function = bob.learn.boosting.LogitLoss()
weak_trainer = bob.learn.boosting.LUTTrainer(256)
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
weight = 15.46452387
machine = booster.train(
inputs.astype(numpy.uint16), aligned, number_of_rounds=1
)
self.assertEqual(machine.weights.shape, (1, 1))
self.assertTrue(numpy.allclose(machine.weights, -weight))
self.assertEqual(len(machine.weak_machines), 1)
self.assertEqual(machine.indices, [379])
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.LUTMachine))
self.assertEqual(weak.lut.shape, (256, 1))
# check first training image
single = machine(inputs[0].astype(numpy.uint16))
self.assertAlmostEqual(single, weight)
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 40 (out of 40) labels are correctly classified by a single feature position
self.assertTrue(numpy.allclose(labels * scores, weight))
self.assertEqual(numpy.count_nonzero(labels == aligned), 40)
def test03_multi_shared(self):
# get test input data
digits = [1, 4, 7, 9]
inputs, targets = self._data(digits)
aligned = self._align_multi(targets, digits)
# for stump trainers, the logit loss function is preferred
loss_function = bob.learn.boosting.LogitLoss()
weak_trainer = bob.learn.boosting.LUTTrainer(256, len(digits), "shared")
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
weights = numpy.array([2.5123104, 2.19725677, 2.34455412, 1.94584326])
machine = booster.train(
inputs.astype(numpy.uint16), aligned, number_of_rounds=1
)
self.assertEqual(machine.weights.shape, (1, len(digits)))
self.assertTrue(numpy.allclose(machine.weights, -weights, rtol=10e-3))
self.assertEqual(len(machine.weak_machines), 1)
self.assertEqual(machine.indices, [437])
weak = machine.weak_machines[0]
self.assertTrue(isinstance(weak, bob.learn.boosting.LUTMachine))
self.assertEqual(weak.lut.shape, (256, 4))
# check first training image
score = numpy.ndarray(4)
machine(inputs[0].astype(numpy.uint16), score)
self.assertTrue(
numpy.allclose(
score, weights * numpy.array([1.0, -1.0, -1.0, -1.0]), rtol=10e-3
)
)
# check all training images
scores = numpy.ndarray(aligned.shape)
labels = numpy.ndarray(aligned.shape)
machine(inputs.astype(numpy.uint16), scores, labels)
# assert that 286 (out of 360) labels are correctly classified by a single feature position
self.assertTrue(
all(
[
numpy.allclose(numpy.abs(scores[i]), weights, rtol=10e-3)
for i in range(labels.shape[0])
]
)
)
self.assertEqual(numpy.count_nonzero(labels == aligned), 286)
def test04_multi_independent(self):
# get test input data
digits = [1, 4, 7, 9]
inputs, targets = self._data(digits)
aligned = self._align_multi(targets, digits)
# for stump trainers, the logit loss function is preferred
loss_function = bob.learn.boosting.LogitLoss()
weak_trainer = bob.learn.boosting.LUTTrainer(256, len(digits), "independent")
booster = bob.learn.boosting.Boosting(weak_trainer, loss_function)
# perform boosting
weights = numpy.array([2.94443872, 2.70805517, 2.34454354, 2.94443872])
machine = booster.train(
inputs.astype(numpy.uint16), aligned, number_of_rounds=1
)
self.assertEqual(machine.weights.shape, (1, len(digits)))
self.assertTrue(numpy.allclose(machine.weights,</