Commit ba649671 authored by Manuel Günther's avatar Manuel Günther
Browse files

Unified losses; added jesorsky loss; implemented weighted hisdtogram in C++;...

Unified losses; added jesorsky loss; implemented weighted hisdtogram in C++; implemented missing recall.
parent 0208357b
......@@ -5,7 +5,6 @@
[buildout]
parts = xbob.boosting scripts
eggs = xbob.boosting
xbob.db.banca
xbob.db.mnist
newest = false
......@@ -19,4 +18,4 @@ recipe = xbob.buildout:develop
[scripts]
recipe = xbob.buildout:scripts
dependent-scripts = true
dependent-scripts = false
......@@ -124,15 +124,15 @@ setup(
# scripts should be declared using this entry:
'console_scripts': [
'mnist_binary_all.py = xbob.boosting.scripts.mnist_binary_all:main',
'mnist_binary_one.py = xbob.boosting.scripts.mnist_binary_one:main',
'mnist_multi.py = xbob.boosting.scripts.mnist_multi:main',
'mnist_lbp.py = xbob.boosting.scripts.mnist_lbp:main',
'mnist_multi_lbp.py = xbob.boosting.scripts.mnist_multi_lbp:main',
'mnist_onevsall.py = xbob.boosting.scripts.mnist_onevsall:main',
'mnist_onevsall_lbp.py = xbob.boosting.scripts.mnist_onevsall_lbp:main',
'mnist_onevsall_block_lbp.py = xbob.boosting.scripts.mnist_onevsall_block_lbp:main',
'mnist_multi_block_lbp.py = xbob.boosting.scripts.mnist_multi_block_lbp:main',
# 'mnist_binary_all.py = xbob.boosting.scripts.mnist_binary_all:main',
# 'mnist_binary_one.py = xbob.boosting.scripts.mnist_binary_one:main',
# 'mnist_multi.py = xbob.boosting.scripts.mnist_multi:main',
# 'mnist_lbp.py = xbob.boosting.scripts.mnist_lbp:main',
# 'mnist_multi_lbp.py = xbob.boosting.scripts.mnist_multi_lbp:main',
# 'mnist_onevsall.py = xbob.boosting.scripts.mnist_onevsall:main',
# 'mnist_onevsall_lbp.py = xbob.boosting.scripts.mnist_onevsall_lbp:main',
# 'mnist_onevsall_block_lbp.py = xbob.boosting.scripts.mnist_onevsall_block_lbp:main',
# 'mnist_multi_block_lbp.py = xbob.boosting.scripts.mnist_multi_block_lbp:main',
],
# tests that are _exported_ (that can be executed by other packages) can
......
# import the C++ stuff
from ._boosting import StumpMachine, LUTMachine, BoostedMachine
from ._boosting import StumpMachine, LUTMachine, BoostedMachine, weighted_histogram
import core
import features
......
......@@ -180,9 +180,14 @@ class Boost:
targets = targets[:,numpy.newaxis]
num_op = targets.shape[1]
machine = BoostedMachine() if boosted_machine is None else boosted_machine
num_samp = fset.shape[0]
pred_scores = numpy.zeros([num_samp,num_op])
if boosted_machine is not None:
machine = boosted_machine
machine(fset, pred_scores)
else:
machine = BoostedMachine()
loss_class = losses.LOSS_FUNCTIONS[self.loss_type]
loss_func = loss_class()
......@@ -203,26 +208,26 @@ class Boost:
logger.info("Starting %d rounds of boosting" % self.num_rnds)
for r in range(self.num_rnds):
logger.debug("Starting round %d" % (r+1))
# Compute the gradient of the loss function, l'(y,f(x)) using loss_class
loss_grad = loss_func.update_loss_grad(targets,pred_scores)
loss_grad = loss_func.loss_gradient(targets,pred_scores)
# Select the best weak machine for current round of boosting
curr_weak_machine = weak_trainer.compute_weak_trainer(fset, loss_grad)
# Compute the classification scores of the samples based only on the current round weak classifier (g_r)
curr_pred_scores = numpy.zeros([num_samp,num_op], numpy.float64)
curr_weak_machine(fset, curr_pred_scores)
# Initialize the start point for lbfgs minimization
init_point = numpy.zeros(num_op)
# Perform lbfgs minimization and compute the scale (alpha_r) for current weak trainer
lbfgs_struct = scipy.optimize.fmin_l_bfgs_b(loss_func.loss_sum, init_point, fprime = loss_func.loss_grad_sum, args = (targets, pred_scores, curr_pred_scores))
alpha = lbfgs_struct[0]
# Update the prediction score after adding the score from the current weak classifier f(x) = f(x) + alpha_r*g_r
pred_scores = pred_scores + alpha*curr_pred_scores
......@@ -230,7 +235,7 @@ class Boost:
# Add the current trainer into the boosting machine
machine.add_weak_machine(curr_weak_machine, alpha)
logger.debug("Finished round %d / %r" % (r+1, self.num_rnds))
logger.info("Finished round %d / %d" % (r+1, self.num_rnds))
return machine
......
......@@ -2,280 +2,214 @@ import numpy
import math
class LossFunction:
class ExpLossFunction():
""" The class to implement the exponential loss function for the boosting framework.
def loss(self, targets, scores):
raise NotImplementedError("This is a pure abstract function. Please implement that in your derived class.")
def loss_gradient(self, targets, scores):
raise NotImplementedError("This is a pure abstract function. Please implement that in your derived class.")
def loss_sum(self, alpha, targets, prediction_scores, weak_scores):
"""The function computes the sum of the loss which is used to find the optimized values of alpha (x).
The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
This function is given as the input for the lbfgs optimization function.
Inputs:
alpha: The current value of the alpha.
type: float
targets: The targets for the samples
type: numpy array (# number of samples x # number of outputs)
pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
type: numpy array (# number of samples x # number of outputs)
curr_scores: The prediction scores of the samples for the current round of the boosting.
type: numpy array (# number of samples x # number of outputs)
Return:
sum_loss: The sum of the loss values for the current value of the alpha
type: float
"""
# compute the scores and loss for the current alpha
curr_scores = prediction_scores + alpha * weak_scores
loss = self.loss(targets, curr_scores)
# compute the sum of the loss
return numpy.sum(loss, 0)
def loss_grad_sum(self, alpha, targets, prediction_scores, weak_scores):
"""The function computes the gradient as the sum of the derivatives per sample which is used to find the optimized values of alpha (x).
The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
This function is given as the input for the lbfgs optimization function.
Inputs:
alpha: The current value of the alpha.
type: float
targets: The targets for the samples
type: numpy array (# number of samples x # number of outputs)
pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
type: numpy array (# number of samples x # number of outputs)
curr_scores: The prediction scores of the samples for the current round of the boosting.
type: numpy array (# number of samples x # number of outputs)
Returns
The sum of the loss gradient values for the current value of the alpha
type: numpy array (# number of outputs)
"""
# compute the loss gradient for the updated score
curr_scores = prediction_scores + alpha * weak_scores
loss_grad = self.loss_gradient(targets, curr_scores)
# take the sum of the loss gradient values
return numpy.sum(loss_grad * weak_scores, 0)
class ExpLossFunction(LossFunction):
""" The class to implement the exponential loss function for the boosting framework.
"""
def loss(self, targets, scores):
"""The function computes the exponential loss values using prediction scores and targets.
def update_loss(self, targets, scores):
"""The function computes the exponential loss values using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples)
Return:
loss: The loss values for the samples """
return numpy.exp(-(targets * scores))
#return loss
def update_loss_grad(self, targets, scores):
"""The function computes the gradient of the exponential loss function using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples)
Return:
gradient: The loss gradient values for the samples """
loss = numpy.exp(-(targets * scores))
loss_grad = -targets * loss
return loss_grad
#return loss_grad
#def loss_sum(self, *args):
def loss_sum(self, alpha, targets, prediction_scores, weak_scores):
"""The function computes the sum of the exponential loss which is used to find the optimized values of alpha (x).
The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
This function is given as the input for the lbfgs optimization function.
Inputs:
alpha: The current value of the alpha.
type: float
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
type: numpy array (# number of samples)
curr_scores: The prediction scores of the samples for the current round of the boosting.
type: numpy array (# number of samples)
Return:
sum_loss: The sum of the loss values for the current value of the alpha
type: float"""
"""
# initialize the values
x = args[0]
targets = args[1]
pred_scores = args[2]
weak_scores = args[3]
"""
# compute the scores and loss for the current alpha
curr_scores = prediction_scores + alpha * weak_scores
loss = self.update_loss(targets, curr_scores)
# compute the sum of the loss
sum_loss = numpy.sum(loss,0)
return sum_loss
def loss_grad_sum(self, alpha, targets, prediction_scores, weak_scores):
"""The function computes the sum of the exponential loss which is used to find the optimized values of alpha (x).
The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
This function is given as the input for the lbfgs optimization function.
Inputs:
x: The current value of the alpha.
type: float
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
type: numpy array (# number of samples)
curr_scores: The prediction scores of the samples for the current round of the boosting.
type: numpy array (# number of samples)
Return:
sum_loss: The sum of the loss gradient values for the current value of the alpha
type: float"""
"""
# initialize the values
x = args[0]
targets = args[1]
pred_scores = args[2]
weak_scores = args[3]
"""
# compute the loss gradient for the updated score
curr_scores = prediction_scores + alpha *weak_scores
loss_grad = self.update_loss_grad(targets, curr_scores)
# take the sum of the loss gradient values
sum_grad = numpy.sum(loss_grad*weak_scores, 0)
return sum_grad
"""Log loss function """
class LogLossFunction():
""" The class to implement the logit loss function for the boosting framework.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x # number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples x # number of outputs)
Returns
The loss values for the samples
"""
def update_loss(self, targets, scores):
"""The function computes the exponential loss values using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples)
Return:
loss: The loss values for the samples """
e = numpy.exp(-(targets * scores))
return numpy.log(1 + e)
def update_loss_grad(self, targets, scores):
"""The function computes the gradient of the exponential loss function using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples x # number of outputs)
Return:
gradient: The loss gradient values for the samples """
e = numpy.exp(-(targets * scores))
denom = 1/(1 + e)
return - targets* e* denom
def loss_sum(self, alpha, targets, prediction_scores, weak_scores):
"""The function computes the sum of the logit loss which is used to find the optimized values of alpha (x).
The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
This function is given as the input for the lbfgs optimization function.
Inputs:
x: The current value of the alpha.
type: float
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
prediction_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
type: numpy array (# number of samples)
weak_scores: The prediction scores of the samples for the current round of the boosting.
type: numpy array (# number of samples)
Return:
sum_loss: The sum of the loss values for the current value of the alpha
type: float"""
"""
x = args[0]
targets = args[1]
pred_scores = args[2]
weak_scores = args[3]
"""
curr_scores = prediction_scores + alpha*weak_scores
loss = self.update_loss(targets, curr_scores)
sum_loss = numpy.sum(loss,0)
return sum_loss
def loss_grad_sum(self, alpha, targets, prediction_scores, weak_scores):
"""The function computes the sum of the logit loss gradient which is used to find the optimized values of alpha (x).
The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
This function is given as the input for the lbfgs optimization function.
Inputs:
x: The current value of the alpha.
type: float
targets: The targets for the samples
type: numpy array (# number of samples x #number of outputs)
pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
type: numpy array (# number of samples)
curr_scores: The prediction scores of the samples for the current round of the boosting.
type: numpy array (# number of samples)
Return:
sum_loss: The sum of the loss gradient values for the current value of the alpha
type: float"""
"""
x = args[0]
targets = args[1]
pred_scores = args[2]
weak_scores = args[3]
"""
curr_scores = prediction_scores + alpha*weak_scores
loss_grad = self.update_loss_grad( targets, curr_scores)
sum_grad = numpy.sum(loss_grad*weak_scores, 0)
return sum_grad
"""def loss_sum(self, targets, scores):
loss = self.update_loss(self,targets, scores)
return np.sum(loss, 0)
def loss_grad_sum(self, targets, scores)
loss_grad = self.update_loss_grad(self, targets, scores)"""
"""Tangent loss function """
return numpy.exp(-(targets * scores))
def loss_gradient(self, targets, scores):
"""The function computes the gradient of the exponential loss function using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x # number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples x # number of outputs)
Returns
The loss gradient values for the samples
"""
loss = numpy.exp(-(targets * scores))
return -targets * loss
class LogLossFunction(LossFunction):
""" The class to implement the logit loss function for the boosting framework.
"""
def loss(self, targets, scores):
"""The function computes the exponential loss values using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x # number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples x # number of outputs)
Returns
The loss values for the samples
"""
e = numpy.exp(-(targets * scores))
return numpy.log(1 + e)
def loss_gradient(self, targets, scores):
"""The function computes the gradient of the exponential loss function using prediction scores and targets.
Inputs:
targets: The targets for the samples
type: numpy array (# number of samples x # number of outputs)
scores: The current prediction scores for the samples.
type: numpy array (# number of samples x # number of outputs)
Returns
The loss gradient values for the samples
"""
e = numpy.exp(-(targets * scores))
denom = 1./(1. + e)
return - targets * e * denom
class TangLossFunction():
def update_loss(self, targets, scores):
loss = (2* numpy.arctan(targets * scores) -1)**2
return loss
def update_loss_grad(self, targets, scores):
m = targets*scores
numer = 4*(2*numpy.arctan(m) -1)
denom = 1 + m**2
loss_grad = numer/denom
return loss_grad
def loss_sum(self, *args):
x = args[0]
targets = args[1]
pred_scores = args[2]
weak_scores = args[3]
curr_scores_x = pred_scores + x*weak_scores
loss = self.update_loss(targets, curr_scores_x)
return numpy.sum(loss, 0)
#@abstractmethod
def loss_grad_sum(self, *args):
x = args[0]
targets = args[1]
pred_scores = args[2]
weak_scores = args[3]
curr_scores_x = pred_scores + x*weak_scores
loss_grad = self.update_loss_grad( targets, curr_scores_x)
return numpy.sum(loss_grad*weak_scores, 0)
"""Tangent loss function """
def loss(self, targets, scores):
return (2. * numpy.arctan(targets * scores) -1)**2
def loss_gradient(self, targets, scores):
m = targets*scores
numer = 4.*(2. * numpy.arctan(m) - 1.)
denom = 1. + m**2
return numer/denom
class JesorskyLossFunction (LossFunction):
def _inter_eye_distance(self, targets):
"""Computes the inter eye distance from the given target vector.
It assumes that the eyes are stored as the first two elements in the vector,
as: [0]: re_y [1]: re_x, [2]: le_y, [3]: re_x
"""
return math.sqrt((targets[0] - targets[2])**2 + (targets[1] - targets[3])**2)
def loss(self, targets, scores):
"""Computes the jesorsky loss for the given target and score vectors."""
"""
errors = numpy.ndarray(targets.shape[0], numpy.float)
for i in range(targets.shape[0]):
scale = 0.5/self._inter_eye_distance(targets[i])
errors[i] = math.sqrt(numpy.sum((targets[i] - scores[i])**2)) * scale
"""
errors = numpy.zeros((targets.shape[0],1), numpy.float)
for i in range(targets.shape[0]):
scale = 0.5/self._inter_eye_distance(targets[i])
for j in range(0, targets.shape[1], 2):
dx = scores[i,j] - targets[i,j]
dy = scores[i,j+1] - targets[i,j+1]
errors[i,0] += math.sqrt(dx**2 + dy**2) * scale
return errors
def loss_gradient(self, targets, scores):
"""Computes the gradient of the jesorsky loss."""
gradient = numpy.ndarray(targets.shape, numpy.float)
for i in range(targets.shape[0]):
scale = 0.5/self._inter_eye_distance(targets[i])
for j in range(0, targets.shape[1], 2):
dx = scores[i,j] - targets[i,j]
dy = scores[i,j+1] - targets[i,j+1]
error = math.sqrt(dx**2 + dy**2)
gradient[i,j] = dx * scale / error
gradient[i,j+1] = dy * scale / error
return gradient
LOSS_FUNCTIONS = {'log':LogLossFunction,
'exp':ExpLossFunction,
'tang':TangLossFunction}
'tang':TangLossFunction,
'jesorsky':JesorskyLossFunction}
......@@ -250,7 +250,7 @@ class LutMachine():
from .. import LUTMachine
from .. import LUTMachine, weighted_histogram
class LutTrainer():
""" The LutTrainer class contain methods to learn weak trainer using LookUp Tables.
......@@ -381,7 +381,7 @@ class LutTrainer():
for feature_index in range(num_fea):
for output_index in range(self.num_outputs):
hist_grad = self.compute_grad_hist(loss_grad[:,output_index],fea[:,feature_index])
sum_loss[feature_index,output_index] = - sum(abs(hist_grad))
sum_loss[feature_index,output_index] = - numpy.sum(numpy.abs(hist_grad))
return sum_loss
......@@ -400,16 +400,8 @@ class LutTrainer():
fval: single feature selected for all samples. No. of samples x 1
return: hist_grad: The sum of the loss gradient"""
# initialize the values
# hist_grad = numpy.zeros([self.num_entries])
# compute the sum of the gradient
hist_grad, bin_val = numpy.histogram(features, bins = self.num_entries, range = (0,self.num_entries-1), weights = loss_grado)
# hist_grad = [sum(loss_grado[features == feature_value]) for feature_value in xrange(self.num_entries)]
#for feature_value in range(self.num_entries):
# hist_grad[feature_value] = sum(loss_grado[features == feature_value])
hist_grad = weighted_histogram(features, loss_grado, self.num_entries)
return hist_grad
......
......@@ -8,11 +8,14 @@ class WeakMachine{
virtual double forward1(const blitz::Array<uint16_t, 1>& features) const {throw std::runtime_error("This function is not implemented for the given data type in the current class.");}
virtual double forward1(const blitz::Array<double, 1>& features) const {throw std::runtime_error("This function is not implemented for the given data type in the current class.");}