Unified losses; added jesorsky loss; implemented weighted hisdtogram in C++;...

Unified losses; added jesorsky loss; implemented weighted hisdtogram in C++; implemented missing recall.

buildout.cfg

@@ -5,7 +5,6 @@
 [buildout]
 parts = xbob.boosting scripts
 eggs = xbob.boosting
-       xbob.db.banca
        xbob.db.mnist
 newest = false
 
@@ -19,4 +18,4 @@ recipe = xbob.buildout:develop
 
 [scripts]
 recipe = xbob.buildout:scripts
-dependent-scripts = true
+dependent-scripts = false

setup.py

@@ -124,15 +124,15 @@ setup(
 
       # scripts should be declared using this entry:
       'console_scripts': [
-        'mnist_binary_all.py = xbob.boosting.scripts.mnist_binary_all:main',
-        'mnist_binary_one.py = xbob.boosting.scripts.mnist_binary_one:main',
-        'mnist_multi.py = xbob.boosting.scripts.mnist_multi:main',
-        'mnist_lbp.py = xbob.boosting.scripts.mnist_lbp:main',
-        'mnist_multi_lbp.py = xbob.boosting.scripts.mnist_multi_lbp:main',
-        'mnist_onevsall.py = xbob.boosting.scripts.mnist_onevsall:main',
-        'mnist_onevsall_lbp.py = xbob.boosting.scripts.mnist_onevsall_lbp:main',
-        'mnist_onevsall_block_lbp.py = xbob.boosting.scripts.mnist_onevsall_block_lbp:main',
-        'mnist_multi_block_lbp.py = xbob.boosting.scripts.mnist_multi_block_lbp:main',
+#        'mnist_binary_all.py = xbob.boosting.scripts.mnist_binary_all:main',
+#        'mnist_binary_one.py = xbob.boosting.scripts.mnist_binary_one:main',
+#        'mnist_multi.py = xbob.boosting.scripts.mnist_multi:main',
+#        'mnist_lbp.py = xbob.boosting.scripts.mnist_lbp:main',
+#        'mnist_multi_lbp.py = xbob.boosting.scripts.mnist_multi_lbp:main',
+#        'mnist_onevsall.py = xbob.boosting.scripts.mnist_onevsall:main',
+#        'mnist_onevsall_lbp.py = xbob.boosting.scripts.mnist_onevsall_lbp:main',
+#        'mnist_onevsall_block_lbp.py = xbob.boosting.scripts.mnist_onevsall_block_lbp:main',
+#        'mnist_multi_block_lbp.py = xbob.boosting.scripts.mnist_multi_block_lbp:main',
         ],
 
       # tests that are _exported_ (that can be executed by other packages) can

xbob/boosting/__init__.py

 # import the C++ stuff
-from ._boosting import StumpMachine, LUTMachine, BoostedMachine
+from ._boosting import StumpMachine, LUTMachine, BoostedMachine, weighted_histogram
 
 import core
 import features

xbob/boosting/core/boosting.py

@@ -180,9 +180,14 @@ class Boost:
             targets = targets[:,numpy.newaxis]
 
         num_op = targets.shape[1]
-        machine = BoostedMachine() if boosted_machine is None else boosted_machine
         num_samp = fset.shape[0]
         pred_scores = numpy.zeros([num_samp,num_op])
+        if boosted_machine is not None:
+          machine =  boosted_machine
+          machine(fset, pred_scores)
+        else:
+          machine = BoostedMachine()
+
         loss_class = losses.LOSS_FUNCTIONS[self.loss_type]
         loss_func = loss_class()
 
@@ -203,26 +208,26 @@ class Boost:
         logger.info("Starting %d rounds of boosting" % self.num_rnds)
         for r in range(self.num_rnds):
 
+            logger.debug("Starting round %d" % (r+1))
 
             # Compute the gradient of the loss function, l'(y,f(x)) using loss_class
-            loss_grad = loss_func.update_loss_grad(targets,pred_scores)
+            loss_grad = loss_func.loss_gradient(targets,pred_scores)
 
             # Select the best weak machine for current round of boosting
             curr_weak_machine = weak_trainer.compute_weak_trainer(fset, loss_grad)
 
             # Compute the classification scores of the samples based only on the current round weak classifier (g_r)
             curr_pred_scores = numpy.zeros([num_samp,num_op], numpy.float64)
+
             curr_weak_machine(fset, curr_pred_scores)
 
             # Initialize the start point for lbfgs minimization
             init_point = numpy.zeros(num_op)
 
-
             # Perform lbfgs minimization and compute the scale (alpha_r) for current weak trainer
             lbfgs_struct = scipy.optimize.fmin_l_bfgs_b(loss_func.loss_sum, init_point, fprime = loss_func.loss_grad_sum, args = (targets, pred_scores, curr_pred_scores))
             alpha = lbfgs_struct[0]
 
-
             # Update the prediction score after adding the score from the current weak classifier f(x) = f(x) + alpha_r*g_r
             pred_scores = pred_scores + alpha*curr_pred_scores
 
@@ -230,7 +235,7 @@ class Boost:
             # Add the current trainer into the boosting machine
             machine.add_weak_machine(curr_weak_machine, alpha)
 
-            logger.debug("Finished round %d / %r" % (r+1, self.num_rnds))
+            logger.info("Finished round %d / %d" % (r+1, self.num_rnds))
 
         return machine
 

xbob/boosting/core/losses.py

@@ -2,280 +2,214 @@ import numpy
 import math
 
 
+class LossFunction:
 
-class ExpLossFunction():
-    """ The class to implement the exponential loss function for the boosting framework. 
+  def loss(self, targets, scores):
+    raise NotImplementedError("This is a pure abstract function. Please implement that in your derived class.")
+
+  def loss_gradient(self, targets, scores):
+    raise NotImplementedError("This is a pure abstract function. Please implement that in your derived class.")
+
+
+  def loss_sum(self, alpha, targets, prediction_scores, weak_scores):
+    """The function computes the sum of the loss which is used to find the optimized values of alpha (x).
+
+    The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
+    This function is given as the input for the lbfgs optimization function.
+
+    Inputs:
+    alpha: The current value of the alpha.
+       type: float
+
+    targets: The targets for the samples
+             type: numpy array (# number of samples x # number of outputs)
+
+    pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
+             type: numpy array (# number of samples x # number of outputs)
+
+    curr_scores: The prediction scores of the samples for the current round of the boosting.
+             type: numpy array (# number of samples x # number of outputs)
+
+
+    Return:
+    sum_loss: The sum of the loss values for the current value of the alpha
+             type: float
+    """
+
+    # compute the scores and loss for the current alpha
+    curr_scores = prediction_scores + alpha * weak_scores
+    loss = self.loss(targets, curr_scores)
+
+    # compute the sum of the loss
+    return numpy.sum(loss, 0)
+
+
+  def loss_grad_sum(self, alpha, targets, prediction_scores, weak_scores):
+    """The function computes the gradient as the sum of the derivatives per sample which is used to find the optimized values of alpha (x).
+
+    The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
+    This function is given as the input for the lbfgs optimization function.
+
+    Inputs:
+    alpha: The current value of the alpha.
+       type: float
+
+    targets: The targets for the samples
+             type: numpy array (# number of samples x # number of outputs)
+
+    pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
+             type: numpy array (# number of samples x # number of outputs)
+
+    curr_scores: The prediction scores of the samples for the current round of the boosting.
+             type: numpy array (# number of samples x # number of outputs)
+
+
+    Returns
+      The sum of the loss gradient values for the current value of the alpha
+      type: numpy array (# number of outputs)
     """
 
+    # compute the loss gradient for the updated score
+    curr_scores = prediction_scores + alpha * weak_scores
+    loss_grad = self.loss_gradient(targets, curr_scores)
+
+    # take the sum of the loss gradient values
+    return numpy.sum(loss_grad * weak_scores, 0)
+
+
+class ExpLossFunction(LossFunction):
+  """ The class to implement the exponential loss function for the boosting framework.
+  """
+  def loss(self, targets, scores):
+    """The function computes the exponential loss values using prediction scores and targets.
 
-    def update_loss(self, targets, scores):
-        """The function computes the exponential loss values using prediction scores and targets.
-
-        Inputs: 
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        scores: The current prediction scores for the samples.
-                type: numpy array (# number of samples) 
-
-        Return:
-        loss: The loss values for the samples     """
-
-        return numpy.exp(-(targets * scores))
-        #return loss 
-
-    def update_loss_grad(self, targets, scores):
-        """The function computes the gradient of the exponential loss function using prediction scores and targets.
-
-        Inputs: 
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        scores: The current prediction scores for the samples.
-                type: numpy array (# number of samples) 
-
-        Return:
-        gradient: The loss gradient values for the samples     """
-        loss = numpy.exp(-(targets * scores))
-        loss_grad = -targets * loss
-        return loss_grad
-        #return loss_grad
-
-    #def loss_sum(self, *args):
-    def loss_sum(self, alpha, targets, prediction_scores, weak_scores):
-        """The function computes the sum of the exponential loss which is used to find the optimized values of alpha (x).
-         
-        The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
-        This function is given as the input for the lbfgs optimization function. 
-
-        Inputs: 
-        alpha: The current value of the alpha.
-           type: float
-
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
-                 type: numpy array (# number of samples) 
-
-        curr_scores: The prediction scores of the samples for the current round of the boosting.
-                 type: numpy array (# number of samples) 
-
-
-        Return:
-        sum_loss: The sum of the loss values for the current value of the alpha    
-                 type: float"""
-        """
-        # initialize the values
-        x = args[0]
-        targets = args[1]
-        pred_scores = args[2]
-        weak_scores = args[3]
-        """
-        
-        # compute the scores and loss for the current alpha
-        curr_scores = prediction_scores + alpha * weak_scores
-        loss = self.update_loss(targets, curr_scores)
-
-        # compute the sum of the loss
-        sum_loss = numpy.sum(loss,0)
-        return sum_loss
-        
-
-    def loss_grad_sum(self, alpha, targets, prediction_scores, weak_scores):
-        """The function computes the sum of the exponential loss which is used to find the optimized values of alpha (x).
-         
-        The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
-        This function is given as the input for the lbfgs optimization function. 
-
-        Inputs: 
-        x: The current value of the alpha.
-           type: float
-
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
-                 type: numpy array (# number of samples) 
-
-        curr_scores: The prediction scores of the samples for the current round of the boosting.
-                 type: numpy array (# number of samples) 
-
-
-        Return:
-        sum_loss: The sum of the loss gradient values for the current value of the alpha    
-                 type: float"""
-
-        """
-        # initialize the values
-        x = args[0]
-        targets = args[1]
-        pred_scores = args[2]
-        weak_scores = args[3]
-        """
-
-        # compute the loss gradient for the updated score
-        curr_scores = prediction_scores + alpha *weak_scores
-        loss_grad = self.update_loss_grad(targets, curr_scores)
-
-        # take the sum of the loss gradient values
-        sum_grad = numpy.sum(loss_grad*weak_scores, 0)
-        return sum_grad
-
-
-
-
-"""Log loss function """
-class LogLossFunction():
-    """ The class to implement the logit loss function for the boosting framework. 
+    Inputs:
+    targets: The targets for the samples
+             type: numpy array (# number of samples x # number of outputs)
+
+    scores: The current prediction scores for the samples.
+            type: numpy array (# number of samples x # number of outputs)
+
+    Returns
+      The loss values for the samples
     """
-    def update_loss(self, targets, scores):
-        """The function computes the exponential loss values using prediction scores and targets.
-
-        Inputs: 
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        scores: The current prediction scores for the samples.
-                type: numpy array (# number of samples) 
-
-        Return:
-        loss: The loss values for the samples     """
-        e = numpy.exp(-(targets * scores))
-        return numpy.log(1 + e)
-
-
-    def update_loss_grad(self, targets, scores):
-        """The function computes the gradient of the exponential loss function using prediction scores and targets.
-
-        Inputs: 
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        scores: The current prediction scores for the samples.
-                type: numpy array (# number of samples x # number of outputs) 
-
-        Return:
-        gradient: The loss gradient values for the samples     """
-        e = numpy.exp(-(targets * scores))
-        denom = 1/(1 + e)
-        return - targets* e* denom
-
-    def loss_sum(self, alpha, targets, prediction_scores, weak_scores):
-        """The function computes the sum of the logit loss which is used to find the optimized values of alpha (x).
-         
-        The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
-        This function is given as the input for the lbfgs optimization function. 
-
-        Inputs: 
-        x: The current value of the alpha.
-           type: float
-
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        prediction_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
-                 type: numpy array (# number of samples) 
-
-        weak_scores: The prediction scores of the samples for the current round of the boosting.
-                 type: numpy array (# number of samples) 
-
-
-        Return:
-        sum_loss: The sum of the loss values for the current value of the alpha    
-                 type: float"""
-
-        """
-        x = args[0]
-        targets = args[1]
-        pred_scores = args[2]
-        weak_scores = args[3]
-        """
-        curr_scores = prediction_scores + alpha*weak_scores
-        loss = self.update_loss(targets, curr_scores)
-        sum_loss = numpy.sum(loss,0)
-        return sum_loss
-        
-    def loss_grad_sum(self, alpha, targets, prediction_scores, weak_scores):
-        """The function computes the sum of the logit loss gradient which is used to find the optimized values of alpha (x).
-         
-        The functions computes sum of loss values which is required during the linesearch step for the optimization of the alpha.
-        This function is given as the input for the lbfgs optimization function. 
-
-        Inputs: 
-        x: The current value of the alpha.
-           type: float
-
-        targets: The targets for the samples
-                 type: numpy array (# number of samples x #number of outputs)
-        
-        pred_scores: The cumulative prediction scores of the samples until the previous round of the boosting.
-                 type: numpy array (# number of samples) 
-
-        curr_scores: The prediction scores of the samples for the current round of the boosting.
-                 type: numpy array (# number of samples) 
-
-
-        Return:
-        sum_loss: The sum of the loss gradient values for the current value of the alpha    
-                 type: float"""
-        """
-        x = args[0]
-        targets = args[1]
-        pred_scores = args[2]
-        weak_scores = args[3]
-        """
-        curr_scores = prediction_scores + alpha*weak_scores
-        loss_grad = self.update_loss_grad( targets, curr_scores)
-        sum_grad = numpy.sum(loss_grad*weak_scores, 0)
-        return sum_grad
-
-
-    """def loss_sum(self, targets, scores):
-        loss = self.update_loss(self,targets, scores)
-        return np.sum(loss, 0)
-
-    def loss_grad_sum(self, targets, scores)
-        loss_grad = self.update_loss_grad(self, targets, scores)"""
-
-
-"""Tangent loss function """
+    return numpy.exp(-(targets * scores))
+
+  def loss_gradient(self, targets, scores):
+    """The function computes the gradient of the exponential loss function using prediction scores and targets.
+
+    Inputs:
+    targets: The targets for the samples
+             type: numpy array (# number of samples x # number of outputs)
+
+    scores: The current prediction scores for the samples.
+            type: numpy array (# number of samples x # number of outputs)
+
+    Returns
+      The loss gradient values for the samples
+    """
+    loss = numpy.exp(-(targets * scores))
+    return -targets * loss
+
+
+class LogLossFunction(LossFunction):
+  """ The class to implement the logit loss function for the boosting framework.
+  """
+  def loss(self, targets, scores):
+    """The function computes the exponential loss values using prediction scores and targets.
+
+    Inputs:
+    targets: The targets for the samples
+             type: numpy array (# number of samples x # number of outputs)
+
+    scores: The current prediction scores for the samples.
+            type: numpy array (# number of samples x # number of outputs)
+
+    Returns
+      The loss values for the samples
+    """
+    e = numpy.exp(-(targets * scores))
+    return numpy.log(1 + e)
+
+
+  def loss_gradient(self, targets, scores):
+    """The function computes the gradient of the exponential loss function using prediction scores and targets.
+
+    Inputs:
+    targets: The targets for the samples
+             type: numpy array (# number of samples x # number of outputs)
+
+    scores: The current prediction scores for the samples.
+            type: numpy array (# number of samples x # number of outputs)
+
+    Returns
+      The loss gradient values for the samples
+    """
+    e = numpy.exp(-(targets * scores))
+    denom = 1./(1. + e)
+    return - targets * e * denom
+
+
 
 class TangLossFunction():
-    def update_loss(self, targets, scores):
-        loss = (2* numpy.arctan(targets * scores) -1)**2
-        return loss
-
-    def update_loss_grad(self, targets, scores):
-        m = targets*scores
-        numer = 4*(2*numpy.arctan(m) -1)
-        denom = 1 + m**2
-        loss_grad = numer/denom
-        return loss_grad
-
-    def loss_sum(self, *args):
-        x = args[0]
-        targets = args[1]
-        pred_scores = args[2]
-        weak_scores = args[3]
-        curr_scores_x = pred_scores + x*weak_scores
-        loss = self.update_loss(targets, curr_scores_x)
-        return numpy.sum(loss, 0)
-        
-    #@abstractmethod
-    def loss_grad_sum(self, *args):
-        x = args[0]
-        targets = args[1]
-        pred_scores = args[2]
-        weak_scores = args[3]
-        curr_scores_x = pred_scores + x*weak_scores
-        loss_grad = self.update_loss_grad( targets, curr_scores_x)
-        return numpy.sum(loss_grad*weak_scores, 0)
+  """Tangent loss function """
+  def loss(self, targets, scores):
+      return (2. * numpy.arctan(targets * scores) -1)**2
+
+  def loss_gradient(self, targets, scores):
+    m = targets*scores
+    numer = 4.*(2. * numpy.arctan(m) - 1.)
+    denom = 1. + m**2
+    return numer/denom
+
+
 
+class JesorskyLossFunction (LossFunction):
+
+  def _inter_eye_distance(self, targets):
+    """Computes the inter eye distance from the given target vector.
+    It assumes that the eyes are stored as the first two elements in the vector,
+    as: [0]: re_y [1]: re_x, [2]: le_y, [3]: re_x
+    """
+    return math.sqrt((targets[0] - targets[2])**2 + (targets[1] - targets[3])**2)
+
+  def loss(self, targets, scores):
+    """Computes the jesorsky loss for the given target and score vectors."""
+
+    """
+    errors = numpy.ndarray(targets.shape[0], numpy.float)
+    for i in range(targets.shape[0]):
+      scale = 0.5/self._inter_eye_distance(targets[i])
+      errors[i] = math.sqrt(numpy.sum((targets[i] - scores[i])**2)) * scale
+    """
+    errors = numpy.zeros((targets.shape[0],1), numpy.float)
+    for i in range(targets.shape[0]):
+      scale = 0.5/self._inter_eye_distance(targets[i])
+      for j in range(0, targets.shape[1], 2):
+        dx = scores[i,j] - targets[i,j]
+        dy = scores[i,j+1] - targets[i,j+1]
+        errors[i,0] += math.sqrt(dx**2 + dy**2) * scale
+
+    return errors
+
+  def loss_gradient(self, targets, scores):
+    """Computes the gradient of the jesorsky loss."""
+    gradient = numpy.ndarray(targets.shape, numpy.float)
+    for i in range(targets.shape[0]):
+      scale = 0.5/self._inter_eye_distance(targets[i])
+      for j in range(0, targets.shape[1], 2):
+        dx = scores[i,j] - targets[i,j]
+        dy = scores[i,j+1] - targets[i,j+1]
+        error = math.sqrt(dx**2 + dy**2)
+        gradient[i,j] = dx * scale / error
+        gradient[i,j+1] = dy * scale / error
+
+    return gradient
 
 
 LOSS_FUNCTIONS = {'log':LogLossFunction,
                   'exp':ExpLossFunction,
-                  'tang':TangLossFunction}
+                  'tang':TangLossFunction,
+                  'jesorsky':JesorskyLossFunction}
 
 

xbob/boosting/core/trainers.py

@@ -250,7 +250,7 @@ class LutMachine():
 
 
 
-from .. import LUTMachine
+from .. import LUTMachine, weighted_histogram
 
 class LutTrainer():
     """ The LutTrainer class contain methods to learn weak trainer using LookUp Tables.
@@ -381,7 +381,7 @@ class LutTrainer():
         for feature_index in range(num_fea):
             for output_index in range(self.num_outputs):
                 hist_grad = self.compute_grad_hist(loss_grad[:,output_index],fea[:,feature_index])
-                sum_loss[feature_index,output_index] = - sum(abs(hist_grad))
+                sum_loss[feature_index,output_index] = - numpy.sum(numpy.abs(hist_grad))
 
 
         return sum_loss
@@ -400,16 +400,8 @@ class LutTrainer():
                fval: single feature selected for all samples. No. of samples x 1
 
         return: hist_grad: The sum of the loss gradient"""
-        # initialize the values
-        # hist_grad = numpy.zeros([self.num_entries])
-
-
-
         # compute the sum of the gradient
-        hist_grad, bin_val = numpy.histogram(features, bins = self.num_entries, range = (0,self.num_entries-1), weights = loss_grado)
-        # hist_grad = [sum(loss_grado[features == feature_value]) for feature_value in xrange(self.num_entries)]
-        #for feature_value in range(self.num_entries):
-        #    hist_grad[feature_value] = sum(loss_grado[features == feature_value])
+        hist_grad = weighted_histogram(features, loss_grado, self.num_entries)
         return hist_grad
 
 

xbob/boosting/cpp/Machines.h

@@ -8,11 +8,14 @@ class WeakMachine{
     virtual double forward1(const blitz::Array<uint16_t, 1>& features) const {throw std::runtime_error("This function is not implemented for the given data type in the current class.");}
     virtual double forward1(const blitz::Array<double, 1>& features) const {throw std::runtime_error("This function is not implemented for the given data type in the current class.");}