Commit 6789e102 authored by Manuel Günther's avatar Manuel Günther
Browse files

Implemented strong and weak classifiers (only LUT so far) in C++.

parent 9558aba2
......@@ -3,12 +3,16 @@
; Mon 16 Apr 08:29:18 2012 CEST
[buildout]
prefixes = /idiap/group/torch5spro/releases/bob-1.2.0/install/linux-x86_64-release
parts = scripts
develop = .
parts = xbob.boosting scripts
eggs = xbob.boosting
newest = false
prefixes = /idiap/group/torch5spro/releases/bob-1.2.0/install/linux-x86_64-release
[xbob.boosting]
recipe = xbob.buildout:develop
[scripts]
recipe = xbob.buildout:scripts
dependent-scripts = true
File deleted
......@@ -19,7 +19,9 @@
# allows you to test your package with new python dependencies w/o requiring
# administrative interventions.
from setuptools import setup, find_packages
from setuptools import setup, find_packages, dist
dist.Distribution(dict(setup_requires='xbob.extension'))
from xbob.extension import Extension, build_ext
# The only thing we do in this file is to call the setup() function with all
# parameters that define our package.
......@@ -29,7 +31,7 @@ setup(
# information before releasing code publicly.
name='xbob.boosting',
version='0.1',
description='Boosting framework for the BOB',
description='Boosting framework for Bob',
url='http://github.com/bioidiap/xbob.boosting',
license='GPLv3',
......@@ -44,6 +46,10 @@ setup(
packages=find_packages(),
include_package_data=True,
setup_requires=[
'xbob.extension',
],
# This line defines which packages should be installed when you "install"
# this package. All packages that are mentioned here, but are not installed
# on the current system will be installed locally and only visible to the
......@@ -53,10 +59,41 @@ setup(
'setuptools',
'bob', # base signal proc./machine learning library
'xbob.db.mnist',
'xbob.db.banca',
'xbob.db.banca'
],
cmdclass={
'build_ext': build_ext,
},
ext_modules = [
Extension(
'xbob.boosting._boosting',
[
"xbob/boosting/cpp/lutmachine.cpp",
"xbob/boosting/cpp/boosted_machine.cpp",
"xbob/boosting/cpp/bindings.cpp",
],
pkgconfig = [
'bob-io',
],
include_dirs = [
"xbob/boosting/cpp"
],
# STUFF for DEBUGGING goes here (requires DEBUG bob version...):
# extra_compile_args = [
# '-ggdb',
# ],
# define_macros = [
# ('BZ_DEBUG', 1)
# ],
# undef_macros=[
# 'NDEBUG'
# ]
)
],
# Your project should be called something like 'xbob.<foo>' or
# Your project should be called something like 'xbob.<foo>' or
# 'xbob.<foo>.<bar>'. To implement this correctly and still get all your
# packages to be imported w/o problems, you need to implement namespaces
# on the various levels of the package and declare them here. See more
......
# import the C++ stuff
from ._boosting import BoostedMachine, LUTMachine
import core
import features
import util
import tests
......@@ -29,7 +29,10 @@ import losses
import scipy.optimize
import itertools
import logging
logger = logging.getLogger('bob')
from .. import BoostedMachine
class Boost:
......@@ -141,7 +144,7 @@ class Boost:
def train(self, fset, targets):
def train(self, fset, targets, boosted_machine = None):
""" The function to train a boosting machine.
The function boosts the discrete features (fset) and returns a strong classifier
......@@ -164,8 +167,11 @@ class Boost:
[-1, -1, -1, 1]] #Predicted class is 3
There can be only single 1 in a row and the index of 1 indicates the class.
boosted_machine: the machine to add the weak machines to. If not given, a new machine is created.
Type: BoostMachine (with valid output dimension)
Return:
machine: The boosting machine that is combination of the weak classifiers.
machine: The boosted machine that is combination of the weak classifiers.
"""
......@@ -174,7 +180,7 @@ class Boost:
targets = targets[:,numpy.newaxis]
num_op = targets.shape[1]
machine = BoostMachine()
machine = BoostedMachine() if boosted_machine is None else boosted_machine
num_samp = fset.shape[0]
pred_scores = numpy.zeros([num_samp,num_op])
loss_class = losses.LOSS_FUNCTIONS[self.loss_type]
......@@ -194,17 +200,19 @@ class Boost:
# Start boosting iterations for num_rnds rounds
logger.info("Starting %d rounds of boosting" % self.num_rnds)
for r in range(self.num_rnds):
# Compute the gradient of the loss function, l'(y,f(x)) using loss_class
loss_grad = loss_func.update_loss_grad(targets,pred_scores)
# Select the best weak trainer for current round of boosting
curr_weak_trainer = weak_trainer.compute_weak_trainer(fset, loss_grad)
# Select the best weak machine for current round of boosting
curr_weak_machine = weak_trainer.compute_weak_trainer(fset, loss_grad)
# Compute the classification scores of the samples based only on the current round weak classifier (g_r)
curr_pred_scores = curr_weak_trainer.get_weak_scores(fset)
curr_pred_scores = numpy.zeros([num_samp,num_op], numpy.float64)
curr_weak_machine(fset, curr_pred_scores)
# Initialize the start point for lbfgs minimization
init_point = numpy.zeros(num_op)
......@@ -220,8 +228,9 @@ class Boost:
# Add the current trainer into the boosting machine
machine.add_weak_trainer(curr_weak_trainer, alpha)
machine.add_weak_machine(curr_weak_machine, alpha)
logger.debug("Finished round %d / %r" % (r+1, self.num_rnds))
return machine
......@@ -236,13 +245,22 @@ class BoostMachine():
""" The class to perform the classification using the set of weak trainer """
def __init__(self, number_of_outputs = 1):
def __init__(self, number_of_outputs = 1, hdf5file = None):
""" Initialize the set of weak trainers and the alpha values (scale)"""
self.alpha = []
self.weak_trainer = []
self.number_of_outputs = number_of_outputs
self.selected_indices = set()
if hdf5file is not None:
self.load(hdf5file)
else:
self.alpha = []
self.weak_trainer = []
self.number_of_outputs = number_of_outputs
self.selected_indices = set()
self._update()
def _update(self):
""" Initializes internal variables."""
self.selected_indices = set([weak_trainer.selected_indices[i] for weak_trainer in self.weak_trainer for i in range(self.number_of_outputs)])
self._weak_results = numpy.ndarray((len(self.weak_trainer),), numpy.float64)
def add_weak_trainer(self, curr_trainer, curr_alpha):
......@@ -255,7 +273,7 @@ class BoostMachine():
"""
self.alpha.append(curr_alpha)
self.weak_trainer.append(curr_trainer)
self.selected_indices |= set([curr_trainer.selected_indices[i] for i in range(self.number_of_outputs)])
self._update()
def feature_indices(self):
......@@ -270,7 +288,10 @@ class BoostMachine():
Output: A single floating point number
"""
return sum([a * weak.get_weak_score(feature) for (a, weak) in itertools.izip(self.alpha, self.weak_trainer)])
# iterate over the weak classifiers
for index in xrange(len(self.weak_trainer)):
self._weak_results[index] = self.alpha[index] * self.weak_trainer[index].get_weak_score(feature)
return numpy.sum(self._weak_results)
def classify(self, test_features):
......@@ -326,12 +347,11 @@ class BoostMachine():
def save(self, hdf5File):
# hdf5File.set_attribute("MachineType", self.weak_trainer_type)
hdf5File.set_attribute("version", 0)
hdf5File.set("Weights", self.alpha)
hdf5File.set("Outputs", self.number_of_outputs)
for i in range(len(self.weak_trainer)):
dir_name = "WeakMachine%d"%i
dir_name = "WeakMachine_%d"%i
hdf5File.create_group(dir_name)
hdf5File.cd(dir_name)
hdf5File.set_attribute("MachineType", self.weak_trainer[i].__class__.__name__)
......@@ -340,13 +360,12 @@ class BoostMachine():
def load(self, hdf5File):
# self.weak_trainer_type = hdf5File.get_attribute("MachineType")
self.alpha = hdf5File.read("Weights")
self.number_of_outputs = hdf5File.read("Outputs")
self.weak_trainer = []
self.selected_indices = set()
for i in range(len(self.alpha)):
dir_name = "WeakMachine%d"%i
dir_name = "WeakMachine_%d"%i
hdf5File.cd(dir_name)
weak_machine_type = hdf5File.get_attribute("MachineType")
weak_machine = {
......@@ -357,5 +376,6 @@ class BoostMachine():
self.weak_trainer.append(weak_machine)
self.selected_indices |= set([weak_machine.selected_indices[i] for i in range(self.number_of_outputs)])
hdf5File.cd('..')
self._update()
......@@ -182,8 +182,6 @@ class StumpTrainer():
class LutMachine():
""" The LUT machine consist of the core elements of the LUT weak classfier i.e. the LUT and
the feature index corresponding to the weak classifier. """
......@@ -253,6 +251,8 @@ class LutMachine():
self.selected_indices = numpy.array([self.selected_indices], dtype=numpy.int)
from .. import LUTMachine
class LutTrainer():
""" The LutTrainer class contain methods to learn weak trainer using LookUp Tables.
It can be used for multi-variate binary classification """
......@@ -287,7 +287,6 @@ class LutTrainer():
def compute_weak_trainer(self, fea, loss_grad):
""" The function to learn the weak LutTrainer.
......@@ -311,7 +310,8 @@ class LutTrainer():
# Initializations
# num_outputs = loss_grad.shape[1]
fea_grad = numpy.zeros([self.num_entries, self.num_outputs])
lut_machine = LutMachine(self.num_outputs, self.num_entries)
luts = numpy.ones((self.num_entries, self.num_outputs), numpy.float64)
selected_indices = numpy.ndarray((self.num_outputs,), numpy.uint64)
# Compute the sum of the gradient based on the feature values or the loss associated with each
# feature index
......@@ -332,7 +332,7 @@ class LutTrainer():
for output_index in range(self.num_outputs):
curr_id = sum_loss[:,output_index].argmin()
fea_grad[:,output_index] = self.compute_grad_hist(loss_grad[:,output_index],fea[:,curr_id])
lut_machine.selected_indices[output_index] = curr_id
selected_indices[output_index] = curr_id
elif self.selection_type == 'shared':
......@@ -342,15 +342,18 @@ class LutTrainer():
accum_loss = numpy.sum(sum_loss,1)
selected_findex = accum_loss.argmin()
lut_machine.selected_indices = selected_findex*numpy.ones([self.num_outputs,1],'int16')
selected_indices = selected_findex*numpy.ones([self.num_outputs,1],'int16')
for output_index in range(self.num_outputs):
fea_grad[:,output_index] = self.compute_grad_hist(loss_grad[:,output_index],fea[:,selected_findex])
# Assign the values to LookUp Table
lut_machine.luts[fea_grad <= 0.0] = -1
return lut_machine
luts[fea_grad <= 0.0] = -1
return LUTMachine(luts, selected_indices)
......
#include <bob/io/HDF5File.h>
#include <boost/shared_ptr.hpp>
#include <set>
class WeakMachine{
public:
WeakMachine(){}
virtual double forward1(const blitz::Array<uint16_t, 1>& features) const = 0;
virtual void forward2(const blitz::Array<uint16_t, 2>& features, blitz::Array<double,1> predictions) const = 0;
virtual void forward3(const blitz::Array<uint16_t, 2>& features, blitz::Array<double,2> predictions) const = 0;
virtual blitz::Array<uint64_t,1> getIndices() const = 0;
virtual void save(bob::io::HDF5File& file) const = 0;
virtual void load(bob::io::HDF5File& file) = 0;
};
class LUTMachine : public WeakMachine{
public:
LUTMachine(const blitz::Array<double,2> look_up_tables, const blitz::Array<uint64_t,1> indices);
// LUTMachine(const blitz::Array<double,1>& look_up_table, uint64_t index);
LUTMachine(bob::io::HDF5File& file);
virtual double forward1(const blitz::Array<uint16_t, 1>& features) const;
virtual void forward2(const blitz::Array<uint16_t, 2>& features, blitz::Array<double,1> predictions) const;
virtual void forward3(const blitz::Array<uint16_t, 2>& features, blitz::Array<double,2> predictions) const;
virtual blitz::Array<uint64_t,1> getIndices() const;
virtual void save(bob::io::HDF5File& file) const;
virtual void load(bob::io::HDF5File& file);
const blitz::Array<double, 2> getLut() const{return m_look_up_tables;}
private:
// the LUT for the multi-variate case
blitz::Array<double,2> m_look_up_tables;
blitz::Array<uint64_t,1> m_indices;
// for speed reasons, we also keep the LUT for the uni-variate case
blitz::Array<double,1> m_look_up_table;
uint64_t m_index;
};
inline boost::shared_ptr<WeakMachine> loadWeakMachine(bob::io::HDF5File& file){
std::string machine_type;
file.getAttribute(".", "MachineType", machine_type);
if (machine_type == "LutMachine" || machine_type == "LUTMachine"){
return boost::shared_ptr<WeakMachine>(new LUTMachine(file));
}
throw std::runtime_error("Weak machine type '" + machine_type + "' is not known or supported.");
}
class BoostedMachine{
public:
BoostedMachine();
BoostedMachine(bob::io::HDF5File& file);
// adds the machine
void add_weak_machine1(const boost::shared_ptr<WeakMachine> weak_machine, const blitz::Array<double,1> weights);
void add_weak_machine2(const boost::shared_ptr<WeakMachine> weak_machine, const double weight);
// predicts the output for the given single feature
double forward1(const blitz::Array<uint16_t, 1>& features) const;
// predicts the output and the labels for the given features (uni-variate case)
void forward2(const blitz::Array<uint16_t, 2>& features, blitz::Array<double,1> predictions, blitz::Array<double,1> labels) const;
// predicts the output and the labels for the given features (multi-variate case)
void forward3(const blitz::Array<uint16_t, 2>& features, blitz::Array<double,2> predictions, blitz::Array<double,2> labels) const;
blitz::Array<uint64_t,1> getIndices() const;
const blitz::Array<double,2> getWeights() const {return m_weights;}
// writes the machine to file
void save(bob::io::HDF5File& file) const;
// loads the machine from file
void load(bob::io::HDF5File& file);
private:
std::vector<boost::shared_ptr<WeakMachine> > m_weak_machines;
blitz::Array<double,2> m_weights;
blitz::Array<double,1> _weights;
// shortcut to avoid allocating memory for each call of 'forward'
mutable blitz::Array<double,1> _predictions1;
mutable blitz::Array<double,2> _predictions2;
};
#include <boost/python.hpp>
#include <bob/config.h>
#include <bob/python/ndarray.h>
#include "Machines.h"
using namespace boost::python;
BOOST_PYTHON_MODULE(_boosting) {
bob::python::setup_python("Bindings for the xbob.boosting machines.");
class_<WeakMachine, boost::shared_ptr<WeakMachine>, boost::noncopyable>("WeakMachine", "Pure virtual base class for weak machines", no_init);
class_<LUTMachine, boost::shared_ptr<LUTMachine>, bases<WeakMachine> >("LUTMachine", "A machine containing a Look-Up-Table.", no_init)
.def(init<const blitz::Array<double,2>&, const blitz::Array<uint64_t,1>&>((arg("self"), arg("look_up_tables"), arg("indices")), "Creates a LUTMachine with the given look-up-table and the feature indices, for which the LUT is valid."))
.def(init<bob::io::HDF5File&>((arg("self"),arg("file")), "Creates a new machine from file."))
.def("__call__", &LUTMachine::forward1, (arg("self"), arg("features")), "Returns the prediction for the given feature vector.")
.def("__call__", &LUTMachine::forward2, (arg("self"), arg("features"), arg("predictions")), "Computes the predictions for the given feature set (uni-variate).")
.def("__call__", &LUTMachine::forward3, (arg("self"), arg("features"), arg("predictions")), "Computes the predictions for the given feature set (multi-variate).")
.def("load", &LUTMachine::load, "Reads a Machine from file")
.def("save", &LUTMachine::save, "Writes the machine to file")
.add_property("lut", &LUTMachine::getLut, "The look up table of the machine.")
.add_property("selected_indices", &LUTMachine::getIndices, "The indices into the feature vector required by this machine.")
;
class_<BoostedMachine, boost::shared_ptr<BoostedMachine> >("BoostedMachine", "A machine containing of several weak machines", no_init)
.def(init<>(arg("self"), "Creates an empty machine."))
.def(init<bob::io::HDF5File&>((arg("self"), arg("file")), "Creates a new machine from file"))
.def("add_weak_machine", &BoostedMachine::add_weak_machine1, (arg("self"), arg("machine"), arg("weight")), "Adds the given weak machine with the given weight (uni-variate)")
.def("add_weak_machine", &BoostedMachine::add_weak_machine2, (arg("self"), arg("machine"), arg("weights")), "Adds the given weak machine with the given weights (multi-variate)")
.def("__call__", &BoostedMachine::forward1, (arg("self"), arg("features")), "Returns the prediction for the given feature vector.")
.def("__call__", &BoostedMachine::forward2, (arg("self"), arg("features"), arg("predictions"), arg("labels")), "Computes the predictions and the labels for the given feature set (uni-variate).")
.def("__call__", &BoostedMachine::forward3, (arg("self"), arg("features"), arg("predictions"), arg("labels")), "Computes the predictions and the labels for the given feature set (multi-variate).")
.def("load", &BoostedMachine::load, "Reads a Machine from file")
.def("save", &BoostedMachine::save, "Writes the machine to file")
.def("feature_indices", &BoostedMachine::getIndices, (arg("self")), "Returns the indices required for this machine.")
.def("alpha", &BoostedMachine::getWeights, (arg("self")), "Returns the weights for the weak machines.")
;
}
#include "Machines.h"
#include <sstream>
BoostedMachine::BoostedMachine() :
m_weak_machines(),
m_weights()
{
}
BoostedMachine::BoostedMachine(bob::io::HDF5File& file) :
m_weak_machines(),
m_weights()
{
load(file);
}
void BoostedMachine::add_weak_machine1(const boost::shared_ptr<WeakMachine> weak_machine, const blitz::Array<double,1> weights){
m_weak_machines.push_back(weak_machine);
m_weights.resizeAndPreserve(m_weak_machines.size(), weights.extent(0));
m_weights(m_weights.extent(0)-1, blitz::Range::all()) = weights;
_weights.reference(m_weights(blitz::Range::all(), 0));
}
void BoostedMachine::add_weak_machine2(const boost::shared_ptr<WeakMachine> weak_machine, const double weight){
m_weak_machines.push_back(weak_machine);
m_weights.resizeAndPreserve(m_weak_machines.size(), 1);
m_weights(m_weights.extent(0)-1, 0) = weight;
_weights.reference(m_weights(blitz::Range::all(), 0));
}
double BoostedMachine::forward1(const blitz::Array<uint16_t,1>& features) const{
double sum = 0.;
//TODO: optimize using STL
for (int i = m_weak_machines.size(); i--;){
sum += _weights(i) * m_weak_machines[i]->forward1(features);
}
return sum;
}
void BoostedMachine::forward2(const blitz::Array<uint16_t,2>& features, blitz::Array<double,1> predictions, blitz::Array<double,1> labels) const{
// initialize the predictions since they will be overwritten
_predictions1.resize(predictions.shape());
predictions = 0.;
for (int i = m_weak_machines.size(); i--;){
// predict locally
m_weak_machines[i]->forward2(features, _predictions1);
predictions += _weights(i) * _predictions1;
}
// get the labels
for (int i = predictions.extent(0); i--;)
labels(i) = (predictions(i) > 0) * 2 - 1;
}
void BoostedMachine::forward3(const blitz::Array<uint16_t,2>& features, blitz::Array<double,2> predictions, blitz::Array<double,2> labels) const{
// initialize the predictions since they will be overwritten
_predictions2.resize(predictions.shape());
predictions = 0.;
for (int i = m_weak_machines.size(); i--;){
// predict locally
m_weak_machines[i]->forward3(features, _predictions2);
predictions += m_weights(i) * _predictions2;
}
// get the labels
labels = -1;
for (int i = predictions.extent(0); i--;){
labels(i, blitz::maxIndex(predictions(i, blitz::Range::all()))[0]) = 1;
}
}
blitz::Array<uint64_t,1> BoostedMachine::getIndices() const{
std::set<uint64_t> indices;
for (unsigned i = 0; i < m_weak_machines.size(); ++i){
const blitz::Array<uint64_t,1>& ind = m_weak_machines[i]->getIndices();
indices.insert(ind.begin(), ind.end());
}
blitz::Array<uint64_t,1> ret(indices.size());
std::copy(indices.begin(), indices.end(), ret.begin());
return ret;
}
// writes the machine to file
void BoostedMachine::save(bob::io::HDF5File& file) const{
file.setAttribute(".", "version", 2);
file.setArray("Weights", m_weights);
for (unsigned i = 0; i < m_weights.size(); ++i){
std::ostringstream fns;
fns << "WeakMachine_" << i;
file.createGroup(fns.str());
file.cd(fns.str());
m_weak_machines[i]->save(file);
file.cd("..");
}
}
// loads the machine from file
void BoostedMachine::load(bob::io::HDF5File& file){
m_weak_machines.clear();
// the weights
m_weights.reference(file.readArray<double,2>("Weights"));
_weights.reference(m_weights(blitz::Range::all(), 0));
// name of the first machine
std::string machine_name("WeakMachine_0");
while (file.hasGroup(machine_name)){
// load weight and machine
file.cd(machine_name);
m_weak_machines.push_back(loadWeakMachine(file));
file.cd("..");
// get name of the next machine
std::ostringstream fns;
fns << "WeakMachine_" << m_weak_machines.size();
machine_name = fns.str();
}
}
#include <Machines.h>
#include <bob/core/cast.h>
#include <assert.h>
LUTMachine::LUTMachine(const blitz::Array<double,2> look_up_tables, const blitz::Array<uint64_t,1> indices):
m_look_up_tables(look_up_tables.shape()),
m_indices(indices.shape()),
m_look_up_table(),
m_index(0)
{
// we have to copy the array, otherwise weird things happen
m_look_up_tables = look_up_tables;
m_indices = indices;
// for the shortcut, we just reference the first row of the the look up tables
m_look_up_table.reference(m_look_up_tables(blitz::Range::all(),0));
m_index = m_indices(0);
}
LUTMachine::LUTMachine(bob::io::HDF5File& file):
m_look_up_tables(),
m_indices(),