30 AdaBoost::AdaBoost(
const WeakClassifier &weakClassifier,
bool useScaling,
bool useNullRejection,
double nullRejectionCoeff,UINT numBoostingIterations,UINT predictionMethod)
32 setWeakClassifier( weakClassifier );
33 this->useScaling = useScaling;
34 this->useNullRejection = useNullRejection;
35 this->nullRejectionCoeff = nullRejectionCoeff;
36 this->numBoostingIterations = numBoostingIterations;
37 this->predictionMethod = predictionMethod;
38 classType =
"AdaBoost";
39 classifierType = classType;
40 classifierMode = STANDARD_CLASSIFIER_MODE;
41 debugLog.setProceedingText(
"[DEBUG AdaBoost]");
42 errorLog.setProceedingText(
"[ERROR AdaBoost]");
43 trainingLog.setProceedingText(
"[TRAINING AdaBoost]");
44 warningLog.setProceedingText(
"[WARNING AdaBoost]");
48 classifierType =
"AdaBoost";
49 classifierMode = STANDARD_CLASSIFIER_MODE;
50 debugLog.setProceedingText(
"[DEBUG AdaBoost]");
51 errorLog.setProceedingText(
"[ERROR AdaBoost]");
52 trainingLog.setProceedingText(
"[TRAINING AdaBoost]");
53 warningLog.setProceedingText(
"[WARNING AdaBoost]");
57 AdaBoost::~AdaBoost(
void)
60 clearWeakClassifiers();
66 clearWeakClassifiers();
68 this->numBoostingIterations = rhs.numBoostingIterations;
69 this->predictionMethod = rhs.predictionMethod;
70 this->models = rhs.models;
72 if( rhs.weakClassifiers.size() > 0 ){
73 for(UINT i=0; i<rhs.weakClassifiers.size(); i++){
74 WeakClassifier *weakClassiferPtr = rhs.weakClassifiers[i]->createNewInstance();
75 weakClassifiers.push_back( weakClassiferPtr );
85 bool AdaBoost::deepCopyFrom(
const Classifier *classifier){
87 if( classifier == NULL ){
88 errorLog <<
"deepCopyFrom(const Classifier *classifier) - The classifier pointer is NULL!" << endl;
97 clearWeakClassifiers();
99 this->numBoostingIterations = ptr->numBoostingIterations;
100 this->predictionMethod = ptr->predictionMethod;
101 this->models = ptr->models;
103 if( ptr->weakClassifiers.size() > 0 ){
104 for(UINT i=0; i<ptr->weakClassifiers.size(); i++){
105 WeakClassifier *weakClassiferPtr = ptr->weakClassifiers[i]->createNewInstance();
106 weakClassifiers.push_back( weakClassiferPtr );
111 return copyBaseVariables( classifier );
122 errorLog <<
"train_(ClassificationData &trainingData) - There are not enough training samples to train a model! Number of samples: " << trainingData.
getNumSamples() << endl;
129 const UINT POSITIVE_LABEL = WEAK_CLASSIFIER_POSITIVE_CLASS_LABEL;
130 const UINT NEGATIVE_LABEL = WEAK_CLASSIFIER_NEGATIVE_CLASS_LABEL;
132 const double beta = 0.001;
136 const UINT K = (UINT)weakClassifiers.size();
138 errorLog <<
"train_(ClassificationData &trainingData) - No weakClassifiers have been set. You need to set at least one weak classifier first." << endl;
142 classLabels.resize(numClasses);
143 models.resize(numClasses);
148 trainingData.
scale(ranges,0,1);
152 VectorDouble weights(M);
157 for(UINT classIter=0; classIter<numClasses; classIter++){
160 classLabels[classIter] = trainingData.
getClassLabels()[classIter];
163 models[ classIter ].setClassLabel( classLabels[classIter] );
168 for(UINT i=0; i<M; i++){
169 UINT label = trainingData[i].getClassLabel()==classLabels[classIter] ? POSITIVE_LABEL : NEGATIVE_LABEL;
170 VectorDouble trainingSample = trainingData[i].getSample();
171 classData.
addSample(label,trainingSample);
175 std::fill(weights.begin(),weights.end(),1.0/M);
178 bool keepBoosting =
true;
181 while( keepBoosting ){
184 UINT bestClassifierIndex = 0;
185 double minError = numeric_limits<double>::max();
186 for(UINT k=0; k<K; k++){
191 if( !weakLearner->
train(classData,weights) ){
192 errorLog <<
"Failed to train weakLearner!" << endl;
199 double numCorrect = 0;
200 double numIncorrect = 0;
201 for(UINT i=0; i<M; i++){
203 double prediction = weakLearner->
predict( classData[i].getSample() );
205 if( (prediction == positiveLabel && classData[i].getClassLabel() != POSITIVE_LABEL) ||
206 (prediction != positiveLabel && classData[i].getClassLabel() == POSITIVE_LABEL) ){
208 errorMatrix[k][i] = 1;
211 errorMatrix[k][i] = 0;
216 trainingLog <<
"PositiveClass: " << classLabels[classIter] <<
" Boosting Iter: " << t <<
" Classifier: " << k <<
" WeightedError: " << e <<
" NumCorrect: " << numCorrect/M <<
" NumIncorrect: " <<numIncorrect/M << endl;
220 bestClassifierIndex = k;
228 alpha = 0.5 * log( (1.0-epsilon)/epsilon );
230 trainingLog <<
"PositiveClass: " << classLabels[classIter] <<
" Boosting Iter: " << t <<
" Best Classifier Index: " << bestClassifierIndex <<
" MinError: " << minError <<
" Alpha: " << alpha << endl;
232 if( grt_isinf(alpha) ){ keepBoosting =
false; trainingLog <<
"Alpha is INF. Stopping boosting for current class" << endl; }
233 if( 0.5 - epsilon <= beta ){ keepBoosting =
false; trainingLog <<
"Epsilon <= Beta. Stopping boosting for current class" << endl; }
234 if( ++t >= numBoostingIterations ) keepBoosting =
false;
237 trainingResults.push_back(trainingResult);
238 trainingResultsObserverManager.notifyObservers( trainingResult );
243 models[ classIter ].addClassifierToCommitee( weakClassifiers[bestClassifierIndex], alpha );
246 double reWeight = (1.0 - epsilon) / epsilon;
249 for(UINT i=0; i<M; i++){
250 oldSum += weights[i];
252 if( errorMatrix[bestClassifierIndex][i] == 1 ) weights[i] *= reWeight;
253 newSum += weights[i];
259 reWeight = oldSum/newSum;
260 for(UINT i=0; i<M; i++){
261 weights[i] *= reWeight;
265 trainingLog <<
"Stopping boosting training at iteration : " << t-1 <<
" with an error of " << epsilon << endl;
268 if( grt_isinf(alpha) ){ alpha = 1; }
269 models[ classIter ].addClassifierToCommitee( weakClassifiers[bestClassifierIndex], alpha );
277 for(UINT k=0; k<numClasses; k++){
278 models[k].normalizeWeights();
285 predictedClassLabel = 0;
287 classLikelihoods.resize(numClasses);
288 classDistances.resize(numClasses);
293 bool AdaBoost::predict_(VectorDouble &inputVector){
295 predictedClassLabel = 0;
296 maxLikelihood = -10000;
299 errorLog <<
"predict_(VectorDouble &inputVector) - AdaBoost Model Not Trained!" << endl;
303 if( inputVector.size() != numInputDimensions ){
304 errorLog <<
"predict_(VectorDouble &inputVector) - The size of the input vector (" << inputVector.size() <<
") does not match the num features in the model (" << numInputDimensions << endl;
309 for(UINT n=0; n<numInputDimensions; n++){
310 inputVector[n] = scale(inputVector[n], ranges[n].minValue, ranges[n].maxValue, 0, 1);
314 if( classLikelihoods.size() != numClasses ) classLikelihoods.resize(numClasses,0);
315 if( classDistances.size() != numClasses ) classDistances.resize(numClasses,0);
317 UINT bestClassIndex = 0;
318 UINT numPositivePredictions = 0;
319 bestDistance = -numeric_limits<double>::max();
320 double worstDistance = numeric_limits<double>::max();
322 for(UINT k=0; k<numClasses; k++){
323 double result = models[k].predict( inputVector );
325 switch ( predictionMethod ) {
326 case MAX_POSITIVE_VALUE:
328 if( result > bestDistance ){
329 bestDistance = result;
332 numPositivePredictions++;
333 classLikelihoods[k] = result;
334 }
else classLikelihoods[k] = 0;
336 classDistances[k] = result;
337 sum += classLikelihoods[k];
341 if( result > bestDistance ){
342 bestDistance = result;
345 if( result < worstDistance ){
346 worstDistance = result;
348 numPositivePredictions++;
349 classLikelihoods[k] = result;
350 classDistances[k] = result;
354 errorLog <<
"predict_(VectorDouble &inputVector) - Unknown prediction method!" << endl;
359 if( predictionMethod == MAX_VALUE ){
361 worstDistance = fabs( worstDistance );
362 for(UINT k=0; k<numClasses; k++){
363 classLikelihoods[k] += worstDistance;
364 sum += classLikelihoods[k];
370 for(UINT k=0; k<numClasses; k++)
371 classLikelihoods[k] /= sum;
373 maxLikelihood = classLikelihoods[ bestClassIndex ];
375 if( numPositivePredictions == 0 ){
376 predictedClassLabel = GRT_DEFAULT_NULL_CLASS_LABEL;
377 }
else predictedClassLabel = classLabels[ bestClassIndex ];
382 bool AdaBoost::recomputeNullRejectionThresholds(){
391 bool AdaBoost::setNullRejectionCoeff(
double nullRejectionCoeff){
393 if( nullRejectionCoeff > 0 ){
394 this->nullRejectionCoeff = nullRejectionCoeff;
395 recomputeNullRejectionThresholds();
401 bool AdaBoost::saveModelToFile(fstream &file)
const{
405 errorLog <<
"saveModelToFile(fstream &file) - The file is not open!" << endl;
410 file<<
"GRT_ADABOOST_MODEL_FILE_V2.0\n";
413 if( !Classifier::saveBaseSettingsToFile(file) ){
414 errorLog <<
"saveModelToFile(fstream &file) - Failed to save classifier base settings to file!" << endl;
419 file <<
"PredictionMethod: " << predictionMethod << endl;
423 file <<
"Models: " << endl;
424 for(UINT i=0; i<models.size(); i++){
425 if( !models[i].saveModelToFile( file ) ){
426 errorLog <<
"saveModelToFile(fstream &file) - Failed to write model " << i <<
" to file!" << endl;
436 bool AdaBoost::loadModelFromFile(fstream &file){
442 errorLog <<
"loadModelFromFile(string filename) - Could not open file to load model!" << endl;
450 if( word ==
"GRT_ADABOOST_MODEL_FILE_V1.0" ){
451 return loadLegacyModelFromFile( file );
454 if( word !=
"GRT_ADABOOST_MODEL_FILE_V2.0" ){
455 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read file header!" << endl;
456 errorLog << word << endl;
461 if( !Classifier::loadBaseSettingsFromFile(file) ){
462 errorLog <<
"loadModelFromFile(string filename) - Failed to load base settings from file!" << endl;
467 if( word !=
"PredictionMethod:" ){
468 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read PredictionMethod header!" << endl;
471 file >> predictionMethod;
475 if( word !=
"Models:" ){
476 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read Models header!" << endl;
481 models.resize( numClasses );
482 for(UINT i=0; i<models.size(); i++){
483 if( !models[i].loadModelFromFile( file ) ){
484 errorLog <<
"loadModelFromFile(fstream &file) - Failed to load model " << i <<
" from file!" << endl;
491 recomputeNullRejectionThresholds();
494 maxLikelihood = DEFAULT_NULL_LIKELIHOOD_VALUE;
495 bestDistance = DEFAULT_NULL_DISTANCE_VALUE;
496 classLikelihoods.resize(numClasses,DEFAULT_NULL_LIKELIHOOD_VALUE);
497 classDistances.resize(numClasses,DEFAULT_NULL_DISTANCE_VALUE);
503 bool AdaBoost::clear(){
517 clearWeakClassifiers();
521 weakClassifiers.push_back( weakClassiferPtr );
529 weakClassifiers.push_back( weakClassiferPtr );
534 bool AdaBoost::clearWeakClassifiers(){
536 for(UINT i=0; i<weakClassifiers.size(); i++){
537 if( weakClassifiers[i] != NULL ){
538 delete weakClassifiers[i];
539 weakClassifiers[i] = NULL;
542 weakClassifiers.clear();
546 bool AdaBoost::setNumBoostingIterations(UINT numBoostingIterations){
547 if( numBoostingIterations > 0 ){
548 this->numBoostingIterations = numBoostingIterations;
554 bool AdaBoost::setPredictionMethod(UINT predictionMethod){
555 if( predictionMethod != MAX_POSITIVE_VALUE && predictionMethod != MAX_VALUE ){
558 this->predictionMethod = predictionMethod;
562 void AdaBoost::printModel(){
564 cout <<
"AdaBoostModel: \n";
565 cout<<
"NumFeatures: " << numInputDimensions << endl;
566 cout<<
"NumClasses: " << numClasses << endl;
567 cout <<
"UseScaling: " << useScaling << endl;
568 cout<<
"UseNullRejection: " << useNullRejection << endl;
570 for(UINT k=0; k<numClasses; k++){
571 cout <<
"Class: " << k+1 <<
" ClassLabel: " << classLabels[k] << endl;
577 bool AdaBoost::loadLegacyModelFromFile( fstream &file ){
582 if( word !=
"NumFeatures:" ){
583 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read NumFeatures header!" << endl;
586 file >> numInputDimensions;
589 if( word !=
"NumClasses:" ){
590 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read NumClasses header!" << endl;
596 if( word !=
"UseScaling:" ){
597 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read UseScaling header!" << endl;
603 if( word !=
"UseNullRejection:" ){
604 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read UseNullRejection header!" << endl;
607 file >> useNullRejection;
611 if( word !=
"Ranges:" ){
612 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read Ranges header!" << endl;
615 ranges.resize( numInputDimensions );
617 for(UINT n=0; n<ranges.size(); n++){
618 file >> ranges[n].minValue;
619 file >> ranges[n].maxValue;
624 if( word !=
"Trained:" ){
625 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read Trained header!" << endl;
631 if( word !=
"PredictionMethod:" ){
632 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read PredictionMethod header!" << endl;
635 file >> predictionMethod;
639 if( word !=
"Models:" ){
640 errorLog <<
"loadModelFromFile(fstream &file) - Failed to read Models header!" << endl;
645 models.resize( numClasses );
646 classLabels.resize( numClasses );
647 for(UINT i=0; i<models.size(); i++){
648 if( !models[i].loadModelFromFile( file ) ){
649 errorLog <<
"loadModelFromFile(fstream &file) - Failed to load model " << i <<
" from file!" << endl;
655 classLabels[i] = models[i].getClassLabel();
660 recomputeNullRejectionThresholds();
663 maxLikelihood = DEFAULT_NULL_LIKELIHOOD_VALUE;
664 bestDistance = DEFAULT_NULL_DISTANCE_VALUE;
665 classLikelihoods.resize(numClasses,DEFAULT_NULL_LIKELIHOOD_VALUE);
666 classDistances.resize(numClasses,DEFAULT_NULL_DISTANCE_VALUE);
vector< UINT > getClassLabels() const
virtual double predict(const VectorDouble &x)
UINT getNumDimensions() const
bool setNumDimensions(UINT numDimensions)
This class contains the AdaBoost classifier. AdaBoost (Adaptive Boosting) is a powerful classifier th...
UINT getNumSamples() const
virtual bool train(ClassificationData &trainingData, VectorDouble &weights)
UINT getNumClasses() const
bool scale(const double minTarget, const double maxTarget)
WeakClassifier * createNewInstance() const
virtual void print() const
virtual double getPositiveClassLabel() const
bool setClassificationResult(unsigned int trainingIteration, double accuracy, MLBase *trainer)
vector< MinMax > getRanges() const
string getClassifierType() const
bool addSample(UINT classLabel, const VectorDouble &sample)