GestureRecognitionToolkit  Version: 1.0 Revision: 04-03-15
The Gesture Recognition Toolkit (GRT) is a cross-platform, open-source, c++ machine learning library for real-time gesture recognition.
MLP.cpp
1 /*
2 GRT MIT License
3 Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>
4 
5 Permission is hereby granted, free of charge, to any person obtaining a copy of this software
6 and associated documentation files (the "Software"), to deal in the Software without restriction,
7 including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
9 subject to the following conditions:
10 
11 The above copyright notice and this permission notice shall be included in all copies or substantial
12 portions of the Software.
13 
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
15 LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
16 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
17 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
18 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19 */
20 
21 #include "MLP.h"
22 
23 namespace GRT{
24 
25 //Register the MLP module with the Regressifier base class
26 RegisterRegressifierModule< MLP > MLP::registerModule("MLP");
27 
29  inputLayerActivationFunction = Neuron::LINEAR;
30  hiddenLayerActivationFunction = Neuron::LINEAR;
31  outputLayerActivationFunction = Neuron::LINEAR;
32  minNumEpochs = 10;
33  numRandomTrainingIterations = 10;
34  validationSetSize = 20; //20% of the training data will be set aside for the validation set
35  trainingMode = ONLINE_GRADIENT_DESCENT;
36  momentum = 0.5;
37  gamma = 2.0;
38  trainingError = 0;
39  nullRejectionCoeff = 0.9;
40  nullRejectionThreshold = 0;
41  useValidationSet = true;
42  randomiseTrainingOrder = false;
43  useScaling = true;
44  trained = false;
45  initialized = false;
46  classificationModeActive = false;
47  useNullRejection = true;
48  clear();
49  classType = "MLP";
50  regressifierType = classType;
51  debugLog.setProceedingText("[DEBUG MLP]");
52  errorLog.setProceedingText("[ERROR MLP]");
53  trainingLog.setProceedingText("[TRAINING MLP]");
54  warningLog.setProceedingText("[WARNING MLP]");
55 }
56 
57 MLP::MLP(const MLP &rhs){
58  classType = "MLP";
59  regressifierType = classType;
60  debugLog.setProceedingText("[DEBUG MLP]");
61  errorLog.setProceedingText("[ERROR MLP]");
62  trainingLog.setProceedingText("[TRAINING MLP]");
63  warningLog.setProceedingText("[WARNING MLP]");
64 
65  *this = rhs;
66 }
67 
69  clear();
70 }
71 
72 MLP& MLP::operator=(const MLP &rhs){
73  if( this != &rhs ){
74  //MLP variables
75  this->numInputNeurons = rhs.numInputNeurons;
76  this->numHiddenNeurons = rhs.numHiddenNeurons;
77  this->numOutputNeurons = rhs.numOutputNeurons;
78  this->inputLayerActivationFunction = rhs.inputLayerActivationFunction;
79  this->hiddenLayerActivationFunction = rhs.hiddenLayerActivationFunction;
80  this->outputLayerActivationFunction = rhs.outputLayerActivationFunction;
81  this->numRandomTrainingIterations = rhs.numRandomTrainingIterations;
82  this->trainingMode = rhs.trainingMode;
83  this->momentum = rhs.momentum;
84  this->trainingError = rhs.trainingError;
85  this->gamma = rhs.gamma;
86  this->initialized = rhs.initialized;
87  this->inputLayer = rhs.inputLayer;
88  this->hiddenLayer = rhs.hiddenLayer;
89  this->outputLayer = rhs.outputLayer;
90  this->inputVectorRanges = rhs.inputVectorRanges;
91  this->targetVectorRanges = rhs.targetVectorRanges;
92  this->trainingErrorLog = rhs.trainingErrorLog;
93 
94  this->classificationModeActive = rhs.classificationModeActive;
95  this->useNullRejection = rhs.useNullRejection;
96  this->predictedClassLabel = rhs.predictedClassLabel;
97  this->nullRejectionCoeff = rhs.nullRejectionCoeff;
98  this->nullRejectionThreshold = rhs.nullRejectionThreshold;
99  this->maxLikelihood = rhs.maxLikelihood;
100  this->classLikelihoods = rhs.classLikelihoods;
101 
102  //Copy the base variables
104  }
105  return *this;
106 }
107 
108 bool MLP::deepCopyFrom(const Regressifier *regressifier){
109 
110  if( regressifier == NULL ){
111  errorLog << "deepCopyFrom(const Regressifier *regressifier) - regressifier is NULL!" << endl;
112  return false;
113  }
114 
115  if( this->getRegressifierType() != regressifier->getRegressifierType() ){
116  errorLog << "deepCopyFrom(const Regressifier *regressifier) - regressifier is not the correct type!" << endl;
117  return false;
118  }
119 
120  *this = *dynamic_cast<const MLP*>(regressifier);
121 
122  return true;
123 }
124 
125 //Classifier interface
126 bool MLP::train_(ClassificationData &trainingData){
127 
128  if( !initialized ){
129  errorLog << "train_(ClassificationData trainingData) - The MLP has not been initialized!" << endl;
130  return false;
131  }
132 
133  if( trainingData.getNumDimensions() != numInputNeurons ){
134  errorLog << "train_(ClassificationData trainingData) - The number of input dimensions in the training data (" << trainingData.getNumDimensions() << ") does not match that of the MLP (" << numInputNeurons << ")" << endl;
135  return false;
136  }
137  if( trainingData.getNumClasses() != numOutputNeurons ){
138  errorLog << "train_(ClassificationData trainingData) - The number of classes in the training data (" << trainingData.getNumClasses() << ") does not match that of the MLP (" << numOutputNeurons << ")" << endl;
139  return false;
140  }
141 
142  //Reformat the LabelledClassificationData as LabelledRegressionData
143  RegressionData regressionData = trainingData.reformatAsRegressionData();
144 
145  //Flag that the MLP is being used for classification, not regression
146  classificationModeActive = true;
147 
148  return trainModel(regressionData);
149 }
150 
151 bool MLP::train_(RegressionData &trainingData){
152 
153  //Flag that the MLP is being used for regression, not classification
154  classificationModeActive = false;
155 
156  return trainModel(trainingData);
157 }
158 
159 //Classifier interface
160 bool MLP::predict_(VectorDouble &inputVector){
161 
162  if( !trained ){
163  errorLog << "predict_(VectorDouble &inputVector) - Model not trained!" << endl;
164  return false;
165  }
166 
167  if( inputVector.size() != numInputNeurons ){
168  errorLog << "predict_(VectorDouble &inputVector) - The sie of the input vector (" << int(inputVector.size()) << ") does not match that of the number of input dimensions (" << numInputNeurons << ") " << endl;
169  return false;
170  }
171 
172  //Set the mapped data as the classLikelihoods
173  regressionData = feedforward( inputVector );
174 
175  if( classificationModeActive ){
176 
177  //Estimate the class likelihoods
178  const UINT K = (UINT)regressionData.size();
179  classLikelihoods = regressionData;
180 
181  //Make sure all the values are greater than zero, we do this by finding the min value and adding this onto all the values
182  double minValue = Util::getMin( classLikelihoods );
183  for(UINT i=0; i<K; i++){
184  classLikelihoods[i] += minValue;
185  }
186 
187  //Normalize the likelihoods so they sum to 1
188  double sum = Util::sum(classLikelihoods);
189  if( sum > 0 ){
190  for(UINT i=0; i<K; i++){
191  classLikelihoods[i] /= sum;
192  }
193  }
194 
195  //Find the best value
196  double bestValue = classLikelihoods[0];
197  UINT bestIndex = 0;
198  for(UINT i=1; i<K; i++){
199  if( classLikelihoods[i] > bestValue ){
200  bestValue = classLikelihoods[i];
201  bestIndex = i;
202  }
203  }
204 
205  //Set the maximum likelihood and predicted class label
206  maxLikelihood = bestValue;
207  predictedClassLabel = bestIndex+1;
208 
209  if( useNullRejection ){
210  if( maxLikelihood < nullRejectionCoeff ){
211  predictedClassLabel = 0;
212  }
213  }
214  }
215 
216  return true;
217 }
218 
219 bool MLP::init(const UINT numInputNeurons, const UINT numHiddenNeurons, const UINT numOutputNeurons){
220  return init(numInputNeurons, numHiddenNeurons, numOutputNeurons, inputLayerActivationFunction, hiddenLayerActivationFunction, outputLayerActivationFunction );
221 }
222 
223 bool MLP::init(const UINT numInputNeurons,
224  const UINT numHiddenNeurons,
225  const UINT numOutputNeurons,
226  const UINT inputLayerActivationFunction,
227  const UINT hiddenLayerActivationFunction,
228  const UINT outputLayerActivationFunction){
229 
230  //Clear any previous models
231  clear();
232 
233  //Initialize the random seed
234  random.setSeed( (UINT)time(NULL) );
235 
236  if( numInputNeurons == 0 || numHiddenNeurons == 0 || numOutputNeurons == 0 ){
237  if( numInputNeurons == 0 ){ errorLog << "init(...) - The number of input neurons is zero!" << endl; }
238  if( numHiddenNeurons == 0 ){ errorLog << "init(...) - The number of hidden neurons is zero!" << endl; }
239  if( numOutputNeurons == 0 ){ errorLog << "init(...) - The number of output neurons is zero!" << endl; }
240  return false;
241  }
242 
243  //Validate the activation functions
244  if( !validateActivationFunction(inputLayerActivationFunction) || !validateActivationFunction(hiddenLayerActivationFunction) || !validateActivationFunction(outputLayerActivationFunction) ){
245  errorLog << "init(...) - One Of The Activation Functions Failed The Validation Check" << endl;
246  return false;
247  }
248 
249  //Set the size of the MLP
250  this->numInputNeurons = numInputNeurons;
251  this->numHiddenNeurons = numHiddenNeurons;
252  this->numOutputNeurons = numOutputNeurons;
253 
254  //Set the regression IO
255  this->numInputDimensions = numInputNeurons;
256  this->numOutputDimensions = numOutputNeurons;
257 
258  //Set the validation layers
259  this->inputLayerActivationFunction = inputLayerActivationFunction;
260  this->hiddenLayerActivationFunction = hiddenLayerActivationFunction;
261  this->outputLayerActivationFunction = outputLayerActivationFunction;
262 
263  //Setup the neurons for each of the layers
264  inputLayer.resize(numInputNeurons);
265  hiddenLayer.resize(numHiddenNeurons);
266  outputLayer.resize(numOutputNeurons);
267 
268  //Init the neuron memory for each of the layers
269  for(UINT i=0; i<numInputNeurons; i++){
270  inputLayer[i].init(1,inputLayerActivationFunction);
271  inputLayer[i].weights[0] = 1.0; //The weights for the input layer should always be 1
272  inputLayer[i].bias = 0.0; //The bias for the input layer should always be 0
273  inputLayer[i].gamma = gamma;
274  }
275 
276  for(UINT i=0; i<numHiddenNeurons; i++){
277  hiddenLayer[i].init(numInputNeurons,hiddenLayerActivationFunction);
278  hiddenLayer[i].gamma = gamma;
279  }
280 
281  for(UINT i=0; i<numOutputNeurons; i++){
282  outputLayer[i].init(numHiddenNeurons,outputLayerActivationFunction);
283  outputLayer[i].gamma = gamma;
284  }
285 
286  initialized = true;
287 
288  return true;
289 }
290 
291 bool MLP::clear(){
292 
293  //Clear the base class
295 
296  numInputNeurons = 0;
297  numHiddenNeurons = 0;
298  numOutputNeurons = 0;
299  inputLayer.clear();
300  hiddenLayer.clear();
301  outputLayer.clear();
302  initialized = false;
303 
304  return true;
305 }
306 
307 bool MLP::print() const{
308  printNetwork();
309  return true;
310 }
311 
312 bool MLP::trainModel(RegressionData &trainingData){
313 
314  trained = false;
315 
316  if( !initialized ){
317  errorLog << "train(RegressionData trainingData) - The MLP has not be initialized!" << endl;
318  return false;
319  }
320 
321  if( trainingData.getNumSamples() == 0 ){
322  errorLog << "train(RegressionData trainingData) - The training data is empty!" << endl;
323  return false;
324  }
325 
326  //Create a validation dataset, if needed
327  RegressionData validationData;
328  if( useValidationSet ){
329  validationData = trainingData.partition( 100 - validationSetSize );
330  }
331 
332  const UINT N = trainingData.getNumInputDimensions();
333  const UINT T = trainingData.getNumTargetDimensions();
334 
335  if( N != numInputNeurons ){
336  errorLog << "train(LabelledRegressionData trainingData) - The number of input dimensions in the training data (" << N << ") does not match that of the MLP (" << numInputNeurons << ")" << endl;
337  return false;
338  }
339  if( T != numOutputNeurons ){
340  errorLog << "train(LabelledRegressionData trainingData) - The number of target dimensions in the training data (" << T << ") does not match that of the MLP (" << numOutputNeurons << ")" << endl;
341  return false;
342  }
343 
344  //Set the Regressifier input and output dimensions
345  numInputDimensions = numInputNeurons;
346  numOutputDimensions = numOutputNeurons;
347 
348  //Scale the training and validation data, if needed
349  if( useScaling ){
350  //Find the ranges for the input data
351  inputVectorRanges = trainingData.getInputRanges();
352 
353  //Find the ranges for the target data
354  targetVectorRanges = trainingData.getTargetRanges();
355 
356  //Now scale the training data and the validation data if required
357  trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
358 
359  if( useValidationSet ){
360  validationData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
361  }
362  }
363  //If scaling is enabled then the training and validation data will be scaled - so turn it off so the we do not need to scale the data again
364  //The actual scaling state will be reset at the end of traiing
365  bool tempScalingState = useScaling;
366  useScaling = false;
367 
368  //Setup the memory
369  trainingErrorLog.clear();
370  inputNeuronsOuput.resize(numInputNeurons);
371  hiddenNeuronsOutput.resize(numHiddenNeurons);
372  outputNeuronsOutput.resize(numOutputNeurons);
373  deltaO.resize(numOutputNeurons);
374  deltaH.resize(numHiddenNeurons);
375 
376  //Call the main training function
377  switch( trainingMode ){
378  case ONLINE_GRADIENT_DESCENT:
379  if( classificationModeActive ){
380  trained = trainOnlineGradientDescentClassification( trainingData, validationData );
381  }else{
382  trained = trainOnlineGradientDescentRegression( trainingData, validationData );
383  }
384  break;
385  default:
386  useScaling = tempScalingState;
387  errorLog << "train(RegressionData trainingData) - Uknown training mode!" << endl;
388  return false;
389  break;
390  }
391 
392  //Reset the scaling state so the prediction data will be scaled if needed
393  useScaling = tempScalingState;
394 
395  return true;
396 }
397 
398 bool MLP::trainOnlineGradientDescentClassification(const RegressionData &trainingData,const RegressionData &validationData){
399 
400  const UINT M = trainingData.getNumSamples();
401  const UINT T = trainingData.getNumTargetDimensions();
402  const UINT numTestingExamples = useValidationSet ? validationData.getNumSamples() : M;
403 
404  //Setup the training loop
405  bool keepTraining = true;
406  UINT epoch = 0;
407  double alpha = learningRate;
408  double beta = momentum;
409  UINT bestIter = 0;
410  MLP bestNetwork;
411  totalSquaredTrainingError = 0;
412  rootMeanSquaredTrainingError = 0;
413  trainingError = 0;
414  double error = 0;
415  double lastError = 0;
416  double accuracy = 0;
417  double trainingSetAccuracy = 0;
418  double trainingSetTotalSquaredError = 0;
419  double bestError = numeric_limits< double >::max();
420  double bestTSError = numeric_limits< double >::max();
421  double bestRMSError = numeric_limits< double >::max();
422  double bestAccuracy = 0;
423  double delta = 0;
424  vector< UINT > indexList(M);
425  vector< vector< double > > tempTrainingErrorLog;
426  TrainingResult result;
427  trainingResults.reserve(M);
428 
429  //Reset the indexList, this is used to randomize the order of the training examples, if needed
430  for(UINT i=0; i<M; i++) indexList[i] = i;
431 
432  for(UINT iter=0; iter<numRandomTrainingIterations; iter++){
433 
434  epoch = 0;
435  keepTraining = true;
436  tempTrainingErrorLog.clear();
437 
438  //Randomise the start values of the neurons
439  init(numInputNeurons,numHiddenNeurons,numOutputNeurons,inputLayerActivationFunction,hiddenLayerActivationFunction,outputLayerActivationFunction);
440 
441  if( randomiseTrainingOrder ){
442  for(UINT i=0; i<M; i++){
443  SWAP(indexList[ i ], indexList[ random.getRandomNumberInt(0, M) ]);
444  }
445  }
446 
447  while( keepTraining ){
448 
449  //Perform one training epoch
450  accuracy = 0;
451  totalSquaredTrainingError = 0;
452 
453  for(UINT i=0; i<M; i++){
454  //Get the i'th training and target vectors
455  const VectorDouble &trainingExample = trainingData[ indexList[i] ].getInputVector();
456  const VectorDouble &targetVector = trainingData[ indexList[i] ].getTargetVector();
457 
458  //Perform the back propagation
459  double backPropError = back_prop(trainingExample,targetVector,alpha,beta);
460 
461  //debugLog << "i: " << i << " backPropError: " << backPropError << endl;
462 
463  if( isNAN(backPropError) ){
464  keepTraining = false;
465  errorLog << "train(LabelledRegressionData trainingData) - NaN found!" << endl;
466  }
467 
468  //Compute the error for the i'th example
469  if( classificationModeActive ){
470  VectorDouble y = feedforward(trainingExample);
471 
472  //Get the class label
473  double bestValue = targetVector[0];
474  UINT bestIndex = 0;
475  for(UINT i=1; i<targetVector.size(); i++){
476  if( targetVector[i] > bestValue ){
477  bestValue = targetVector[i];
478  bestIndex = i;
479  }
480  }
481  UINT classLabel = bestIndex + 1;
482 
483  //Get the predicted class label
484  bestValue = y[0];
485  bestIndex = 0;
486  for(UINT i=1; i<numOutputNeurons; i++){
487  if( y[i] > bestValue ){
488  bestValue = y[i];
489  bestIndex = i;
490  }
491  }
492  predictedClassLabel = bestIndex+1;
493 
494  if( classLabel == predictedClassLabel ){
495  accuracy++;
496  }
497 
498  }else{
499  totalSquaredTrainingError += backPropError; //The backPropError is already squared
500  }
501  }
502 
503  if( checkForNAN() ){
504  keepTraining = false;
505  errorLog << "train(LabelledRegressionData trainingData) - NaN found!" << endl;
506  break;
507  }
508 
509  //Compute the error over all the training/validation examples
510  if( useValidationSet ){
511  trainingSetAccuracy = accuracy;
512  trainingSetTotalSquaredError = totalSquaredTrainingError;
513  accuracy = 0;
514  totalSquaredTrainingError = 0;
515 
516  //Iterate over the validation samples
517  UINT numValidationSamples = validationData.getNumSamples();
518  for(UINT i=0; i<numValidationSamples; i++){
519  const VectorDouble &trainingExample = validationData[i].getInputVector();
520  const VectorDouble &targetVector = validationData[i].getTargetVector();
521 
522  VectorDouble y = feedforward(trainingExample);
523 
524  if( classificationModeActive ){
525  //Get the class label
526  double bestValue = targetVector[0];
527  UINT bestIndex = 0;
528  for(UINT i=1; i<numInputNeurons; i++){
529  if( targetVector[i] > bestValue ){
530  bestValue = targetVector[i];
531  bestIndex = i;
532  }
533  }
534  UINT classLabel = bestIndex + 1;
535 
536  //Get the predicted class label
537  bestValue = y[0];
538  bestIndex = 0;
539  for(UINT i=1; i<numOutputNeurons; i++){
540  if( y[i] > bestValue ){
541  bestValue = y[i];
542  bestIndex = i;
543  }
544  }
545  predictedClassLabel = bestIndex+1;
546 
547  if( classLabel == predictedClassLabel ){
548  accuracy++;
549  }
550 
551  }else{
552  //Update the total squared error
553  for(UINT j=0; j<T; j++){
554  totalSquaredTrainingError += SQR( targetVector[j]-y[j] );
555  }
556  }
557  }
558 
559  accuracy = (accuracy/double(numValidationSamples))*double(numValidationSamples);
560  rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(numValidationSamples) );
561 
562  }else{//We are not using a validation set
563  accuracy = (accuracy/double(M))*double(M);
564  rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
565  }
566 
567  //Store the errors
568  VectorDouble temp(2);
569  temp[0] = 100.0 - trainingSetAccuracy;
570  temp[1] = 100.0 - accuracy;
571  tempTrainingErrorLog.push_back( temp );
572 
573  error = 100.0 - accuracy;
574 
575  //Store the training results
576  result.setClassificationResult(iter,accuracy,this);
577  trainingResults.push_back( result );
578 
579  delta = fabs( error - lastError );
580 
581  trainingLog << "Random Training Iteration: " << iter+1 << " Epoch: " << epoch << " Error: " << error << " Delta: " << delta << endl;
582 
583  //Check to see if we should stop training
584  if( ++epoch >= maxNumEpochs ){
585  keepTraining = false;
586  }
587  if( delta <= minChange && epoch >= minNumEpochs ){
588  keepTraining = false;
589  }
590 
591  //Update the last error
592  lastError = error;
593 
594  //Notify any observers of the new training result
595  trainingResultsObserverManager.notifyObservers( result );
596 
597  }//End of While( keepTraining )
598 
599  if( lastError < bestError ){
600  bestIter = iter;
601  bestError = lastError;
602  bestTSError = totalSquaredTrainingError;
603  bestRMSError = rootMeanSquaredTrainingError;
604  bestAccuracy = accuracy;
605  bestNetwork = *this;
606  trainingErrorLog = tempTrainingErrorLog;
607  }
608 
609  }//End of For( numRandomTrainingIterations )
610 
611  trainingLog << "Best Accuracy: " << bestAccuracy << " in Random Training Iteration: " << bestIter+1 << endl;
612 
613  //Check to make sure the best network has not got any NaNs in it
614  if( checkForNAN() ){
615  errorLog << "train(LabelledRegressionData trainingData) - NAN Found!" << endl;
616  return false;
617  }
618 
619  //Set the MLP model to the model that best during training
620  *this = bestNetwork;
621  trainingError = bestAccuracy;
622 
623  //Compute the rejection threshold
624  double averageValue = 0;
625  VectorDouble classificationPredictions;
626 
627  for(UINT i=0; i<numTestingExamples; i++){
628  VectorDouble inputVector = useValidationSet ? validationData[i].getInputVector() : trainingData[i].getInputVector();
629  VectorDouble targetVector = useValidationSet ? validationData[i].getTargetVector() : trainingData[i].getTargetVector();
630 
631  //Make the prediction
632  VectorDouble y = feedforward(inputVector);
633 
634  //Get the class label
635  double bestValue = targetVector[0];
636  UINT bestIndex = 0;
637  for(UINT i=1; i<targetVector.size(); i++){
638  if( targetVector[i] > bestValue ){
639  bestValue = targetVector[i];
640  bestIndex = i;
641  }
642  }
643  UINT classLabel = bestIndex + 1;
644 
645  //Get the predicted class label
646  bestValue = y[0];
647  bestIndex = 0;
648  for(UINT i=1; i<y.size(); i++){
649  if( y[i] > bestValue ){
650  bestValue = y[i];
651  bestIndex = i;
652  }
653  }
654  predictedClassLabel = bestIndex+1;
655 
656  //Only add the max value if the prediction is correct
657  if( classLabel == predictedClassLabel ){
658  classificationPredictions.push_back( bestValue );
659  averageValue += bestValue;
660  }
661  }
662 
663  averageValue /= double(classificationPredictions.size());
664  double stdDev = 0;
665  for(UINT i=0; i<classificationPredictions.size(); i++){
666  stdDev += SQR(classificationPredictions[i]-averageValue);
667  }
668  stdDev = sqrt( stdDev / double(classificationPredictions.size()-1) );
669 
670  nullRejectionThreshold = averageValue-(stdDev*nullRejectionCoeff);
671 
672  //Return true to flag that the model was trained OK
673  return true;
674 }
675 
676 bool MLP::trainOnlineGradientDescentRegression(const RegressionData &trainingData,const RegressionData &validationData){
677 
678  const UINT M = trainingData.getNumSamples();
679  const UINT T = trainingData.getNumTargetDimensions();
680  const UINT numValidationSamples = useValidationSet ? validationData.getNumSamples() : M;
681 
682  //Setup the training loop
683  bool keepTraining = true;
684  UINT epoch = 0;
685  double alpha = learningRate;
686  double beta = momentum;
687  UINT bestIter = 0;
688  MLP bestNetwork;
689  totalSquaredTrainingError = 0;
690  rootMeanSquaredTrainingError = 0;
691  trainingError = 0;
692  double error = 0;
693  double lastError = 0;
694  double trainingSetTotalSquaredError = 0;
695  double bestError = numeric_limits< double >::max();
696  double bestTSError = numeric_limits< double >::max();
697  double bestRMSError = numeric_limits< double >::max();
698  double delta = 0;
699  vector< UINT > indexList(M);
700  vector< vector< double > > tempTrainingErrorLog;
701  TrainingResult result;
702  trainingResults.reserve(M);
703 
704  //Reset the indexList, this is used to randomize the order of the training examples, if needed
705  for(UINT i=0; i<M; i++) indexList[i] = i;
706 
707  for(UINT iter=0; iter<numRandomTrainingIterations; iter++){
708 
709  epoch = 0;
710  keepTraining = true;
711  tempTrainingErrorLog.clear();
712 
713  //Randomise the start values of the neurons
714  init(numInputNeurons,numHiddenNeurons,numOutputNeurons,inputLayerActivationFunction,hiddenLayerActivationFunction,outputLayerActivationFunction);
715 
716  if( randomiseTrainingOrder ){
717  for(UINT i=0; i<M; i++){
718  SWAP(indexList[ i ], indexList[ random.getRandomNumberInt(0, M) ]);
719  }
720  }
721 
722  while( keepTraining ){
723 
724  //Perform one training epoch
725  totalSquaredTrainingError = 0;
726 
727  for(UINT i=0; i<M; i++){
728  //Get the i'th training and target vectors
729  const VectorDouble &trainingExample = trainingData[ indexList[i] ].getInputVector();
730  const VectorDouble &targetVector = trainingData[ indexList[i] ].getTargetVector();
731 
732  //Perform the back propagation
733  double backPropError = back_prop(trainingExample,targetVector,alpha,beta);
734 
735  //debugLog << "i: " << i << " backPropError: " << backPropError << endl;
736 
737  if( isNAN(backPropError) ){
738  keepTraining = false;
739  errorLog << "train(RegressionData trainingData) - NaN found!" << endl;
740  }
741 
742  //Compute the error for the i'th example
743  totalSquaredTrainingError += backPropError; //The backPropError is already squared
744  }
745 
746  if( checkForNAN() ){
747  keepTraining = false;
748  errorLog << "train(RegressionData trainingData) - NaN found!" << endl;
749  break;
750  }
751 
752  //Compute the error over all the training/validation examples
753  if( useValidationSet ){
754  trainingSetTotalSquaredError = totalSquaredTrainingError;
755  totalSquaredTrainingError = 0;
756 
757  //Iterate over the validation samples
758  for(UINT i=0; i<numValidationSamples; i++){
759  const VectorDouble &trainingExample = validationData[i].getInputVector();
760  const VectorDouble &targetVector = validationData[i].getTargetVector();
761 
762  VectorDouble y = feedforward(trainingExample);
763 
764  //Update the total squared error
765  for(UINT j=0; j<T; j++){
766  totalSquaredTrainingError += SQR( targetVector[j]-y[j] );
767  }
768 
769  }
770 
771  rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(numValidationSamples) );
772 
773  }else{//We are not using a validation set
774  rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
775  }
776 
777  //Store the errors
778  VectorDouble temp(2);
779  temp[0] = trainingSetTotalSquaredError;
780  temp[1] = rootMeanSquaredTrainingError;
781  tempTrainingErrorLog.push_back( temp );
782 
783  error = rootMeanSquaredTrainingError;
784 
785  //Store the training results
786  result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError,this);
787  trainingResults.push_back( result );
788 
789  delta = fabs( error - lastError );
790 
791  trainingLog << "Random Training Iteration: " << iter+1 << " Epoch: " << epoch << " Error: " << error << " Delta: " << delta << endl;
792 
793  //Check to see if we should stop training
794  if( ++epoch >= maxNumEpochs ){
795  keepTraining = false;
796  }
797  if( delta <= minChange && epoch >= minNumEpochs ){
798  keepTraining = false;
799  }
800 
801  //Update the last error
802  lastError = error;
803 
804  //Notify any observers of the new training result
805  trainingResultsObserverManager.notifyObservers( result );
806 
807  }//End of While( keepTraining )
808 
809  //Check to see if this is the best model so far
810  if( lastError < bestError ){
811  bestIter = iter;
812  bestError = lastError;
813  bestTSError = totalSquaredTrainingError;
814  bestRMSError = rootMeanSquaredTrainingError;
815  bestNetwork = *this;
816  trainingErrorLog = tempTrainingErrorLog;
817  }
818 
819  }//End of For( numRandomTrainingIterations )
820 
821  trainingLog << "Best RMSError: " << bestRMSError << " in Random Training Iteration: " << bestIter+1 << endl;
822 
823  //Check to make sure the best network has not got any NaNs in it
824  if( checkForNAN() ){
825  errorLog << "train(RegressionData trainingData) - NAN Found!" << endl;
826  return false;
827  }
828 
829  //Set the MLP model to the model that best during training
830  *this = bestNetwork;
831  trainingError = bestRMSError;
832 
833  //Return true to flag that the model was trained OK
834  return true;
835 }
836 
837 double MLP::back_prop(const VectorDouble &trainingExample,const VectorDouble &targetVector,const double alpha,const double beta){
838 
839  double update = 0;
840 
841  //Forward propagation
842  feedforward(trainingExample,inputNeuronsOuput,hiddenNeuronsOutput,outputNeuronsOutput);
843 
844  //Compute the error of the output layer: the derivative of the function times the error of the output
845  for(UINT i=0; i<numOutputNeurons; i++){
846  deltaO[i] = outputLayer[i].getDerivative( outputNeuronsOutput[i] ) * (targetVector[i]-outputNeuronsOutput[i]);
847  }
848 
849  //Compute the error of the hidden layer
850  for(UINT i=0; i<numHiddenNeurons; i++){
851  double sum = 0;
852  for(UINT j=0; j<numOutputNeurons; j++){
853  sum += outputLayer[j].weights[i] * deltaO[j];
854  }
855  deltaH[i] = hiddenLayer[i].getDerivative( hiddenNeuronsOutput[i] ) * sum;
856  }
857 
858  //Update the hidden weights: old hidden weights + (learningRate * inputToTheHiddenNeuron * deltaHidden )
859  for(UINT i=0; i<numHiddenNeurons; i++){
860  for(UINT j=0; j<numInputNeurons; j++){
861  //Compute the update
862  update = alpha * (beta * hiddenLayer[i].previousUpdate[j] + (1.0 - beta) * inputNeuronsOuput[j] * deltaH[i]);
863 
864  //Update the weights
865  hiddenLayer[i].weights[j] += update;
866 
867  //Store the previous update
868  hiddenLayer[i].previousUpdate[j] = update;
869  }
870  }
871 
872  //Update the output weights
873  for(UINT i=0; i<numOutputNeurons; i++){
874  for(UINT j=0; j<numHiddenNeurons; j++){
875  //Compute the update
876  update = alpha * (beta * outputLayer[i].previousUpdate[j] + (1.0 - beta) * hiddenNeuronsOutput[j] * deltaO[i]);
877 
878  //Update the weights
879  outputLayer[i].weights[j] += update;
880 
881  //Store the update
882  outputLayer[i].previousUpdate[j] = update;
883 
884  }
885  }
886 
887  //Update the hidden bias
888  for(UINT i=0; i<numHiddenNeurons; i++){
889  //Compute the update
890  update = alpha * (beta * hiddenLayer[i].previousBiasUpdate + (1.0 - beta) * deltaH[i]);
891 
892  //Update the bias
893  hiddenLayer[i].bias += update;
894 
895  //Store the update
896  hiddenLayer[i].previousBiasUpdate = update;
897  }
898 
899  //Update the output bias
900  for(UINT i=0; i<numOutputNeurons; i++){
901  //Compute the update
902  update = alpha * (beta * outputLayer[i].previousBiasUpdate + (1.0 - beta) * deltaO[i]);
903 
904  //Update the bias
905  outputLayer[i].bias += update;
906 
907  //Stire the update
908  outputLayer[i].previousBiasUpdate = update;
909  }
910 
911  //Compute the squared error between the output of the network and the target vector
912  double error = 0;
913  for(UINT i=0; i<numOutputNeurons; i++){
914  error += SQR( targetVector[i] - outputNeuronsOutput[i] );
915  }
916 
917  return error;
918 }
919 
920 VectorDouble MLP::feedforward(VectorDouble trainingExample){
921 
922  if( inputNeuronsOuput.size() != numInputNeurons ) inputNeuronsOuput.resize(numInputNeurons,0);
923  if( hiddenNeuronsOutput.size() != numHiddenNeurons ) hiddenNeuronsOutput.resize(numHiddenNeurons,0);
924  if( outputNeuronsOutput.size() != numOutputNeurons ) outputNeuronsOutput.resize(numOutputNeurons,0);
925 
926  //Scale the input vector if required
927  if( useScaling ){
928  for(UINT i=0; i<numInputNeurons; i++){
929  trainingExample[i] = scale(trainingExample[i],inputVectorRanges[i].minValue,inputVectorRanges[i].maxValue,0.0,1.0);
930  }
931  }
932 
933  //Input layer
934  VectorDouble input(1);
935  for(UINT i=0; i<numInputNeurons; i++){
936  input[0] = trainingExample[i];
937  inputNeuronsOuput[i] = inputLayer[i].fire( input );
938  }
939 
940  //Hidden Layer
941  for(UINT i=0; i<numHiddenNeurons; i++){
942  hiddenNeuronsOutput[i] = hiddenLayer[i].fire( inputNeuronsOuput );
943  }
944 
945  //Output Layer
946  for(UINT i=0; i<numOutputNeurons; i++){
947  outputNeuronsOutput[i] = outputLayer[i].fire( hiddenNeuronsOutput );
948  }
949 
950  //Scale the output vector if required
951  if( useScaling ){
952  for(unsigned int i=0; i<numOutputNeurons; i++){
953  outputNeuronsOutput[i] = scale(outputNeuronsOutput[i],0.0,1.0,targetVectorRanges[i].minValue,targetVectorRanges[i].maxValue);
954  }
955  }
956 
957  return outputNeuronsOutput;
958 
959 }
960 
961 void MLP::feedforward(const VectorDouble &trainingExample,VectorDouble &inputNeuronsOuput,VectorDouble &hiddenNeuronsOutput,VectorDouble &outputNeuronsOutput){
962 
963  if( inputNeuronsOuput.size() != numInputNeurons ) inputNeuronsOuput.resize(numInputNeurons,0);
964  if( hiddenNeuronsOutput.size() != numHiddenNeurons ) hiddenNeuronsOutput.resize(numHiddenNeurons,0);
965  if( outputNeuronsOutput.size() != numOutputNeurons ) outputNeuronsOutput.resize(numOutputNeurons,0);
966 
967  //Input layer
968  VectorDouble input(1);
969  for(UINT i=0; i<numInputNeurons; i++){
970  input[0] = trainingExample[i];
971  inputNeuronsOuput[i] = inputLayer[i].fire( input );
972  }
973 
974  //Hidden Layer
975  for(UINT i=0; i<numHiddenNeurons; i++){
976  hiddenNeuronsOutput[i] = hiddenLayer[i].fire( inputNeuronsOuput );
977  }
978 
979  //Output Layer
980  for(UINT i=0; i<numOutputNeurons; i++){
981  outputNeuronsOutput[i] = outputLayer[i].fire( hiddenNeuronsOutput );
982  }
983 
984 }
985 
986 void MLP::printNetwork() const{
987  cout<<"***************** MLP *****************\n";
988  cout<<"NumInputNeurons: "<<numInputNeurons<<endl;
989  cout<<"NumHiddenNeurons: "<<numHiddenNeurons<<endl;
990  cout<<"NumOutputNeurons: "<<numOutputNeurons<<endl;
991 
992  cout<<"InputWeights:\n";
993  for(UINT i=0; i<numInputNeurons; i++){
994  cout<<"Neuron: "<<i<<" Bias: " << inputLayer[i].bias << " Weights: ";
995  for(UINT j=0; j<inputLayer[i].weights.size(); j++){
996  cout<<inputLayer[i].weights[j]<<"\t";
997  }cout<<endl;
998  }
999 
1000  cout<<"HiddenWeights:\n";
1001  for(UINT i=0; i<numHiddenNeurons; i++){
1002  cout<<"Neuron: "<<i<<" Bias: " << hiddenLayer[i].bias << " Weights: ";
1003  for(UINT j=0; j<hiddenLayer[i].weights.size(); j++){
1004  cout<<hiddenLayer[i].weights[j]<<"\t";
1005  }cout<<endl;
1006  }
1007 
1008  cout<<"OutputWeights:\n";
1009  for(UINT i=0; i<numOutputNeurons; i++){
1010  cout<<"Neuron: "<<i<<" Bias: " << outputLayer[i].bias << " Weights: ";
1011  for(UINT j=0; j<outputLayer[i].weights.size(); j++){
1012  cout<<outputLayer[i].weights[j]<<"\t";
1013  }cout<<endl;
1014  }
1015 
1016 }
1017 
1018 bool MLP::checkForNAN() const{
1019 
1020  for(UINT i=0; i<numInputNeurons; i++){
1021  if( isNAN(inputLayer[i].bias) ) return true;
1022  for(UINT j=0; j<inputLayer[i].weights.size(); j++){
1023  if( isNAN(inputLayer[i].weights[j]) ) return true;
1024  }
1025  }
1026 
1027  for(UINT i=0; i<numHiddenNeurons; i++){
1028  if( isNAN(hiddenLayer[i].bias) ) return true;
1029  for(UINT j=0; j<hiddenLayer[i].weights.size(); j++){
1030  if( isNAN(hiddenLayer[i].weights[j]) ) return true;
1031  }
1032  }
1033 
1034  for(UINT i=0; i<numOutputNeurons; i++){
1035  if( isNAN(outputLayer[i].bias) ) return true;
1036  for(UINT j=0; j<outputLayer[i].weights.size(); j++){
1037  if( isNAN(outputLayer[i].weights[j]) ) return true;
1038  }
1039  }
1040 
1041  return false;
1042 }
1043 
1044 bool inline MLP::isNAN(const double v) const{
1045  if( v != v ) return true;
1046  return false;
1047 }
1048 
1049 bool MLP::saveModelToFile(fstream &file) const{
1050 
1051  if( !file.is_open() ){
1052  errorLog << "saveModelToFile(fstream &file) - File is not open!" << endl;
1053  return false;
1054  }
1055 
1056  file << "GRT_MLP_FILE_V2.0\n";
1057 
1058  //Write the regressifier settings to the file
1060  errorLog <<"saveModelToFile(fstream &file) - Failed to save Regressifier base settings to file!" << endl;
1061  return false;
1062  }
1063 
1064  file << "NumInputNeurons: "<<numInputNeurons<<endl;
1065  file << "NumHiddenNeurons: "<<numHiddenNeurons<<endl;
1066  file << "NumOutputNeurons: "<<numOutputNeurons<<endl;
1067  file << "InputLayerActivationFunction: " <<activationFunctionToString(inputLayerActivationFunction)<< endl;
1068  file << "HiddenLayerActivationFunction: " <<activationFunctionToString(hiddenLayerActivationFunction)<< endl;
1069  file << "OutputLayerActivationFunction: " <<activationFunctionToString(outputLayerActivationFunction)<< endl;
1070  file << "NumRandomTrainingIterations: " << numRandomTrainingIterations << endl;
1071  file << "Momentum: " << momentum << endl;
1072  file << "Gamma: " << gamma << endl;
1073  file << "ClassificationMode: " << classificationModeActive << endl;
1074  file << "UseNullRejection: " << useNullRejection << endl;
1075  file << "RejectionThreshold: " << nullRejectionThreshold << endl;
1076 
1077  if( trained ){
1078  file << "InputLayer: \n";
1079  for(UINT i=0; i<numInputNeurons; i++){
1080  file << "InputNeuron: " << i+1 << endl;
1081  file << "NumInputs: " << inputLayer[i].numInputs << endl;
1082  file << "Bias: " << inputLayer[i].bias << endl;
1083  file << "Gamma: " << inputLayer[i].gamma << endl;
1084  file << "Weights: " << endl;
1085  for(UINT j=0; j<inputLayer[i].numInputs; j++){
1086  file << inputLayer[i].weights[j] << "\t";
1087  }
1088  file << endl;
1089  }
1090  file << "\n";
1091 
1092  file << "HiddenLayer: \n";
1093  for(UINT i=0; i<numHiddenNeurons; i++){
1094  file << "HiddenNeuron: " << i+1 << endl;
1095  file << "NumInputs: " << hiddenLayer[i].numInputs << endl;
1096  file << "Bias: " << hiddenLayer[i].bias << endl;
1097  file << "Gamma: " << hiddenLayer[i].gamma << endl;
1098  file << "Weights: " << endl;
1099  for(UINT j=0; j<hiddenLayer[i].numInputs; j++){
1100  file << hiddenLayer[i].weights[j] << "\t";
1101  }
1102  file << endl;
1103  }
1104  file << "\n";
1105 
1106  file << "OutputLayer: \n";
1107  for(UINT i=0; i<numOutputNeurons; i++){
1108  file << "OutputNeuron: " << i+1 << endl;
1109  file << "NumInputs: " << outputLayer[i].numInputs << endl;
1110  file << "Bias: " << outputLayer[i].bias << endl;
1111  file << "Gamma: " << outputLayer[i].gamma << endl;
1112  file << "Weights: " << endl;
1113  for(UINT j=0; j<outputLayer[i].numInputs; j++){
1114  file << outputLayer[i].weights[j] << "\t";
1115  }
1116  file << endl;
1117  }
1118  }
1119 
1120  return true;
1121 }
1122 
1123 bool MLP::loadModelFromFile(fstream &file){
1124 
1125  string activationFunction;
1126 
1127  //Clear any previous models
1128  clear();
1129 
1130  if( !file.is_open() ){
1131  errorLog << "loadModelFromFile(fstream &file) - File is not open!" << endl;
1132  return false;
1133  }
1134 
1135  string word;
1136 
1137 
1138  file >> word;
1139 
1140  //See if we should load a legacy file
1141  if( word == "GRT_MLP_FILE_V1.0" ){
1142  return loadLegacyModelFromFile( file );
1143  }
1144 
1145  //Check to make sure this is a file with the MLP File Format
1146  if( word != "GRT_MLP_FILE_V2.0" ){
1147  file.close();
1148  errorLog << "loadModelFromFile(fstream &file) - Failed to find file header!" << endl;
1149  return false;
1150  }
1151 
1152  //Load the base settings from the file
1154  file.close();
1155  errorLog << "loadModelFromFile(fstream &file) - Failed to load regressifier base settings from file!" << endl;
1156  return false;
1157  }
1158 
1159  file >> word;
1160  if(word != "NumInputNeurons:"){
1161  file.close();
1162  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputNeurons!" << endl;
1163  return false;
1164  }
1165  file >> numInputNeurons;
1166  numInputDimensions = numInputNeurons;
1167 
1168  file >> word;
1169  if(word != "NumHiddenNeurons:"){
1170  file.close();
1171  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumHiddenNeurons!" << endl;
1172  return false;
1173  }
1174  file >> numHiddenNeurons;
1175 
1176  file >> word;
1177  if(word != "NumOutputNeurons:"){
1178  file.close();
1179  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumOutputNeurons!" << endl;
1180  return false;
1181  }
1182  file >> numOutputNeurons;
1183 
1184  file >> word;
1185  if(word != "InputLayerActivationFunction:"){
1186  file.close();
1187  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputLayerActivationFunction!" << endl;
1188  return false;
1189  }
1190  file >> activationFunction;
1191  inputLayerActivationFunction = activationFunctionFromString(activationFunction);
1192 
1193  file >> word;
1194  if(word != "HiddenLayerActivationFunction:"){
1195  file.close();
1196  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenLayerActivationFunction!" << endl;
1197  return false;
1198  }
1199  file >> activationFunction;
1200  hiddenLayerActivationFunction = activationFunctionFromString(activationFunction);
1201 
1202  file >> word;
1203  if(word != "OutputLayerActivationFunction:"){
1204  file.close();
1205  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputLayerActivationFunction!" << endl;
1206  return false;
1207  }
1208  file >> activationFunction;
1209  outputLayerActivationFunction = activationFunctionFromString(activationFunction);
1210 
1211  file >> word;
1212  if(word != "NumRandomTrainingIterations:"){
1213  file.close();
1214  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumRandomTrainingIterations!" << endl;
1215  return false;
1216  }
1217  file >> numRandomTrainingIterations;
1218 
1219  file >> word;
1220  if(word != "Momentum:"){
1221  file.close();
1222  errorLog << "loadModelFromFile(fstream &file) - Failed to find Momentum!" << endl;
1223  return false;
1224  }
1225  file >> momentum;
1226 
1227  file >> word;
1228  if(word != "Gamma:"){
1229  file.close();
1230  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1231  return false;
1232  }
1233  file >> gamma;
1234 
1235  file >> word;
1236  if(word != "ClassificationMode:"){
1237  file.close();
1238  errorLog << "loadModelFromFile(fstream &file) - Failed to find ClassificationMode!" << endl;
1239  return false;
1240  }
1241  file >> classificationModeActive;
1242 
1243  file >> word;
1244  if(word != "UseNullRejection:"){
1245  file.close();
1246  errorLog << "loadModelFromFile(fstream &file) - Failed to find UseNullRejection!" << endl;
1247  return false;
1248  }
1249  file >> useNullRejection;
1250 
1251  file >> word;
1252  if(word != "RejectionThreshold:"){
1253  file.close();
1254  errorLog << "loadModelFromFile(fstream &file) - Failed to find RejectionThreshold!" << endl;
1255  return false;
1256  }
1257  file >> nullRejectionThreshold;
1258 
1259  if( trained ) initialized = true;
1260  else init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
1261 
1262  if( trained ){
1263 
1264  //Resize the layers
1265  inputLayer.resize( numInputNeurons );
1266  hiddenLayer.resize( numHiddenNeurons );
1267  outputLayer.resize( numOutputNeurons );
1268 
1269  //Load the neuron data
1270  file >> word;
1271  if(word != "InputLayer:"){
1272  file.close();
1273  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputLayer!" << endl;
1274  return false;
1275  }
1276 
1277  for(UINT i=0; i<numInputNeurons; i++){
1278  UINT tempNeuronID = 0;
1279 
1280  file >> word;
1281  if(word != "InputNeuron:"){
1282  file.close();
1283  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputNeuron!" << endl;
1284  return false;
1285  }
1286  file >> tempNeuronID;
1287 
1288  if( tempNeuronID != i+1 ){
1289  file.close();
1290  errorLog << "loadModelFromFile(fstream &file) - InputNeuron ID does not match!" << endl;
1291  return false;
1292  }
1293 
1294  file >> word;
1295  if(word != "NumInputs:"){
1296  file.close();
1297  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputs!" << endl;
1298  return false;
1299  }
1300  file >> inputLayer[i].numInputs;
1301 
1302  //Resize the buffers
1303  inputLayer[i].weights.resize( inputLayer[i].numInputs );
1304 
1305  file >> word;
1306  if(word != "Bias:"){
1307  file.close();
1308  errorLog << "loadModelFromFile(fstream &file) - Failed to find Bias!" << endl;
1309  return false;
1310  }
1311  file >> inputLayer[i].bias;
1312 
1313  file >> word;
1314  if(word != "Gamma:"){
1315  file.close();
1316  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1317  return false;
1318  }
1319  file >> inputLayer[i].gamma;
1320 
1321  file >> word;
1322  if(word != "Weights:"){
1323  file.close();
1324  errorLog << "loadModelFromFile(fstream &file) - Failed to find Weights!" << endl;
1325  return false;
1326  }
1327 
1328  for(UINT j=0; j<inputLayer[i].numInputs; j++){
1329  file >> inputLayer[i].weights[j];
1330  }
1331  }
1332 
1333  //Load the Hidden Layer
1334  file >> word;
1335  if(word != "HiddenLayer:"){
1336  file.close();
1337  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenLayer!" << endl;
1338  return false;
1339  }
1340 
1341  for(UINT i=0; i<numHiddenNeurons; i++){
1342  UINT tempNeuronID = 0;
1343 
1344  file >> word;
1345  if(word != "HiddenNeuron:"){
1346  file.close();
1347  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenNeuron!" << endl;
1348  return false;
1349  }
1350  file >> tempNeuronID;
1351 
1352  if( tempNeuronID != i+1 ){
1353  file.close();
1354  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenNeuron ID does not match!" << endl;
1355  return false;
1356  }
1357 
1358  file >> word;
1359  if(word != "NumInputs:"){
1360  file.close();
1361  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputs!" << endl;
1362  return false;
1363  }
1364  file >> hiddenLayer[i].numInputs;
1365 
1366  //Resize the buffers
1367  hiddenLayer[i].weights.resize( hiddenLayer[i].numInputs );
1368 
1369  file >> word;
1370  if(word != "Bias:"){
1371  file.close();
1372  errorLog << "loadModelFromFile(fstream &file) - Failed to find Bias!" << endl;
1373  return false;
1374  }
1375  file >> hiddenLayer[i].bias;
1376 
1377  file >> word;
1378  if(word != "Gamma:"){
1379  file.close();
1380  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1381  return false;
1382  }
1383  file >> hiddenLayer[i].gamma;
1384 
1385  file >> word;
1386  if(word != "Weights:"){
1387  file.close();
1388  errorLog << "loadModelFromFile(fstream &file) - Failed to find Weights!" << endl;
1389  return false;
1390  }
1391 
1392  for(unsigned int j=0; j<hiddenLayer[i].numInputs; j++){
1393  file >> hiddenLayer[i].weights[j];
1394  }
1395  }
1396 
1397  //Load the Output Layer
1398  file >> word;
1399  if(word != "OutputLayer:"){
1400  file.close();
1401  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputLayer!" << endl;
1402  return false;
1403  }
1404 
1405  for(UINT i=0; i<numOutputNeurons; i++){
1406  UINT tempNeuronID = 0;
1407 
1408  file >> word;
1409  if(word != "OutputNeuron:"){
1410  file.close();
1411  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputNeuron!" << endl;
1412  return false;
1413  }
1414  file >> tempNeuronID;
1415 
1416  if( tempNeuronID != i+1 ){
1417  file.close();
1418  errorLog << "loadModelFromFile(fstream &file) - Failed to find OuputNeuron ID does not match!!" << endl;
1419  return false;
1420  }
1421 
1422  file >> word;
1423  if(word != "NumInputs:"){
1424  file.close();
1425  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputs!" << endl;
1426  return false;
1427  }
1428  file >> outputLayer[i].numInputs;
1429 
1430  //Resize the buffers
1431  outputLayer[i].weights.resize( outputLayer[i].numInputs );
1432 
1433  file >> word;
1434  if(word != "Bias:"){
1435  file.close();
1436  errorLog << "loadModelFromFile(fstream &file) - Failed to find Bias!" << endl;
1437  return false;
1438  }
1439  file >> outputLayer[i].bias;
1440 
1441  file >> word;
1442  if(word != "Gamma:"){
1443  file.close();
1444  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1445  return false;
1446  }
1447  file >> outputLayer[i].gamma;
1448 
1449  file >> word;
1450  if(word != "Weights:"){
1451  file.close();
1452  errorLog << "loadModelFromFile(fstream &file) - Failed to find Weights!" << endl;
1453  return false;
1454  }
1455 
1456  for(UINT j=0; j<outputLayer[i].numInputs; j++){
1457  file >> outputLayer[i].weights[j];
1458  }
1459  }
1460 
1461  }
1462 
1463  return true;
1464 }
1465 
1466 UINT MLP::getNumClasses() const{
1467  if( classificationModeActive )
1468  return numOutputNeurons;
1469  return 0;
1470 }
1471 
1473  return numInputNeurons;
1474 }
1475 
1477  return numHiddenNeurons;
1478 }
1479 
1481  return numOutputNeurons;
1482 }
1483 
1485  return inputLayerActivationFunction;
1486 }
1487 
1489  return hiddenLayerActivationFunction;
1490 }
1491 
1493  return outputLayerActivationFunction;
1494 }
1495 
1497  return numRandomTrainingIterations;
1498 }
1499 
1500 double MLP::getTrainingRate() const{
1501  return learningRate;
1502 }
1503 
1504 double MLP::getMomentum() const{
1505  return momentum;
1506 }
1507 
1508 double MLP::getGamma() const{
1509  return gamma;
1510 }
1511 
1512 double MLP::getTrainingError() const{
1513  return trainingError;
1514 }
1515 
1517  return classificationModeActive;
1518 }
1519 
1521  return !classificationModeActive;
1522 }
1523 
1524 vector< Neuron > MLP::getInputLayer() const{
1525  return inputLayer;
1526 }
1527 
1528 vector< Neuron > MLP::getHiddenLayer() const{
1529  return hiddenLayer;
1530 }
1531 
1532 vector< Neuron > MLP::getOutputLayer() const{
1533  return outputLayer;
1534 }
1535 
1536 vector< vector< double > > MLP::getTrainingLog() const{
1537  return trainingErrorLog;
1538 }
1539 
1541  return useNullRejection;
1542 }
1543 
1545  return nullRejectionCoeff;
1546 }
1547 
1549  return nullRejectionThreshold;
1550 }
1551 
1553  if( trained ) return maxLikelihood;
1554  return DEFAULT_NULL_LIKELIHOOD_VALUE;
1555 }
1556 
1557 VectorDouble MLP::getClassLikelihoods() const{
1558  if( trained && classificationModeActive ) return classLikelihoods;
1559  return VectorDouble();
1560 }
1561 
1562 VectorDouble MLP::getClassDistances() const{
1563  //The class distances is simply the regression data
1564  if( trained && classificationModeActive ) return regressionData;
1565  return VectorDouble();
1566 }
1567 
1569  if( trained && classificationModeActive ) return predictedClassLabel;
1570  return 0;
1571 }
1572 
1573 string MLP::activationFunctionToString(const UINT activationFunction) const{
1574  string activationName;
1575 
1576  switch(activationFunction){
1577  case(Neuron::LINEAR):
1578  activationName = "LINEAR";
1579  break;
1580  case(Neuron::SIGMOID):
1581  activationName = "SIGMOID";
1582  break;
1583  case(Neuron::BIPOLAR_SIGMOID):
1584  activationName = "BIPOLAR_SIGMOID";
1585  break;
1586  default:
1587  activationName = "UNKNOWN";
1588  break;
1589  }
1590 
1591  return activationName;
1592 }
1593 
1594 UINT MLP::activationFunctionFromString(const string activationName) const{
1595  UINT activationFunction = 0;
1596 
1597  if(activationName == "LINEAR" ){
1598  activationFunction = 0;
1599  return activationFunction;
1600  }
1601  if(activationName == "SIGMOID" ){
1602  activationFunction = 1;
1603  return activationFunction;
1604  }
1605  if(activationName == "BIPOLAR_SIGMOID" ){
1606  activationFunction = 2;
1607  return activationFunction;
1608  }
1609  return activationFunction;
1610 }
1611 
1612 bool MLP::validateActivationFunction(const UINT actvationFunction) const{
1613  if( actvationFunction >= Neuron::LINEAR && actvationFunction < Neuron::NUMBER_OF_ACTIVATION_FUNCTIONS ) return true;
1614  return false;
1615 }
1616 
1617 bool MLP::setInputLayerActivationFunction(const UINT activationFunction){
1618 
1619  if( !validateActivationFunction(activationFunction) ){
1620  warningLog << "setInputLayerActivationFunction(const UINT activationFunction) - The activation function is not valid. It should be one of the Neuron ActivationFunctions enums." << endl;
1621  }
1622 
1623  this->inputLayerActivationFunction = activationFunction;
1624 
1625  if( initialized ){
1626  return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
1627  }
1628 
1629  return true;
1630 }
1631 
1632 
1633 bool MLP::setHiddenLayerActivationFunction(const UINT activationFunction){
1634 
1635  if( !validateActivationFunction(activationFunction) ){
1636  warningLog << "setHiddenLayerActivationFunction(const UINT activationFunction) - The activation function is not valid. It should be one of the Neuron ActivationFunctions enums." << endl;
1637  }
1638 
1639  this->hiddenLayerActivationFunction = activationFunction;
1640 
1641  if( initialized ){
1642  return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
1643  }
1644 
1645  return true;
1646 }
1647 
1648 
1649 bool MLP::setOutputLayerActivationFunction(const UINT activationFunction){
1650 
1651  if( !validateActivationFunction(activationFunction) ){
1652  warningLog << "setOutputLayerActivationFunction(const UINT activationFunction) - The activation function is not valid. It should be one of the Neuron ActivationFunctions enums." << endl;
1653  }
1654 
1655  this->outputLayerActivationFunction = activationFunction;
1656 
1657  if( initialized ){
1658  return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
1659  }
1660 
1661  return true;
1662 }
1663 
1664 bool MLP::setTrainingRate(const double trainingRate){
1665  return setLearningRate( trainingRate );
1666 }
1667 
1668 bool MLP::setMomentum(const double momentum){
1669  if( momentum >= 0 && momentum <= 1.0 ){
1670  this->momentum = momentum;
1671  return true;
1672  }
1673  return false;
1674 }
1675 
1676 bool MLP::setGamma(const double gamma){
1677 
1678  if( gamma < 0 ){
1679  warningLog << "setGamma(const double gamma) - Gamma must be greater than zero!" << endl;
1680  }
1681 
1682  this->gamma = gamma;
1683 
1684  if( initialized ){
1685  return init(numInputNeurons,numHiddenNeurons,numOutputNeurons);
1686  }
1687 
1688  return true;
1689 }
1690 
1691 bool MLP::setNumRandomTrainingIterations(const UINT numRandomTrainingIterations){
1692  if( numRandomTrainingIterations > 0 ){
1693  this->numRandomTrainingIterations = numRandomTrainingIterations;
1694  return true;
1695  }
1696  return false;
1697 }
1698 
1699 bool MLP::setNullRejection(const bool useNullRejection){
1700  this->useNullRejection = useNullRejection;
1701  return true;
1702 }
1703 
1704 bool MLP::setNullRejectionCoeff(const double nullRejectionCoeff){
1705  if( nullRejectionCoeff > 0 ){
1706  this->nullRejectionCoeff = nullRejectionCoeff;
1707  return true;
1708  }
1709  return false;
1710 }
1711 
1712 bool MLP::loadLegacyModelFromFile( fstream &file ){
1713 
1714  string word;
1715 
1716  file >> word;
1717  if(word != "NumInputNeurons:"){
1718  file.close();
1719  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputNeurons!" << endl;
1720  return false;
1721  }
1722  file >> numInputNeurons;
1723  numInputDimensions = numInputNeurons;
1724 
1725  file >> word;
1726  if(word != "NumHiddenNeurons:"){
1727  file.close();
1728  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumHiddenNeurons!" << endl;
1729  return false;
1730  }
1731  file >> numHiddenNeurons;
1732 
1733  file >> word;
1734  if(word != "NumOutputNeurons:"){
1735  file.close();
1736  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumOutputNeurons!" << endl;
1737  return false;
1738  }
1739  file >> numOutputNeurons;
1740 
1741  file >> word;
1742  if(word != "InputLayerActivationFunction:"){
1743  file.close();
1744  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputLayerActivationFunction!" << endl;
1745  return false;
1746  }
1747  file >> word;
1748  inputLayerActivationFunction = activationFunctionFromString(word);
1749 
1750  file >> word;
1751  if(word != "HiddenLayerActivationFunction:"){
1752  file.close();
1753  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenLayerActivationFunction!" << endl;
1754  return false;
1755  }
1756  file >> word;
1757  hiddenLayerActivationFunction = activationFunctionFromString(word);
1758 
1759  file >> word;
1760  if(word != "OutputLayerActivationFunction:"){
1761  file.close();
1762  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputLayerActivationFunction!" << endl;
1763  return false;
1764  }
1765  file >> word;
1766  outputLayerActivationFunction = activationFunctionFromString(word);
1767 
1768  file >> word;
1769  if(word != "MinNumEpochs:"){
1770  file.close();
1771  errorLog << "loadModelFromFile(fstream &file) - Failed to find MinNumEpochs!" << endl;
1772  return false;
1773  }
1774  file >> minNumEpochs;
1775 
1776  file >> word;
1777  if(word != "MaxNumEpochs:"){
1778  file.close();
1779  errorLog << "loadModelFromFile(fstream &file) - Failed to find MaxNumEpochs!" << endl;
1780  return false;
1781  }
1782  file >> maxNumEpochs;
1783 
1784  file >> word;
1785  if(word != "NumRandomTrainingIterations:"){
1786  file.close();
1787  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumRandomTrainingIterations!" << endl;
1788  return false;
1789  }
1790  file >> numRandomTrainingIterations;
1791 
1792  file >> word;
1793  if(word != "ValidationSetSize:"){
1794  file.close();
1795  errorLog << "loadModelFromFile(fstream &file) - Failed to find ValidationSetSize!" << endl;
1796  return false;
1797  }
1798  file >> validationSetSize;
1799 
1800  file >> word;
1801  if(word != "MinChange:"){
1802  file.close();
1803  errorLog << "loadModelFromFile(fstream &file) - Failed to find MinChange!" << endl;
1804  return false;
1805  }
1806  file >> minChange;
1807 
1808  file >> word;
1809  if(word != "TrainingRate:"){
1810  file.close();
1811  errorLog << "loadModelFromFile(fstream &file) - Failed to find TrainingRate!" << endl;
1812  return false;
1813  }
1814  file >> learningRate;
1815 
1816  file >> word;
1817  if(word != "Momentum:"){
1818  file.close();
1819  errorLog << "loadModelFromFile(fstream &file) - Failed to find Momentum!" << endl;
1820  return false;
1821  }
1822  file >> momentum;
1823 
1824  file >> word;
1825  if(word != "Gamma:"){
1826  file.close();
1827  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1828  return false;
1829  }
1830  file >> gamma;
1831 
1832  file >> word;
1833  if(word != "UseValidationSet:"){
1834  file.close();
1835  errorLog << "loadModelFromFile(fstream &file) - Failed to find UseValidationSet!" << endl;
1836  return false;
1837  }
1838  file >> useValidationSet;
1839 
1840  file >> word;
1841  if(word != "RandomiseTrainingOrder:"){
1842  file.close();
1843  errorLog << "loadModelFromFile(fstream &file) - Failed to find RandomiseTrainingOrder!" << endl;
1844  return false;
1845  }
1846  file >> randomiseTrainingOrder;
1847 
1848  file >> word;
1849  if(word != "UseScaling:"){
1850  file.close();
1851  errorLog << "loadModelFromFile(fstream &file) - Failed to find UseScaling!" << endl;
1852  return false;
1853  }
1854  file >> useScaling;
1855 
1856  file >> word;
1857  if(word != "ClassificationMode:"){
1858  file.close();
1859  errorLog << "loadModelFromFile(fstream &file) - Failed to find ClassificationMode!" << endl;
1860  return false;
1861  }
1862  file >> classificationModeActive;
1863 
1864  file >> word;
1865  if(word != "UseNullRejection:"){
1866  file.close();
1867  errorLog << "loadModelFromFile(fstream &file) - Failed to find UseNullRejection!" << endl;
1868  return false;
1869  }
1870  file >> useNullRejection;
1871 
1872  file >> word;
1873  if(word != "RejectionThreshold:"){
1874  file.close();
1875  errorLog << "loadModelFromFile(fstream &file) - Failed to find RejectionThreshold!" << endl;
1876  return false;
1877  }
1878  file >> nullRejectionThreshold;
1879 
1880  //Resize the layers
1881  inputLayer.resize( numInputNeurons );
1882  hiddenLayer.resize( numHiddenNeurons );
1883  outputLayer.resize( numOutputNeurons );
1884 
1885  //Load the neuron data
1886  file >> word;
1887  if(word != "InputLayer:"){
1888  file.close();
1889  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputLayer!" << endl;
1890  return false;
1891  }
1892 
1893  for(UINT i=0; i<numInputNeurons; i++){
1894  UINT tempNeuronID = 0;
1895 
1896  file >> word;
1897  if(word != "InputNeuron:"){
1898  file.close();
1899  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputNeuron!" << endl;
1900  return false;
1901  }
1902  file >> tempNeuronID;
1903 
1904  if( tempNeuronID != i+1 ){
1905  file.close();
1906  errorLog << "loadModelFromFile(fstream &file) - InputNeuron ID does not match!" << endl;
1907  return false;
1908  }
1909 
1910  file >> word;
1911  if(word != "NumInputs:"){
1912  file.close();
1913  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputs!" << endl;
1914  return false;
1915  }
1916  file >> inputLayer[i].numInputs;
1917 
1918  //Resize the buffers
1919  inputLayer[i].weights.resize( inputLayer[i].numInputs );
1920 
1921  file >> word;
1922  if(word != "Bias:"){
1923  file.close();
1924  errorLog << "loadModelFromFile(fstream &file) - Failed to find Bias!" << endl;
1925  return false;
1926  }
1927  file >> inputLayer[i].bias;
1928 
1929  file >> word;
1930  if(word != "Gamma:"){
1931  file.close();
1932  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1933  return false;
1934  }
1935  file >> inputLayer[i].gamma;
1936 
1937  file >> word;
1938  if(word != "Weights:"){
1939  file.close();
1940  errorLog << "loadModelFromFile(fstream &file) - Failed to find Weights!" << endl;
1941  return false;
1942  }
1943 
1944  for(UINT j=0; j<inputLayer[i].numInputs; j++){
1945  file >> inputLayer[i].weights[j];
1946  }
1947  }
1948 
1949  //Load the Hidden Layer
1950  file >> word;
1951  if(word != "HiddenLayer:"){
1952  file.close();
1953  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenLayer!" << endl;
1954  return false;
1955  }
1956 
1957  for(UINT i=0; i<numHiddenNeurons; i++){
1958  UINT tempNeuronID = 0;
1959 
1960  file >> word;
1961  if(word != "HiddenNeuron:"){
1962  file.close();
1963  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenNeuron!" << endl;
1964  return false;
1965  }
1966  file >> tempNeuronID;
1967 
1968  if( tempNeuronID != i+1 ){
1969  file.close();
1970  errorLog << "loadModelFromFile(fstream &file) - Failed to find HiddenNeuron ID does not match!" << endl;
1971  return false;
1972  }
1973 
1974  file >> word;
1975  if(word != "NumInputs:"){
1976  file.close();
1977  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputs!" << endl;
1978  return false;
1979  }
1980  file >> hiddenLayer[i].numInputs;
1981 
1982  //Resize the buffers
1983  hiddenLayer[i].weights.resize( hiddenLayer[i].numInputs );
1984 
1985  file >> word;
1986  if(word != "Bias:"){
1987  file.close();
1988  errorLog << "loadModelFromFile(fstream &file) - Failed to find Bias!" << endl;
1989  return false;
1990  }
1991  file >> hiddenLayer[i].bias;
1992 
1993  file >> word;
1994  if(word != "Gamma:"){
1995  file.close();
1996  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
1997  return false;
1998  }
1999  file >> hiddenLayer[i].gamma;
2000 
2001  file >> word;
2002  if(word != "Weights:"){
2003  file.close();
2004  errorLog << "loadModelFromFile(fstream &file) - Failed to find Weights!" << endl;
2005  return false;
2006  }
2007 
2008  for(unsigned int j=0; j<hiddenLayer[i].numInputs; j++){
2009  file >> hiddenLayer[i].weights[j];
2010  }
2011  }
2012 
2013  //Load the Output Layer
2014  file >> word;
2015  if(word != "OutputLayer:"){
2016  file.close();
2017  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputLayer!" << endl;
2018  return false;
2019  }
2020 
2021  for(UINT i=0; i<numOutputNeurons; i++){
2022  UINT tempNeuronID = 0;
2023 
2024  file >> word;
2025  if(word != "OutputNeuron:"){
2026  file.close();
2027  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputNeuron!" << endl;
2028  return false;
2029  }
2030  file >> tempNeuronID;
2031 
2032  if( tempNeuronID != i+1 ){
2033  file.close();
2034  errorLog << "loadModelFromFile(fstream &file) - Failed to find OuputNeuron ID does not match!!" << endl;
2035  return false;
2036  }
2037 
2038  file >> word;
2039  if(word != "NumInputs:"){
2040  file.close();
2041  errorLog << "loadModelFromFile(fstream &file) - Failed to find NumInputs!" << endl;
2042  return false;
2043  }
2044  file >> outputLayer[i].numInputs;
2045 
2046  //Resize the buffers
2047  outputLayer[i].weights.resize( outputLayer[i].numInputs );
2048 
2049  file >> word;
2050  if(word != "Bias:"){
2051  file.close();
2052  errorLog << "loadModelFromFile(fstream &file) - Failed to find Bias!" << endl;
2053  return false;
2054  }
2055  file >> outputLayer[i].bias;
2056 
2057  file >> word;
2058  if(word != "Gamma:"){
2059  file.close();
2060  errorLog << "loadModelFromFile(fstream &file) - Failed to find Gamma!" << endl;
2061  return false;
2062  }
2063  file >> outputLayer[i].gamma;
2064 
2065  file >> word;
2066  if(word != "Weights:"){
2067  file.close();
2068  errorLog << "loadModelFromFile(fstream &file) - Failed to find Weights!" << endl;
2069  return false;
2070  }
2071 
2072  for(UINT j=0; j<outputLayer[i].numInputs; j++){
2073  file >> outputLayer[i].weights[j];
2074  }
2075  }
2076 
2077  if( useScaling ){
2078  //Resize the ranges buffers
2079  inputVectorRanges.resize( numInputNeurons );
2080  targetVectorRanges.resize( numOutputNeurons );
2081 
2082  //Load the ranges
2083  file >> word;
2084  if(word != "InputVectorRanges:"){
2085  file.close();
2086  errorLog << "loadModelFromFile(fstream &file) - Failed to find InputVectorRanges!" << endl;
2087  return false;
2088  }
2089  for(UINT j=0; j<inputVectorRanges.size(); j++){
2090  file >> inputVectorRanges[j].minValue;
2091  file >> inputVectorRanges[j].maxValue;
2092  }
2093 
2094  file >> word;
2095  if(word != "OutputVectorRanges:"){
2096  file.close();
2097  errorLog << "loadModelFromFile(fstream &file) - Failed to find OutputVectorRanges!" << endl;
2098  return false;
2099  }
2100  for(UINT j=0; j<targetVectorRanges.size(); j++){
2101  file >> targetVectorRanges[j].minValue;
2102  file >> targetVectorRanges[j].maxValue;
2103  }
2104  }
2105 
2106  initialized = true;
2107  trained = true;
2108 
2109  return true;
2110 }
2111 
2112 } //End of namespace GRT
UINT getInputLayerActivationFunction() const
Definition: MLP.cpp:1484
virtual bool saveModelToFile(fstream &file) const
Definition: MLP.cpp:1049
Definition: MLP.h:41
bool setNumRandomTrainingIterations(const UINT numRandomTrainingIterations)
Definition: MLP.cpp:1691
bool copyBaseVariables(const Regressifier *regressifier)
virtual bool train_(ClassificationData &trainingData)
Definition: MLP.cpp:126
double getMomentum() const
Definition: MLP.cpp:1504
bool getNullRejectionEnabled() const
Definition: MLP.cpp:1540
vector< Neuron > getHiddenLayer() const
Definition: MLP.cpp:1528
double getTrainingRate() const
Definition: MLP.cpp:1500
string activationFunctionToString(const UINT activationFunction) const
Definition: MLP.cpp:1573
virtual bool deepCopyFrom(const Regressifier *regressifier)
Definition: MLP.cpp:108
double getGamma() const
Definition: MLP.cpp:1508
double getNullRejectionCoeff() const
Definition: MLP.cpp:1544
Definition: AdaBoost.cpp:25
bool getRegressionModeActive() const
Definition: MLP.cpp:1520
UINT getNumClasses() const
Definition: MLP.cpp:1466
virtual bool loadModelFromFile(fstream &file)
Definition: MLP.cpp:1123
bool loadBaseSettingsFromFile(fstream &file)
bool setMomentum(const double momentum)
Definition: MLP.cpp:1668
bool setHiddenLayerActivationFunction(const UINT activationFunction)
Definition: MLP.cpp:1633
static double getMin(const std::vector< double > &x)
Definition: Util.cpp:241
virtual ~MLP()
Definition: MLP.cpp:68
UINT getOutputLayerActivationFunction() const
Definition: MLP.cpp:1492
double getMaximumLikelihood() const
Definition: MLP.cpp:1552
vector< MinMax > getInputRanges() const
double getNullRejectionThreshold() const
Definition: MLP.cpp:1548
VectorDouble feedforward(VectorDouble trainingExample)
Definition: MLP.cpp:920
bool saveBaseSettingsToFile(fstream &file) const
UINT getNumRandomTrainingIterations() const
Definition: MLP.cpp:1496
bool setTrainingRate(const double trainingRate)
Definition: MLP.cpp:1664
virtual bool predict_(VectorDouble &inputVector)
Definition: MLP.cpp:160
VectorDouble getClassDistances() const
Definition: MLP.cpp:1562
UINT getHiddenLayerActivationFunction() const
Definition: MLP.cpp:1488
bool init(const UINT numInputNeurons, const UINT numHiddenNeurons, const UINT numOutputNeurons)
Definition: MLP.cpp:219
bool setOutputLayerActivationFunction(const UINT activationFunction)
Definition: MLP.cpp:1649
UINT getNumSamples() const
bool setGamma(const double gamma)
Definition: MLP.cpp:1676
void setSeed(unsigned long long seed=0)
Definition: Random.h:67
double scale(const double &x, const double &minSource, const double &maxSource, const double &minTarget, const double &maxTarget, const bool constrain=false)
Definition: MLBase.h:339
vector< Neuron > getOutputLayer() const
Definition: MLP.cpp:1532
This class implements a Multilayer Perceptron Artificial Neural Network.
virtual bool print() const
Definition: MLP.cpp:307
void printNetwork() const
Definition: MLP.cpp:986
RegressionData reformatAsRegressionData() const
string getRegressifierType() const
bool setNullRejection(const bool useNullRejection)
Definition: MLP.cpp:1699
double back_prop(const VectorDouble &trainingExample, const VectorDouble &targetVector, const double alpha, const double beta)
Definition: MLP.cpp:837
vector< Neuron > getInputLayer() const
Definition: MLP.cpp:1524
UINT getNumInputNeurons() const
Definition: MLP.cpp:1472
int getRandomNumberInt(int minRange, int maxRange)
Definition: Random.h:87
vector< MinMax > getTargetRanges() const
UINT getPredictedClassLabel() const
Definition: MLP.cpp:1568
UINT getNumHiddenNeurons() const
Definition: MLP.cpp:1476
bool checkForNAN() const
Definition: MLP.cpp:1018
virtual bool clear()
RegressionData partition(const UINT trainingSizePercentage)
UINT getNumTargetDimensions() const
UINT activationFunctionFromString(const string activationName) const
Definition: MLP.cpp:1594
virtual bool clear()
Definition: MLP.cpp:291
MLP & operator=(const MLP &rhs)
Definition: MLP.cpp:72
MLP()
Definition: MLP.cpp:28
UINT getNumInputDimensions() const
bool setLearningRate(double learningRate)
Definition: MLBase.cpp:260
bool setInputLayerActivationFunction(const UINT activationFunction)
Definition: MLP.cpp:1617
static double sum(const std::vector< double > &x)
Definition: Util.cpp:155
VectorDouble getClassLikelihoods() const
Definition: MLP.cpp:1557
double getTrainingError() const
Definition: MLP.cpp:1512
UINT getNumOutputNeurons() const
Definition: MLP.cpp:1480
bool scale(const double minTarget, const double maxTarget)
vector< VectorDouble > getTrainingLog() const
Definition: MLP.cpp:1536
bool validateActivationFunction(const UINT avactivationFunction) const
Definition: MLP.cpp:1612
bool setNullRejectionCoeff(const double nullRejectionCoeff)
Definition: MLP.cpp:1704
bool getClassificationModeActive() const
Definition: MLP.cpp:1516