GestureRecognitionToolkit  Version: 1.0 Revision: 04-03-15
The Gesture Recognition Toolkit (GRT) is a cross-platform, open-source, c++ machine learning library for real-time gesture recognition.
LogisticRegression.cpp
1 /*
2 GRT MIT License
3 Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>
4 
5 Permission is hereby granted, free of charge, to any person obtaining a copy of this software
6 and associated documentation files (the "Software"), to deal in the Software without restriction,
7 including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
9 subject to the following conditions:
10 
11 The above copyright notice and this permission notice shall be included in all copies or substantial
12 portions of the Software.
13 
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
15 LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
16 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
17 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
18 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19 */
20 
21 #include "LogisticRegression.h"
22 
23 using namespace std;
24 
25 namespace GRT{
26 
27 //Register the LogisticRegression module with the Classifier base class
28 RegisterRegressifierModule< LogisticRegression > LogisticRegression::registerModule("LogisticRegression");
29 
30 LogisticRegression::LogisticRegression(const bool useScaling)
31 {
32  this->useScaling = useScaling;
33  minChange = 1.0e-5;
34  maxNumEpochs = 500;
35  learningRate = 0.01;
36  classType = "LogisticRegression";
37  regressifierType = classType;
38  debugLog.setProceedingText("[DEBUG LogisticRegression]");
39  errorLog.setProceedingText("[ERROR LogisticRegression]");
40  trainingLog.setProceedingText("[TRAINING LogisticRegression]");
41  warningLog.setProceedingText("[WARNING LogisticRegression]");
42 }
43 
44 LogisticRegression::~LogisticRegression(void)
45 {
46 }
47 
48 LogisticRegression& LogisticRegression::operator=(const LogisticRegression &rhs){
49  if( this != &rhs ){
50  this->w0 = rhs.w0;
51  this->w = rhs.w;
52 
53  //Copy the base variables
54  copyBaseVariables( (Regressifier*)&rhs );
55  }
56  return *this;
57 }
58 
59 bool LogisticRegression::deepCopyFrom(const Regressifier *regressifier){
60 
61  if( regressifier == NULL ) return false;
62 
63  if( this->getRegressifierType() == regressifier->getRegressifierType() ){
64  const LogisticRegression *ptr = dynamic_cast<const LogisticRegression*>(regressifier);
65 
66  this->w0 = ptr->w0;
67  this->w = ptr->w;
68 
69  //Copy the base variables
70  return copyBaseVariables( regressifier );
71  }
72  return false;
73 }
74 
75 bool LogisticRegression::train_(RegressionData &trainingData){
76 
77  const unsigned int M = trainingData.getNumSamples();
78  const unsigned int N = trainingData.getNumInputDimensions();
79  const unsigned int K = trainingData.getNumTargetDimensions();
80  trained = false;
81  trainingResults.clear();
82 
83  if( M == 0 ){
84  errorLog << "train_(RegressionData trainingData) - Training data has zero samples!" << endl;
85  return false;
86  }
87 
88  if( K == 0 ){
89  errorLog << "train_(RegressionData trainingData) - The number of target dimensions is not 1!" << endl;
90  return false;
91  }
92 
93  numInputDimensions = N;
94  numOutputDimensions = 1; //Logistic Regression will have 1 output
95  inputVectorRanges.clear();
96  targetVectorRanges.clear();
97 
98  //Scale the training and validation data, if needed
99  if( useScaling ){
100  //Find the ranges for the input data
101  inputVectorRanges = trainingData.getInputRanges();
102 
103  //Find the ranges for the target data
104  targetVectorRanges = trainingData.getTargetRanges();
105 
106  //Scale the training data
107  trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
108  }
109 
110  //Reset the weights
111  Random rand;
112  w0 = rand.getRandomNumberUniform(-0.1,0.1);
113  w.resize(N);
114  for(UINT j=0; j<N; j++){
115  w[j] = rand.getRandomNumberUniform(-0.1,0.1);
116  }
117 
118  double error = 0;
119  double lastSquaredError = 0;
120  double delta = 0;
121  UINT iter = 0;
122  bool keepTraining = true;
123  Random random;
124  vector< UINT > randomTrainingOrder(M);
125  TrainingResult result;
126  trainingResults.reserve(M);
127 
128  //In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
129  //This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
130  //training samples. This random order is then used at each epoch.
131  for(UINT i=0; i<M; i++){
132  randomTrainingOrder[i] = i;
133  }
134  std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
135 
136  //Run the main stochastic gradient descent training algorithm
137  while( keepTraining ){
138 
139  //Run one epoch of training using stochastic gradient descent
140  totalSquaredTrainingError = 0;
141  for(UINT m=0; m<M; m++){
142 
143  //Select the random sample
144  UINT i = randomTrainingOrder[m];
145 
146  //Compute the error, given the current weights
147  VectorDouble x = trainingData[i].getInputVector();
148  VectorDouble y = trainingData[i].getTargetVector();
149  double h = w0;
150  for(UINT j=0; j<N; j++){
151  h += x[j] * w[j];
152  }
153  error = y[0] - sigmoid( h );
154  totalSquaredTrainingError += SQR(error);
155 
156  //Update the weights
157  for(UINT j=0; j<N; j++){
158  w[j] += learningRate * error * x[j];
159  }
160  w0 += learningRate * error;
161  }
162 
163  //Compute the error
164  delta = fabs( totalSquaredTrainingError-lastSquaredError );
165  lastSquaredError = totalSquaredTrainingError;
166 
167  //Check to see if we should stop
168  if( delta <= minChange ){
169  keepTraining = false;
170  }
171 
172  if( ++iter >= maxNumEpochs ){
173  keepTraining = false;
174  }
175 
176  if( grt_isinf( totalSquaredTrainingError ) || grt_isnan( totalSquaredTrainingError ) ){
177  errorLog << "train_(RegressionData &trainingData) - Training failed! Total squared error is NAN. If scaling is not enabled then you should try to scale your data and see if this solves the issue." << endl;
178  return false;
179  }
180 
181  //Store the training results
182  rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
183  result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError,this);
184  trainingResults.push_back( result );
185 
186  //Notify any observers of the new result
187  trainingResultsObserverManager.notifyObservers( result );
188 
189  trainingLog << "Epoch: " << iter << " SSE: " << totalSquaredTrainingError << " Delta: " << delta << endl;
190  }
191 
192  //Flag that the algorithm has been trained
193  regressionData.resize(1,0);
194  trained = true;
195  return trained;
196 }
197 
198 bool LogisticRegression::predict_(VectorDouble &inputVector){
199 
200  if( !trained ){
201  errorLog << "predict_(VectorDouble &inputVector) - Model Not Trained!" << endl;
202  return false;
203  }
204 
205  if( !trained ) return false;
206 
207  if( inputVector.size() != numInputDimensions ){
208  errorLog << "predict_(VectorDouble &inputVector) - The size of the input vector (" << int(inputVector.size()) << ") does not match the num features in the model (" << numInputDimensions << endl;
209  return false;
210  }
211 
212  if( useScaling ){
213  for(UINT n=0; n<numInputDimensions; n++){
214  inputVector[n] = scale(inputVector[n], inputVectorRanges[n].minValue, inputVectorRanges[n].maxValue, 0, 1);
215  }
216  }
217 
218  regressionData[0] = w0;
219  for(UINT j=0; j<numInputDimensions; j++){
220  regressionData[0] += inputVector[j] * w[j];
221  }
222  regressionData[0] = sigmoid( regressionData[0] );
223 
224  if( useScaling ){
225  for(UINT n=0; n<numOutputDimensions; n++){
226  regressionData[n] = scale(regressionData[n], 0, 1, targetVectorRanges[n].minValue, targetVectorRanges[n].maxValue);
227  }
228  }
229 
230  return true;
231 }
232 
233 bool LogisticRegression::saveModelToFile(fstream &file) const{
234 
235  if(!file.is_open())
236  {
237  errorLog << "loadModelFromFile(fstream &file) - The file is not open!" << endl;
238  return false;
239  }
240 
241  //Write the header info
242  file<<"GRT_LOGISTIC_REGRESSION_MODEL_FILE_V2.0\n";
243 
244  //Write the regressifier settings to the file
245  if( !Regressifier::saveBaseSettingsToFile(file) ){
246  errorLog <<"saveModelToFile(fstream &file) - Failed to save Regressifier base settings to file!" << endl;
247  return false;
248  }
249 
250  if( trained ){
251  file << "Weights: ";
252  file << w0;
253  for(UINT j=0; j<numInputDimensions; j++){
254  file << " " << w[j];
255  }
256  file << endl;
257  }
258 
259  return true;
260 }
261 
262 bool LogisticRegression::loadModelFromFile(fstream &file){
263 
264  trained = false;
265  numInputDimensions = 0;
266  w0 = 0;
267  w.clear();
268 
269  if(!file.is_open())
270  {
271  errorLog << "loadModelFromFile(string filename) - Could not open file to load model" << endl;
272  return false;
273  }
274 
275  std::string word;
276 
277  //Find the file type header
278  file >> word;
279 
280  //Check to see if we should load a legacy file
281  if( word == "GRT_LOGISTIC_REGRESSION_MODEL_FILE_V1.0" ){
282  return loadLegacyModelFromFile( file );
283  }
284 
285  if( word != "GRT_LOGISTIC_REGRESSION_MODEL_FILE_V2.0" ){
286  errorLog << "loadModelFromFile( fstream &file ) - Could not find Model File Header" << endl;
287  return false;
288  }
289 
290  //Load the regressifier settings from the file
291  if( !Regressifier::loadBaseSettingsFromFile(file) ){
292  errorLog <<"loadModelFromFile( fstream &file ) - Failed to save Regressifier base settings to file!" << endl;
293  return false;
294  }
295 
296  if( trained ){
297 
298  //Resize the weights
299  w.resize(numInputDimensions);
300 
301  //Load the weights
302  file >> word;
303  if(word != "Weights:"){
304  errorLog << "loadModelFromFile( fstream &file ) - Could not find the Weights!" << endl;
305  return false;
306  }
307 
308  file >> w0;
309  for(UINT j=0; j<numInputDimensions; j++){
310  file >> w[j];
311 
312  }
313  }
314 
315  return true;
316 }
317 
318 UINT LogisticRegression::getMaxNumIterations() const{
319  return getMaxNumEpochs();
320 }
321 
322 bool LogisticRegression::setMaxNumIterations(const UINT maxNumIterations){
323 return setMaxNumEpochs( maxNumIterations );
324 }
325 
326 double LogisticRegression::sigmoid(const double x) const{
327  return 1.0 / (1 + exp(-x));
328 }
329 
330 bool LogisticRegression::loadLegacyModelFromFile( fstream &file ){
331 
332  string word;
333 
334  file >> word;
335  if(word != "NumFeatures:"){
336  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find NumFeatures!" << endl;
337  return false;
338  }
339  file >> numInputDimensions;
340 
341  file >> word;
342  if(word != "NumOutputDimensions:"){
343  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find NumOutputDimensions!" << endl;
344  return false;
345  }
346  file >> numOutputDimensions;
347 
348  file >> word;
349  if(word != "UseScaling:"){
350  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find UseScaling!" << endl;
351  return false;
352  }
353  file >> useScaling;
354 
356  if( useScaling ){
357  //Resize the ranges buffer
358  inputVectorRanges.resize(numInputDimensions);
359  targetVectorRanges.resize(numOutputDimensions);
360 
361  //Load the ranges
362  file >> word;
363  if(word != "InputVectorRanges:"){
364  file.close();
365  errorLog << "loadLegacyModelFromFile( fstream &file ) - Failed to find InputVectorRanges!" << endl;
366  return false;
367  }
368  for(UINT j=0; j<inputVectorRanges.size(); j++){
369  file >> inputVectorRanges[j].minValue;
370  file >> inputVectorRanges[j].maxValue;
371  }
372 
373  file >> word;
374  if(word != "OutputVectorRanges:"){
375  file.close();
376  errorLog << "loadLegacyModelFromFile( fstream &file ) - Failed to find OutputVectorRanges!" << endl;
377  return false;
378  }
379  for(UINT j=0; j<targetVectorRanges.size(); j++){
380  file >> targetVectorRanges[j].minValue;
381  file >> targetVectorRanges[j].maxValue;
382  }
383  }
384 
385  //Resize the weights
386  w.resize(numInputDimensions);
387 
388  //Load the weights
389  file >> word;
390  if(word != "Weights:"){
391  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find the Weights!" << endl;
392  return false;
393  }
394 
395  file >> w0;
396  for(UINT j=0; j<numInputDimensions; j++){
397  file >> w[j];
398 
399  }
400 
401  //Resize the regression data vector
402  regressionData.resize(1,0);
403 
404  //Flag that the model has been trained
405  trained = true;
406 
407  return true;
408 }
409 
410 } //End of namespace GRT
411 
bool setRegressionResult(unsigned int trainingIteration, double totalSquaredTrainingError, double rootMeanSquaredTrainingError, MLBase *trainer)
Definition: AdaBoost.cpp:25
This class implements the Logistic Regression algorithm. Logistic Regression is a simple but effectiv...
vector< MinMax > getInputRanges() const
UINT getNumSamples() const
string getRegressifierType() const
vector< MinMax > getTargetRanges() const
double getRandomNumberUniform(double minRange=0.0, double maxRange=1.0)
Definition: Random.h:197
UINT getNumTargetDimensions() const
UINT getNumInputDimensions() const
bool scale(const double minTarget, const double maxTarget)