GestureRecognitionToolkit  Version: 1.0 Revision: 04-03-15
The Gesture Recognition Toolkit (GRT) is a cross-platform, open-source, c++ machine learning library for real-time gesture recognition.
LinearRegression.cpp
1 /*
2 GRT MIT License
3 Copyright (c) <2012> <Nicholas Gillian, Media Lab, MIT>
4 
5 Permission is hereby granted, free of charge, to any person obtaining a copy of this software
6 and associated documentation files (the "Software"), to deal in the Software without restriction,
7 including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
9 subject to the following conditions:
10 
11 The above copyright notice and this permission notice shall be included in all copies or substantial
12 portions of the Software.
13 
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
15 LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
16 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
17 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
18 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19 */
20 
21 #include "LinearRegression.h"
22 
23 using namespace std;
24 
25 namespace GRT{
26 
27 //Register the LinearRegression module with the Classifier base class
28 RegisterRegressifierModule< LinearRegression > LinearRegression::registerModule("LinearRegression");
29 
30 LinearRegression::LinearRegression(bool useScaling)
31 {
32  this->useScaling = useScaling;
33  minChange = 1.0e-5;
34  maxNumEpochs = 500;
35  learningRate = 0.01;
36  classType = "LinearRegression";
37  regressifierType = classType;
38  debugLog.setProceedingText("[DEBUG LinearRegression]");
39  errorLog.setProceedingText("[ERROR LinearRegression]");
40  trainingLog.setProceedingText("[TRAINING LinearRegression]");
41  warningLog.setProceedingText("[WARNING LinearRegression]");
42 }
43 
44 LinearRegression::~LinearRegression(void)
45 {
46 }
47 
48 LinearRegression& LinearRegression::operator=(const LinearRegression &rhs){
49  if( this != &rhs ){
50  this->w0 = rhs.w0;
51  this->w = rhs.w;
52 
53  //Copy the base variables
54  copyBaseVariables( (Regressifier*)&rhs );
55  }
56  return *this;
57 }
58 
59 bool LinearRegression::deepCopyFrom(const Regressifier *regressifier){
60 
61  if( regressifier == NULL ) return false;
62 
63  if( this->getRegressifierType() == regressifier->getRegressifierType() ){
64 
65  const LinearRegression *ptr = dynamic_cast<const LinearRegression*>(regressifier);
66 
67  this->w0 = ptr->w0;
68  this->w = ptr->w;
69 
70  //Copy the base variables
71  return copyBaseVariables( regressifier );
72  }
73  return false;
74 }
75 
76 bool LinearRegression::train_(RegressionData &trainingData){
77 
78  const unsigned int M = trainingData.getNumSamples();
79  const unsigned int N = trainingData.getNumInputDimensions();
80  const unsigned int K = trainingData.getNumTargetDimensions();
81  trained = false;
82  trainingResults.clear();
83 
84  if( M == 0 ){
85  errorLog << "train_(RegressionData &trainingData) - Training data has zero samples!" << endl;
86  return false;
87  }
88 
89  if( K == 0 ){
90  errorLog << "train_(RegressionData &trainingData) - The number of target dimensions is not 1!" << endl;
91  return false;
92  }
93 
94  numInputDimensions = N;
95  numOutputDimensions = 1; //Logistic Regression will have 1 output
96  inputVectorRanges.clear();
97  targetVectorRanges.clear();
98 
99  //Scale the training and validation data, if needed
100  if( useScaling ){
101  //Find the ranges for the input data
102  inputVectorRanges = trainingData.getInputRanges();
103 
104  //Find the ranges for the target data
105  targetVectorRanges = trainingData.getTargetRanges();
106 
107  //Scale the training data
108  trainingData.scale(inputVectorRanges,targetVectorRanges,0.0,1.0);
109  }
110 
111  //Reset the weights
112  Random rand;
113  w0 = rand.getRandomNumberUniform(-0.1,0.1);
114  w.resize(N);
115  for(UINT j=0; j<N; j++){
116  w[j] = rand.getRandomNumberUniform(-0.1,0.1);
117  }
118 
119  double error = 0;
120  double lastError = 0;
121  double delta = 0;
122  UINT iter = 0;
123  bool keepTraining = true;
124  vector< UINT > randomTrainingOrder(M);
125  TrainingResult result;
126  trainingResults.reserve(M);
127 
128  //In most cases, the training data is grouped into classes (100 samples for class 1, followed by 100 samples for class 2, etc.)
129  //This can cause a problem for stochastic gradient descent algorithm. To avoid this issue, we randomly shuffle the order of the
130  //training samples. This random order is then used at each epoch.
131  for(UINT i=0; i<M; i++){
132  randomTrainingOrder[i] = i;
133  }
134  std::random_shuffle(randomTrainingOrder.begin(), randomTrainingOrder.end());
135 
136  //Run the main stochastic gradient descent training algorithm
137  while( keepTraining ){
138 
139  //Run one epoch of training using stochastic gradient descent
140  totalSquaredTrainingError = 0;
141  for(UINT m=0; m<M; m++){
142 
143  //Select the random sample
144  UINT i = randomTrainingOrder[m];
145 
146  //Compute the error, given the current weights
147  VectorDouble x = trainingData[i].getInputVector();
148  VectorDouble y = trainingData[i].getTargetVector();
149  double h = w0;
150  for(UINT j=0; j<N; j++){
151  h += x[j] * w[j];
152  }
153  error = y[0] - h;
154  totalSquaredTrainingError += SQR( error );
155 
156  //Update the weights
157  for(UINT j=0; j<N; j++){
158  w[j] += learningRate * error * x[j];
159  }
160  w0 += learningRate * error;
161  }
162 
163  //Compute the error
164  delta = fabs( totalSquaredTrainingError-lastError );
165  lastError = totalSquaredTrainingError;
166 
167  //Check to see if we should stop
168  if( delta <= minChange ){
169  keepTraining = false;
170  }
171 
172  if( grt_isinf( totalSquaredTrainingError ) || grt_isnan( totalSquaredTrainingError ) ){
173  errorLog << "train_(RegressionData &trainingData) - Training failed! Total squared training error is NAN. If scaling is not enabled then you should try to scale your data and see if this solves the issue." << endl;
174  return false;
175  }
176 
177  if( ++iter >= maxNumEpochs ){
178  keepTraining = false;
179  }
180 
181  //Store the training results
182  rootMeanSquaredTrainingError = sqrt( totalSquaredTrainingError / double(M) );
183  result.setRegressionResult(iter,totalSquaredTrainingError,rootMeanSquaredTrainingError,this);
184  trainingResults.push_back( result );
185 
186  //Notify any observers of the new result
187  trainingResultsObserverManager.notifyObservers( result );
188 
189  trainingLog << "Epoch: " << iter << " SSE: " << totalSquaredTrainingError << " Delta: " << delta << endl;
190  }
191 
192  //Flag that the algorithm has been trained
193  regressionData.resize(1,0);
194  trained = true;
195  return trained;
196 }
197 
198 bool LinearRegression::predict_(VectorDouble &inputVector){
199 
200  if( !trained ){
201  errorLog << "predict_(VectorDouble &inputVector) - Model Not Trained!" << endl;
202  return false;
203  }
204 
205  if( !trained ) return false;
206 
207  if( inputVector.size() != numInputDimensions ){
208  errorLog << "predict_(VectorDouble &inputVector) - The size of the input vector (" << int( inputVector.size() ) << ") does not match the num features in the model (" << numInputDimensions << endl;
209  return false;
210  }
211 
212  if( useScaling ){
213  for(UINT n=0; n<numInputDimensions; n++){
214  inputVector[n] = scale(inputVector[n], inputVectorRanges[n].minValue, inputVectorRanges[n].maxValue, 0, 1);
215  }
216  }
217 
218  regressionData[0] = w0;
219  for(UINT j=0; j<numInputDimensions; j++){
220  regressionData[0] += inputVector[j] * w[j];
221  }
222 
223  if( useScaling ){
224  for(UINT n=0; n<numOutputDimensions; n++){
225  regressionData[n] = scale(regressionData[n], 0, 1, targetVectorRanges[n].minValue, targetVectorRanges[n].maxValue);
226  }
227  }
228 
229  return true;
230 }
231 
232 bool LinearRegression::saveModelToFile(fstream &file) const{
233 
234  if(!file.is_open())
235  {
236  errorLog << "loadModelFromFile(fstream &file) - The file is not open!" << endl;
237  return false;
238  }
239 
240  //Write the header info
241  file<<"GRT_LINEAR_REGRESSION_MODEL_FILE_V2.0\n";
242 
243  //Write the regressifier settings to the file
244  if( !Regressifier::saveBaseSettingsToFile(file) ){
245  errorLog <<"saveModelToFile(fstream &file) - Failed to save Regressifier base settings to file!" << endl;
246  return false;
247  }
248 
249  if( trained ){
250  file << "Weights: ";
251  file << w0;
252  for(UINT j=0; j<numInputDimensions; j++){
253  file << " " << w[j];
254  }
255  file << endl;
256  }
257 
258  return true;
259 }
260 
261 bool LinearRegression::loadModelFromFile(fstream &file){
262 
263  clear();
264 
265  if(!file.is_open())
266  {
267  errorLog << "loadModelFromFile( fstream &file ) - Could not open file to load model" << endl;
268  return false;
269  }
270 
271  std::string word;
272 
273  //Find the file type header
274  file >> word;
275 
276  //Check to see if we should load a legacy file
277  if( word == "GRT_LINEAR_REGRESSION_MODEL_FILE_V1.0" ){
278  return loadLegacyModelFromFile( file );
279  }
280 
281  if( word != "GRT_LINEAR_REGRESSION_MODEL_FILE_V2.0" ){
282  errorLog << "loadModelFromFile( fstream &file ) - Could not find Model File Header" << endl;
283  return false;
284  }
285 
286  //Load the regressifier settings from the file
287  if( !Regressifier::loadBaseSettingsFromFile(file) ){
288  errorLog <<"loadModelFromFile( fstream &file ) - Failed to save Regressifier base settings to file!" << endl;
289  return false;
290  }
291 
292  if( trained ){
293 
294  //Resize the weights
295  w.resize(numInputDimensions);
296 
297  //Load the weights
298  file >> word;
299  if(word != "Weights:"){
300  errorLog << "loadModelFromFile( fstream &file ) - Could not find the Weights!" << endl;
301  return false;
302  }
303 
304  file >> w0;
305  for(UINT j=0; j<numInputDimensions; j++){
306  file >> w[j];
307 
308  }
309  }
310 
311  return true;
312 }
313 
314 bool LinearRegression::setMaxNumIterations(const UINT maxNumIterations){
315  return setMaxNumEpochs( maxNumIterations );
316 }
317 
318 UINT LinearRegression::getMaxNumIterations() const{
319  return getMaxNumEpochs();
320 }
321 
322 bool LinearRegression::loadLegacyModelFromFile( fstream &file ){
323 
324  string word;
325 
326  file >> word;
327  if(word != "NumFeatures:"){
328  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find NumFeatures!" << endl;
329  return false;
330  }
331  file >> numInputDimensions;
332 
333  file >> word;
334  if(word != "NumOutputDimensions:"){
335  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find NumOutputDimensions!" << endl;
336  return false;
337  }
338  file >> numOutputDimensions;
339 
340  file >> word;
341  if(word != "UseScaling:"){
342  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find UseScaling!" << endl;
343  return false;
344  }
345  file >> useScaling;
346 
348  if( useScaling ){
349  //Resize the ranges buffer
350  inputVectorRanges.resize(numInputDimensions);
351  targetVectorRanges.resize(numOutputDimensions);
352 
353  //Load the ranges
354  file >> word;
355  if(word != "InputVectorRanges:"){
356  file.close();
357  errorLog << "loadLegacyModelFromFile( fstream &file ) - Failed to find InputVectorRanges!" << endl;
358  return false;
359  }
360  for(UINT j=0; j<inputVectorRanges.size(); j++){
361  file >> inputVectorRanges[j].minValue;
362  file >> inputVectorRanges[j].maxValue;
363  }
364 
365  file >> word;
366  if(word != "OutputVectorRanges:"){
367  file.close();
368  errorLog << "loadLegacyModelFromFile( fstream &file ) - Failed to find OutputVectorRanges!" << endl;
369  return false;
370  }
371  for(UINT j=0; j<targetVectorRanges.size(); j++){
372  file >> targetVectorRanges[j].minValue;
373  file >> targetVectorRanges[j].maxValue;
374  }
375  }
376 
377  //Resize the weights
378  w.resize(numInputDimensions);
379 
380  //Load the weights
381  file >> word;
382  if(word != "Weights:"){
383  errorLog << "loadLegacyModelFromFile( fstream &file ) - Could not find the Weights!" << endl;
384  return false;
385  }
386 
387  file >> w0;
388  for(UINT j=0; j<numInputDimensions; j++){
389  file >> w[j];
390 
391  }
392 
393  //Resize the regression data vector
394  regressionData.resize(1,0);
395 
396  //Flag that the model has been trained
397  trained = true;
398 
399  return true;
400 }
401 
402 } //End of namespace GRT
403 
This class implements the Linear Regression algorithm. Linear Regression is a simple but effective re...
bool setRegressionResult(unsigned int trainingIteration, double totalSquaredTrainingError, double rootMeanSquaredTrainingError, MLBase *trainer)
Definition: AdaBoost.cpp:25
vector< MinMax > getInputRanges() const
UINT getNumSamples() const
string getRegressifierType() const
vector< MinMax > getTargetRanges() const
double getRandomNumberUniform(double minRange=0.0, double maxRange=1.0)
Definition: Random.h:197
UINT getNumTargetDimensions() const
UINT getNumInputDimensions() const
bool scale(const double minTarget, const double maxTarget)