Fawkes API  Fawkes Development Version
siftpp.cpp
1 
2 /***************************************************************************
3  * siftpp.cpp - siftpp based classifier
4  *
5  * Created: Sat Apr 12 10:15:23 2008
6  * Copyright 2008 Stefan Schiffer [stefanschiffer.de]
7  *
8  ****************************************************************************/
9 
10 /* This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version. A runtime exception applies to
14  * this software (see LICENSE.GPL_WRE file mentioned below for details).
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * GNU Library General Public License for more details.
20  *
21  * Read the full text in the LICENSE.GPL_WRE file in the doc directory.
22  */
23 
24 #include <iostream>
25 #include <vector>
26 
27 #include <fvclassifiers/siftpp.h>
28 
29 //#ifdef SIFTPP_TIMETRACKER
30 #include <utils/time/clock.h>
31 #include <utils/time/tracker.h>
32 //#endif
33 
34 #include <core/exception.h>
35 #include <core/exceptions/software.h>
36 #include <fvutils/color/colorspaces.h>
37 #include <fvutils/color/conversions.h>
38 #include <fvutils/readers/png.h>
39 //#include <fvutils/writers/pnm.h>
40 //#include <fvutils/writers/png.h>
41 
42 //using namespace fawkes;
43 using namespace fawkes;
44 
45 namespace firevision {
46 #if 0 /* just to make Emacs auto-indent happy */
47 }
48 #endif
49 
50 /** @class SiftppClassifier <fvclassifiers/siftpp.h>
51  * SIFTPP classifier.
52  *
53  * This class provides a classifier that uses SIFTPP to detect objects in a given
54  * image by matching features. The objects are reported back as regions of interest.
55  * Each ROI contains an object. ROIs with 11x11 are matched features.
56  *
57  * This code uses siftpp from http://vision.ucla.edu/~vedaldi/code/siftpp/siftpp.html
58  * and is partly based on code from their package.
59  *
60  * @author Stefan Schiffer
61  */
62 
63 /** Constructor.
64  * @param object_file file that contains an image of the object to detect
65  * @param samplingStep Initial sampling step
66  * @param octaves Number of analysed octaves
67  * @param levels Number of levels per octave
68  * @param magnif Keypoint magnification (default = 3)
69  * @param noorient rotation invariance (0) or upright (1)
70  * @param unnormalized Normalization of features (default 0)
71  */
72 SiftppClassifier::SiftppClassifier( const char * object_file,
73  int samplingStep, int octaves, int levels,
74  float magnif, int noorient, int unnormalized)
75  : Classifier("SiftppClassifier")
76 {
77  // params for FastHessian
78  __samplingStep = samplingStep;
79  __octaves = octaves;
80  __levels = levels;
81  // params for Descriptors
82  __first = -1 ;
83  __threshold = 0.04f / __levels / 2.0f ;
84  __edgeThreshold = 10.0f;
85  __magnif = magnif;
86  __noorient = noorient;
87  __unnormalized = unnormalized;
88 
89  // descriptor vector length
90  __vlen = 128;
91 
92 
93  //#ifdef SIFTPP_TIMETRACKER
94  __tt = new TimeTracker();
95  __loop_count = 0;
96  __ttc_objconv = __tt->add_class("ObjectConvert");
97  __ttc_objfeat = __tt->add_class("ObjectFeatures");
98  __ttc_imgconv = __tt->add_class("ImageConvert");
99  __ttc_imgfeat = __tt->add_class("ImageFeatures");
100  __ttc_matchin = __tt->add_class("Matching");
101  __ttc_roimerg = __tt->add_class("MergeROIs");
102  //#endif
103 
104  //#ifdef SIFTPP_TIMETRACKER
105  __tt->ping_start(__ttc_objconv);
106  //#endif
107 
108  PNGReader pngr( object_file );
109  unsigned char* buf = malloc_buffer( pngr.colorspace(), pngr.pixel_width(), pngr.pixel_height() );
110  pngr.set_buffer( buf );
111  pngr.read();
112 
113  unsigned int lwidth = pngr.pixel_width();
114  unsigned int lheight = pngr.pixel_height();
115  VL::pixel_t * im_pt = new VL::pixel_t [lwidth * lheight ];
116  VL::pixel_t * start = im_pt;
117  //VL::pixel_t* end = start + lwidth*lheight ;
118  for (unsigned int h = 0; h < lheight; ++h) {
119  for (unsigned int w = 0; w < lwidth ; ++w) {
120  int i = (buf[h * lwidth + w] );
121  VL::pixel_t norm = VL::pixel_t( 255 );
122  *start++ = VL::pixel_t( i ) / norm;
123  }
124  }
125  // make image
126  __obj_img = new VL::PgmBuffer();
127  __obj_img->width = lwidth;
128  __obj_img->height = lheight;
129  __obj_img->data = im_pt;
130 
131  if ( ! __obj_img ) {
132  throw Exception("Could not load object file");
133  }
134 
135  //#ifdef SIFTPP_TIMETRACKER
136  __tt->ping_end(__ttc_objconv);
137  //#endif
138 
139  // save object image for debugging
140  //
141 
142  //#ifdef SIFTPP_TIMETRACKER
143  __tt->ping_start(__ttc_objfeat);
144  //#endif
145 
146  // COMPUTE OBJECT FEATURES
147  __obj_features.clear();
148  //__obj_features.reserve(1000);
149  __obj_num_features = 0;
150 
151  __sigman = .5 ;
152  __sigma0 = 1.6 * powf(2.0f, 1.0f / __levels) ;
153 
154  std::cout << "SiftppClassifier(ctor): init scalespace" << std::endl;
155  // initialize scalespace
156  VL::Sift sift(__obj_img->data, __obj_img->width, __obj_img->height,
157  __sigman, __sigma0, __octaves, __levels, __first, -1, __levels+1) ;
158 
159  std::cout << "SiftppClassifier(ctor): detect object keypoints" << std::endl;
160  // Run SIFTPP detector
161  sift.detectKeypoints(__threshold, __edgeThreshold) ;
162  // Number of keypoints
163  __obj_num_features = sift.keypointsEnd() - sift.keypointsBegin();
164  std::cout << "SiftppClassifier(ctor): computed '" << __obj_num_features << "' object-keypoints" << std::endl;
165 
166  // set descriptor options
167  sift.setNormalizeDescriptor( ! __unnormalized ) ;
168  sift.setMagnification( __magnif ) ;
169 
170  std::cout << "SiftppClassifier(ctor): run detector, compute ori and des ..." << std::endl;
171  // Run detector, compute orientations and descriptors
172  for( VL::Sift::KeypointsConstIter iter = sift.keypointsBegin() ;
173  iter != sift.keypointsEnd() ; ++iter ) {
174 
175  //Feature * feat = new Feature();
176  Feature feat;
177 
178  //std::cout << "SiftppClassifier(ctor): saving keypoint" << std::endl;
179  feat.key = (*iter);
180 
181  // detect orientations
182  VL::float_t angles [4] ;
183  int nangles ;
184  if( ! __noorient ) {
185  nangles = sift.computeKeypointOrientations(angles, *iter) ;
186  } else {
187  nangles = 1;
188  angles[0] = VL::float_t(0) ;
189  }
190  feat.number_of_desc = nangles;
191  feat.descs = new VL::float_t*[nangles];
192 
193  //std::cout << "SiftppClassifier(ctor): computing '" << nangles << "' descriptors" << std::endl;
194  // compute descriptors
195  for(int a = 0 ; a < nangles ; ++a) {
196  // out << setprecision(2) << iter->x << ' ' << setprecision(2) << iter->y << ' '
197  // << setprecision(2) << iter->sigma << ' ' << setprecision(3) << angles[a] ;
198  // compute descriptor
199  feat.descs[a] = new VL::float_t[__vlen];
200  sift.computeKeypointDescriptor(feat.descs[a], *iter, angles[a]) ;
201  } // next angle
202  //std::cout << "SiftppClassifier(ctor): computed '" << feat.number_of_desc << "' descriptors." << std::endl;
203 
204  // save feature
205  __obj_features.push_back( feat );
206 
207  } // next keypoint
208 
209  __obj_num_features = __obj_features.size();
210  if ( ! __obj_num_features > 0 ) {
211  throw Exception("Could not compute object features");
212  }
213  std::cout << "SiftppClassifier(ctor): computed '" << __obj_num_features << "' features from object" << std::endl;
214 
215  //#ifdef SIFTPP_TIMETRACKER
216  __tt->ping_end(__ttc_objfeat);
217  //#endif
218 
219 }
220 
221 
222 /** Destructor. */
224 {
225  //
226  delete __obj_img;
227  __obj_features.clear();
228  //
229  //delete __image;
230  __img_features.clear();
231 }
232 
233 
234 std::list< ROI > *
236 {
237  //#ifdef SIFTPP_TIMETRACKER
238  __tt->ping_start(0);
239  //#endif
240 
241  // list of ROIs to return
242  std::list< ROI > *rv = new std::list< ROI >();
243 
244  // for ROI calculation
245  int x_min = _width;
246  int y_min = _height;
247  int x_max = 0;
248  int y_max = 0;
249 
250  //#ifdef SIFTPP_TIMETRACKER
251  __tt->ping_start(__ttc_imgconv);
252  //#endif
253  std::cout << "SiftppClassifier(classify): copy imgdat to SIFTPP Image" << std::endl;
254 
255  VL::pixel_t * im_pt = new VL::pixel_t [_width * _height ];
256  VL::pixel_t * start = im_pt;
257  for (unsigned int h = 0; h < _height; ++h) {
258  for (unsigned int w = 0; w < _width ; ++w) {
259  int i = (_src[h * _width + w] );
260  VL::pixel_t norm = VL::pixel_t( 255 );
261  *start++ = VL::pixel_t( i ) / norm;
262  }
263  }
264  // make image
265  __image = new VL::PgmBuffer();
266  __image->width = _width;
267  __image->height = _height;
268  __image->data = im_pt;
269 
270  //#ifdef SIFTPP_TIMETRACKER
271  __tt->ping_end(__ttc_imgconv);
272  //#endif
273 
274  /// Write image to verify correct operation
275  // nothing yet
276 
277  //#ifdef SIFTPP_TIMETRACKER
278  __tt->ping_start(__ttc_imgfeat);
279  //#endif
280 
281  // COMPUTE IMAGE FEATURES
282  __img_features.clear();
283  __img_num_features = 0;
284  //__img_features.reserve(1000);
285 
286  std::cout << "SiftppClassifier(classify): init scalespace" << std::endl;
287  // initialize scalespace
288  VL::Sift sift(__image->data, __image->width, __image->height,
289  __sigman, __sigma0, __octaves, __levels, __first, -1, __levels+1) ;
290 
291  std::cout << "SiftppClassifier(classify): detect image keypoints" << std::endl;
292  // Run SIFTPP detector
293  sift.detectKeypoints(__threshold, __edgeThreshold) ;
294 
295  // Number of keypoints
296  __img_num_features = sift.keypointsEnd() - sift.keypointsBegin();
297  std::cout << "SiftppClassifier(classify): Extracted '" << __img_num_features << "' image keypoints" << std::endl;
298 
299  // set descriptor options
300  sift.setNormalizeDescriptor( ! __unnormalized ) ;
301  sift.setMagnification( __magnif ) ;
302 
303  std::cout << "SiftppClassifier(classify): run detector, compute ori and des ..." << std::endl;
304  // Run detector, compute orientations and descriptors
305  for( VL::Sift::KeypointsConstIter iter = sift.keypointsBegin() ;
306  iter != sift.keypointsEnd() ; ++iter ) {
307 
308  Feature feat; // = new Feature();
309 
310  //std::cout << "SiftppClassifier(classify): saving keypoint" << std::endl;
311  feat.key = (*iter);
312 
313  //std::cout << "SiftppClassifier(classify): detect orientations" << std::endl;
314  // detect orientations
315  VL::float_t angles [4] ;
316  int nangles ;
317  if( ! __noorient ) {
318  nangles = sift.computeKeypointOrientations(angles, *iter) ;
319  } else {
320  nangles = 1;
321  angles[0] = VL::float_t(0) ;
322  }
323  feat.number_of_desc = nangles;
324  feat.descs = new VL::float_t*[nangles];
325 
326  //std::cout << "SiftppClassifier(classify): computing '" << nangles << "' descriptors" << std::endl;
327  // compute descriptors
328  for(int a = 0 ; a < nangles ; ++a) {
329  // compute descriptor
330  feat.descs[a] = new VL::float_t[__vlen] ;
331  sift.computeKeypointDescriptor(feat.descs[a], *iter, angles[a]) ;
332  } // next angle
333  //std::cout << "SiftppClassifier(classify): computed '" << feat.number_of_desc << "' descriptors." << std::endl;
334 
335  // save feature
336  __img_features.push_back( feat );
337 
338  } // next keypoint
339 
340  // Number of feature
341  __img_num_features = __img_features.size();
342 
343  //#ifdef SIFTPP_TIMETRACKER
344  __tt->ping_end(__ttc_imgfeat);
345  //#endif
346 
347  std::cout << "SiftppClassifier(classify): Extracted '" << __img_num_features << "' image features" << std::endl;
348 
349  //#ifdef SIFTPP_TIMETRACKER
350  __tt->ping_start(__ttc_matchin);
351  //#endif
352  std::cout << "SiftppClassifier(classify): matching ..." << std::endl;
353 
354  std::vector< int > matches(__obj_features.size());
355  int m = 0;
356  for (unsigned i = 0; i < __obj_features.size(); i++) {
357  int match = findMatch(__obj_features[i], __img_features);
358  matches[i] = match;
359  if (match != -1) {
360  std::cout << "SiftppClassifier(classify): Matched feature " << i << " in object image with feature " << match << " in image." << std::endl;
361  /// adding feature-ROI
362  ROI r( (int)(__img_features[matches[i]].key.x)-5, (int)(__img_features[matches[i]].key.y )-5, 11, 11, _width, _height);
363  rv->push_back(r);
364  // increment feature-match-count
365  ++m;
366  }
367  }
368 
369  //#ifdef SIFTPP_TIMETRACKER
370  __tt->ping_end(__ttc_matchin);
371  //#endif
372  std::cout << "SiftppClassifier(classify) matched '" << m << "' of '" << __obj_features.size() << "' features in scene." << std::endl;
373 
374  std::cout << "SiftppClassifier(classify): computing ROI" << std::endl;
375  //#ifdef SIFTPP_TIMETRACKER
376  __tt->ping_start(__ttc_roimerg);
377  //#endif
378 
379  for (unsigned i = 0; i < matches.size(); i++) {
380  if (matches[i] != -1) {
381  if( (int)__img_features[matches[i]].key.x < x_min )
382  x_min = (int)__img_features[matches[i]].key.x;
383  if( (int)__img_features[matches[i]].key.y < y_min )
384  y_min = (int)__img_features[matches[i]].key.y;
385  if( (int)__img_features[matches[i]].key.x > x_max )
386  x_max = (int)__img_features[matches[i]].key.x;
387  if( (int)__img_features[matches[i]].key.y > y_max )
388  y_max = (int)__img_features[matches[i]].key.y;
389  }
390  }
391  if( m != 0 ) {
392  ROI r(x_min, y_min, x_max-x_min, y_max-y_min, _width, _height);
393  rv->push_back(r);
394  }
395 
396  //#ifdef SIFTPP_TIMETRACKER
397  __tt->ping_end(__ttc_roimerg);
398  //#endif
399 
400  //#ifdef SIFTPP_TIMETRACKER
401  __tt->ping_end(0);
402  //#endif
403 
404  //#ifdef SIFTPP_TIMETRACKER
405  // print timetracker statistics
406  __tt->print_to_stdout();
407  //#endif
408 
409  delete __image;
410 
411  std::cout << "SiftppClassifier(classify): done ... returning '" << rv->size() << "' ROIs." << std::endl;
412  return rv;
413 }
414 
415 int
416 SiftppClassifier::findMatch(const Feature & ip1, const std::vector< Feature > & ipts) {
417  double mind = 1e100, second = 1e100;
418  int match = -1;
419 
420  for (unsigned i = 0; i < ipts.size(); i++) {
421 
422  if (ipts[i].number_of_desc != ip1.number_of_desc)
423  continue;
424  //std::cout << "SiftppClassifier(findMatch): number_of_desc matched!" << std::endl;
425  for ( int j = 0; j < ip1.number_of_desc; ++j ) {
426  double d = distSquare(ipts[i].descs[j], ip1.descs[j], __vlen);
427 
428  if (d < mind) {
429  second = mind;
430  mind = d;
431  match = i;
432  } else if (d < second) {
433  second = d;
434  }
435  }
436  }
437 
438  if (mind < 0.5 * second)
439  return match;
440 
441  return -1;
442 }
443 
444 
445 double
446 SiftppClassifier::distSquare(VL::float_t *v1, VL::float_t *v2, int n) {
447  double dsq = 0.;
448  while (n--) {
449  dsq += (v1[n-1] - v2[n-1]) * (v1[n-1] - v2[n-1]);
450  }
451  //std::cout << " dsq: '" << dsq << "'" << std::endl;
452  return dsq;
453 }
454 
455 } // end namespace firevision
virtual unsigned int pixel_height()
Get height of read image in pixels.
Definition: png.cpp:207
virtual ~SiftppClassifier()
Destructor.
Definition: siftpp.cpp:223
void ping_start(unsigned int cls)
Start of given class task.
Definition: tracker.cpp:228
Siftpp Feature struct.
Definition: siftpp.h:66
VL::float_t ** descs
descriptors
Definition: siftpp.h:69
Fawkes library namespace.
virtual unsigned int pixel_width()
Get width of read image in pixels.
Definition: png.cpp:196
virtual void read()
Read data from file.
Definition: png.cpp:218
virtual void set_buffer(unsigned char *yuv422planar_buffer)
Set buffer that the read image should be written to.
Definition: png.cpp:182
unsigned int _width
Width in pixels of _src buffer.
Definition: classifier.h:54
Region of interest.
Definition: roi.h:58
VL::Sift::Keypoint key
keypoint
Definition: siftpp.h:67
unsigned int _height
Height in pixels of _src buffer.
Definition: classifier.h:56
PNG file reader.
Definition: png.h:36
int number_of_desc
number of descriptors
Definition: siftpp.h:68
Base class for exceptions in Fawkes.
Definition: exception.h:36
virtual std::list< ROI > * classify()
Classify image.
Definition: siftpp.cpp:235
unsigned int add_class(std::string name)
Add a new class.
Definition: tracker.cpp:156
Time tracking utility.
Definition: tracker.h:38
void ping_end(unsigned int cls)
End of given class task.
Definition: tracker.cpp:254
void print_to_stdout()
Print results to stdout.
Definition: tracker.cpp:317
Classifier to extract regions of interest.
Definition: classifier.h:37
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
Definition: classifier.h:52
virtual colorspace_t colorspace()
Get colorspace from the just read image.
Definition: png.cpp:189