Fawkes API  Fawkes Development Version
sift.cpp
1 
2 /***************************************************************************
3  * sift.cpp - Feature-based classifier using OpenCV structures
4  *
5  * Created: Mon Mar 15 15:47:11 2008
6  * Copyright 2008 Stefan Schiffer [stefanschiffer.de]
7  *
8  ****************************************************************************/
9 
10 /* This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version. A runtime exception applies to
14  * this software (see LICENSE.GPL_WRE file mentioned below for details).
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * GNU Library General Public License for more details.
20  *
21  * Read the full text in the LICENSE.GPL_WRE file in the doc directory.
22  */
23 
24 #include <iostream>
25 #include <vector>
26 
27 #include <fvclassifiers/sift.h>
28 //#ifdef SIFT_TIMETRACKER
29 #include <utils/time/clock.h>
30 #include <utils/time/tracker.h>
31 //#endif
32 
33 extern "C" {
34 #include <sift/sift.h>
35 #include <sift/imgfeatures.h>
36 #include <sift/kdtree.h>
37 #include <sift/utils.h>
38 #include <sift/xform.h>
39 }
40 
41 #include <core/exception.h>
42 #include <core/exceptions/software.h>
43 #include <fvutils/color/colorspaces.h>
44 #include <fvutils/color/conversions.h>
45 
46 #include <opencv/cv.h>
47 #include <opencv/cxcore.h>
48 #include <opencv/highgui.h>
49 
50 using namespace fawkes;
51 
52 namespace firevision {
53 #if 0 /* just to make Emacs auto-indent happy */
54 }
55 #endif
56 
57 /** @class SiftClassifier <fvclassifiers/sift.h>
58  * SIFT classifier.
59  *
60  * This class provides a classifier that uses OpenCV to detect objects in a given
61  * image by matching features using SIFT. The objects are reported back as regions
62  * of interest. Each ROI contains an object.
63  *
64  * This code is based on the sift package provided by Rob Hess.
65  * at http://web.engr.oregonstate.edu/~hess/
66  *
67  * @author Stefan Schiffer
68  */
69 
70 /** Constructor.
71  * @param object_file file that contains the object to detect
72  * @param pixel_width width of images that will be processed
73  * @param pixel_height height of images that will be processed
74  * @param kdtree_bbf_max_nn_chks maximum number of keypoint NN candidates to check during BBF search
75  * @param nn_sq_dist_ratio_thr threshold on squared ratio of distances between NN and 2nd NN
76  * @param flags flags, not used yet.
77  */
78 SiftClassifier::SiftClassifier( const char * object_file,
79  unsigned int pixel_width, unsigned int pixel_height,
80  int kdtree_bbf_max_nn_chks, float nn_sq_dist_ratio_thr, int flags)
81  : Classifier("SiftClassifier")
82 {
83  __kdtree_bbf_max_nn_chks = kdtree_bbf_max_nn_chks;
84  __nn_sq_dist_ratio_thr = nn_sq_dist_ratio_thr;
85  __flags = flags;
86 
87 
88  //#ifdef SIFT_TIMETRACKER
89  __tt = new TimeTracker();
90  __loop_count = 0;
91  __ttc_objconv = __tt->add_class("ObjectConvert");
92  __ttc_objfeat = __tt->add_class("ObjectFeatures");
93  __ttc_imgconv = __tt->add_class("ImageConvert");
94  __ttc_imgfeat = __tt->add_class("ImageFeatures");
95  __ttc_matchin = __tt->add_class("Matching");
96  __ttc_roimerg = __tt->add_class("MergeROIs");
97  //#endif
98 
99  //#ifdef SIFT_TIMETRACKER
100  __tt->ping_start(__ttc_objconv);
101  //#endif
102  __obj_img = cvLoadImage( object_file, 1 );
103  if ( ! __obj_img ) {
104  throw Exception("Could not load object file");
105  }
106  //#ifdef SIFT_TIMETRACKER
107  __tt->ping_end(__ttc_objconv);
108  //#endif
109 
110  //#ifdef SIFT_TIMETRACKER
111  __tt->ping_start(__ttc_objfeat);
112  //#endif
113  __obj_num_features = 0;
114  __obj_num_features = sift_features( __obj_img, &__obj_features );
115  if ( ! __obj_num_features > 0 ) {
116  throw Exception("Could not compute object features");
117  }
118  std::cout << "SiftClassifier(classify): computed '" << __obj_num_features << "' features from object" << std::endl;
119  //cvReleaseImage(&__obj_img);
120  //#ifdef SIFT_TIMETRACKER
121  __tt->ping_end(__ttc_objfeat);
122  //#endif
123 
124  // create space for OpenCV image
125  __image = cvCreateImage(cvSize(pixel_width, pixel_height), IPL_DEPTH_8U, 3);
126 
127 }
128 
129 
130 /** Destructor. */
132 {
133  //
134  cvReleaseImage(&__obj_img);
135  cvReleaseImage(&__image);
136 }
137 
138 
139 std::list< ROI > *
141 {
142  //#ifdef SIFT_TIMETRACKER
143  __tt->ping_start(0);
144  //#endif
145 
146  // list of ROIs to return
147  std::list< ROI > *rv = new std::list< ROI >();
148 
149  struct feature * feat;
150  struct feature** nbrs;
151  struct kd_node* kd_root;
152  CvPoint pt1, pt2;
153 
154  // for ROI calculation
155  CvPoint ftpt;
156  std::vector< CvPoint > ftlist;
157  //= new std::vector< CvPoint >();
158  int x_min = _width;
159  int y_min = _height;
160  int x_max = 0;
161  int y_max = 0;
162 
163  double d0, d1;// = 0.0;
164  int k, m = 0;
165 
166  //#ifdef SIFT_TIMETRACKER
167  __tt->ping_start(__ttc_imgconv);
168  //#endif
169  //std::cout << "SiftClassifier(classify): convert frame to IplImage" << std::endl;
170  convert(YUV422_PLANAR, BGR, _src, (unsigned char *)__image->imageData, _width, _height);
171  //#ifdef SIFT_TIMETRACKER
172  __tt->ping_end(__ttc_imgconv);
173  //#endif
174 
175  //#ifdef SIFT_TIMETRACKER
176  __tt->ping_start(__ttc_imgfeat);
177  //#endif
178  //std::cout << "SiftClassifier(classify): compute features on current frame " << std::endl;
179  int num_img_ft = sift_features( __image, &__img_features );
180  kd_root = kdtree_build( __img_features, num_img_ft );
181  //#ifdef SIFT_TIMETRACKER
182  __tt->ping_end(__ttc_imgfeat);
183  //#endif
184 
185  if( ! kd_root ) {
186  std::cerr << "SiftClassifier(classify): KD-Root NULL!" << std::endl;
187  }
188 
189  //#ifdef SIFT_TIMETRACKER
190  __tt->ping_start(__ttc_matchin);
191  //#endif
192  std::cout << "SiftClassifier(classify): matching ..." << std::endl;
193  for( int i = 0; i < __obj_num_features; ++i ) {
194  //std::cout << "SiftClassifier(classify): ... feature '" << i << "'" << std::endl;
195  feat = __obj_features + i;
196  k = kdtree_bbf_knn( kd_root, feat, 2, &nbrs, __kdtree_bbf_max_nn_chks );
197  if( k == 2 )
198  {
199  d0 = descr_dist_sq( feat, nbrs[0] );
200  d1 = descr_dist_sq( feat, nbrs[1] );
201  if( d0 < d1 * __nn_sq_dist_ratio_thr )
202  {
203  pt1 = cvPoint( cvRound( feat->x ), cvRound( feat->y ) );
204  pt2 = cvPoint( cvRound( nbrs[0]->x ), cvRound( nbrs[0]->y ) );
205  m++;
206  __obj_features[i].fwd_match = nbrs[0];
207  // save matched feature points
208  ftpt = cvPoint( cvRound( nbrs[0]->x), cvRound( nbrs[0]->y ) );
209  ftlist.push_back(ftpt);
210  // save matched features as ROIs
211  ROI r( pt2.x-5, pt2.y-5, 11, 11, _width, _height);
212  rv->push_back(r);
213  }
214  }
215  free( nbrs );
216  }
217  std::cout << "SiftClassifier(classify): found '" << m << "' matches" << std::endl;
218  kdtree_release( kd_root );
219  //#ifdef SIFT_TIMETRACKER
220  __tt->ping_end(__ttc_matchin);
221  //#endif
222 
223  //#ifdef SIFT_TIMETRACKER
224  __tt->ping_start(__ttc_roimerg);
225  //#endif
226  std::cout << "SiftClassifier(classify): computing ROI" << std::endl;
227  //for ( int i = 0; i < m; ++i) {
228  for ( std::vector< CvPoint >::size_type i = 0; i < ftlist.size(); ++i) {
229  if( ftlist[i].x < x_min )
230  x_min = ftlist[i].x;
231  if( ftlist[i].y < y_min )
232  y_min = ftlist[i].y;
233  if( ftlist[i].x > x_max )
234  x_max = ftlist[i].x;
235  if( ftlist[i].y > y_max )
236  y_max = ftlist[i].y;
237  }
238  if( m != 0 ) {
239  ROI r(x_min, y_min, x_max-x_min, y_max-y_min, _width, _height);
240  rv->push_back(r);
241  }
242  //#ifdef SIFT_TIMETRACKER
243  __tt->ping_end(__ttc_roimerg);
244  //#endif
245 
246  //#ifdef SIFT_TIMETRACKER
247  __tt->ping_end(0);
248  //#endif
249 
250  //#ifdef SIFT_TIMETRACKER
251  __tt->print_to_stdout();
252  //#endif
253 
254  std::cout << "SiftClassifier(classify): done ... returning '" << rv->size() << "' ROIs." << std::endl;
255  return rv;
256 }
257 
258 } // end namespace firevision
void ping_start(unsigned int cls)
Start of given class task.
Definition: tracker.cpp:228
Fawkes library namespace.
unsigned int _width
Width in pixels of _src buffer.
Definition: classifier.h:54
Region of interest.
Definition: roi.h:58
unsigned int _height
Height in pixels of _src buffer.
Definition: classifier.h:56
Base class for exceptions in Fawkes.
Definition: exception.h:36
unsigned int add_class(std::string name)
Add a new class.
Definition: tracker.cpp:156
Time tracking utility.
Definition: tracker.h:38
virtual ~SiftClassifier()
Destructor.
Definition: sift.cpp:131
void ping_end(unsigned int cls)
End of given class task.
Definition: tracker.cpp:254
void print_to_stdout()
Print results to stdout.
Definition: tracker.cpp:317
virtual std::list< ROI > * classify()
Classify image.
Definition: sift.cpp:140
Classifier to extract regions of interest.
Definition: classifier.h:37
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
Definition: classifier.h:52