Fawkes API Fawkes Development Version
siftpp.cpp
1
2/***************************************************************************
3 * siftpp.cpp - siftpp based classifier
4 *
5 * Created: Sat Apr 12 10:15:23 2008
6 * Copyright 2008 Stefan Schiffer [stefanschiffer.de]
7 *
8 ****************************************************************************/
9
10/* This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version. A runtime exception applies to
14 * this software (see LICENSE.GPL_WRE file mentioned below for details).
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU Library General Public License for more details.
20 *
21 * Read the full text in the LICENSE.GPL_WRE file in the doc directory.
22 */
23
24#include <fvclassifiers/siftpp.h>
25
26#include <iostream>
27#include <vector>
28
29//#ifdef SIFTPP_TIMETRACKER
30#include <utils/time/clock.h>
31#include <utils/time/tracker.h>
32//#endif
33
34#include <core/exception.h>
35#include <core/exceptions/software.h>
36#include <fvutils/color/colorspaces.h>
37#include <fvutils/color/conversions.h>
38#include <fvutils/readers/png.h>
39//#include <fvutils/writers/pnm.h>
40//#include <fvutils/writers/png.h>
41
42//using namespace fawkes;
43using namespace fawkes;
44
45namespace firevision {
46
47/** @class SiftppClassifier <fvclassifiers/siftpp.h>
48 * SIFTPP classifier.
49 *
50 * This class provides a classifier that uses SIFTPP to detect objects in a given
51 * image by matching features. The objects are reported back as regions of interest.
52 * Each ROI contains an object. ROIs with 11x11 are matched features.
53 *
54 * This code uses siftpp from http://vision.ucla.edu/~vedaldi/code/siftpp/siftpp.html
55 * and is partly based on code from their package.
56 *
57 * @author Stefan Schiffer
58 */
59
60/** Constructor.
61 * @param object_file file that contains an image of the object to detect
62 * @param samplingStep Initial sampling step
63 * @param octaves Number of analysed octaves
64 * @param levels Number of levels per octave
65 * @param magnif Keypoint magnification (default = 3)
66 * @param noorient rotation invariance (0) or upright (1)
67 * @param unnormalized Normalization of features (default 0)
68 */
69SiftppClassifier::SiftppClassifier(const char *object_file,
70 int samplingStep,
71 int octaves,
72 int levels,
73 float magnif,
74 int noorient,
75 int unnormalized)
76: Classifier("SiftppClassifier")
77{
78 // params for FastHessian
79 samplingStep_ = samplingStep;
80 octaves_ = octaves;
81 levels_ = levels;
82 // params for Descriptors
83 first_ = -1;
84 threshold_ = 0.04f / levels_ / 2.0f;
85 edgeThreshold_ = 10.0f;
86 magnif_ = magnif;
87 noorient_ = noorient;
88 unnormalized_ = unnormalized;
89
90 // descriptor vector length
91 vlen_ = 128;
92
93 //#ifdef SIFTPP_TIMETRACKER
94 tt_ = new TimeTracker();
95 loop_count_ = 0;
96 ttc_objconv_ = tt_->add_class("ObjectConvert");
97 ttc_objfeat_ = tt_->add_class("ObjectFeatures");
98 ttc_imgconv_ = tt_->add_class("ImageConvert");
99 ttc_imgfeat_ = tt_->add_class("ImageFeatures");
100 ttc_matchin_ = tt_->add_class("Matching");
101 ttc_roimerg_ = tt_->add_class("MergeROIs");
102 //#endif
103
104 //#ifdef SIFTPP_TIMETRACKER
105 tt_->ping_start(ttc_objconv_);
106 //#endif
107
108 PNGReader pngr(object_file);
109 unsigned char *buf = malloc_buffer(pngr.colorspace(), pngr.pixel_width(), pngr.pixel_height());
110 pngr.set_buffer(buf);
111 pngr.read();
112
113 unsigned int lwidth = pngr.pixel_width();
114 unsigned int lheight = pngr.pixel_height();
115 VL::pixel_t *im_pt = new VL::pixel_t[lwidth * lheight];
116 VL::pixel_t *start = im_pt;
117 //VL::pixel_t* end = start + lwidth*lheight ;
118 for (unsigned int h = 0; h < lheight; ++h) {
119 for (unsigned int w = 0; w < lwidth; ++w) {
120 int i = (buf[h * lwidth + w]);
121 VL::pixel_t norm = VL::pixel_t(255);
122 *start++ = VL::pixel_t(i) / norm;
123 }
124 }
125 // make image
126 obj_img_ = new VL::PgmBuffer();
127 obj_img_->width = lwidth;
128 obj_img_->height = lheight;
129 obj_img_->data = im_pt;
130
131 if (!obj_img_) {
132 throw Exception("Could not load object file");
133 }
134
135 //#ifdef SIFTPP_TIMETRACKER
136 tt_->ping_end(ttc_objconv_);
137 //#endif
138
139 // save object image for debugging
140 //
141
142 //#ifdef SIFTPP_TIMETRACKER
143 tt_->ping_start(ttc_objfeat_);
144 //#endif
145
146 // COMPUTE OBJECT FEATURES
147 obj_features_.clear();
148 //obj_features_.reserve(1000);
149 obj_num_features_ = 0;
150
151 sigman_ = .5;
152 sigma0_ = 1.6 * powf(2.0f, 1.0f / levels_);
153
154 std::cout << "SiftppClassifier(ctor): init scalespace" << std::endl;
155 // initialize scalespace
156 VL::Sift sift(obj_img_->data,
157 obj_img_->width,
158 obj_img_->height,
159 sigman_,
160 sigma0_,
161 octaves_,
162 levels_,
163 first_,
164 -1,
165 levels_ + 1);
166
167 std::cout << "SiftppClassifier(ctor): detect object keypoints" << std::endl;
168 // Run SIFTPP detector
169 sift.detectKeypoints(threshold_, edgeThreshold_);
170 // Number of keypoints
171 obj_num_features_ = sift.keypointsEnd() - sift.keypointsBegin();
172 std::cout << "SiftppClassifier(ctor): computed '" << obj_num_features_ << "' object-keypoints"
173 << std::endl;
174
175 // set descriptor options
176 sift.setNormalizeDescriptor(!unnormalized_);
177 sift.setMagnification(magnif_);
178
179 std::cout << "SiftppClassifier(ctor): run detector, compute ori and des ..." << std::endl;
180 // Run detector, compute orientations and descriptors
181 for (VL::Sift::KeypointsConstIter iter = sift.keypointsBegin(); iter != sift.keypointsEnd();
182 ++iter) {
183 //Feature * feat = new Feature();
184 Feature feat;
185
186 //std::cout << "SiftppClassifier(ctor): saving keypoint" << std::endl;
187 feat.key = (*iter);
188
189 // detect orientations
190 VL::float_t angles[4];
191 int nangles;
192 if (!noorient_) {
193 nangles = sift.computeKeypointOrientations(angles, *iter);
194 } else {
195 nangles = 1;
196 angles[0] = VL::float_t(0);
197 }
198 feat.number_of_desc = nangles;
199 feat.descs = new VL::float_t *[nangles];
200
201 //std::cout << "SiftppClassifier(ctor): computing '" << nangles << "' descriptors" << std::endl;
202 // compute descriptors
203 for (int a = 0; a < nangles; ++a) {
204 // out << setprecision(2) << iter->x << ' ' << setprecision(2) << iter->y << ' '
205 // << setprecision(2) << iter->sigma << ' ' << setprecision(3) << angles[a] ;
206 // compute descriptor
207 feat.descs[a] = new VL::float_t[vlen_];
208 sift.computeKeypointDescriptor(feat.descs[a], *iter, angles[a]);
209 } // next angle
210 //std::cout << "SiftppClassifier(ctor): computed '" << feat.number_of_desc << "' descriptors." << std::endl;
211
212 // save feature
213 obj_features_.push_back(feat);
214
215 } // next keypoint
216
217 obj_num_features_ = obj_features_.size();
218 if (!obj_num_features_ > 0) {
219 throw Exception("Could not compute object features");
220 }
221 std::cout << "SiftppClassifier(ctor): computed '" << obj_num_features_ << "' features from object"
222 << std::endl;
223
224 //#ifdef SIFTPP_TIMETRACKER
225 tt_->ping_end(ttc_objfeat_);
226 //#endif
227}
228
229/** Destructor. */
231{
232 //
233 delete obj_img_;
234 obj_features_.clear();
235 //
236 //delete image_;
237 img_features_.clear();
238}
239
240std::list<ROI> *
242{
243 //#ifdef SIFTPP_TIMETRACKER
244 tt_->ping_start(0);
245 //#endif
246
247 // list of ROIs to return
248 std::list<ROI> *rv = new std::list<ROI>();
249
250 // for ROI calculation
251 int x_min = _width;
252 int y_min = _height;
253 int x_max = 0;
254 int y_max = 0;
255
256 //#ifdef SIFTPP_TIMETRACKER
257 tt_->ping_start(ttc_imgconv_);
258 //#endif
259 std::cout << "SiftppClassifier(classify): copy imgdat to SIFTPP Image" << std::endl;
260
261 VL::pixel_t *im_pt = new VL::pixel_t[_width * _height];
262 VL::pixel_t *start = im_pt;
263 for (unsigned int h = 0; h < _height; ++h) {
264 for (unsigned int w = 0; w < _width; ++w) {
265 int i = (_src[h * _width + w]);
266 VL::pixel_t norm = VL::pixel_t(255);
267 *start++ = VL::pixel_t(i) / norm;
268 }
269 }
270 // make image
271 image_ = new VL::PgmBuffer();
272 image_->width = _width;
273 image_->height = _height;
274 image_->data = im_pt;
275
276 //#ifdef SIFTPP_TIMETRACKER
277 tt_->ping_end(ttc_imgconv_);
278 //#endif
279
280 /// Write image to verify correct operation
281 // nothing yet
282
283 //#ifdef SIFTPP_TIMETRACKER
284 tt_->ping_start(ttc_imgfeat_);
285 //#endif
286
287 // COMPUTE IMAGE FEATURES
288 img_features_.clear();
289 img_num_features_ = 0;
290 //img_features_.reserve(1000);
291
292 std::cout << "SiftppClassifier(classify): init scalespace" << std::endl;
293 // initialize scalespace
294 VL::Sift sift(image_->data,
295 image_->width,
296 image_->height,
297 sigman_,
298 sigma0_,
299 octaves_,
300 levels_,
301 first_,
302 -1,
303 levels_ + 1);
304
305 std::cout << "SiftppClassifier(classify): detect image keypoints" << std::endl;
306 // Run SIFTPP detector
307 sift.detectKeypoints(threshold_, edgeThreshold_);
308
309 // Number of keypoints
310 img_num_features_ = sift.keypointsEnd() - sift.keypointsBegin();
311 std::cout << "SiftppClassifier(classify): Extracted '" << img_num_features_ << "' image keypoints"
312 << std::endl;
313
314 // set descriptor options
315 sift.setNormalizeDescriptor(!unnormalized_);
316 sift.setMagnification(magnif_);
317
318 std::cout << "SiftppClassifier(classify): run detector, compute ori and des ..." << std::endl;
319 // Run detector, compute orientations and descriptors
320 for (VL::Sift::KeypointsConstIter iter = sift.keypointsBegin(); iter != sift.keypointsEnd();
321 ++iter) {
322 Feature feat; // = new Feature();
323
324 //std::cout << "SiftppClassifier(classify): saving keypoint" << std::endl;
325 feat.key = (*iter);
326
327 //std::cout << "SiftppClassifier(classify): detect orientations" << std::endl;
328 // detect orientations
329 VL::float_t angles[4];
330 int nangles;
331 if (!noorient_) {
332 nangles = sift.computeKeypointOrientations(angles, *iter);
333 } else {
334 nangles = 1;
335 angles[0] = VL::float_t(0);
336 }
337 feat.number_of_desc = nangles;
338 feat.descs = new VL::float_t *[nangles];
339
340 //std::cout << "SiftppClassifier(classify): computing '" << nangles << "' descriptors" << std::endl;
341 // compute descriptors
342 for (int a = 0; a < nangles; ++a) {
343 // compute descriptor
344 feat.descs[a] = new VL::float_t[vlen_];
345 sift.computeKeypointDescriptor(feat.descs[a], *iter, angles[a]);
346 } // next angle
347 //std::cout << "SiftppClassifier(classify): computed '" << feat.number_of_desc << "' descriptors." << std::endl;
348
349 // save feature
350 img_features_.push_back(feat);
351
352 } // next keypoint
353
354 // Number of feature
355 img_num_features_ = img_features_.size();
356
357 //#ifdef SIFTPP_TIMETRACKER
358 tt_->ping_end(ttc_imgfeat_);
359 //#endif
360
361 std::cout << "SiftppClassifier(classify): Extracted '" << img_num_features_ << "' image features"
362 << std::endl;
363
364 //#ifdef SIFTPP_TIMETRACKER
365 tt_->ping_start(ttc_matchin_);
366 //#endif
367 std::cout << "SiftppClassifier(classify): matching ..." << std::endl;
368
369 std::vector<int> matches(obj_features_.size());
370 int m = 0;
371 for (unsigned i = 0; i < obj_features_.size(); i++) {
372 int match = findMatch(obj_features_[i], img_features_);
373 matches[i] = match;
374 if (match != -1) {
375 std::cout << "SiftppClassifier(classify): Matched feature " << i
376 << " in object image with feature " << match << " in image." << std::endl;
377 /// adding feature-ROI
378 ROI r((int)(img_features_[matches[i]].key.x) - 5,
379 (int)(img_features_[matches[i]].key.y) - 5,
380 11,
381 11,
382 _width,
383 _height);
384 rv->push_back(r);
385 // increment feature-match-count
386 ++m;
387 }
388 }
389
390 //#ifdef SIFTPP_TIMETRACKER
391 tt_->ping_end(ttc_matchin_);
392 //#endif
393 std::cout << "SiftppClassifier(classify) matched '" << m << "' of '" << obj_features_.size()
394 << "' features in scene." << std::endl;
395
396 std::cout << "SiftppClassifier(classify): computing ROI" << std::endl;
397 //#ifdef SIFTPP_TIMETRACKER
398 tt_->ping_start(ttc_roimerg_);
399 //#endif
400
401 for (unsigned i = 0; i < matches.size(); i++) {
402 if (matches[i] != -1) {
403 if ((int)img_features_[matches[i]].key.x < x_min)
404 x_min = (int)img_features_[matches[i]].key.x;
405 if ((int)img_features_[matches[i]].key.y < y_min)
406 y_min = (int)img_features_[matches[i]].key.y;
407 if ((int)img_features_[matches[i]].key.x > x_max)
408 x_max = (int)img_features_[matches[i]].key.x;
409 if ((int)img_features_[matches[i]].key.y > y_max)
410 y_max = (int)img_features_[matches[i]].key.y;
411 }
412 }
413 if (m != 0) {
414 ROI r(x_min, y_min, x_max - x_min, y_max - y_min, _width, _height);
415 rv->push_back(r);
416 }
417
418 //#ifdef SIFTPP_TIMETRACKER
419 tt_->ping_end(ttc_roimerg_);
420 //#endif
421
422 //#ifdef SIFTPP_TIMETRACKER
423 tt_->ping_end(0);
424 //#endif
425
426 //#ifdef SIFTPP_TIMETRACKER
427 // print timetracker statistics
428 tt_->print_to_stdout();
429 //#endif
430
431 delete image_;
432
433 std::cout << "SiftppClassifier(classify): done ... returning '" << rv->size() << "' ROIs."
434 << std::endl;
435 return rv;
436}
437
438int
439SiftppClassifier::findMatch(const Feature &ip1, const std::vector<Feature> &ipts)
440{
441 double mind = 1e100, second = 1e100;
442 int match = -1;
443
444 for (unsigned i = 0; i < ipts.size(); i++) {
445 if (ipts[i].number_of_desc != ip1.number_of_desc)
446 continue;
447 //std::cout << "SiftppClassifier(findMatch): number_of_desc matched!" << std::endl;
448 for (int j = 0; j < ip1.number_of_desc; ++j) {
449 double d = distSquare(ipts[i].descs[j], ip1.descs[j], vlen_);
450
451 if (d < mind) {
452 second = mind;
453 mind = d;
454 match = i;
455 } else if (d < second) {
456 second = d;
457 }
458 }
459 }
460
461 if (mind < 0.5 * second)
462 return match;
463
464 return -1;
465}
466
467double
468SiftppClassifier::distSquare(VL::float_t *v1, VL::float_t *v2, int n)
469{
470 double dsq = 0.;
471 while (n--) {
472 dsq += (v1[n - 1] - v2[n - 1]) * (v1[n - 1] - v2[n - 1]);
473 }
474 //std::cout << " dsq: '" << dsq << "'" << std::endl;
475 return dsq;
476}
477
478} // end namespace firevision
Base class for exceptions in Fawkes.
Definition: exception.h:36
Time tracking utility.
Definition: tracker.h:37
void print_to_stdout()
Print results to stdout.
Definition: tracker.cpp:307
unsigned int add_class(std::string name)
Add a new class.
Definition: tracker.cpp:149
void ping_end(unsigned int cls)
End of given class task.
Definition: tracker.cpp:243
void ping_start(unsigned int cls)
Start of given class task.
Definition: tracker.cpp:218
Classifier to extract regions of interest.
Definition: classifier.h:36
unsigned int _height
Height in pixels of _src buffer.
Definition: classifier.h:53
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
Definition: classifier.h:49
unsigned int _width
Width in pixels of _src buffer.
Definition: classifier.h:51
PNG file reader.
Definition: png.h:34
virtual void set_buffer(unsigned char *yuv422planar_buffer)
Set buffer that the read image should be written to.
Definition: png.cpp:178
virtual unsigned int pixel_height()
Get height of read image in pixels.
Definition: png.cpp:200
virtual colorspace_t colorspace()
Get colorspace from the just read image.
Definition: png.cpp:184
virtual void read()
Read data from file.
Definition: png.cpp:210
virtual unsigned int pixel_width()
Get width of read image in pixels.
Definition: png.cpp:190
Region of interest.
Definition: roi.h:55
SiftppClassifier(const char *features_file, int samplingStep=2, int octaves=4, int levels=3, float magnif=3.0, int noorient=0, int unnormalized=0)
Constructor.
Definition: siftpp.cpp:69
virtual std::list< ROI > * classify()
Classify image.
Definition: siftpp.cpp:241
virtual ~SiftppClassifier()
Destructor.
Definition: siftpp.cpp:230
Fawkes library namespace.
Siftpp Feature struct.
Definition: siftpp.h:64
VL::Sift::Keypoint key
keypoint
Definition: siftpp.h:65
VL::float_t ** descs
descriptors
Definition: siftpp.h:67
int number_of_desc
number of descriptors
Definition: siftpp.h:66