Fawkes API  Fawkes Development Version
sift.cpp
00001 
00002 /***************************************************************************
00003  *  sift.cpp - Feature-based classifier using OpenCV structures
00004  *
00005  *  Created: Mon Mar 15 15:47:11 2008
00006  *  Copyright 2008 Stefan Schiffer [stefanschiffer.de]
00007  *
00008  ****************************************************************************/
00009 
00010 /*  This program is free software; you can redistribute it and/or modify
00011  *  it under the terms of the GNU General Public License as published by
00012  *  the Free Software Foundation; either version 2 of the License, or
00013  *  (at your option) any later version. A runtime exception applies to
00014  *  this software (see LICENSE.GPL_WRE file mentioned below for details).
00015  *
00016  *  This program is distributed in the hope that it will be useful,
00017  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
00018  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00019  *  GNU Library General Public License for more details.
00020  *
00021  *  Read the full text in the LICENSE.GPL_WRE file in the doc directory.
00022  */
00023 
00024 #include <iostream>
00025 #include <vector>
00026 
00027 #include <classifiers/sift.h>
00028 //#ifdef SIFT_TIMETRACKER
00029 #include <utils/time/clock.h>
00030 #include <utils/time/tracker.h>
00031 //#endif
00032 
00033 extern "C" {
00034 #include <sift/sift.h>
00035 #include <sift/imgfeatures.h>
00036 #include <sift/kdtree.h>
00037 #include <sift/utils.h>
00038 #include <sift/xform.h>
00039 }
00040 
00041 #include <core/exception.h>
00042 #include <core/exceptions/software.h>
00043 #include <fvutils/color/colorspaces.h>
00044 #include <fvutils/color/conversions.h>
00045 
00046 #include <opencv/cv.h>
00047 #include <opencv/cxcore.h>
00048 #include <opencv/highgui.h>
00049 
00050 using namespace fawkes;
00051 
00052 namespace firevision {
00053 #if 0 /* just to make Emacs auto-indent happy */
00054 }
00055 #endif
00056 
00057 /** @class SiftClassifier <classifiers/sift.h>
00058  * SIFT classifier.
00059  *
00060  * This class provides a classifier that uses OpenCV to detect objects in a given
00061  * image by matching features using SIFT. The objects are reported back as regions
00062  * of interest.  Each ROI contains an object.
00063  *
00064  * This code is based on the sift package provided by Rob Hess.
00065  * at http://web.engr.oregonstate.edu/~hess/
00066  *
00067  * @author Stefan Schiffer
00068  */
00069 
00070 /** Constructor.
00071  * @param object_file file that contains the object to detect
00072  * @param pixel_width width of images that will be processed
00073  * @param pixel_height height of images that will be processed
00074  * @param kdtree_bbf_max_nn_chks maximum number of keypoint NN candidates to check during BBF search
00075  * @param nn_sq_dist_ratio_thr threshold on squared ratio of distances between NN and 2nd NN
00076  * @param flags flags, not used yet.
00077  */
00078 SiftClassifier::SiftClassifier( const char * object_file,
00079                                       unsigned int pixel_width, unsigned int pixel_height,
00080                                       int kdtree_bbf_max_nn_chks, float nn_sq_dist_ratio_thr, int flags)
00081   : Classifier("SiftClassifier")
00082 {
00083   __kdtree_bbf_max_nn_chks = kdtree_bbf_max_nn_chks;
00084   __nn_sq_dist_ratio_thr = nn_sq_dist_ratio_thr;
00085   __flags = flags;
00086 
00087 
00088   //#ifdef SIFT_TIMETRACKER
00089   __tt = new TimeTracker();
00090   __loop_count = 0;
00091   __ttc_objconv = __tt->add_class("ObjectConvert");
00092   __ttc_objfeat = __tt->add_class("ObjectFeatures");
00093   __ttc_imgconv = __tt->add_class("ImageConvert");
00094   __ttc_imgfeat = __tt->add_class("ImageFeatures");
00095   __ttc_matchin = __tt->add_class("Matching");
00096   __ttc_roimerg = __tt->add_class("MergeROIs");
00097   //#endif
00098 
00099   //#ifdef SIFT_TIMETRACKER
00100   __tt->ping_start(__ttc_objconv);
00101   //#endif
00102   __obj_img = cvLoadImage( object_file, 1 );
00103   if ( ! __obj_img ) {
00104     throw Exception("Could not load object file");
00105   }
00106   //#ifdef SIFT_TIMETRACKER
00107   __tt->ping_end(__ttc_objconv);
00108   //#endif
00109 
00110   //#ifdef SIFT_TIMETRACKER
00111   __tt->ping_start(__ttc_objfeat);
00112   //#endif
00113   __obj_num_features = 0;
00114   __obj_num_features = sift_features( __obj_img, &__obj_features );
00115   if ( ! __obj_num_features > 0 ) {
00116     throw Exception("Could not compute object features");
00117   }
00118   std::cout << "SiftClassifier(classify): computed '" << __obj_num_features << "' features from object" << std::endl;
00119   //cvReleaseImage(&__obj_img);
00120   //#ifdef SIFT_TIMETRACKER
00121   __tt->ping_end(__ttc_objfeat);
00122   //#endif
00123 
00124   // create space for OpenCV image
00125   __image = cvCreateImage(cvSize(pixel_width, pixel_height), IPL_DEPTH_8U, 3);
00126 
00127 }
00128 
00129 
00130 /** Destructor. */
00131 SiftClassifier::~SiftClassifier()
00132 {
00133   //
00134   cvReleaseImage(&__obj_img);
00135   cvReleaseImage(&__image);
00136 }
00137 
00138 
00139 std::list< ROI > *
00140 SiftClassifier::classify()
00141 {
00142   //#ifdef SIFT_TIMETRACKER
00143   __tt->ping_start(0);
00144   //#endif
00145 
00146   // list of ROIs to return
00147   std::list< ROI > *rv = new std::list< ROI >();
00148 
00149   struct feature * feat;
00150   struct feature** nbrs;
00151   struct kd_node* kd_root;
00152   CvPoint pt1, pt2;
00153 
00154   // for ROI calculation
00155   CvPoint ftpt;
00156   std::vector< CvPoint > ftlist;
00157   //= new std::vector< CvPoint >();
00158   int x_min = _width;
00159   int y_min = _height;
00160   int x_max = 0;
00161   int y_max = 0;
00162 
00163   double d0, d1;// = 0.0;
00164   int k, m = 0;
00165 
00166   //#ifdef SIFT_TIMETRACKER
00167   __tt->ping_start(__ttc_imgconv);
00168   //#endif
00169   //std::cout << "SiftClassifier(classify): convert frame to IplImage" << std::endl;
00170   convert(YUV422_PLANAR, BGR, _src, (unsigned char *)__image->imageData, _width, _height);
00171   //#ifdef SIFT_TIMETRACKER
00172   __tt->ping_end(__ttc_imgconv);
00173   //#endif
00174 
00175   //#ifdef SIFT_TIMETRACKER
00176   __tt->ping_start(__ttc_imgfeat);
00177   //#endif
00178   //std::cout << "SiftClassifier(classify): compute features on current frame " << std::endl;
00179   int num_img_ft = sift_features( __image, &__img_features );
00180   kd_root = kdtree_build( __img_features, num_img_ft );
00181   //#ifdef SIFT_TIMETRACKER
00182   __tt->ping_end(__ttc_imgfeat);
00183   //#endif
00184 
00185   if( ! kd_root ) {
00186     std::cerr << "SiftClassifier(classify): KD-Root NULL!" << std::endl;
00187   }
00188 
00189   //#ifdef SIFT_TIMETRACKER
00190   __tt->ping_start(__ttc_matchin);
00191   //#endif
00192   std::cout << "SiftClassifier(classify): matching ..." << std::endl;
00193   for( int i = 0; i < __obj_num_features; ++i ) {
00194     //std::cout << "SiftClassifier(classify): ... feature '" << i << "'" << std::endl;
00195     feat = __obj_features + i;
00196     k = kdtree_bbf_knn( kd_root, feat, 2, &nbrs, __kdtree_bbf_max_nn_chks );
00197     if( k == 2 )
00198       {
00199         d0 = descr_dist_sq( feat, nbrs[0] );
00200         d1 = descr_dist_sq( feat, nbrs[1] );
00201         if( d0 < d1 * __nn_sq_dist_ratio_thr )
00202           {
00203             pt1 = cvPoint( cvRound( feat->x ), cvRound( feat->y ) );
00204             pt2 = cvPoint( cvRound( nbrs[0]->x ), cvRound( nbrs[0]->y ) );
00205             m++;
00206             __obj_features[i].fwd_match = nbrs[0];
00207             // save matched feature points
00208             ftpt = cvPoint( cvRound( nbrs[0]->x), cvRound( nbrs[0]->y ) );
00209             ftlist.push_back(ftpt);
00210             // save matched features as ROIs
00211             ROI r( pt2.x-5, pt2.y-5, 11, 11, _width, _height);
00212             rv->push_back(r);
00213           }
00214        }
00215      free( nbrs );
00216   }
00217   std::cout << "SiftClassifier(classify): found '" << m << "' matches" << std::endl;
00218   kdtree_release( kd_root );
00219   //#ifdef SIFT_TIMETRACKER
00220   __tt->ping_end(__ttc_matchin);
00221   //#endif
00222 
00223   //#ifdef SIFT_TIMETRACKER
00224   __tt->ping_start(__ttc_roimerg);
00225   //#endif
00226   std::cout << "SiftClassifier(classify): computing ROI" << std::endl;
00227   //for ( int i = 0; i < m; ++i) {
00228   for ( std::vector< CvPoint >::size_type i = 0; i < ftlist.size(); ++i) {
00229     if( ftlist[i].x < x_min )
00230       x_min = ftlist[i].x;
00231     if( ftlist[i].y < y_min )
00232       y_min = ftlist[i].y;
00233     if( ftlist[i].x > x_max )
00234       x_max = ftlist[i].x;
00235     if( ftlist[i].y > y_max )
00236       y_max = ftlist[i].y;
00237   }
00238   if( m != 0 ) {
00239     ROI r(x_min, y_min, x_max-x_min, y_max-y_min, _width, _height);
00240     rv->push_back(r);
00241   }
00242   //#ifdef SIFT_TIMETRACKER
00243   __tt->ping_end(__ttc_roimerg);
00244   //#endif
00245 
00246   //#ifdef SIFT_TIMETRACKER
00247   __tt->ping_end(0);
00248   //#endif
00249 
00250   //#ifdef SIFT_TIMETRACKER
00251   __tt->print_to_stdout();
00252   //#endif
00253 
00254   std::cout << "SiftClassifier(classify): done ... returning '" << rv->size() << "' ROIs." << std::endl;
00255   return rv;
00256 }
00257 
00258 } // end namespace firevision