[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [46706] branches/soc-2011-tomato: Add new planar tracker features and use the new planar API

Keir Mierle mierle at gmail.com
Thu May 17 04:31:54 CEST 2012


Revision: 46706
          http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=46706
Author:   keir
Date:     2012-05-17 02:31:52 +0000 (Thu, 17 May 2012)
Log Message:
-----------
Add new planar tracker features and use the new planar API

This commit removes the use of the legacy RegionTracker API from
Blender, and replaces it with the new TrackRegion API. This also
adds several features to the planar tracker in libmv:

- Do a brute-force initialization of tracking similar to "Hybrid"
  mode in the stable release, but using all floats. This is slower
  but more accurate. It is still necessary to evaluate if the
  performance loss is worth it. In particular, this change is
  necessary to support high bit depth imagery.

- Add support for masks over the search window. This is a step
  towards supporting user-defined tracker masks. The tracker masks
  will make it easy for users to make a mask for e.g. a ball.

- Add Pearson product moment correlation coefficient checking (aka
  "Correlation" in the UI. This causes tracking failure if the
  tracked patch is not linearly related to the template.

- Add support for warping a few points in addition to the supplied
  points. This is useful because the tracking code deliberately
  does not expose the underlying warp representation. Instead,
  warps are specified in an aparametric way via the correspondences.

- Remove the "num_samples_xy" concept and replace it with
  automatic determination of the number of samples. This makes the
  API easier for users.

- Fix various bugs in the parameterizations.

There remains a bug with subpixel precision tracking when in
"keyframe" mode; this will get fixed shortly.

Modified Paths:
--------------
    branches/soc-2011-tomato/extern/libmv/libmv/tracking/esm_region_tracker.cc
    branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc
    branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.h
    branches/soc-2011-tomato/extern/libmv/libmv-capi.cpp
    branches/soc-2011-tomato/extern/libmv/libmv-capi.h
    branches/soc-2011-tomato/source/blender/blenkernel/intern/tracking.c
    branches/soc-2011-tomato/source/blender/makesdna/DNA_tracking_types.h

Modified: branches/soc-2011-tomato/extern/libmv/libmv/tracking/esm_region_tracker.cc
===================================================================
--- branches/soc-2011-tomato/extern/libmv/libmv/tracking/esm_region_tracker.cc	2012-05-17 01:42:06 UTC (rev 46705)
+++ branches/soc-2011-tomato/extern/libmv/libmv/tracking/esm_region_tracker.cc	2012-05-17 02:31:52 UTC (rev 46706)
@@ -98,8 +98,6 @@
 
   TrackRegionOptions options;
   options.mode = TrackRegionOptions::TRANSLATION;
-  options.num_samples_x = 2 * half_window_size + 1;
-  options.num_samples_y = 2 * half_window_size + 1;
   options.max_iterations = 20;
   options.sigma = sigma;
   options.use_esm = true;

Modified: branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc
===================================================================
--- branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc	2012-05-17 01:42:06 UTC (rev 46705)
+++ branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc	2012-05-17 02:31:52 UTC (rev 46706)
@@ -38,6 +38,19 @@
 
 namespace libmv {
 
+TrackRegionOptions::TrackRegionOptions()
+    : mode(TRANSLATION),
+      minimum_correlation(0),
+      max_iterations(20),
+      use_esm(true),
+      use_brute_initialization(true),
+      sigma(0.9),
+      num_extra_points(0),
+      image1_mask(NULL) {
+}
+
+namespace {
+
 // TODO(keir): Consider adding padding.
 template<typename T>
 bool InBounds(const FloatImage &image,
@@ -161,22 +174,26 @@
                   const FloatImage &image_and_gradient1,
                   const FloatImage &image_and_gradient2,
                   const Mat3 &canonical_to_image1,
+                  int num_samples_x,
+                  int num_samples_y,
                   const Warp &warp)
       : options_(options),
         image_and_gradient1_(image_and_gradient1),       
         image_and_gradient2_(image_and_gradient2),       
         canonical_to_image1_(canonical_to_image1),
+        num_samples_x_(num_samples_x),
+        num_samples_y_(num_samples_y),
         warp_(warp) {}
 
- template<typename T>
- bool operator()(const T *warp_parameters, T *residuals) const {
-   for (int i = 0; i < Warp::NUM_PARAMETERS; ++i) {
-     VLOG(2) << "warp_parameters[" << i << "]: " << warp_parameters[i];
-   }
+  template<typename T>
+  bool operator()(const T *warp_parameters, T *residuals) const {
+    for (int i = 0; i < Warp::NUM_PARAMETERS; ++i) {
+      VLOG(2) << "warp_parameters[" << i << "]: " << warp_parameters[i];
+    }
 
-   int cursor = 0;
-   for (int r = 0; r < options_.num_samples_y; ++r) {
-     for (int c = 0; c < options_.num_samples_x; ++c) {
+    int cursor = 0;
+    for (int r = 0; r < num_samples_y_; ++r) {
+      for (int c = 0; c < num_samples_x_; ++c) {
         // Compute the location of the source pixel (via homography).
         Vec3 image1_position = canonical_to_image1_ * Vec3(c, r, 1);
         image1_position /= image1_position(2);
@@ -223,17 +240,102 @@
         }
 
         // The difference is the error.
+        T error = src_sample - dst_sample;
+
+        // Weight the error by the mask, if one is present.
+        if (options_.image1_mask != NULL) {
+          error *= T(AutoDiff<double>::Sample(*options_.image1_mask,
+                                              image1_position[0],
+                                              image1_position[1]));
+        }
         residuals[cursor++] = src_sample - dst_sample;
       }
     }
     return true;
   }
 
+ // TODO(keir): Consider also computing the cost here.
+ double PearsonProductMomentCorrelationCoefficient(
+     const double *warp_parameters) const {
+   for (int i = 0; i < Warp::NUM_PARAMETERS; ++i) {
+     VLOG(2) << "Correlation warp_parameters[" << i << "]: "
+             << warp_parameters[i];
+   }
+
+   // The single-pass PMCC computation is somewhat numerically unstable, but
+   // it's sufficient for the tracker.
+   double sX = 0, sY = 0, sXX = 0, sYY = 0, sXY = 0;
+
+   // Due to masking, it's important to account for fractional samples.
+   // Samples with a 50% mask get counted as a half sample.
+   double num_samples = 0;
+
+   for (int r = 0; r < num_samples_y_; ++r) {
+     for (int c = 0; c < num_samples_x_; ++c) {
+        // Compute the location of the source pixel (via homography).
+        // TODO(keir): Cache these projections.
+        Vec3 image1_position = canonical_to_image1_ * Vec3(c, r, 1);
+        image1_position /= image1_position(2);
+        
+        // Compute the location of the destination pixel.
+        double image2_position[2];
+        warp_.Forward(warp_parameters,
+                      image1_position[0],
+                      image1_position[1],
+                      &image2_position[0],
+                      &image2_position[1]);
+
+        double x = AutoDiff<double>::Sample(image_and_gradient2_,
+                                            image2_position[0],
+                                            image2_position[1]);
+
+        double y = AutoDiff<double>::Sample(image_and_gradient1_,
+                                            image1_position[0],
+                                            image1_position[1]);
+
+        // Weight the signals by the mask, if one is present.
+        if (options_.image1_mask != NULL) {
+          double mask_value = AutoDiff<double>::Sample(*options_.image1_mask,
+                                                       image1_position[0],
+                                                       image1_position[1]);
+          x *= mask_value;
+          y *= mask_value;
+          num_samples += mask_value;
+        } else {
+          num_samples++;
+        }
+        sX += x;
+        sY += y;
+        sXX += x*x;
+        sYY += y*y;
+        sXY += x*y;
+      }
+    }
+    // Normalize.
+    sX /= num_samples;
+    sY /= num_samples;
+    sXX /= num_samples;
+    sYY /= num_samples;
+    sXY /= num_samples;
+
+    double var_x = sXX - sX*sX;
+    double var_y = sYY - sY*sY;
+    double covariance_xy = sXY - sX*sY;
+
+    double correlation = covariance_xy / sqrt(var_x * var_y);
+    LG << "Covariance xy: " << covariance_xy
+       << ", var 1: " << var_x << ", var 2: " << var_y
+       << ", correlation: " << correlation;
+    return correlation;
+  }
+
  private:
   const TrackRegionOptions &options_;
   const FloatImage &image_and_gradient1_;
   const FloatImage &image_and_gradient2_;
   const Mat3 &canonical_to_image1_;
+  int num_samples_x_;
+  int num_samples_y_;
   const Warp &warp_;
 };
 
@@ -260,9 +362,7 @@
 
 class Quad {
  public:
-  Quad(const double *x, const double *y)
-      : x_(x), y_(y) {
-
+  Quad(const double *x, const double *y) : x_(x), y_(y) {
     // Compute the centroid and store it.
     centroid_ = Vec2(0.0, 0.0);
     for (int i = 0; i < 4; ++i) {
@@ -298,7 +398,7 @@
 struct TranslationWarp {
   TranslationWarp(const double *x1, const double *y1,
                   const double *x2, const double *y2) {
-    Vec2 t = Quad(x1, y1).Centroid() - Quad(x2, y2).Centroid();
+    Vec2 t = Quad(x2, y2).Centroid() - Quad(x1, y1).Centroid() ;
     parameters[0] = t[0];
     parameters[1] = t[1];
   }
@@ -310,6 +410,13 @@
     *y2 = y1 + warp_parameters[1];
   }
 
+  template<typename T>
+  void Backward(const T *warp_parameters,
+                const T &x2, const T& y2, T *x1, T* y1) const {
+    *x1 = x2 - warp_parameters[0];
+    *y1 = y2 - warp_parameters[1];
+  }
+
   // Translation x, translation y.
   enum { NUM_PARAMETERS = 2 };
   double parameters[NUM_PARAMETERS];
@@ -322,7 +429,7 @@
     Quad q2(x2, y2);
 
     // The difference in centroids is the best guess for translation.
-    Vec2 t = q1.Centroid() - q2.Centroid();
+    Vec2 t = q2.Centroid() - q1.Centroid();
     parameters[0] = t[0];
     parameters[1] = t[1];
 
@@ -354,6 +461,12 @@
     *y2 = y1_scaled + warp_parameters[1];
   }
 
+  template<typename T>
+  void Backward(const T *warp_parameters,
+                const T &x2, const T& y2, T *x1, T* y1) const {
+    // XXX
+  }
+
   // Translation x, translation y, scale.
   enum { NUM_PARAMETERS = 3 };
   double parameters[NUM_PARAMETERS];
@@ -375,7 +488,7 @@
     Quad q2(x2, y2);
 
     // The difference in centroids is the best guess for translation.
-    Vec2 t = q1.Centroid() - q2.Centroid();
+    Vec2 t = q2.Centroid() - q1.Centroid();
     parameters[0] = t[0];
     parameters[1] = t[1];
 
@@ -387,6 +500,10 @@
     }
     Mat2 R = OrthogonalProcrustes(correlation_matrix);
     parameters[2] = acos(R(0, 0));
+
+    std::cout << "correlation_matrix:\n" << correlation_matrix << "\n";
+    std::cout << "R:\n" << R << "\n";
+    std::cout << "theta:" << parameters[2] << "\n";
   }
 
   // The strange way of parameterizing the translation and rotation is to make
@@ -422,6 +539,12 @@
     *y2 = y1_rotated + warp_parameters[1];
   }
 
+  template<typename T>
+  void Backward(const T *warp_parameters,
+                const T &x2, const T& y2, T *x1, T* y1) const {
+    // XXX
+  }
+
   // Translation x, translation y, rotation about the center of Q1 degrees.
   enum { NUM_PARAMETERS = 3 };
   double parameters[NUM_PARAMETERS];
@@ -436,7 +559,7 @@
     Quad q2(x2, y2);
 
     // The difference in centroids is the best guess for translation.
-    Vec2 t = q1.Centroid() - q2.Centroid();
+    Vec2 t = q2.Centroid() - q1.Centroid();
     parameters[0] = t[0];
     parameters[1] = t[1];
 
@@ -449,8 +572,11 @@
       correlation_matrix += q1.CornerRelativeToCentroid(i) * 
                             q2.CornerRelativeToCentroid(i).transpose();
     }
+    std::cout << "correlation_matrix:\n" << correlation_matrix << "\n";
     Mat2 R = OrthogonalProcrustes(correlation_matrix);
+    std::cout << "R:\n" << R << "\n";
     parameters[3] = acos(R(0, 0));
+    std::cout << "theta:" << parameters[3] << "\n";
   }
 
   // The strange way of parameterizing the translation and rotation is to make
@@ -471,7 +597,7 @@
     const T y1_origin = y1 - q1.Centroid()(1);
 
     // Rotate about the origin (i.e. centroid of Q1).
-    const T theta = warp_parameters[2];
+    const T theta = warp_parameters[3];
     const T costheta = cos(theta);
     const T sintheta = sin(theta);
     const T x1_origin_rotated = costheta * x1_origin - sintheta * y1_origin;
@@ -491,6 +617,12 @@
     *y2 = y1_rotated_scaled + warp_parameters[1];
   }
 
+  template<typename T>
+  void Backward(const T *warp_parameters,
+                const T &x2, const T& y2, T *x1, T* y1) const {
+    // XXX
+  }
+

@@ Diff output truncated at 10240 characters. @@



More information about the Bf-blender-cvs mailing list