[Bf-blender-cvs] SVN commit: /data/svn/bf-blender [46756] branches/soc-2011-tomato: Add light-normalized tracking to the planar tracker
Keir Mierle
mierle at gmail.com
Fri May 18 04:12:51 CEST 2012
Revision: 46756
http://projects.blender.org/scm/viewvc.php?view=rev&root=bf-blender&revision=46756
Author: keir
Date: 2012-05-18 02:12:47 +0000 (Fri, 18 May 2012)
Log Message:
-----------
Add light-normalized tracking to the planar tracker
This commit adds the ability to normalize patterns by their
average value while tracking, to make them invariant to global
illumination changes.
To see this in action, check out the "Lobby" scene from Hollywood
VFX. If you track the markers that are shadowed by the actress,
previously they would not track. With the scale adaption on, the
tracker would shrink the area to compensate for the changed
illumination, losing the track. With "Normalize" turned on, the
patch is correctly tracked and scale is maintained.
A remaining problem is that only the Ceres cost function is
updated to handle the normalization. The brute translation search
does not take this into account. Perhaps "Prepass" (see below)
should get disabled if normalization is enabled until I fix the
prepass to normalize as well.
There are a few other changes:
- Cleanups in tracking RNA comments.
- Bail out of the sampling loop early if the mask is zero; this
saves expensive samples of the image derivatives.
- Rename the wordy "Translation initialization" to "Prepass" at
Sebastian's suggestion.
- Fix a bug where the mask was ignored when sampling in the cost
functor.
Modified Paths:
--------------
branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc
branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.h
branches/soc-2011-tomato/extern/libmv/libmv-capi.cpp
branches/soc-2011-tomato/extern/libmv/libmv-capi.h
branches/soc-2011-tomato/release/scripts/startup/bl_ui/space_clip.py
branches/soc-2011-tomato/source/blender/blenkernel/intern/tracking.c
branches/soc-2011-tomato/source/blender/makesdna/DNA_tracking_types.h
branches/soc-2011-tomato/source/blender/makesrna/intern/rna_tracking.c
Modified: branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc
===================================================================
--- branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc 2012-05-18 01:02:49 UTC (rev 46755)
+++ branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.cc 2012-05-18 02:12:47 UTC (rev 46756)
@@ -44,6 +44,7 @@
max_iterations(20),
use_esm(true),
use_brute_initialization(true),
+ use_normalized_intensities(false),
sigma(0.9),
num_extra_points(0),
image1_mask(NULL) {
@@ -191,6 +192,14 @@
VLOG(2) << "warp_parameters[" << i << "]: " << warp_parameters[i];
}
+ T src_mean = T(1.0);
+ T dst_mean = T(1.0);
+ if (options_.use_normalized_intensities) {
+ ComputeNormalizingCoefficients(warp_parameters,
+ &src_mean,
+ &dst_mean);
+ }
+
int cursor = 0;
for (int r = 0; r < num_samples_y_; ++r) {
for (int c = 0; c < num_samples_x_; ++c) {
@@ -198,6 +207,19 @@
Vec3 image1_position = canonical_to_image1_ * Vec3(c, r, 1);
image1_position /= image1_position(2);
+ // Sample the mask early; if it's zero, this pixel has no effect. This
+ // allows early bailout from the expensive sampling that happens below.
+ double mask_value = 1.0;
+ if (options_.image1_mask != NULL) {
+ mask_value = AutoDiff<double>::Sample(*options_.image1_mask,
+ image1_position[0],
+ image1_position[1]);
+ if (mask_value == 0.0) {
+ residuals[cursor++] = T(0.0);
+ continue;
+ }
+ }
+
// Compute the location of the destination pixel.
T image2_position[2];
warp_.Forward(warp_parameters,
@@ -206,7 +228,6 @@
&image2_position[0],
&image2_position[1]);
-
// Sample the destination, propagating derivatives.
T dst_sample = AutoDiff<T>::Sample(image_and_gradient2_,
image2_position[0],
@@ -239,6 +260,15 @@
image1_position[1]));
}
+ // Normalize the samples by the mean values of each signal. The typical
+ // light model assumes multiplicative intensity changes with changing
+ // light, so this is a reasonable choice. Note that dst_mean has
+ // derivative information attached thanks to autodiff.
+ if (options_.use_normalized_intensities) {
+ src_sample /= src_mean;
+ dst_sample /= dst_mean;
+ }
+
// The difference is the error.
T error = src_sample - dst_sample;
@@ -248,12 +278,80 @@
image1_position[0],
image1_position[1]));
}
- residuals[cursor++] = src_sample - dst_sample;
+ residuals[cursor++] = error;
}
}
return true;
}
+ // For normalized matching, the average and
+ template<typename T>
+ void ComputeNormalizingCoefficients(const T *warp_parameters,
+ T *src_mean,
+ T *dst_mean) const {
+
+ *src_mean = T(0.0);
+ *dst_mean = T(0.0);
+ double num_samples = 0.0;
+ for (int r = 0; r < num_samples_y_; ++r) {
+ for (int c = 0; c < num_samples_x_; ++c) {
+ // Compute the location of the source pixel (via homography).
+ Vec3 image1_position = canonical_to_image1_ * Vec3(c, r, 1);
+ image1_position /= image1_position(2);
+
+ // Sample the mask early; if it's zero, this pixel has no effect. This
+ // allows early bailout from the expensive sampling that happens below.
+ double mask_value = 1.0;
+ if (options_.image1_mask != NULL) {
+ mask_value = AutoDiff<double>::Sample(*options_.image1_mask,
+ image1_position[0],
+ image1_position[1]);
+ if (mask_value == 0.0) {
+ continue;
+ }
+ }
+
+ // Compute the location of the destination pixel.
+ T image2_position[2];
+ warp_.Forward(warp_parameters,
+ T(image1_position[0]),
+ T(image1_position[1]),
+ &image2_position[0],
+ &image2_position[1]);
+
+
+ // Sample the destination, propagating derivatives.
+ // TODO(keir): This accumulation can, surprisingly, be done as a
+ // pre-pass by using integral images. This is complicated by the need
+ // to store the jets in the integral image, but it is possible.
+ T dst_sample = AutoDiff<T>::Sample(image_and_gradient2_,
+ image2_position[0],
+ image2_position[1]);
+
+ // Sample the source.
+ // TODO(keir): There is no reason to do this inside the loop;
+ // precompute this and reuse it.
+ T src_sample = T(AutoDiff<double>::Sample(image_and_gradient1_,
+ image1_position[0],
+ image1_position[1]));
+
+ // Weight the sample by the mask, if one is present.
+ if (options_.image1_mask != NULL) {
+ src_sample *= T(mask_value);
+ dst_sample *= T(mask_value);
+ }
+
+ *src_mean += src_sample;
+ *dst_mean += dst_sample;
+ num_samples += mask_value;
+ }
+ }
+ *src_mean /= T(num_samples);
+ *dst_mean /= T(num_samples);
+ std::cout << "Normalization for src:\n" << *src_mean << "\n";
+ std::cout << "Normalization for dst:\n" << *dst_mean << "\n";
+ }
+
// TODO(keir): Consider also computing the cost here.
double PearsonProductMomentCorrelationCoefficient(
const double *warp_parameters) const {
Modified: branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.h
===================================================================
--- branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.h 2012-05-18 01:02:49 UTC (rev 46755)
+++ branches/soc-2011-tomato/extern/libmv/libmv/tracking/track_region.h 2012-05-18 02:12:47 UTC (rev 46756)
@@ -57,6 +57,16 @@
// that the nearby minima is correct, or the search area is too small.
bool use_brute_initialization;
+ // If true, normalize the image patches by their mean before doing the sum of
+ // squared error calculation. This is reasonable since the effect of
+ // increasing light intensity is multiplicative on the pixel intensities.
+ //
+ // Note: This does nearly double the solving time, so it is not advised to
+ // turn this on all the time.
+ bool use_normalized_intensities;
+
+ // The size in pixels of the blur kernel used to both smooth the image and
+ // take the image derivative.
double sigma;
// Extra points that should get transformed by the warp. This is useful
Modified: branches/soc-2011-tomato/extern/libmv/libmv-capi.cpp
===================================================================
--- branches/soc-2011-tomato/extern/libmv/libmv-capi.cpp 2012-05-18 01:02:49 UTC (rev 46755)
+++ branches/soc-2011-tomato/extern/libmv/libmv-capi.cpp 2012-05-18 02:12:47 UTC (rev 46756)
@@ -381,6 +381,7 @@
track_region_options.num_extra_points = 1;
track_region_options.image1_mask = NULL;
track_region_options.use_brute_initialization = options->use_brute;
+ track_region_options.use_normalized_intensities = options->use_normalization;
/* Convert from raw float buffers to libmv's FloatImage. */
libmv::FloatImage old_patch, new_patch;
Modified: branches/soc-2011-tomato/extern/libmv/libmv-capi.h
===================================================================
--- branches/soc-2011-tomato/extern/libmv/libmv-capi.h 2012-05-18 01:02:49 UTC (rev 46755)
+++ branches/soc-2011-tomato/extern/libmv/libmv-capi.h 2012-05-18 02:12:47 UTC (rev 46756)
@@ -55,6 +55,7 @@
int motion_model;
int num_iterations;
int use_brute;
+ int use_normalization;
double minimum_correlation;
double sigma;
};
Modified: branches/soc-2011-tomato/release/scripts/startup/bl_ui/space_clip.py
===================================================================
--- branches/soc-2011-tomato/release/scripts/startup/bl_ui/space_clip.py 2012-05-18 01:02:49 UTC (rev 46755)
+++ branches/soc-2011-tomato/release/scripts/startup/bl_ui/space_clip.py 2012-05-18 02:12:47 UTC (rev 46756)
@@ -189,6 +189,7 @@
col.label(text="Tracker:")
col.prop(settings, "default_motion_model")
col.prop(settings, "default_use_brute")
+ col.prop(settings, "default_use_normalization")
col.prop(settings, "default_correlation_min")
col.separator()
@@ -488,6 +489,7 @@
if active:
col.prop(active, "motion_model")
col.prop(active, "use_brute")
+ col.prop(active, "use_normalization")
col.prop(active, "correlation_min")
col.separator()
Modified: branches/soc-2011-tomato/source/blender/blenkernel/intern/tracking.c
===================================================================
--- branches/soc-2011-tomato/source/blender/blenkernel/intern/tracking.c 2012-05-18 01:02:49 UTC (rev 46755)
+++ branches/soc-2011-tomato/source/blender/blenkernel/intern/tracking.c 2012-05-18 02:12:47 UTC (rev 46756)
@@ -1735,6 +1735,7 @@
/* Configure the tracker */
options.motion_model = track->motion_model;
options.use_brute = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) == 0);
+ options.use_normalization = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) == 0);
options.num_iterations = 50;
options.minimum_correlation = track->minimum_correlation;
options.sigma = 0.9;
Modified: branches/soc-2011-tomato/source/blender/makesdna/DNA_tracking_types.h
===================================================================
--- branches/soc-2011-tomato/source/blender/makesdna/DNA_tracking_types.h 2012-05-18 01:02:49 UTC (rev 46755)
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list