[Bf-blender-cvs] [b8e3394] soc-2016-multiview: Add keyframe selection in multi-view solver mode
Tianwei Shen
noreply at git.blender.org
Mon Jul 11 04:50:58 CEST 2016
Commit: b8e339470eebb0f598b2aa7e85a7863957d2ec81
Author: Tianwei Shen
Date: Mon Jul 11 10:46:09 2016 +0800
Branches: soc-2016-multiview
https://developer.blender.org/rBb8e339470eebb0f598b2aa7e85a7863957d2ec81
Add keyframe selection in multi-view solver mode
===================================================================
M intern/libmv/CMakeLists.txt
M intern/libmv/intern/reconstructionN.cc
M intern/libmv/libmv/autotrack/intersect.cc
A intern/libmv/libmv/autotrack/keyframe_selection.cc
A intern/libmv/libmv/autotrack/keyframe_selection.h
M intern/libmv/libmv/autotrack/reconstruction.cc
M intern/libmv/libmv/autotrack/reconstruction.h
M intern/libmv/libmv/autotrack/tracks.cc
M intern/libmv/libmv/autotrack/tracks.h
===================================================================
diff --git a/intern/libmv/CMakeLists.txt b/intern/libmv/CMakeLists.txt
index f1cc270..e277405 100644
--- a/intern/libmv/CMakeLists.txt
+++ b/intern/libmv/CMakeLists.txt
@@ -77,6 +77,7 @@ if(WITH_LIBMV)
libmv/autotrack/autotrack.cc
libmv/autotrack/bundle.cc
libmv/autotrack/intersect.cc
+ libmv/autotrack/keyframe_selection.cc
libmv/autotrack/pipeline.cc
libmv/autotrack/predict_tracks.cc
libmv/autotrack/reconstruction.cc
@@ -134,6 +135,7 @@ if(WITH_LIBMV)
libmv/autotrack/callbacks.h
libmv/autotrack/frame_accessor.h
libmv/autotrack/intersect.h
+ libmv/autotrack/keyframe_selection.h
libmv/autotrack/marker.h
libmv/autotrack/model.h
libmv/autotrack/pipeline.h
diff --git a/intern/libmv/intern/reconstructionN.cc b/intern/libmv/intern/reconstructionN.cc
index e51f689..c9b88e6 100644
--- a/intern/libmv/intern/reconstructionN.cc
+++ b/intern/libmv/intern/reconstructionN.cc
@@ -33,6 +33,7 @@
#include "libmv/autotrack/autotrack.h"
#include "libmv/autotrack/bundle.h"
#include "libmv/autotrack/frame_accessor.h"
+#include "libmv/autotrack/keyframe_selection.h"
#include "libmv/autotrack/marker.h"
#include "libmv/autotrack/model.h"
#include "libmv/autotrack/pipeline.h"
@@ -105,7 +106,7 @@ bool ReconstructionUpdateFixedIntrinsics(libmv_ReconstructionN **all_libmv_recon
Reconstruction *reconstruction)
{
int clip_num = tracks->GetClipNum();
- for(int i = 0; i < clip_num; i++) {
+ for (int i = 0; i < clip_num; i++) {
CameraIntrinsics *camera_intrinsics = all_libmv_reconstruction[i]->intrinsics;
int cam_intrinsic_index = reconstruction->AddCameraIntrinsics(camera_intrinsics);
assert(cam_intrinsic_index == i);
@@ -164,6 +165,75 @@ void finishMultiviewReconstruction(
camera_intrinsics);
}
+bool selectTwoClipKeyframesBasedOnGRICAndVariance(
+ const int clip_index,
+ Tracks& tracks,
+ Tracks& normalized_tracks,
+ CameraIntrinsics& camera_intrinsics,
+ int& keyframe1,
+ int& keyframe2) {
+ libmv::vector<int> keyframes;
+
+ /* Get list of all keyframe candidates first. */
+ mv::SelectClipKeyframesBasedOnGRICAndVariance(clip_index,
+ normalized_tracks,
+ camera_intrinsics,
+ keyframes);
+
+ if (keyframes.size() < 2) {
+ LG << "Not enough keyframes detected by GRIC";
+ return false;
+ } else if (keyframes.size() == 2) {
+ keyframe1 = keyframes[0];
+ keyframe2 = keyframes[1];
+ return true;
+ }
+
+ /* Now choose two keyframes with minimal reprojection error after initial
+ * reconstruction choose keyframes with the least reprojection error after
+ * solving from two candidate keyframes.
+ *
+ * In fact, currently libmv returns single pair only, so this code will
+ * not actually run. But in the future this could change, so let's stay
+ * prepared.
+ */
+ int previous_keyframe = keyframes[0];
+ double best_error = std::numeric_limits<double>::max();
+ for (int i = 1; i < keyframes.size(); i++) {
+ Reconstruction reconstruction;
+ int current_keyframe = keyframes[i];
+ libmv::vector<mv::Marker> keyframe_markers;
+ normalized_tracks.GetMarkersForTracksInBothFrames(clip_index, previous_keyframe,
+ clip_index, current_keyframe,
+ &keyframe_markers);
+
+ Tracks keyframe_tracks(keyframe_markers);
+
+ /* get a solution from two keyframes only */
+ mv::ReconstructTwoFrames(keyframe_markers, 0, &reconstruction);
+ mv::EuclideanBundleAll(keyframe_tracks, &reconstruction);
+ mv::EuclideanCompleteMultiviewReconstruction(keyframe_tracks, &reconstruction, NULL);
+
+ double current_error = mv::EuclideanReprojectionError(tracks,
+ reconstruction,
+ camera_intrinsics);
+
+ LG << "Error between " << previous_keyframe
+ << " and " << current_keyframe
+ << ": " << current_error;
+
+ if (current_error < best_error) {
+ best_error = current_error;
+ keyframe1 = previous_keyframe;
+ keyframe2 = current_keyframe;
+ }
+
+ previous_keyframe = current_keyframe;
+ }
+
+ return true;
+}
+
// re-apply camera intrinsics on the normalized 2d points
Marker libmv_projectMarker(const mv::Point& point,
const mv::CameraPose& camera,
@@ -226,7 +296,7 @@ libmv_ReconstructionN** libmv_solveMultiviewReconstruction(
///* keyframe selection. */
keyframe1 = libmv_reconstruction_options->keyframe1;
keyframe2 = libmv_reconstruction_options->keyframe2;
- normalized_tracks.GetMarkersForTracksInBothImages(i, keyframe1, i, keyframe2, &keyframe_markers);
+ normalized_tracks.GetMarkersForTracksInBothFrames(i, keyframe1, i, keyframe2, &keyframe_markers);
}
}
// make reconstrution on the primary clip reconstruction
@@ -247,33 +317,35 @@ libmv_ReconstructionN** libmv_solveMultiviewReconstruction(
MultiviewReconstructUpdateCallback(progress_update_callback,
callback_customdata);
- // TODO(tianwei): skip the automatic keyframe selection for now
- //if (libmv_reconstruction_options->select_keyframes) {
- // LG << "Using automatic keyframe selection";
+ if (libmv_reconstruction_options->select_keyframes) {
+ LG << "Using automatic keyframe selection";
- // update_callback.invoke(0, "Selecting keyframes");
+ update_callback.invoke(0, "Selecting keyframes");
- // selectTwoKeyframesBasedOnGRICAndVariance(tracks,
- // normalized_tracks,
- // *camera_intrinsics,
- // keyframe1,
- // keyframe2);
+ // select two keyframes from the primary camera (camera_index == 0)
+ selectTwoClipKeyframesBasedOnGRICAndVariance(0,
+ all_tracks,
+ all_normalized_tracks,
+ *all_libmv_reconstruction[0]->intrinsics,
+ keyframe1,
+ keyframe2);
- // /* so keyframes in the interface would be updated */
- // libmv_reconstruction_options->keyframe1 = keyframe1;
- // libmv_reconstruction_options->keyframe2 = keyframe2;
- //}
+ /* so keyframes in the interface would be updated */
+ libmv_reconstruction_options->keyframe1 = keyframe1;
+ libmv_reconstruction_options->keyframe2 = keyframe2;
+ }
///* Actual reconstruction. */
update_callback.invoke(0, "Initial reconstruction");
// update intrinsics mapping from (clip, frame) -> intrinsics
// TODO(tianwei): in the future we may support varing focal length,
- // thus each (clip, frame) should have a unique intrinsics index
+ // thus each (clip, frame) should have a unique intrinsics index.
+ // This function has to be called before ReconstructTwoFrames.
ReconstructionUpdateFixedIntrinsics(all_libmv_reconstruction, &all_normalized_tracks, &reconstruction);
// reconstruct two views from the main clip
- if (!mv::ReconstructTwoFrames(keyframe_markers, 0, *(all_libmv_reconstruction[0]->intrinsics), &reconstruction)) {
+ if (!mv::ReconstructTwoFrames(keyframe_markers, 0, &reconstruction)) {
LG << "mv::ReconstrucTwoFrames failed\n";
all_libmv_reconstruction[0]->is_valid = false;
return all_libmv_reconstruction;
diff --git a/intern/libmv/libmv/autotrack/intersect.cc b/intern/libmv/libmv/autotrack/intersect.cc
index e6d5139..67649b5 100644
--- a/intern/libmv/libmv/autotrack/intersect.cc
+++ b/intern/libmv/libmv/autotrack/intersect.cc
@@ -86,7 +86,7 @@ bool EuclideanIntersect(const vector<Marker> &markers,
vector<Mat34> cameras;
Mat34 P;
for (int i = 0; i < markers.size(); ++i) {
- LG << "[Intersect] marker clip frame: " << markers[i].clip << " " << markers[i].frame << std::endl;
+ LG << "marker clip and frame: " << markers[i].clip << " " << markers[i].frame << std::endl;
CameraPose *camera = reconstruction->CameraPoseForFrame(markers[i].clip, markers[i].frame);
libmv::P_From_KRt(K, camera->R, camera->t, &P);
cameras.push_back(P);
diff --git a/intern/libmv/libmv/autotrack/keyframe_selection.cc b/intern/libmv/libmv/autotrack/keyframe_selection.cc
new file mode 100644
index 0000000..cb8f5bd
--- /dev/null
+++ b/intern/libmv/libmv/autotrack/keyframe_selection.cc
@@ -0,0 +1,455 @@
+// Copyright (c) 2016 libmv authors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal in the Software without restriction, including without limitation the
+// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+// sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+// IN THE SOFTWARE.
+//
+// Author: Tianwei Shen <shentianweipku at gmail.com>
+// adapted from simple_pipeline/keyframe_selection.cc
+
+#include "libmv/autotrack/keyframe_selection.h"
+
+#include "libmv/numeric/numeric.h"
+#include "ceres/ceres.h"
+#include "libmv/logging/logging.h"
+#inclu
@@ Diff output truncated at 10240 characters. @@
More information about the Bf-blender-cvs
mailing list