Prelimary working version of multi-frame reconstruction in an frame iterator

CMakeLists.txt

@@ -75,7 +75,7 @@ set(HEADERS
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/Landmark.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/LandmarkMapper.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/Mesh.hpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/BufferedVideoIterator.hpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/video/BufferedVideoIterator.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/landmark_utils.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/morphablemodel/PcaModel.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/morphablemodel/MorphableModel.hpp

include/eos/core/landmark_utils.hpp

@@ -10,6 +10,7 @@
 
 #include "eos/morphablemodel/morphablemodel.hpp"
 #include "eos/core/LandmarkMapper.hpp"
+#include "eos/core/Landmark.hpp"
 
 #include "boost/filesystem.hpp"
 
@@ -38,10 +39,11 @@ namespace fs = boost::filesystem;
  */
 namespace eos {
 	namespace core {
-		eos::core::LandmarkCollection <cv::Vec2f> read_pts_landmarks(std::string filename) {
+		template<typename vec2f>
+		eos::core::LandmarkCollection <vec2f> read_pts_landmarks(std::string filename) {
 			using std::getline;
 
-			LandmarkCollection <Vec2f> landmarks;
+			LandmarkCollection<vec2f> landmarks;
 			landmarks.reserve(68);
 
 			std::ifstream file(filename);
@@ -62,7 +64,7 @@ namespace eos {
 				}
 
 				std::stringstream lineStream(line);
-				Landmark <Vec2f> landmark;
+				Landmark <vec2f> landmark;
 				landmark.name = std::to_string(ibugId);
 
 				if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
@@ -90,10 +92,11 @@ namespace eos {
 		 * @param landmark_mapper
 		 * @return std::pair<std::vector<Vec4f>, std::vector<int> model_points and vertex_indices.
 		 */
-		std::pair <std::vector<Eigen::Vector3f>, std::vector<int>>
-		load_model_data(eos::core::LandmarkCollection<cv::Vec2f> landmarks,
+		template<typename vec2f, typename vec3f>
+		std::pair <std::vector<vec2f>, std::vector<int>>
+		load_model_data(eos::core::LandmarkCollection<vec2f> landmarks,
 		                morphablemodel::MorphableModel morphable_model, eos::core::LandmarkMapper landmark_mapper) {
-			std::vector <Eigen::Vector3f> model_points;
+			std::vector <vec3f> model_points;
 			std::vector<int> vertex_indices;
 
 			// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
@@ -103,7 +106,7 @@ namespace eos {
 					continue;
 				}
 				int vertex_idx = std::stoi(converted_name.get());
-				Eigen::Vector3f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
+				vec3f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
 				model_points.emplace_back(vertex);
 				vertex_indices.emplace_back(vertex_idx);
 			}
@@ -119,34 +122,33 @@ namespace eos {
 		* @throws std::runtime_error in case of faulty annotation file
 		* @return std::vector<std::vector<cv::Vec2f>> image_points in a vector of OpenCV float pairs (Vec2f).
 		*/
-		std::vector <std::vector<cv::Vec2f>> load_annotations(std::vector <std::string> annotations, fs::path mappingsfile) {
-			std::vector <std::vector<cv::Vec2f>> image_points; // the corresponding 2D landmark points of all annotation files.
+		template<typename vec2f, typename vec3f>
+		std::vector<core::LandmarkCollection<vec2f>> load_annotations(std::vector <std::string> annotations, fs::path mappingsfile) {
+			//std::vector <std::vector<vec2f>> image_points; // the corresponding 2D landmark points of all annotation files.
 			eos::core::LandmarkMapper landmark_mapper = eos::core::LandmarkMapper(mappingsfile);
+			std::vector<core::LandmarkCollection<vec2f>> landmark_collection_list;
 
 			// These will be the final 2D and 3D points used for the fitting:
-			std::vector <Vec4f> model_points; // the points in the 3D shape model
+			std::vector <vec3f> model_points; // the points in the 3D shape model
 			std::vector<int> vertex_indices; // their vertex indices
 
 			for (int i = 0; i < annotations.size(); i++) {
-				eos::core::LandmarkCollection <cv::Vec2f> landmarks;
+				eos::core::LandmarkCollection <vec2f> landmarks;
 
 				try {
-					landmarks = read_pts_landmarks(annotations[i]);
-				}
-				catch (const std::runtime_error &e) {
+					landmarks = read_pts_landmarks<vec2f>(annotations[i]);
+				} catch (const std::runtime_error &e) {
 					std::cout << e.what() << std::endl;
 					throw std::runtime_error("Error reading the landmarks:  " + annotations[i]);
 				}
 
-				std::vector <cv::Vec2f> image_points_tmp;
-
 				// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
 				for (int j = 0; j < landmarks.size(); j++) {
-					image_points_tmp.emplace_back(landmarks[i].coordinates);
+					landmark_collection_list.emplace_back(landmarks);
 				}
 			}
 
-			return image_points;
+			return landmark_collection_list;
 		}
 	}
 }

include/eos/fitting/FittingResult.hpp

+/*
+ * eos - A 3D Morphable Model fitting library written in modern C++11/14.
+ *
+ * File: include/eos/fitting/FittingResult.hpp
+ *
+ * Copyright 2016 Patrik Huber
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef FITTINGRESULT_HPP_
+#define FITTINGRESULT_HPP_
+
+#include "eos/fitting/RenderingParameters.hpp"
+
+#include <vector>
+
+namespace eos {
+	namespace fitting {
+
+/**
+ * @brief A.
+ *
+ * A.
+ * Note: Any of them can be empty (=default constructed?).
+ * For example when we fit to a video, and want to save the
+ * final result, which will only have PCA shape coeffs, without the rest.
+ * What happens in serialisation?
+ * Or in that case, we should actually just use save_coefficients?
+ */
+struct FittingResult
+{
+	RenderingParameters rendering_parameters;
+	std::vector<float> pca_shape_coefficients;
+	std::vector<float> blendshape_coefficients;
+};
+
+	} /* namespace fitting */
+} /* namespace eos */
+
+#endif /* FITTINGRESULT_HPP_ */

include/eos/fitting/linear_shape_fitting.hpp

@@ -25,6 +25,7 @@
 #include "eos/morphablemodel/MorphableModel.hpp"
 
 #include "Eigen/QR"
+#include "Eigen/Sparse"
 
 #include "opencv2/core/core.hpp"
 
@@ -37,6 +38,138 @@ namespace eos {
 	namespace fitting {
 
 /**
+* Fits the shape of a Morphable Model to given 2D landmarks (i.e. estimates the maximum likelihood solution of the shape coefficients) as proposed in [1].
+* It's a linear, closed-form solution fitting of the shape, with regularisation (prior towards the mean).
+*
+* [1] O. Aldrian & W. Smith, Inverse Rendering of Faces with a 3D Morphable Model, PAMI 2013.
+*
+* Note: Using less than the maximum number of coefficients to fit is not thoroughly tested yet and may contain an error.
+* Note: Returns coefficients following standard normal distribution (i.e. all have similar magnitude). Why? Because we fit using the normalised basis?
+* Note: The standard deviations given should be a vector, i.e. different for each landmark. This is not implemented yet.
+*
+* @param[in] morphable_model The Morphable Model whose shape (coefficients) are estimated.
+* @param[in] affine_camera_matrix A 3x4 affine camera matrix from model to screen-space (should probably be of type CV_32FC1 as all our calculations are done with float).
+* @param[in] landmarks 2D landmarks from an image to fit the model to.
+* @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
+* @param[in] base_face The base or reference face from where the fitting is started. Usually this would be the models mean face, which is what will be used if the parameter is not explicitly specified.
+* @param[in] lambda The regularisation parameter (weight of the prior towards the mean). Gets normalized by the number of images given.
+* @param[in] num_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients.
+* @param[in] detector_standard_deviation The standard deviation of the 2D landmarks given (e.g. of the detector used), in pixels.
+* @param[in] model_standard_deviation The standard deviation of the 3D vertex points in the 3D model, projected to 2D (so the value is in pixels).
+* @return The estimated shape-coefficients (alphas).
+*/
+inline std::vector<float> fit_shape_to_landmarks_linear_multi(const morphablemodel::MorphableModel& morphable_model, std::vector<cv::Mat> affine_camera_matrix, std::vector<std::vector<cv::Vec2f>>& landmarks, std::vector<std::vector<int>>& vertex_ids, std::vector<Eigen::VectorXf> base_face=std::vector<Eigen::VectorXf>(), float lambda=3.0f, boost::optional<int> num_coefficients_to_fit=boost::optional<int>(), boost::optional<float> detector_standard_deviation=boost::optional<float>(), boost::optional<float> model_standard_deviation=boost::optional<float>())
+{
+
+	assert(affine_camera_matrix.size() == landmarks.size() && landmarks.size() == vertex_ids.size()); // same number of instances (i.e. images/frames) for each of them
+	assert(landmarks.size() == vertex_ids.size());
+
+	using Eigen::VectorXf;
+	using Eigen::MatrixXf;
+
+	int num_coeffs_to_fit = num_coefficients_to_fit.get_value_or(morphable_model.get_shape_model().get_num_principal_components());
+	int num_images = affine_camera_matrix.size();
+
+	// the regularisation has to be adjusted when more than one image is given
+	lambda *= num_images;
+
+	int total_num_landmarks_dimension = 0;
+	for (auto&& l : landmarks) {
+		total_num_landmarks_dimension += l.size();
+	}
+
+	// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
+	// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
+	MatrixXf V_hat_h = MatrixXf::Zero(4 * total_num_landmarks_dimension, num_coeffs_to_fit);
+	int V_hat_h_row_index = 0;
+	// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
+	Eigen::SparseMatrix<float> P (3 * total_num_landmarks_dimension, 4 * total_num_landmarks_dimension);
+	std::vector<Eigen::Triplet<float>> P_coefficients; // list of non-zeros coefficients
+	int P_index = 0;
+	// The variances: Add the 2D and 3D standard deviations.
+	// If the user doesn't provide them, we choose the following:
+	// 2D (detector) standard deviation: In pixel, we follow [1] and choose sqrt(3) as the default value.
+	// 3D (model) variance: 0.0f. It only makes sense to set it to something when we have a different variance for different vertices.
+	// The 3D variance has to be projected to 2D (for details, see paper [1]) so the units do match up.
+	float sigma_squared_2D = std::pow(detector_standard_deviation.get_value_or(std::sqrt(3.0f)), 2) + std::pow(model_standard_deviation.get_value_or(0.0f), 2);
+	// We use a VectorXf, and later use .asDiagonal():
+	VectorXf Omega = VectorXf::Constant(3 * total_num_landmarks_dimension, 1.0f / sigma_squared_2D);
+	// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
+	VectorXf y = VectorXf::Ones(3 * total_num_landmarks_dimension);
+	int y_index = 0; // also runs the same as P_index. Should rename to "running_index"?
+	// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
+	VectorXf v_bar = VectorXf::Ones(4 * total_num_landmarks_dimension);
+	int v_bar_index = 0; // also runs the same as P_index. But be careful, if I change it to be only 1 variable, only increment it once! :-)
+	// Well I think that would make it a bit messy since we need to increment inside the for (landmarks...) loop. Try to refactor some other way.
+
+	for (int k = 0; k < num_images; ++k)
+	{
+		// For each image we have, set up the equations and add it to the matrices:
+		assert(landmarks[k].size() == vertex_ids[k].size()); // has to be valid for each img
+
+		int num_landmarks = static_cast<int>(landmarks[k].size());
+
+		if (base_face[k].size()==0)
+		{
+			base_face[k] = morphable_model.get_shape_model().get_mean();
+		}
+
+		// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
+		// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
+		//Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
+		for (int i = 0; i < num_landmarks; ++i) {
+			MatrixXf basis_rows_ = morphable_model.get_shape_model().get_rescaled_pca_basis_at_point(vertex_ids[k][i]); // In the paper, the orthonormal basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the rescaled basis is fine/better.
+			V_hat_h.block(V_hat_h_row_index, 0, 3, V_hat_h.cols()) = basis_rows_.block(0, 0, basis_rows_.rows(), num_coeffs_to_fit);
+			V_hat_h_row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
+		}
+
+		// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
+		for (int i = 0; i < num_landmarks; ++i) {
+			for (int x = 0; x < affine_camera_matrix[k].rows; ++x){
+				for (int y = 0; y < affine_camera_matrix[k].cols; ++y){
+					P_coefficients.push_back(Eigen::Triplet<float>(3 * P_index + x, 4 * P_index + y, affine_camera_matrix[k].at<float>(x,y)));
+				}
+			}
+			++P_index;
+		}
+
+
+		// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
+		//Mat y = Mat::ones(3 * num_landmarks, 1, CV_32FC1);
+		for (int i = 0; i < num_landmarks; ++i) {
+			y(3 * y_index) = landmarks[k][i][0];
+			y((3 * y_index) + 1) = landmarks[k][i][1];
+			//y((3 * i) + 2) = 1; // already 1, stays (homogeneous coordinate)
+			++y_index;
+		}
+		// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
+		//Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
+		for (int i = 0; i < num_landmarks; ++i) {
+			v_bar(4 * v_bar_index) = base_face[k](vertex_ids[k][i] * 3);
+			v_bar((4 * v_bar_index) + 1) = base_face[k](vertex_ids[k][i] * 3 + 1);
+			v_bar((4 * v_bar_index) + 2) = base_face[k](vertex_ids[k][i] * 3 + 2);
+			//v_bar.at<float>((4 * i) + 3) = 1; // already 1, stays (homogeneous coordinate)
+			++v_bar_index;
+		}
+	}
+
+	// fill P with coefficients
+	P.setFromTriplets(P_coefficients.begin(), P_coefficients.end());
+
+	// Bring into standard regularised quadratic form with diagonal distance matrix Omega:
+	const MatrixXf A = P * V_hat_h; // camera matrix times the basis
+	const MatrixXf b = P * v_bar - y; // camera matrix times the mean, minus the landmarks
+	const MatrixXf AtOmegaAReg = A.transpose() * Omega.asDiagonal() * A + lambda * Eigen::MatrixXf::Identity(num_coeffs_to_fit, num_coeffs_to_fit);
+	const MatrixXf rhs = -A.transpose() * Omega.asDiagonal() * b; // It's -A^t*Omega^t*b, but we don't need to transpose Omega, since it's a diagonal matrix, and Omega^t = Omega.
+	// c_s: The 'x' that we solve for. (The variance-normalised shape parameter vector, $c_s = [a_1/sigma_{s,1} , ..., a_m-1/sigma_{s,m-1}]^t$.)
+	// We get coefficients ~ N(0, 1), because we're fitting with the rescaled basis. The coefficients are not multiplied with their eigenvalues.
+	const VectorXf c_s = AtOmegaAReg.colPivHouseholderQr().solve(rhs);
+
+	return std::vector<float>(c_s.data(), c_s.data() + c_s.size());
+};
+
+
+		/**
  * Fits the shape of a Morphable Model to given 2D landmarks (i.e. estimates the maximum likelihood solution of the shape coefficients) as proposed in [1].
  * It's a linear, closed-form solution fitting of the shape, with regularisation (prior towards the mean).
  *

include/eos/core/BufferedVideoIterator.hpp → include/eos/video/BufferedVideoIterator.hpp

@@ -26,8 +26,9 @@
 #include "opencv2/opencv.hpp"
 #include "opencv2/highgui/highgui.hpp"
 
-#include <vector>
 #include <string>
+#include <vector>
+#include <fstream>
 
 using cv::Mat;
 using cv::Vec2f;
@@ -41,100 +42,8 @@ using namespace cv;
 using namespace std;
 
 namespace eos {
-	namespace core {
-		/**
-		 * BufferedVideo Iterator will keep a buffer of the last seen n_frames. By calling .next() it will load a new
-		 * frame from the given video and you will get a pointer to the front of the buffer (which has n_frames).
-		 *
-		 * Just imagine a sliding window accross the video, this is what we aim to implement here.
-		 *
-		 * Example:
-		 *		vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
-		 *
-		 *		std::deque<cv::mat> frames = vid_iterator.next();
-		 *		while(!(frames.empty())) {
-		 *			for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
-		 *				std::cout << ' ' << *it;
-		 *			}
-		 *
-		 *			frames = vid_iterator.next();
-		 *		}
-		 *
-		 * @tparam T
-		 */
-		// Note for this template: later we can use other templates for easy testing (not using cv:Mat but <int> for example).
-		template<typename T>
-		class BufferedVideoIterator {
-		public:
-			BufferedVideoIterator() {};
-
-			// TODO: build support for setting the amount of max_frames in the buffer.
-			BufferedVideoIterator(std::string videoFilePath, std::vector<std::vector<cv::Vec2f>> landmarks) {
-				std::ifstream file(videoFilePath);
-
-				if (!file.is_open()) {
-					throw std::runtime_error("Error opening given file: " + videoFilePath);
-				}
-
-				cv::VideoCapture tmp_cap(videoFilePath); // open video file
-
-				if (!tmp_cap.isOpened()) { // check if we succeeded
-					throw std::runtime_error("Could not play video");
-				}
-
-				cap = tmp_cap;
-				this->landmarks = landmarks;
-			}
-
-			/**
-			 * Set next frame and return frame_buffer.
-			 *
-			 * @return dequeue<Mat> frame buffer.
-			 *
-			 * TODO: build support for returning landmarks AND frames.
-			 */
-			std::deque <Mat> next() {
-				long frame_buffer_length = frame_buffer.size();
-				long landmark_landmark_length = landmark_buffer.size();
-
-				// Get a new frame from camera.
-				Mat frame;
-				cap >> frame;
-
-				// Pop if we exceeded max_frames.
-				if (n_frames > max_frames) {
-					frame_buffer.pop_front();
-				}
-
-				if (frame.empty() == 0) {
-					frame_buffer.push_back(frame);
-				}
-
-				n_frames++;
-
-				return frame_buffer;
-			}
-
-			std::deque <Mat> get_frame_buffer() {
-				return frame_buffer;
-			}
-
-			std::deque <Mat> get_landmark_buffer() {
-				return landmark_buffer;
-			}
-
-		private:
-			cv::VideoCapture cap;
-			std::deque <Mat> frame_buffer;
-			std::deque <Mat> landmark_buffer;
-			std::vector<std::vector<cv::Vec2f>> landmarks;
-
-			// TODO: make set-able
-			// load n_frames at once into the buffer.
-			long n_frames = 1;
-			// keep max_frames into the buffer.
-			long max_frames = 5;
-		};
+	namespace video {
+		// TODO: maybe move video iterator here.. or remove file.
 	}
 }
 

include/eos/video/Keyframe.hpp

@@ -26,9 +26,12 @@
 #include "eos/fitting/RenderingParameters.hpp"
 #include "eos/morphablemodel/Blendshape.hpp"
 #include "eos/morphablemodel/MorphableModel.hpp"
+#include "eos/render/texture_extraction.hpp"
 
 #include "opencv2/core/core.hpp"
 
+#include <deque>
+
 namespace eos {
 namespace video {
 
@@ -37,9 +40,42 @@ namespace video {
  *
  * Contains the original frame, all necessary fitting parameters, and a score.
  */
-struct Keyframe
-{
-    float score; // = 0.0f?
+struct Keyframe {
+public:
+	Keyframe() {}
+	/**
+	 * Only used when only a frame is available.
+	 *
+	 * @param frame
+	 */
+	Keyframe(cv::Mat frame) {
+		this->frame = frame;
+	}
+
+	/**
+	 * Only used when score and frame is available
+	 *
+	 * @param frame
+	 * @param score
+	 */
+	Keyframe(float score, cv::Mat frame) {
+		this->score = score;
+		this->frame = frame;
+	}
+
+	/**
+	 *
+	 * @param score
+	 * @param frame
+	 * @param fitting_result
+	 */
+	Keyframe(float score, cv::Mat frame, fitting::FittingResult fitting_result) {
+		this->frame = frame;
+		this->score = score;
+		this->fitting_result = fitting_result;
+	}
+
+	float score = 0.0f;
 	cv::Mat frame;
 	fitting::FittingResult fitting_result;
 };
@@ -47,11 +83,11 @@ struct Keyframe
 /**
  * @brief A keyframe selection that selects keyframes according to yaw pose and score.
  *
- * Separates the +-90 yaw pose range into 20 intervals (i.e. 90 to 70, ..., -10 to 10, ...), and puts frames
+ * Separates the +-90� yaw pose range into 20� intervals (i.e. 90 to 70, ..., -10 to 10, ...), and puts frames
  * into each bin, until full. Replaces keyframes with better frames if the score is higher than that of
  * current keyframes.
  *
- * The yaw pose bins are currently hard-coded (9 bins, 20 intervals).
+ * The yaw pose bins are currently hard-coded (9 bins, 20� intervals).
  */
 struct PoseBinningKeyframeSelector
 {
@@ -112,7 +148,7 @@ private:
     int frames_per_bin;
 
     // Converts a given yaw angle to an index in the internal bins vector.
-    // Assumes 9 bins and 20 intervals.
+    // Assumes 9 bins and 20� intervals.
     static std::size_t angle_to_index(float yaw_angle)
     {
         if (yaw_angle <= -70.f)
@@ -168,18 +204,21 @@ cv::Mat merge_weighted_mean(const std::vector<Keyframe>& keyframes,
     using std::vector;
 
 	vector<Mat> isomaps;
-    for (const auto& frame_data : keyframes)
-    {
-        const Mat shape =
+	for (const auto& frame_data : keyframes) {
+	//		VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(frame_data.fitting_result.pca_shape_coefficients);
+	//		const current_combined_shape = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Eigen::Map<const Eigen::VectorXf>(frame_data.fitting_result.blendshape_coefficients);
+		const Eigen::VectorXf shape =
 			morphable_model.get_shape_model().draw_sample(frame_data.fitting_result.pca_shape_coefficients) +
-            morphablemodel::to_matrix(blendshapes) * Mat(frame_data.fitting_result.blendshape_coefficients);
+			morphablemodel::to_matrix(blendshapes) * morphablemodel::to_vector(frame_data.fitting_result.blendshape_coefficients);
 		const auto mesh =
-            morphablemodel::sample_to_mesh(shape, {}, morphable_model.get_shape_model().get_triangle_list(),
-                                           {}, morphable_model.get_texture_coordinates());
+			morphablemodel::sample_to_mesh(
+					shape, {}, morphable_model.get_shape_model().get_triangle_list(), {}, morphable_model.get_texture_coordinates());
 		const Mat affine_camera_matrix = fitting::get_3x4_affine_camera_matrix(
-            frame_data.fitting_result.rendering_parameters, frame_data.frame.cols, frame_data.frame.rows);
-        const Mat isomap = render::extract_texture(mesh, affine_camera_matrix, frame_data.frame, true,
-                                                   render::TextureInterpolation::NearestNeighbour, 1024);
+			frame_data.fitting_result.rendering_parameters, frame_data.frame.cols, frame_data.frame.rows
+		);
+		const Mat isomap = render::extract_texture(
+			mesh, affine_camera_matrix, frame_data.frame, true, render::TextureInterpolation::NearestNeighbour, 1024
+		);
 		isomaps.push_back(isomap);
 	}
 
@@ -248,6 +287,113 @@ double variance_of_laplacian(const cv::Mat& image)
     return focus_measure;
 };
 
+/**
+ * BufferedVideo Iterator will keep a buffer of the last seen n_frames. By calling .next() it will load a new
+ * frame from the given video and you will get a pointer to the front of the buffer (which has n_frames).
+ *
+ * Just imagine a sliding window accross the video, this is what we aim to implement here.
+ *
+ * Example:
+ *    vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
+ *
+ *    std::deque<cv::mat> frames = vid_iterator.next();
+ *    while(!(frames.empty())) {
+ *        for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+ *            std::cout << ' ' << *it;
+ *        }
+ *
+ *         frames = vid_iterator.next();
+ *    }
+ *
+ * @tparam T
+ */
+// Note for this template: later we can use other templates for easy testing (not using cv:Mat but <int> for example).
+class BufferedVideoIterator {
+public:
+	int width;
+	int height;
+
+	BufferedVideoIterator() {};
+
+	// TODO: build support for setting the amount of max_frames in the buffer.
+	BufferedVideoIterator(std::string videoFilePath, int max_frames = 5, int min_frames = 4) {
+		std::ifstream file(videoFilePath);
+
+		if (!file.is_open()) {
+			throw std::runtime_error("Error opening given file: " + videoFilePath);
+		}
+
+		cv::VideoCapture cap(videoFilePath); // open video file
+
+		if (!cap.isOpened()) { // check if we succeeded
+			throw std::runtime_error("Could not play video");
+		}
+
+		width = static_cast<int>(cap.get(CV_CAP_PROP_FRAME_HEIGHT));
+		height = static_cast<int>(cap.get(CV_CAP_PROP_FRAME_HEIGHT));
+
+		this->max_frames = max_frames;
+		this->min_frames = min_frames;
+	}
+
+	/**
+	 * Set next frame and return frame_buffer.
+	 *
+	 * @return dequeue<Mat> frame buffer.
+	 *
+	 * TODO: build support for returning landmarks AND frames.
+	 */
+	std::deque <Keyframe> next() {
+		long frame_buffer_length = frame_buffer.size();
+
+		// Get a new frame from camera.
+		cv::Mat frame;
+		cap >> frame;
+
+		// Pop if we exceeded max_frames.
+		if (n_frames > max_frames) {
+			frame_buffer.pop_front();
+		}
+
+		float lap_score = static_cast<float>(variance_of_laplacian(frame));
+
+		if (lap_score < laplacian_threshold) {
+			frame_buffer.push_back(Keyframe(lap_score, frame));
+		}
+
+		n_frames++;
+
+		if(frame_buffer_length + 1 < min_frames) {
+			frame_buffer = next();
+		}
+
+		return frame_buffer;
+	}
+
+	std::deque <Keyframe> get_frame_buffer() {
+		return frame_buffer;
+	}
+
+private:
+	cv::VideoCapture cap;
+	std::deque<Keyframe> frame_buffer;
+
+	// TODO: make set-able
+	// total frames in buffer
+	long n_frames = 1;
+
+	// min frames to load at the start.
+	long min_frames = 3;
+	// keep max_frames into the buffer.
+	long max_frames = 5;
+
+	// Note: these settings are for future use
+	int drop_frames = 0;
+
+	// laplacian threshold
+	double laplacian_threshold = 10000000;
+};
+
 } /* namespace video */
 } /* namespace eos */
 

utils/accuracy-evaluation.cpp

@@ -16,15 +16,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "eos/core/Landmark.hpp"
-#include "eos/core/LandmarkMapper.hpp"
 #include "eos/core/landmark_utils.hpp"
-#include "eos/core/BufferedVideoIterator.hpp"
 #include "eos/morphablemodel/morphablemodel.hpp"
 #include "eos/morphablemodel/blendshape.hpp"
 #include "eos/fitting/fitting.hpp"
 #include "eos/render/utils.hpp"
 #include "eos/render/texture_extraction.hpp"
+#include "eos/video/Keyframe.hpp"
 
 #include "opencv2/core/core.hpp"
 #include "opencv2/opencv.hpp"
@@ -42,7 +40,7 @@ namespace po = boost::program_options;
 namespace fs = boost::filesystem;
 using eos::core::Landmark;
 using eos::core::LandmarkCollection;
-using eos::core::BufferedVideoIterator;
+using eos::video::BufferedVideoIterator;
 using cv::Mat;
 using cv::Vec2f;
 using cv::Vec3f;
@@ -121,83 +119,115 @@ int main(int argc, char *argv[]) {
 		po::variables_map vm;
 		po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
 		if (vm.count("help")) {
-			cout << "Usage: fit-model [options]" << endl;
-			cout << desc;
+			std::cout << "Usage: fit-model [options]" << std::endl;
+			std::cout << desc;
 			return EXIT_SUCCESS;
 		}
 		po::notify(vm);
 	}
 	catch (const po::error &e) {
-		cout << "Error while parsing command-line arguments: " << e.what() << endl;
-		cout << "Use --help to display a list of options." << endl;
+		std::cout << "Error while parsing command-line arguments: " << e.what() << std::endl;
+		std::cout << "Use --help to display a list of options." << std::endl;
 		return EXIT_SUCCESS;
 	}
 
+	// start loading prerequisites
+	morphablemodel::MorphableModel morphable_model;
 	try {
-		vector <vector<Vec2f>> multi_frame_points = eos::core::load_annotations(annotations, mappingsfile);
-	} catch(const std::runtime_error &e) {
-		std::cout << e.what() << std::endl;
+		morphable_model = morphablemodel::load_model(modelfile.string());
+	} catch (const std::runtime_error &e) {
+		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
-	// Load landmarks, LandmarkMapper and the Morphable Model:
-	LandmarkCollection <cv::Vec2f> landmarks;
-	core::LandmarkMapper landmark_mapper = core::LandmarkMapper(mappingsfile);
-
+	// load all annotation files into lists of landmarks
+	vector<core::LandmarkCollection<cv::Vec2f>> landmark_list;
 	try {
-		landmarks = eos::core::read_pts_landmarks(annotations[0]);
-	}
-	catch (const std::runtime_error &e) {
-		cout << "Error reading the landmarks: " << e.what() << endl;
+		landmark_list = eos::core::load_annotations<cv::Vec2f, cv::Vec3f>(annotations, mappingsfile);
+	} catch(const std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
-	morphablemodel::MorphableModel morphable_model;
+	// The expression blendshapes:
+	vector<morphablemodel::Blendshape> blend_shapes = morphablemodel::load_blendshapes(blendshapesfile.string());
 
-	try {
-		morphable_model = morphablemodel::load_model(modelfile.string());
-	} catch (const std::runtime_error &e) {
-		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
-			return EXIT_FAILURE;
-	}
+	// These two are used to fit the front-facing contour to the ibug contour landmarks:
+	fitting::ModelContour model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
+	fitting::ContourLandmarks ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string());
+
+	// The edge topology is used to speed up computation of the occluding face contour fitting:
+	morphablemodel::EdgeTopology edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string());
+
+	// Load landmarks, LandmarkMapper and the Morphable Model:
+	LandmarkCollection <cv::Vec2f> landmarks;
+	core::LandmarkMapper landmark_mapper = core::LandmarkMapper(mappingsfile);
 
 	// These will be the final 2D and 3D points used for the fitting:
-	vector<Eigen::Vector3f > model_points; // the points in the 3D shape model
+	vector<cv::Vec3f> model_points; // the points in the 3D shape model
 	vector<int> vertex_indices; // their vertex indices
-	std::tie(model_points, vertex_indices) = eos::core::load_model_data(landmarks, morphable_model, landmark_mapper);
 
-	BufferedVideoIterator<cv::Mat> vid_iterator;
-	std::vector <std::vector<cv::Vec2f>> landmark_annotation_list = eos::core::load_annotations(annotations, mappingsfile);
+	vector<core::Mesh> meshs;
+	vector<fitting::RenderingParameters> rendering_paramss;
+
+//	std::tie(model_points, vertex_indices) = eos::core::load_model_data<cv::Vec2f, cv::Vec3f>(landmarks, morphable_model, landmark_mapper);
 
+	BufferedVideoIterator vid_iterator;
 	try {
-		vid_iterator = BufferedVideoIterator<cv::Mat>(videofile.string(), landmark_annotation_list);
+		vid_iterator = BufferedVideoIterator(videofile.string());
 	} catch(std::runtime_error &e) {
-		cout << e.what() << endl;
+		std::cout << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
+	// iteration count
+	int frame_width = vid_iterator.width;
+	int frame_height = vid_iterator.height;
 
-	// todo: expand this to really perform some reconstruction, and move this to a test file.
 	// test with loading 10 frames subsequently.
-	// vid_iterator.next() will return a number of frames, depending on
-	std::deque<cv::Mat> frames = vid_iterator.next();
-	int count = 0;
+	std::deque<eos::video::Keyframe> frames = vid_iterator.next();
+
+	std::vector<float> pca_shape_coefficients;
+	std::vector<std::vector<float>> blend_shape_coefficients;
+	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
+
+	int n_frames = static_cast<int>(frames.size());
 	while(!(frames.empty())) {
-		if (count == 10) {
+		if (n_frames == 100) {
 			break;
 		}
-		int frame_count = 0;
-		for (std::deque<cv::Mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
-			//std::cout << ' ' << *it;
-			std::cout << frame_count << " ";
-			frame_count++;
-		}
 
-		std::cout << std::endl << "frames processed: " << count * frame_count << std::endl;
+		int frame_count = 0;
+		std::cout << std::endl << "frames in buffer: " << frames.size() << std::endl;
+		std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
+				morphable_model,
+				blend_shapes,
+				landmark_list,
+				landmark_mapper,
+				frame_width,
+				frame_height,
+				static_cast<int>(frames.size()),
+				edge_topology,
+				ibug_contour,
+				model_contour,
+				50,
+				boost::none,
+				30.0f,
+				boost::none,
+				pca_shape_coefficients,
+				blend_shape_coefficients,
+				fitted_image_points
+		);
 
 		frames = vid_iterator.next();
-		usleep(10);
-		count++;
+
+		n_frames++;
+
+//		// iterator through all frames, not needed persee.
+//		for (std::deque<eos::video::Keyframe>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+//			std::cout << it->score << " ";
+//			frame_count++;
+//		}
 	}
 }