Prelimary working version of multi-frame reconstruction in an frame iterator

CMakeLists.txt

@@ -75,7 +75,7 @@ set(HEADERS
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/Landmark.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/LandmarkMapper.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/Mesh.hpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/BufferedVideoIterator.hpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/video/BufferedVideoIterator.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/core/landmark_utils.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/morphablemodel/PcaModel.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/morphablemodel/MorphableModel.hpp

include/eos/core/landmark_utils.hpp

@@ -10,6 +10,7 @@
 
 #include "eos/morphablemodel/morphablemodel.hpp"
 #include "eos/core/LandmarkMapper.hpp"
+#include "eos/core/Landmark.hpp"
 
 #include "boost/filesystem.hpp"
 
@@ -38,10 +39,11 @@ namespace fs = boost::filesystem;
  */
 namespace eos {
 	namespace core {
-		eos::core::LandmarkCollection <cv::Vec2f> read_pts_landmarks(std::string filename) {
+		template<typename vec2f>
+		eos::core::LandmarkCollection <vec2f> read_pts_landmarks(std::string filename) {
 			using std::getline;
 
-			LandmarkCollection <Vec2f> landmarks;
+			LandmarkCollection<vec2f> landmarks;
 			landmarks.reserve(68);
 
 			std::ifstream file(filename);
@@ -62,7 +64,7 @@ namespace eos {
 				}
 
 				std::stringstream lineStream(line);
-				Landmark <Vec2f> landmark;
+				Landmark <vec2f> landmark;
 				landmark.name = std::to_string(ibugId);
 
 				if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
@@ -90,10 +92,11 @@ namespace eos {
 		 * @param landmark_mapper
 		 * @return std::pair<std::vector<Vec4f>, std::vector<int> model_points and vertex_indices.
 		 */
-		std::pair <std::vector<Eigen::Vector3f>, std::vector<int>>
-		load_model_data(eos::core::LandmarkCollection<cv::Vec2f> landmarks,
+		template<typename vec2f, typename vec3f>
+		std::pair <std::vector<vec2f>, std::vector<int>>
+		load_model_data(eos::core::LandmarkCollection<vec2f> landmarks,
 		                morphablemodel::MorphableModel morphable_model, eos::core::LandmarkMapper landmark_mapper) {
-			std::vector <Eigen::Vector3f> model_points;
+			std::vector <vec3f> model_points;
 			std::vector<int> vertex_indices;
 
 			// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
@@ -103,7 +106,7 @@ namespace eos {
 					continue;
 				}
 				int vertex_idx = std::stoi(converted_name.get());
-				Eigen::Vector3f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
+				vec3f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
 				model_points.emplace_back(vertex);
 				vertex_indices.emplace_back(vertex_idx);
 			}
@@ -119,34 +122,33 @@ namespace eos {
 		* @throws std::runtime_error in case of faulty annotation file
 		* @return std::vector<std::vector<cv::Vec2f>> image_points in a vector of OpenCV float pairs (Vec2f).
 		*/
-		std::vector <std::vector<cv::Vec2f>> load_annotations(std::vector <std::string> annotations, fs::path mappingsfile) {
-			std::vector <std::vector<cv::Vec2f>> image_points; // the corresponding 2D landmark points of all annotation files.
+		template<typename vec2f, typename vec3f>
+		std::vector<core::LandmarkCollection<vec2f>> load_annotations(std::vector <std::string> annotations, fs::path mappingsfile) {
+			//std::vector <std::vector<vec2f>> image_points; // the corresponding 2D landmark points of all annotation files.
 			eos::core::LandmarkMapper landmark_mapper = eos::core::LandmarkMapper(mappingsfile);
+			std::vector<core::LandmarkCollection<vec2f>> landmark_collection_list;
 
 			// These will be the final 2D and 3D points used for the fitting:
-			std::vector <Vec4f> model_points; // the points in the 3D shape model
+			std::vector <vec3f> model_points; // the points in the 3D shape model
 			std::vector<int> vertex_indices; // their vertex indices
 
 			for (int i = 0; i < annotations.size(); i++) {
-				eos::core::LandmarkCollection <cv::Vec2f> landmarks;
+				eos::core::LandmarkCollection <vec2f> landmarks;
 
 				try {
-					landmarks = read_pts_landmarks(annotations[i]);
-				}
-				catch (const std::runtime_error &e) {
+					landmarks = read_pts_landmarks<vec2f>(annotations[i]);
+				} catch (const std::runtime_error &e) {
 					std::cout << e.what() << std::endl;
 					throw std::runtime_error("Error reading the landmarks:  " + annotations[i]);
 				}
 
-				std::vector <cv::Vec2f> image_points_tmp;
-
 				// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
 				for (int j = 0; j < landmarks.size(); j++) {
-					image_points_tmp.emplace_back(landmarks[i].coordinates);
+					landmark_collection_list.emplace_back(landmarks);
 				}
 			}
 
-			return image_points;
+			return landmark_collection_list;
 		}
 	}
 }

include/eos/fitting/FittingResult.hpp

+/*
+ * eos - A 3D Morphable Model fitting library written in modern C++11/14.
+ *
+ * File: include/eos/fitting/FittingResult.hpp
+ *
+ * Copyright 2016 Patrik Huber
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef FITTINGRESULT_HPP_
+#define FITTINGRESULT_HPP_
+
+#include "eos/fitting/RenderingParameters.hpp"
+
+#include <vector>
+
+namespace eos {
+	namespace fitting {
+
+/**
+ * @brief A.
+ *
+ * A.
+ * Note: Any of them can be empty (=default constructed?).
+ * For example when we fit to a video, and want to save the
+ * final result, which will only have PCA shape coeffs, without the rest.
+ * What happens in serialisation?
+ * Or in that case, we should actually just use save_coefficients?
+ */
+struct FittingResult
+{
+	RenderingParameters rendering_parameters;
+	std::vector<float> pca_shape_coefficients;
+	std::vector<float> blendshape_coefficients;
+};
+
+	} /* namespace fitting */
+} /* namespace eos */
+
+#endif /* FITTINGRESULT_HPP_ */

include/eos/fitting/fitting.hpp

@@ -404,6 +404,293 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
 	std::vector<cv::Vec2f> fitted_image_points;
 	return fit_shape_and_pose(morphable_model, blendshapes, landmarks, landmark_mapper, image_width, image_height, edge_topology, contour_landmarks, model_contour, num_iterations, num_shape_coefficients_to_fit, lambda, boost::none, pca_coeffs, blendshape_coeffs, fitted_image_points);
 };
+
+/**
+* @brief Fit the pose (camera), shape model, and expression blendshapes to landmarks,
+* in an iterative way. Can fit to more than one set of landmarks, thus multiple images.
+*
+* Convenience function that fits pose (camera), the shape model, and expression blendshapes
+* to landmarks, in an iterative (alternating) way. It fits both sides of the face contour as well.
+*
+* If \p pca_shape_coefficients and/or \p blendshape_coefficients are given, they are used as
+* starting values in the fitting. When the function returns, they contain the coefficients from
+* the last iteration.
+*
+* Use render::Mesh fit_shape_and_pose(const morphablemodel::MorphableModel&, const std::vector<morphablemodel::Blendshape>&, const core::LandmarkCollection<cv::Vec2f>&, const core::LandmarkMapper&, int, int, const morphablemodel::EdgeTopology&, const fitting::ContourLandmarks&, const fitting::ModelContour&, int, boost::optional<int>, float).
+* for a simpler overload with reasonable defaults and no optional output.
+*
+* \p num_iterations: Results are good for even a single iteration. For single-image fitting and
+* for full convergence of all parameters, it can take up to 300 iterations. In tracking,
+* particularly if initialising with the previous frame, it works well with as low as 1 to 5
+* iterations.
+* \p edge_topology is used for the occluding-edge face contour fitting.
+* \p contour_landmarks and \p model_contour are used to fit the front-facing contour.
+*
+* Todo: Add a convergence criterion.
+*
+* @param[in] morphable_model The 3D Morphable Model used for the shape fitting.
+* @param[in] blendshapes A vector of blendshapes that are being fit to the landmarks in addition to the PCA model.
+* @param[in] landmarks 2D landmarks from an image to fit the model to.
+* @param[in] landmark_mapper Mapping info from the 2D landmark points to 3D vertex indices.
+* @param[in] image_width Width of the input image (needed for the camera model).
+* @param[in] image_height Height of the input image (needed for the camera model).
+* @param[in] edge_topology Precomputed edge topology of the 3D model, needed for fast edge-lookup.
+* @param[in] contour_landmarks 2D image contour ids of left or right side (for example for ibug landmarks).
+* @param[in] model_contour The model contour indices that should be considered to find the closest corresponding 3D vertex.
+* @param[in] num_iterations Number of iterations that the different fitting parts will be alternated for.
+* @param[in] num_shape_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients.
+* @param[in] lambda Regularisation parameter of the PCA shape fitting.
+* @param[in] initial_rendering_params Currently ignored (not used).
+* @param[in,out] pca_shape_coefficients If given, will be used as initial PCA shape coefficients to start the fitting. Will contain the final estimated coefficients.
+* @param[in,out] blendshape_coefficients If given, will be used as initial expression blendshape coefficients to start the fitting. Will contain the final estimated coefficients.
+* @param[out] fitted_image_points Debug parameter: Returns all the 2D points that have been used for the fitting.
+* @return The fitted model shape instance and the final pose.
+*/
+inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi(
+				const morphablemodel::MorphableModel& morphable_model,
+				const std::vector<morphablemodel::Blendshape>& blendshapes,
+				const std::vector<core::LandmarkCollection<cv::Vec2f>>& landmarks,
+				const core::LandmarkMapper& landmark_mapper,
+				int image_width,
+				int image_height,
+				int num_images,
+				const morphablemodel::EdgeTopology& edge_topology,
+				const fitting::ContourLandmarks& contour_landmarks,
+				const fitting::ModelContour& model_contour,
+				int num_iterations,
+				boost::optional<int> num_shape_coefficients_to_fit,
+				float lambda,
+				boost::optional<fitting::RenderingParameters> initial_rendering_params,
+				std::vector<float>& pca_shape_coefficients,
+				std::vector<std::vector<float>>& blendshape_coefficients,
+				std::vector<std::vector<cv::Vec2f>>& fitted_image_points) {
+	assert(blendshapes.size() > 0);
+	assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit?
+	assert(pca_shape_coefficients.size() <= morphable_model.get_shape_model().get_num_principal_components());
+
+	using std::vector;
+	using cv::Vec2f;
+	using cv::Vec4f;
+	using cv::Mat;
+	using Eigen::VectorXf;
+	using Eigen::MatrixXf;
+
+
+	if (!num_shape_coefficients_to_fit) {
+		num_shape_coefficients_to_fit = morphable_model.get_shape_model().get_num_principal_components();
+	}
+
+	if (pca_shape_coefficients.empty()) {
+		pca_shape_coefficients.resize(num_shape_coefficients_to_fit.get());
+	}
+
+	// TODO: This leaves the following case open: num_coeffs given is empty or defined, but the
+	// pca_shape_coefficients given is != num_coeffs or the model's max-coeffs. What to do then? Handle & document!
+
+	if (blendshape_coefficients.size() < num_images) {
+		for (int j = 0; j < num_images; ++j) {
+			std::vector<float> current_blendshape_coefficients;
+			current_blendshape_coefficients.resize(blendshapes.size());
+			blendshape_coefficients.push_back(current_blendshape_coefficients);
+		}
+	}
+
+	MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
+
+	// Current mesh - either from the given coefficients, or the mean:
+	VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
+	vector<VectorXf> current_combined_shapes;
+	vector<eos::core::Mesh> current_meshs;
+
+	for (int j = 0; j < num_images; ++j) {
+		VectorXf current_combined_shape = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
+		current_combined_shapes.push_back(current_combined_shape);
+
+		eos::core::Mesh current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
+		current_meshs.push_back(current_mesh);
+	}
+
+	// The 2D and 3D point correspondences used for the fitting:
+	vector<vector<Vec4f>> model_points; // the points in the 3D shape model of all frames
+	vector<vector<int>> vertex_indices; // their vertex indices of all frames
+	vector<vector<Vec2f>> image_points; // the corresponding 2D landmark points of all frames
+
+	for (int j = 0; j < num_images; ++j) {
+		vector<Vec4f> current_model_points;
+		vector<int> current_vertex_indices;
+		vector<Vec2f> current_image_points;
+
+		// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM),
+		// and get the corresponding model points (mean if given no initial coeffs, from the computed shape otherwise):
+		for (int i = 0; i < landmarks[j].size(); ++i) {
+			auto converted_name = landmark_mapper.convert(landmarks[j][i].name);
+			if (!converted_name) { // no mapping defined for the current landmark
+				continue;
+			}
+			int vertex_idx = std::stoi(converted_name.get());
+			Vec4f vertex(current_meshs[j].vertices[vertex_idx].x, current_meshs[j].vertices[vertex_idx].y, current_meshs[j].vertices[vertex_idx].z, current_meshs[j].vertices[vertex_idx].w);
+			current_model_points.emplace_back(vertex);
+			current_vertex_indices.emplace_back(vertex_idx);
+			current_image_points.emplace_back(landmarks[j][i].coordinates);
+		}
+
+		model_points.push_back(current_model_points);
+		vertex_indices.push_back(current_vertex_indices);
+		image_points.push_back(current_image_points);
+	}
+
+	// Need to do an initial pose fit to do the contour fitting inside the loop.
+	// We'll do an expression fit too, since face shapes vary quite a lot, depending on expressions.
+	vector<fitting::RenderingParameters> rendering_params;
+	for (int j = 0; j < num_images; ++j) {
+		fitting::ScaledOrthoProjectionParameters current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
+		fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height);
+		rendering_params.push_back(current_rendering_params);
+
+		cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
+		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points[j], vertex_indices[j]);
+
+		// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
+		current_combined_shapes[j] = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
+		current_meshs[j] = morphablemodel::sample_to_mesh(current_combined_shapes[j], morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
+	}
+
+	// The static (fixed) landmark correspondences which will stay the same throughout
+	// the fitting (the inner face landmarks):
+	vector<vector<int>> fixed_vertex_indices (vertex_indices);
+	vector<vector<Vec2f>> fixed_image_points (image_points);
+
+	std::vector<cv::Mat> affine_from_orthos;
+	std::vector<VectorXf> mean_plus_blendshapes;
+
+	image_points = fixed_image_points;
+	vertex_indices = fixed_vertex_indices;
+
+	for (int j = 0; j < num_images; ++j) {
+		// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
+		vector<Vec2f> image_points_contour;
+		vector<int> vertex_indices_contour;
+		auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
+		// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
+		std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(landmarks[j], contour_landmarks, model_contour, yaw_angle, current_meshs[j], rendering_params[j].get_modelview(), rendering_params[j].get_projection(), fitting::get_opencv_viewport(image_width, image_height));
+		// Add the contour correspondences to the set of landmarks that we use for the fitting:
+		vertex_indices[j] = fitting::concat(vertex_indices[j], vertex_indices_contour);
+		image_points[j] = fitting::concat(image_points[j], image_points_contour);
+
+		// Fit the occluding (away-facing) contour using the detected contour LMs:
+		vector<Eigen::Vector2f> occluding_contour_landmarks;
+		if (yaw_angle >= 0.0f) // positive yaw = subject looking to the left
+		{ // the left contour is the occluding one we want to use ("away-facing")
+			auto contour_landmarks_ = core::filter(landmarks[j], contour_landmarks.left_contour); // Can do this outside of the loop
+			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
+		}
+		else {
+			auto contour_landmarks_ = core::filter(landmarks[j], contour_landmarks.right_contour);
+			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
+		}
+		auto edge_correspondences = fitting::find_occluding_edge_correspondences(current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f);
+		image_points[j] = fitting::concat(image_points[j], edge_correspondences.first);
+		vertex_indices[j] = fitting::concat(vertex_indices[j], edge_correspondences.second);
+
+		// Get the model points of the current mesh, for all correspondences that we've got:
+		model_points[j].clear();
+		for (const auto& v : vertex_indices[j])
+		{
+			model_points[j].push_back({ current_meshs[j].vertices[v][0], current_meshs[j].vertices[v][1], current_meshs[j].vertices[v][2], current_meshs[j].vertices[v][3] });
+		}
+
+		// Re-estimate the pose, using all correspondences:
+		fitting::ScaledOrthoProjectionParameters current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
+		rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
+
+		cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
+		affine_from_orthos.push_back(affine_from_ortho);
+
+		// Estimate the PCA shape coefficients with the current blendshape coefficients:
+		VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
+		mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
+	}
+
+	pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi(morphable_model, affine_from_orthos, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_shape_coefficients_to_fit);
+
+	// Estimate the blendshape coefficients with the current PCA model estimate:
+	current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
+
+	for (int j = 0; j < num_images; ++j) {
+		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]);
+
+		current_combined_shapes[j] = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
+		current_meshs[j] = morphablemodel::sample_to_mesh(current_combined_shapes[j], morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
+	}
+
+	fitted_image_points = image_points;
+	return { current_meshs, rendering_params }; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices).
+};
+
+/**
+* @brief Fit the pose (camera), shape model, and expression blendshapes to landmarks,
+* in an iterative way.  Can fit to more than one set of landmarks, thus multiple images.
+*
+* Convenience function that fits pose (camera), the shape model, and expression blendshapes
+* to landmarks, in an iterative (alternating) way. It fits both sides of the face contour as well.
+*
+* If you want to access the values of shape or blendshape coefficients, or want to set starting
+* values for them, use the following overload to this function:
+* std::pair<render::Mesh, fitting::RenderingParameters> fit_shape_and_pose(const morphablemodel::MorphableModel&, const std::vector<morphablemodel::Blendshape>&, const core::LandmarkCollection<cv::Vec2f>&, const core::LandmarkMapper&, int, int, const morphablemodel::EdgeTopology&, const fitting::ContourLandmarks&, const fitting::ModelContour&, int, boost::optional<int>, float, boost::optional<fitting::RenderingParameters>, std::vector<float>&, std::vector<float>&, std::vector<cv::Vec2f>&)
+*
+* Todo: Add a convergence criterion.
+*
+* \p num_iterations: Results are good for even a single iteration. For single-image fitting and
+* for full convergence of all parameters, it can take up to 300 iterations. In tracking,
+* particularly if initialising with the previous frame, it works well with as low as 1 to 5
+* iterations.
+* \p edge_topology is used for the occluding-edge face contour fitting.
+* \p contour_landmarks and \p model_contour are used to fit the front-facing contour.
+*
+* @param[in] morphable_model The 3D Morphable Model used for the shape fitting.
+* @param[in] blendshapes A vector of blendshapes that are being fit to the landmarks in addition to the PCA model.
+* @param[in] landmarks 2D landmarks from an image to fit the model to.
+* @param[in] landmark_mapper Mapping info from the 2D landmark points to 3D vertex indices.
+* @param[in] image_width Width of the input image (needed for the camera model).
+* @param[in] image_height Height of the input image (needed for the camera model).
+* @param[in] edge_topology Precomputed edge topology of the 3D model, needed for fast edge-lookup.
+* @param[in] contour_landmarks 2D image contour ids of left or right side (for example for ibug landmarks).
+* @param[in] model_contour The model contour indices that should be considered to find the closest corresponding 3D vertex.
+* @param[in] num_iterations Number of iterations that the different fitting parts will be alternated for.
+* @param[in] num_shape_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients.
+* @param[in] lambda Regularisation parameter of the PCA shape fitting.
+* @return The fitted model shape instance and the final pose.
+*/
+inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi(
+				const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blend_shapes, const std::vector<core::LandmarkCollection<cv::Vec2f>>& landmarks,
+				const core::LandmarkMapper& landmark_mapper, int image_width, int image_height, const morphablemodel::EdgeTopology& edge_topology, const fitting::ContourLandmarks& contour_landmarks,
+				const fitting::ModelContour& model_contour, int num_iterations = 5, boost::optional<int> num_shape_coefficients_to_fit = boost::none, float lambda = 30.0f)
+{
+	std::vector<float> pca_shape_coefficients;
+	std::vector<std::vector<float>> blend_shape_coefficients;
+	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
+
+	return fit_shape_and_pose_multi(
+			morphable_model,
+			blend_shapes,
+			landmarks,
+			landmark_mapper,
+			image_width,
+			image_height,
+			static_cast<int>(landmarks.size()),
+			edge_topology,
+			contour_landmarks,
+			model_contour,
+			num_iterations,
+			num_shape_coefficients_to_fit,
+			lambda,
+			boost::none,
+			pca_shape_coefficients,
+			blend_shape_coefficients,
+			fitted_image_points
+	);
+};
+
 	} /* namespace fitting */
 } /* namespace eos */
 

include/eos/fitting/linear_shape_fitting.hpp

@@ -25,6 +25,7 @@
 #include "eos/morphablemodel/MorphableModel.hpp"
 
 #include "Eigen/QR"
+#include "Eigen/Sparse"
 
 #include "opencv2/core/core.hpp"
 
@@ -37,6 +38,138 @@ namespace eos {
 	namespace fitting {
 
 /**
+* Fits the shape of a Morphable Model to given 2D landmarks (i.e. estimates the maximum likelihood solution of the shape coefficients) as proposed in [1].
+* It's a linear, closed-form solution fitting of the shape, with regularisation (prior towards the mean).
+*
+* [1] O. Aldrian & W. Smith, Inverse Rendering of Faces with a 3D Morphable Model, PAMI 2013.
+*
+* Note: Using less than the maximum number of coefficients to fit is not thoroughly tested yet and may contain an error.
+* Note: Returns coefficients following standard normal distribution (i.e. all have similar magnitude). Why? Because we fit using the normalised basis?
+* Note: The standard deviations given should be a vector, i.e. different for each landmark. This is not implemented yet.
+*
+* @param[in] morphable_model The Morphable Model whose shape (coefficients) are estimated.
+* @param[in] affine_camera_matrix A 3x4 affine camera matrix from model to screen-space (should probably be of type CV_32FC1 as all our calculations are done with float).
+* @param[in] landmarks 2D landmarks from an image to fit the model to.
+* @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
+* @param[in] base_face The base or reference face from where the fitting is started. Usually this would be the models mean face, which is what will be used if the parameter is not explicitly specified.
+* @param[in] lambda The regularisation parameter (weight of the prior towards the mean). Gets normalized by the number of images given.
+* @param[in] num_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients.
+* @param[in] detector_standard_deviation The standard deviation of the 2D landmarks given (e.g. of the detector used), in pixels.
+* @param[in] model_standard_deviation The standard deviation of the 3D vertex points in the 3D model, projected to 2D (so the value is in pixels).
+* @return The estimated shape-coefficients (alphas).
+*/
+inline std::vector<float> fit_shape_to_landmarks_linear_multi(const morphablemodel::MorphableModel& morphable_model, std::vector<cv::Mat> affine_camera_matrix, std::vector<std::vector<cv::Vec2f>>& landmarks, std::vector<std::vector<int>>& vertex_ids, std::vector<Eigen::VectorXf> base_face=std::vector<Eigen::VectorXf>(), float lambda=3.0f, boost::optional<int> num_coefficients_to_fit=boost::optional<int>(), boost::optional<float> detector_standard_deviation=boost::optional<float>(), boost::optional<float> model_standard_deviation=boost::optional<float>())
+{
+
+	assert(affine_camera_matrix.size() == landmarks.size() && landmarks.size() == vertex_ids.size()); // same number of instances (i.e. images/frames) for each of them
+	assert(landmarks.size() == vertex_ids.size());
+
+	using Eigen::VectorXf;
+	using Eigen::MatrixXf;
+
+	int num_coeffs_to_fit = num_coefficients_to_fit.get_value_or(morphable_model.get_shape_model().get_num_principal_components());
+	int num_images = affine_camera_matrix.size();
+
+	// the regularisation has to be adjusted when more than one image is given
+	lambda *= num_images;
+
+	int total_num_landmarks_dimension = 0;
+	for (auto&& l : landmarks) {
+		total_num_landmarks_dimension += l.size();
+	}
+
+	// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
+	// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
+	MatrixXf V_hat_h = MatrixXf::Zero(4 * total_num_landmarks_dimension, num_coeffs_to_fit);
+	int V_hat_h_row_index = 0;
+	// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
+	Eigen::SparseMatrix<float> P (3 * total_num_landmarks_dimension, 4 * total_num_landmarks_dimension);
+	std::vector<Eigen::Triplet<float>> P_coefficients; // list of non-zeros coefficients
+	int P_index = 0;
+	// The variances: Add the 2D and 3D standard deviations.
+	// If the user doesn't provide them, we choose the following:
+	// 2D (detector) standard deviation: In pixel, we follow [1] and choose sqrt(3) as the default value.
+	// 3D (model) variance: 0.0f. It only makes sense to set it to something when we have a different variance for different vertices.
+	// The 3D variance has to be projected to 2D (for details, see paper [1]) so the units do match up.
+	float sigma_squared_2D = std::pow(detector_standard_deviation.get_value_or(std::sqrt(3.0f)), 2) + std::pow(model_standard_deviation.get_value_or(0.0f), 2);
+	// We use a VectorXf, and later use .asDiagonal():
+	VectorXf Omega = VectorXf::Constant(3 * total_num_landmarks_dimension, 1.0f / sigma_squared_2D);
+	// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
+	VectorXf y = VectorXf::Ones(3 * total_num_landmarks_dimension);
+	int y_index = 0; // also runs the same as P_index. Should rename to "running_index"?
+	// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
+	VectorXf v_bar = VectorXf::Ones(4 * total_num_landmarks_dimension);
+	int v_bar_index = 0; // also runs the same as P_index. But be careful, if I change it to be only 1 variable, only increment it once! :-)
+	// Well I think that would make it a bit messy since we need to increment inside the for (landmarks...) loop. Try to refactor some other way.
+
+	for (int k = 0; k < num_images; ++k)
+	{
+		// For each image we have, set up the equations and add it to the matrices:
+		assert(landmarks[k].size() == vertex_ids[k].size()); // has to be valid for each img
+
+		int num_landmarks = static_cast<int>(landmarks[k].size());
+
+		if (base_face[k].size()==0)
+		{
+			base_face[k] = morphable_model.get_shape_model().get_mean();
+		}
+
+		// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
+		// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
+		//Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
+		for (int i = 0; i < num_landmarks; ++i) {
+			MatrixXf basis_rows_ = morphable_model.get_shape_model().get_rescaled_pca_basis_at_point(vertex_ids[k][i]); // In the paper, the orthonormal basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the rescaled basis is fine/better.
+			V_hat_h.block(V_hat_h_row_index, 0, 3, V_hat_h.cols()) = basis_rows_.block(0, 0, basis_rows_.rows(), num_coeffs_to_fit);
+			V_hat_h_row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
+		}
+
+		// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
+		for (int i = 0; i < num_landmarks; ++i) {
+			for (int x = 0; x < affine_camera_matrix[k].rows; ++x){
+				for (int y = 0; y < affine_camera_matrix[k].cols; ++y){
+					P_coefficients.push_back(Eigen::Triplet<float>(3 * P_index + x, 4 * P_index + y, affine_camera_matrix[k].at<float>(x,y)));
+				}
+			}
+			++P_index;
+		}
+
+
+		// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
+		//Mat y = Mat::ones(3 * num_landmarks, 1, CV_32FC1);
+		for (int i = 0; i < num_landmarks; ++i) {
+			y(3 * y_index) = landmarks[k][i][0];
+			y((3 * y_index) + 1) = landmarks[k][i][1];
+			//y((3 * i) + 2) = 1; // already 1, stays (homogeneous coordinate)
+			++y_index;
+		}
+		// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
+		//Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
+		for (int i = 0; i < num_landmarks; ++i) {
+			v_bar(4 * v_bar_index) = base_face[k](vertex_ids[k][i] * 3);
+			v_bar((4 * v_bar_index) + 1) = base_face[k](vertex_ids[k][i] * 3 + 1);
+			v_bar((4 * v_bar_index) + 2) = base_face[k](vertex_ids[k][i] * 3 + 2);
+			//v_bar.at<float>((4 * i) + 3) = 1; // already 1, stays (homogeneous coordinate)
+			++v_bar_index;
+		}
+	}
+
+	// fill P with coefficients
+	P.setFromTriplets(P_coefficients.begin(), P_coefficients.end());
+
+	// Bring into standard regularised quadratic form with diagonal distance matrix Omega:
+	const MatrixXf A = P * V_hat_h; // camera matrix times the basis
+	const MatrixXf b = P * v_bar - y; // camera matrix times the mean, minus the landmarks
+	const MatrixXf AtOmegaAReg = A.transpose() * Omega.asDiagonal() * A + lambda * Eigen::MatrixXf::Identity(num_coeffs_to_fit, num_coeffs_to_fit);
+	const MatrixXf rhs = -A.transpose() * Omega.asDiagonal() * b; // It's -A^t*Omega^t*b, but we don't need to transpose Omega, since it's a diagonal matrix, and Omega^t = Omega.
+	// c_s: The 'x' that we solve for. (The variance-normalised shape parameter vector, $c_s = [a_1/sigma_{s,1} , ..., a_m-1/sigma_{s,m-1}]^t$.)
+	// We get coefficients ~ N(0, 1), because we're fitting with the rescaled basis. The coefficients are not multiplied with their eigenvalues.
+	const VectorXf c_s = AtOmegaAReg.colPivHouseholderQr().solve(rhs);
+
+	return std::vector<float>(c_s.data(), c_s.data() + c_s.size());
+};
+
+
+		/**
  * Fits the shape of a Morphable Model to given 2D landmarks (i.e. estimates the maximum likelihood solution of the shape coefficients) as proposed in [1].
  * It's a linear, closed-form solution fitting of the shape, with regularisation (prior towards the mean).
  *

include/eos/core/BufferedVideoIterator.hpp → include/eos/video/BufferedVideoIterator.hpp

@@ -26,8 +26,9 @@
 #include "opencv2/opencv.hpp"
 #include "opencv2/highgui/highgui.hpp"
 
-#include <vector>
 #include <string>
+#include <vector>
+#include <fstream>
 
 using cv::Mat;
 using cv::Vec2f;
@@ -41,100 +42,8 @@ using namespace cv;
 using namespace std;
 
 namespace eos {
-	namespace core {
-		/**
-		 * BufferedVideo Iterator will keep a buffer of the last seen n_frames. By calling .next() it will load a new
-		 * frame from the given video and you will get a pointer to the front of the buffer (which has n_frames).
-		 *
-		 * Just imagine a sliding window accross the video, this is what we aim to implement here.
-		 *
-		 * Example:
-		 *		vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
-		 *
-		 *		std::deque<cv::mat> frames = vid_iterator.next();
-		 *		while(!(frames.empty())) {
-		 *			for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
-		 *				std::cout << ' ' << *it;
-		 *			}
-		 *
-		 *			frames = vid_iterator.next();
-		 *		}
-		 *
-		 * @tparam T
-		 */
-		// Note for this template: later we can use other templates for easy testing (not using cv:Mat but <int> for example).
-		template<typename T>
-		class BufferedVideoIterator {
-		public:
-			BufferedVideoIterator() {};
-
-			// TODO: build support for setting the amount of max_frames in the buffer.
-			BufferedVideoIterator(std::string videoFilePath, std::vector<std::vector<cv::Vec2f>> landmarks) {
-				std::ifstream file(videoFilePath);
-
-				if (!file.is_open()) {
-					throw std::runtime_error("Error opening given file: " + videoFilePath);
-				}
-
-				cv::VideoCapture tmp_cap(videoFilePath); // open video file
-
-				if (!tmp_cap.isOpened()) { // check if we succeeded
-					throw std::runtime_error("Could not play video");
-				}
-
-				cap = tmp_cap;
-				this->landmarks = landmarks;
-			}
-
-			/**
-			 * Set next frame and return frame_buffer.
-			 *
-			 * @return dequeue<Mat> frame buffer.
-			 *
-			 * TODO: build support for returning landmarks AND frames.
-			 */
-			std::deque <Mat> next() {
-				long frame_buffer_length = frame_buffer.size();
-				long landmark_landmark_length = landmark_buffer.size();
-
-				// Get a new frame from camera.
-				Mat frame;
-				cap >> frame;
-
-				// Pop if we exceeded max_frames.
-				if (n_frames > max_frames) {
-					frame_buffer.pop_front();
-				}
-
-				if (frame.empty() == 0) {
-					frame_buffer.push_back(frame);
-				}
-
-				n_frames++;
-
-				return frame_buffer;
-			}
-
-			std::deque <Mat> get_frame_buffer() {
-				return frame_buffer;
-			}
-
-			std::deque <Mat> get_landmark_buffer() {
-				return landmark_buffer;
-			}
-
-		private:
-			cv::VideoCapture cap;
-			std::deque <Mat> frame_buffer;
-			std::deque <Mat> landmark_buffer;
-			std::vector<std::vector<cv::Vec2f>> landmarks;
-
-			// TODO: make set-able
-			// load n_frames at once into the buffer.
-			long n_frames = 1;
-			// keep max_frames into the buffer.
-			long max_frames = 5;
-		};
+	namespace video {
+		// TODO: maybe move video iterator here.. or remove file.
 	}
 }
 

include/eos/video/Keyframe.hpp

@@ -26,9 +26,12 @@
 #include "eos/fitting/RenderingParameters.hpp"
 #include "eos/morphablemodel/Blendshape.hpp"
 #include "eos/morphablemodel/MorphableModel.hpp"
+#include "eos/render/texture_extraction.hpp"
 
 #include "opencv2/core/core.hpp"
 
+#include <deque>
+
 namespace eos {
 namespace video {
 
@@ -37,9 +40,42 @@ namespace video {
  *
  * Contains the original frame, all necessary fitting parameters, and a score.
  */
-struct Keyframe
-{
-    float score; // = 0.0f?
+struct Keyframe {
+public:
+	Keyframe() {}
+	/**
+	 * Only used when only a frame is available.
+	 *
+	 * @param frame
+	 */
+	Keyframe(cv::Mat frame) {
+		this->frame = frame;
+	}
+
+	/**
+	 * Only used when score and frame is available
+	 *
+	 * @param frame
+	 * @param score
+	 */
+	Keyframe(float score, cv::Mat frame) {
+		this->score = score;
+		this->frame = frame;
+	}
+
+	/**
+	 *
+	 * @param score
+	 * @param frame
+	 * @param fitting_result
+	 */
+	Keyframe(float score, cv::Mat frame, fitting::FittingResult fitting_result) {
+		this->frame = frame;
+		this->score = score;
+		this->fitting_result = fitting_result;
+	}
+
+	float score = 0.0f;
 	cv::Mat frame;
 	fitting::FittingResult fitting_result;
 };
@@ -47,11 +83,11 @@ struct Keyframe
 /**
  * @brief A keyframe selection that selects keyframes according to yaw pose and score.
  *
- * Separates the +-90 yaw pose range into 20 intervals (i.e. 90 to 70, ..., -10 to 10, ...), and puts frames
+ * Separates the +-90� yaw pose range into 20� intervals (i.e. 90 to 70, ..., -10 to 10, ...), and puts frames
  * into each bin, until full. Replaces keyframes with better frames if the score is higher than that of
  * current keyframes.
  *
- * The yaw pose bins are currently hard-coded (9 bins, 20 intervals).
+ * The yaw pose bins are currently hard-coded (9 bins, 20� intervals).
  */
 struct PoseBinningKeyframeSelector
 {
@@ -112,7 +148,7 @@ private:
     int frames_per_bin;
 
     // Converts a given yaw angle to an index in the internal bins vector.
-    // Assumes 9 bins and 20 intervals.
+    // Assumes 9 bins and 20� intervals.
     static std::size_t angle_to_index(float yaw_angle)
     {
         if (yaw_angle <= -70.f)
@@ -168,18 +204,21 @@ cv::Mat merge_weighted_mean(const std::vector<Keyframe>& keyframes,
     using std::vector;
 
 	vector<Mat> isomaps;
-    for (const auto& frame_data : keyframes)
-    {
-        const Mat shape =
+	for (const auto& frame_data : keyframes) {
+	//		VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(frame_data.fitting_result.pca_shape_coefficients);
+	//		const current_combined_shape = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Eigen::Map<const Eigen::VectorXf>(frame_data.fitting_result.blendshape_coefficients);
+		const Eigen::VectorXf shape =
 			morphable_model.get_shape_model().draw_sample(frame_data.fitting_result.pca_shape_coefficients) +
-            morphablemodel::to_matrix(blendshapes) * Mat(frame_data.fitting_result.blendshape_coefficients);
+			morphablemodel::to_matrix(blendshapes) * morphablemodel::to_vector(frame_data.fitting_result.blendshape_coefficients);
 		const auto mesh =
-            morphablemodel::sample_to_mesh(shape, {}, morphable_model.get_shape_model().get_triangle_list(),
-                                           {}, morphable_model.get_texture_coordinates());
+			morphablemodel::sample_to_mesh(
+					shape, {}, morphable_model.get_shape_model().get_triangle_list(), {}, morphable_model.get_texture_coordinates());
 		const Mat affine_camera_matrix = fitting::get_3x4_affine_camera_matrix(
-            frame_data.fitting_result.rendering_parameters, frame_data.frame.cols, frame_data.frame.rows);
-        const Mat isomap = render::extract_texture(mesh, affine_camera_matrix, frame_data.frame, true,
-                                                   render::TextureInterpolation::NearestNeighbour, 1024);
+			frame_data.fitting_result.rendering_parameters, frame_data.frame.cols, frame_data.frame.rows
+		);
+		const Mat isomap = render::extract_texture(
+			mesh, affine_camera_matrix, frame_data.frame, true, render::TextureInterpolation::NearestNeighbour, 1024
+		);
 		isomaps.push_back(isomap);
 	}
 
@@ -248,6 +287,113 @@ double variance_of_laplacian(const cv::Mat& image)
     return focus_measure;
 };
 
+/**
+ * BufferedVideo Iterator will keep a buffer of the last seen n_frames. By calling .next() it will load a new
+ * frame from the given video and you will get a pointer to the front of the buffer (which has n_frames).
+ *
+ * Just imagine a sliding window accross the video, this is what we aim to implement here.
+ *
+ * Example:
+ *    vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
+ *
+ *    std::deque<cv::mat> frames = vid_iterator.next();
+ *    while(!(frames.empty())) {
+ *        for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+ *            std::cout << ' ' << *it;
+ *        }
+ *
+ *         frames = vid_iterator.next();
+ *    }
+ *
+ * @tparam T
+ */
+// Note for this template: later we can use other templates for easy testing (not using cv:Mat but <int> for example).
+class BufferedVideoIterator {
+public:
+	int width;
+	int height;
+
+	BufferedVideoIterator() {};
+
+	// TODO: build support for setting the amount of max_frames in the buffer.
+	BufferedVideoIterator(std::string videoFilePath, int max_frames = 5, int min_frames = 4) {
+		std::ifstream file(videoFilePath);
+
+		if (!file.is_open()) {
+			throw std::runtime_error("Error opening given file: " + videoFilePath);
+		}
+
+		cv::VideoCapture cap(videoFilePath); // open video file
+
+		if (!cap.isOpened()) { // check if we succeeded
+			throw std::runtime_error("Could not play video");
+		}
+
+		width = static_cast<int>(cap.get(CV_CAP_PROP_FRAME_HEIGHT));
+		height = static_cast<int>(cap.get(CV_CAP_PROP_FRAME_HEIGHT));
+
+		this->max_frames = max_frames;
+		this->min_frames = min_frames;
+	}
+
+	/**
+	 * Set next frame and return frame_buffer.
+	 *
+	 * @return dequeue<Mat> frame buffer.
+	 *
+	 * TODO: build support for returning landmarks AND frames.
+	 */
+	std::deque <Keyframe> next() {
+		long frame_buffer_length = frame_buffer.size();
+
+		// Get a new frame from camera.
+		cv::Mat frame;
+		cap >> frame;
+
+		// Pop if we exceeded max_frames.
+		if (n_frames > max_frames) {
+			frame_buffer.pop_front();
+		}
+
+		float lap_score = static_cast<float>(variance_of_laplacian(frame));
+
+		if (lap_score < laplacian_threshold) {
+			frame_buffer.push_back(Keyframe(lap_score, frame));
+		}
+
+		n_frames++;
+
+		if(frame_buffer_length + 1 < min_frames) {
+			frame_buffer = next();
+		}
+
+		return frame_buffer;
+	}
+
+	std::deque <Keyframe> get_frame_buffer() {
+		return frame_buffer;
+	}
+
+private:
+	cv::VideoCapture cap;
+	std::deque<Keyframe> frame_buffer;
+
+	// TODO: make set-able
+	// total frames in buffer
+	long n_frames = 1;
+
+	// min frames to load at the start.
+	long min_frames = 3;
+	// keep max_frames into the buffer.
+	long max_frames = 5;
+
+	// Note: these settings are for future use
+	int drop_frames = 0;
+
+	// laplacian threshold
+	double laplacian_threshold = 10000000;
+};
+
 } /* namespace video */
 } /* namespace eos */
 

utils/accuracy-evaluation.cpp

@@ -16,15 +16,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "eos/core/Landmark.hpp"
-#include "eos/core/LandmarkMapper.hpp"
 #include "eos/core/landmark_utils.hpp"
-#include "eos/core/BufferedVideoIterator.hpp"
 #include "eos/morphablemodel/morphablemodel.hpp"
 #include "eos/morphablemodel/blendshape.hpp"
 #include "eos/fitting/fitting.hpp"
 #include "eos/render/utils.hpp"
 #include "eos/render/texture_extraction.hpp"
+#include "eos/video/Keyframe.hpp"
 
 #include "opencv2/core/core.hpp"
 #include "opencv2/opencv.hpp"
@@ -42,7 +40,7 @@ namespace po = boost::program_options;
 namespace fs = boost::filesystem;
 using eos::core::Landmark;
 using eos::core::LandmarkCollection;
-using eos::core::BufferedVideoIterator;
+using eos::video::BufferedVideoIterator;
 using cv::Mat;
 using cv::Vec2f;
 using cv::Vec3f;
@@ -121,83 +119,115 @@ int main(int argc, char *argv[]) {
 		po::variables_map vm;
 		po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
 		if (vm.count("help")) {
-			cout << "Usage: fit-model [options]" << endl;
-			cout << desc;
+			std::cout << "Usage: fit-model [options]" << std::endl;
+			std::cout << desc;
 			return EXIT_SUCCESS;
 		}
 		po::notify(vm);
 	}
 	catch (const po::error &e) {
-		cout << "Error while parsing command-line arguments: " << e.what() << endl;
-		cout << "Use --help to display a list of options." << endl;
+		std::cout << "Error while parsing command-line arguments: " << e.what() << std::endl;
+		std::cout << "Use --help to display a list of options." << std::endl;
 		return EXIT_SUCCESS;
 	}
 
+	// start loading prerequisites
+	morphablemodel::MorphableModel morphable_model;
 	try {
-		vector <vector<Vec2f>> multi_frame_points = eos::core::load_annotations(annotations, mappingsfile);
-	} catch(const std::runtime_error &e) {
-		std::cout << e.what() << std::endl;
+		morphable_model = morphablemodel::load_model(modelfile.string());
+	} catch (const std::runtime_error &e) {
+		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
-	// Load landmarks, LandmarkMapper and the Morphable Model:
-	LandmarkCollection <cv::Vec2f> landmarks;
-	core::LandmarkMapper landmark_mapper = core::LandmarkMapper(mappingsfile);
-
+	// load all annotation files into lists of landmarks
+	vector<core::LandmarkCollection<cv::Vec2f>> landmark_list;
 	try {
-		landmarks = eos::core::read_pts_landmarks(annotations[0]);
-	}
-	catch (const std::runtime_error &e) {
-		cout << "Error reading the landmarks: " << e.what() << endl;
+		landmark_list = eos::core::load_annotations<cv::Vec2f, cv::Vec3f>(annotations, mappingsfile);
+	} catch(const std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
-	morphablemodel::MorphableModel morphable_model;
+	// The expression blendshapes:
+	vector<morphablemodel::Blendshape> blend_shapes = morphablemodel::load_blendshapes(blendshapesfile.string());
 
-	try {
-		morphable_model = morphablemodel::load_model(modelfile.string());
-	} catch (const std::runtime_error &e) {
-		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
-			return EXIT_FAILURE;
-	}
+	// These two are used to fit the front-facing contour to the ibug contour landmarks:
+	fitting::ModelContour model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
+	fitting::ContourLandmarks ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string());
+
+	// The edge topology is used to speed up computation of the occluding face contour fitting:
+	morphablemodel::EdgeTopology edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string());
+
+	// Load landmarks, LandmarkMapper and the Morphable Model:
+	LandmarkCollection <cv::Vec2f> landmarks;
+	core::LandmarkMapper landmark_mapper = core::LandmarkMapper(mappingsfile);
 
 	// These will be the final 2D and 3D points used for the fitting:
-	vector<Eigen::Vector3f > model_points; // the points in the 3D shape model
+	vector<cv::Vec3f> model_points; // the points in the 3D shape model
 	vector<int> vertex_indices; // their vertex indices
-	std::tie(model_points, vertex_indices) = eos::core::load_model_data(landmarks, morphable_model, landmark_mapper);
 
-	BufferedVideoIterator<cv::Mat> vid_iterator;
-	std::vector <std::vector<cv::Vec2f>> landmark_annotation_list = eos::core::load_annotations(annotations, mappingsfile);
+	vector<core::Mesh> meshs;
+	vector<fitting::RenderingParameters> rendering_paramss;
+
+//	std::tie(model_points, vertex_indices) = eos::core::load_model_data<cv::Vec2f, cv::Vec3f>(landmarks, morphable_model, landmark_mapper);
 
+	BufferedVideoIterator vid_iterator;
 	try {
-		vid_iterator = BufferedVideoIterator<cv::Mat>(videofile.string(), landmark_annotation_list);
+		vid_iterator = BufferedVideoIterator(videofile.string());
 	} catch(std::runtime_error &e) {
-		cout << e.what() << endl;
+		std::cout << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
+	// iteration count
+	int frame_width = vid_iterator.width;
+	int frame_height = vid_iterator.height;
 
-	// todo: expand this to really perform some reconstruction, and move this to a test file.
 	// test with loading 10 frames subsequently.
-	// vid_iterator.next() will return a number of frames, depending on
-	std::deque<cv::Mat> frames = vid_iterator.next();
-	int count = 0;
+	std::deque<eos::video::Keyframe> frames = vid_iterator.next();
+
+	std::vector<float> pca_shape_coefficients;
+	std::vector<std::vector<float>> blend_shape_coefficients;
+	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
+
+	int n_frames = static_cast<int>(frames.size());
 	while(!(frames.empty())) {
-		if (count == 10) {
+		if (n_frames == 100) {
 			break;
 		}
-		int frame_count = 0;
-		for (std::deque<cv::Mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
-			//std::cout << ' ' << *it;
-			std::cout << frame_count << " ";
-			frame_count++;
-		}
 
-		std::cout << std::endl << "frames processed: " << count * frame_count << std::endl;
+		int frame_count = 0;
+		std::cout << std::endl << "frames in buffer: " << frames.size() << std::endl;
+		std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
+				morphable_model,
+				blend_shapes,
+				landmark_list,
+				landmark_mapper,
+				frame_width,
+				frame_height,
+				static_cast<int>(frames.size()),
+				edge_topology,
+				ibug_contour,
+				model_contour,
+				50,
+				boost::none,
+				30.0f,
+				boost::none,
+				pca_shape_coefficients,
+				blend_shape_coefficients,
+				fitted_image_points
+		);
 
 		frames = vid_iterator.next();
-		usleep(10);
-		count++;
+
+		n_frames++;
+
+//		// iterator through all frames, not needed persee.
+//		for (std::deque<eos::video::Keyframe>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+//			std::cout << it->score << " ";
+//			frame_count++;
+//		}
 	}
 }