Incorperated posebinning into the video iterator - made video iterator...

Incorperated posebinning into the video iterator - made video iterator threaded so it will search quickly for good frames

include/eos/core/landmark_utils.hpp

@@ -16,7 +16,6 @@
 
 #include <vector>
 #include <string>
-#include <vector>
 #include <algorithm>
 #include <iostream>
 #include <fstream>
@@ -189,27 +188,83 @@ namespace eos {
 			return std::make_pair(landmark_collection_list, annotations);
 		}
 
+	/**
+     * Get a bounding box around the landmarks
+	 * @param landmarks
+	 * @return
+	 */
+		cv::Rect get_face_roi(vector<Vec2f>image_points, int image_width, int image_height) {
+			cv::Rect bbox = cv::boundingRect(image_points);
+
+			// cap on the image width and height.
+			bbox.width = bbox.x + bbox.width < image_width ? bbox.width: image_width - bbox.x - 1;
+			bbox.height = bbox.y + bbox.height < image_height ? bbox.height: image_height - bbox.y - 1;
+
+			return bbox;
+		}
 		/**
+		 * Get the mesh coordinates.
 		 *
 		 * @tparam vec2f
 		 * @tparam vec4f
-		 * @param landmarks
-		 * @param landmark_mapper
-		 * @param mesh
-		 * @param model_points
-		 * @param vertex_indices
-		 * @param image_points
+		 * @param[in] landmarks
+		 * @param[in] landmark_mapper
+		 * @param[in] mesh
+		 * @param[in,out] model_points
+		 * @param[in,out] vertex_indices
+		 * @param[in,out] image_points
+		 */
+		template <typename vec2f>
+		inline void get_mesh_coordinates(core::LandmarkCollection<vec2f> landmarks,
+						 const core::LandmarkMapper& landmark_mapper,
+						 eos::core::Mesh& mesh,
+						 vector<Vec4f>& model_points,
+						 vector<int>& vertex_indices,
+						 vector<vec2f>& image_points) {
+			for (auto &lm: landmarks) {
+				auto converted_name = landmark_mapper.convert(lm.name);
+				if (!converted_name) { // no mapping defined for the current landmark
+					continue;
+				}
+
+				int vertex_idx = std::stoi(converted_name.get());
+				// todo: see how you can support a template for Vec4f.
+				Vec4f vertex(
+					mesh.vertices[vertex_idx].x,
+					mesh.vertices[vertex_idx].y,
+					mesh.vertices[vertex_idx].z,
+					mesh.vertices[vertex_idx].w
+				);
+
+				model_points.emplace_back(vertex);
+				vertex_indices.emplace_back(vertex_idx);
+				image_points.emplace_back(lm.coordinates);
+			}
+		}
+
+		/**
+		 *
+		 * Get the mesh coordinates, given a set of landmarks.
+		 *
+		 * @tparam vec2f
+		 * @tparam vec4f
+		 * @param[in] landmarks
+		 * @param[in] landmark_mapper
+		 * @param[in] mesh
+		 * @param[in,out] model_points
+		 * @param[in,out] vertex_indices
+		 * @param[in,out] image_points
 		 */
 		template <typename vec2f, typename vec4f>
-		inline void subselect_landmarks(core::LandmarkCollection<vec2f> landmarks,
+		inline void get_landmark_coordinates(core::LandmarkCollection<vec2f> landmarks,
 										const core::LandmarkMapper& landmark_mapper,
 										eos::core::Mesh& mesh,
 										vector<vector<vec4f>>& model_points,
 										vector<vector<int>>& vertex_indices,
 										vector<vector<vec2f>>& image_points) {
-			vector<Vec4f> current_model_points;
+			vector<vec4f> current_model_points;
 			vector<int> current_vertex_indices;
-			vector<Vec2f> current_image_points;
+			vector<vec2f> current_image_points;
 
 			for (auto &lm: landmarks) {
 				auto converted_name = landmark_mapper.convert(lm.name);
@@ -234,60 +289,6 @@ namespace eos {
 			vertex_indices.push_back(current_vertex_indices);
 			image_points.push_back(current_image_points);
 		}
-
-		/**
-		 * As the name suggests, in the multi-frame fitting we can subselect landmarks according to the
-		 * landmark mapper.
-		 *
-		 * @tparam vec2f
-		 * @param[in] landmarks
-		 * @param[in] landmark_mapper
-		 * @param[in] meshs
-		 * @param[out] model_points
-		 * @param[out] vertex_indices
-		 * @param[out] image_points
-		 */
-		template <typename vec2f, typename vec4f>
-		inline void subselect_landmarks_multi(const std::vector<core::LandmarkCollection<vec2f>>& landmarks,
-											  const core::LandmarkMapper& landmark_mapper,
-											  vector<eos::core::Mesh> meshs,
-											  vector<vector<vec4f>>& model_points,
-											  vector<vector<int>>& vertex_indices,
-											  vector<vector<vec2f>>& image_points) {
-
-			vector<Vec4f> current_model_points;
-			vector<int> current_vertex_indices;
-			vector<Vec2f> current_image_points;
-
-			// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM),
-			// and get the corresponding model points (mean if given no initial coeffs, from the computed shape otherwise):
-			for (int i = 0; i < landmarks.size(); i++) {
-				auto landmark_list = landmarks[i];
-				auto mesh = meshs[i];
-
-				for (auto &lm: landmark_list) {
-					auto converted_name = landmark_mapper.convert(lm.name);
-					if (!converted_name) { // no mapping defined for the current landmark
-						continue;
-					}
-					int vertex_idx = std::stoi(converted_name.get());
-					Vec4f vertex(
-						mesh.vertices[vertex_idx].x,
-						mesh.vertices[vertex_idx].y,
-						mesh.vertices[vertex_idx].z,
-						mesh.vertices[vertex_idx].w
-					);
-
-					current_model_points.emplace_back(vertex);
-					current_vertex_indices.emplace_back(vertex_idx);
-					current_image_points.emplace_back(lm.coordinates);
-				}
-
-				model_points.push_back(current_model_points);
-				vertex_indices.push_back(current_vertex_indices);
-				image_points.push_back(current_image_points);
-			}
-		}
 	}
 }
 

include/eos/fitting/FittingResult.hpp

@@ -42,6 +42,7 @@ namespace eos {
 struct FittingResult
 {
 	RenderingParameters rendering_parameters;
+	core::LandmarkCollection<cv::Vec2f> landmarks;
 	std::vector<float> pca_shape_coefficients;
 	std::vector<float> blendshape_coefficients;
 };

include/eos/fitting/ReconstructionData.hpp

-//
-// Created by RA Torenvliet on 12/04/2017.
-//
+#ifndef EOS_RECONSTRUCTIONDATA_HPP_
+#define EOS_RECONSTRUCTIONDATA_HPP_
 
-#ifndef EOS_RECONSTRUCTIONDATA_HPP
-#define EOS_RECONSTRUCTIONDATA_HPP
+#include "eos/core/Landmark.hpp"
+#include "eos/core/LandmarkMapper.hpp"
+#include "eos/morphablemodel/MorphableModel.hpp"
+#include "eos/morphablemodel/Blendshape.hpp"
+#include "eos/morphablemodel/EdgeTopology.hpp"
+#include "eos/fitting/contour_correspondence.hpp"
 
 namespace eos {
 namespace fitting {
 
 struct ReconstructionData {
-  morphablemodel::MorphableModel morphable_model;
-  std::vector <morphablemodel::Blendshape> blendshapes;
-  std::vector <core::LandmarkCollection<cv::Vec2f>> landmarks;
-  std::vector <cv::Mat> affine_camera_matrix;
-
+	morphablemodel::MorphableModel morphable_model;
+	std::vector <morphablemodel::Blendshape> blendshapes;
+	eos::core::LandmarkMapper landmark_mapper;
+	std::vector <core::LandmarkCollection<cv::Vec2f>> landmark_list;
+	eos::fitting::ModelContour model_contour;
+	eos::fitting::ContourLandmarks contour_landmarks;
+	eos::morphablemodel::EdgeTopology edge_topology;
 };
 
 }

include/eos/fitting/fitting.hpp

@@ -22,8 +22,10 @@
 #ifndef FITTING_HPP_
 #define FITTING_HPP_
 
+#include "eos/video/Keyframe.hpp"
 #include "eos/core/Landmark.hpp"
 #include "eos/core/LandmarkMapper.hpp"
+#include "eos/core/landmark_utils.hpp"
 #include "eos/core/Mesh.hpp"
 #include "eos/morphablemodel/MorphableModel.hpp"
 #include "eos/morphablemodel/Blendshape.hpp"
@@ -809,7 +811,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 * @return The fitted model shape instance and the final pose.
 */
 inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi(
-				const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blend_shapes, const std::vector<core::LandmarkCollection<cv::Vec2f>>& landmarks,
+				const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<core::LandmarkCollection<cv::Vec2f>>& landmarks,
 				const core::LandmarkMapper& landmark_mapper, int image_width, int image_height, const morphablemodel::EdgeTopology& edge_topology, const fitting::ContourLandmarks& contour_landmarks,
 				const fitting::ModelContour& model_contour, int num_iterations = 5, boost::optional<int> num_shape_coefficients_to_fit = boost::none, float lambda = 30.0f)
 {
@@ -819,7 +821,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 
 	return fit_shape_and_pose_multi(
 			morphable_model,
-			blend_shapes,
+			blendshapes,
 			landmarks,
 			landmark_mapper,
 			image_width,
@@ -863,10 +865,48 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 //  }
 //}}
 
+
+/**
+ * Generate a new mesh given pca_coefficients and blend_shape_coefficients.
+ * @param pca_shape_coefficients
+ * @param blend_shape_coefficients
+ * @param num_shape_coefficients_to_fit
+ * @return
+ */
+inline eos::core::Mesh generate_new_mesh(
+		morphablemodel::MorphableModel& morphable_model,
+		std::vector<morphablemodel::Blendshape>& blendshapes,
+		std::vector<float>& pca_shape_coefficients,
+		std::vector<float>& blendshape_coefficients,
+		unsigned int num_shape_coefficients_to_fit = 50) {
+
+	eos::core::Mesh mesh;
+	Eigen::MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
+	Eigen::VectorXf pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
+
+	// create new set of blendshape coefficients if empty:
+	if(blendshape_coefficients.empty()) {
+		blendshape_coefficients.resize(blendshapes.size());
+	}
+
+	Eigen::VectorXf combined_shape = pca_shape +
+		blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
+
+	mesh = morphablemodel::sample_to_mesh(
+		combined_shape,
+		morphable_model.get_color_model().get_mean(),
+		morphable_model.get_shape_model().get_triangle_list(),
+		morphable_model.get_color_model().get_triangle_list(),
+		morphable_model.get_texture_coordinates());
+
+	return mesh;
+}
+
+
 inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_2(
 		const morphablemodel::MorphableModel& morphable_model,
 		const std::vector<morphablemodel::Blendshape>& blendshapes,
-		const std::vector<core::LandmarkCollection<cv::Vec2f>>& landmarks,
+		std::vector<eos::video::Keyframe>& key_frames,
 		const core::LandmarkMapper& landmark_mapper,
 		int image_width,
 		int image_height,
@@ -875,7 +915,9 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 		const fitting::ContourLandmarks& contour_landmarks,
 		const fitting::ModelContour& model_contour,
 		int num_iterations,
-		boost::optional<int> num_shape_coefficients_to_fit, float lambda, boost::optional<fitting::RenderingParameters> initial_rendering_params,
+		boost::optional<int> num_shape_coefficients_to_fit,
+		float lambda,
+		boost::optional<fitting::RenderingParameters> initial_rendering_params,
 		std::vector<float>& pca_shape_coefficients,
 		std::vector<std::vector<float>>& blendshape_coefficients,
 		std::vector<std::vector<cv::Vec2f>>& fitted_image_points) {
@@ -901,7 +943,6 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 
 	// TODO: This leaves the following case open: num_coeffs given is empty or defined, but the
 	// pca_shape_coefficients given is != num_coeffs or the model's max-coeffs. What to do then? Handle & document!
-
 	if (blendshape_coefficients.size() < num_images) {
 		for (int j = 0; j < num_images; ++j) {
 			std::vector<float> current_blendshape_coefficients;
@@ -928,37 +969,37 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 	for (int j = 0; j < num_images; ++j) {
 		VectorXf current_combined_shape = current_pca_shape +
 			blendshapes_as_basis *
-			Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size()
-		);
+			Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
 
 		eos::core::Mesh current_mesh = morphablemodel::sample_to_mesh(
 			current_combined_shape,
 			morphable_model.get_color_model().get_mean(),
 			morphable_model.get_shape_model().get_triangle_list(),
 			morphable_model.get_color_model().get_triangle_list(),
-			morphable_model.get_texture_coordinates()
-		);
+			morphable_model.get_texture_coordinates());
 
 		// Get the locations of the model locations of the meshes, vertex_indices and image points
 		// (equal to landmark coordinates), for every image / mesh.
-		core::subselect_landmarks<Vec2f, Vec4f>(
-			landmarks[j],
+		eos::core::get_landmark_coordinates<Vec2f, Vec4f>(
+			key_frames[j].fitting_result.landmarks,
 			landmark_mapper,
 			current_mesh,
 			// output parameters
 			model_points,
 			vertex_indices,
-			image_points
-		);
+			image_points);
 
 		// Start constructing a list of rendering parameters needed for reconstruction.
 		// Get the current points from the last added image points and model points
 		auto current_pose = fitting::estimate_orthographic_projection_linear(
-			image_points.back(), model_points.back(), true, image_height
-		);
+			image_points.back(), model_points.back(), true, image_height);
 
 		fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height);
 		rendering_params.push_back(current_rendering_params);
+
+		// update key frame rendering params
+		key_frames[j].fitting_result.rendering_parameters = current_rendering_params;
+
 		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
 
 		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
@@ -998,60 +1039,107 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 		// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
 		vector<Vec2f> image_points_contour;
 		vector<int> vertex_indices_contour;
+
+		auto curr_keyframe = key_frames[j];
+		auto landmarks = curr_keyframe.fitting_result.landmarks;
 		auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
+
 		// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
-		std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(landmarks[j], contour_landmarks, model_contour, yaw_angle, current_meshs[j], rendering_params[j].get_modelview(), rendering_params[j].get_projection(), fitting::get_opencv_viewport(image_width, image_height));
+		std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(
+			landmarks,
+			contour_landmarks,
+			model_contour,
+			yaw_angle,
+			current_meshs[j],
+			rendering_params[j].get_modelview(),
+			rendering_params[j].get_projection(),
+			fitting::get_opencv_viewport(image_width, image_height)
+		);
+
 		// Add the contour correspondences to the set of landmarks that we use for the fitting:
 		vertex_indices[j] = fitting::concat(vertex_indices[j], vertex_indices_contour);
 		image_points[j] = fitting::concat(image_points[j], image_points_contour);
 
 		// Fit the occluding (away-facing) contour using the detected contour LMs:
 		vector<Eigen::Vector2f> occluding_contour_landmarks;
-		if (yaw_angle >= 0.0f) // positive yaw = subject looking to the left
-		{ // the left contour is the occluding one we want to use ("away-facing")
-			auto contour_landmarks_ = core::filter(landmarks[j], contour_landmarks.left_contour); // Can do this outside of the loop
-			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
-		}
-		else {
-			auto contour_landmarks_ = core::filter(landmarks[j], contour_landmarks.right_contour);
-			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
+
+		// positive yaw = subject looking to the left
+		if (yaw_angle >= 0.0f) {
+			 // the left contour is the occluding one we want to use ("away-facing")
+			auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
+			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
+				occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
+			});
+		} else {
+			auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
+			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
+				occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
+			});
 		}
-		auto edge_correspondences = fitting::find_occluding_edge_correspondences(current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f);
+
+		auto edge_correspondences = fitting::find_occluding_edge_correspondences(
+			current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f
+		);
+
 		image_points[j] = fitting::concat(image_points[j], edge_correspondences.first);
 		vertex_indices[j] = fitting::concat(vertex_indices[j], edge_correspondences.second);
 
 		// Get the model points of the current mesh, for all correspondences that we've got:
 		model_points[j].clear();
-		for (const auto& v : vertex_indices[j])
-		{
-			model_points[j].push_back({ current_meshs[j].vertices[v][0], current_meshs[j].vertices[v][1], current_meshs[j].vertices[v][2], current_meshs[j].vertices[v][3] });
+		for (const auto& v : vertex_indices[j]) {
+			model_points[j].push_back(
+				{
+					current_meshs[j].vertices[v][0],
+					current_meshs[j].vertices[v][1],
+					current_meshs[j].vertices[v][2],
+					current_meshs[j].vertices[v][3]
+				});
 		}
 
 		// Re-estimate the pose, using all correspondences:
-		fitting::ScaledOrthoProjectionParameters current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
+		auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
 		rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
+		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
 
-		cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
 		affine_from_orthos.push_back(affine_from_ortho);
 
 		// Estimate the PCA shape coefficients with the current blendshape coefficients:
-		VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
+		VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
+			blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
 		mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
 	}
 
-	pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi(morphable_model, affine_from_orthos, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_shape_coefficients_to_fit);
+	pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi(
+		morphable_model,
+		affine_from_orthos,
+		image_points,
+		vertex_indices,
+		mean_plus_blendshapes,
+		lambda,
+		num_shape_coefficients_to_fit
+	);
 
 	// Estimate the blendshape coefficients with the current PCA model estimate:
 	current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
 
 	for (int j = 0; j < num_images; ++j) {
-		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]);
+		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
+			blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
+		);
+		current_combined_shapes[j] = current_pca_shape +
+			blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
 
-		current_combined_shapes[j] = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
-		current_meshs[j] = morphablemodel::sample_to_mesh(current_combined_shapes[j], morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
+		current_meshs[j] = morphablemodel::sample_to_mesh(
+			current_combined_shapes[j],
+			morphable_model.get_color_model().get_mean(),
+			morphable_model.get_shape_model().get_triangle_list(),
+			morphable_model.get_color_model().get_triangle_list(),
+			morphable_model.get_texture_coordinates()
+		);
 	}
 
 	fitted_image_points = image_points;
+
 	return { current_meshs, rendering_params }; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices).
 };
 

include/eos/video/BufferedVideoIterator.hpp

@@ -25,10 +25,15 @@
 #include "opencv2/core/core.hpp"
 #include "opencv2/opencv.hpp"
 #include "opencv2/highgui/highgui.hpp"
+#include "eos/video/Keyframe.hpp"
 
 #include <string>
 #include <vector>
 #include <fstream>
+#include <thread>
+#include <mutex>
+#include <atomic>
+#include <unistd.h>
 
 using cv::Mat;
 using cv::Vec2f;
@@ -44,7 +49,435 @@ using namespace std;
 namespace eos {
 	namespace video {
 		// TODO: maybe move video iterator here.. or remove fil.
+/**
+ * @brief Computes the variance of laplacian of the given image or patch.
+ *
+ * This should compute the variance of the laplacian of a given image or patch, according to the 'LAPV'
+ * algorithm of Pech 2000.
+ * It is used as a focus or blurriness measure, i.e. to assess the quality of the given patch.
+ *
+ * @param[in] image Input image or patch.
+ * @return The computed variance of laplacian score.
+ */
+double variance_of_laplacian(const cv::Mat& image)
+{
+	cv::Mat laplacian;
+	cv::Laplacian(image, laplacian, CV_64F);
+
+	cv::Scalar mu, sigma;
+	cv::meanStdDev(laplacian, mu, sigma);
+
+	double focus_measure = sigma.val[0] * sigma.val[0];
+	return focus_measure;
+};
+
+/**
+ *
+ *
+ * @tparam T
+ */
+class BufferedVideoIterator
+{
+public:
+	int width;
+	int height;
+	Mat last_frame;
+	int last_frame_number;
+
+	bool reached_eof;
+
+	std::unique_ptr<std::thread> frame_buffer_worker;
+
+	BufferedVideoIterator() {};
+
+	// TODO: build support for setting the amount of max_frames in the buffer.
+	BufferedVideoIterator(std::string videoFilePath, fitting::ReconstructionData reconstruction_data, boost::property_tree::ptree settings) {
+		std::ifstream file(videoFilePath);
+		std::cout << "video file path: " << videoFilePath << std::endl;
+
+		if (!file.is_open()) {
+			throw std::runtime_error("Error opening given file: " + videoFilePath);
+		}
+
+		cv::VideoCapture tmp_cap(videoFilePath); // open video file
+		if (!tmp_cap.isOpened()) { // check if we succeeded
+			throw std::runtime_error("Could not play video");
+		}
+
+		this->cap = tmp_cap;
+
+		// copy settings from gathered from a .ini file
+		this->min_frames = settings.get<int>("video.min_frames", 5);
+		this->drop_frames = settings.get<int>("video.drop_frames", 0);
+		this->skip_frames = settings.get<int>("video.skip_frames", 0);
+		this->frames_per_bin = settings.get<unsigned int>("video.frames_per_bin", 2);
+		this->num_shape_coefficients_to_fit = settings.get<unsigned int>("video.num_shape_coefficients_to_fit", 50);
+
+		this->reconstruction_data = reconstruction_data;
+
+		// initialize bins
+		this->bins.resize(num_yaw_bins);
+
+		// reset all
+		this->n_frames = 0;
+		this->total_frames = 0;
+
+		std::cout << "Buffered video iter: "
+			"min_frames: " << min_frames << std::endl <<
+				  "drop_frames: " << drop_frames << std::endl <<
+				  "frames_per_bin: " << frames_per_bin << std::endl <<
+				  "num_shape_coefficients_to_fit: " <<  num_shape_coefficients_to_fit << std::endl;
+
+		std::cout << "total frames in video: " << cap.get(CV_CAP_PROP_FRAME_COUNT) << std::endl;
+	}
+
+	bool is_playing() {
+		return cap.isOpened();
+	}
+
+	/**
+	 *
+	 * @param frame
+	 * @return
+	 */
+	Keyframe generate_new_keyframe(cv::Mat frame) {
+		int frame_height = frame.rows;
+		int frame_width = frame.cols;
+
+		// Reached the end of the landmarks (only applicable for annotated videos):
+		if (reconstruction_data.landmark_list.size() <= total_frames) {
+			std::cout << "frame empty? " << frame.empty() << std::endl;
+			std::cout << "Reached end of landmarks(" <<
+				 reconstruction_data.landmark_list.size() << "/" << total_frames << ")" << std::endl;
+
+			return Keyframe();
+		}
+
+		auto landmarks = reconstruction_data.landmark_list[total_frames];
+		auto landmark_mapper = reconstruction_data.landmark_mapper;
+		auto blendshapes = reconstruction_data.blendshapes;
+		auto morphable_model = reconstruction_data.morphable_model;
+
+		vector<cv::Vec4f> model_points;
+		vector<int> vertex_indices;
+		vector<cv::Vec2f> image_points;
+
+		if (num_shape_coefficients_to_fit == 0) {
+			num_shape_coefficients_to_fit = morphable_model.get_shape_model().get_num_principal_components();
+		}
+
+		// make a new one
+		std::vector<float> blend_shape_coefficients;
+
+		if (pca_shape_coefficients.empty()) {
+			pca_shape_coefficients.resize(num_shape_coefficients_to_fit);
+		}
+
+		auto mesh = fitting::generate_new_mesh(
+			morphable_model,
+			blendshapes,
+			pca_shape_coefficients, // current pca_coeff will be the mean for the first iterations.
+			blend_shape_coefficients
+		);
+
+		// Will yield model_points, vertex_indices and frame_points
+		// todo: should this function not come from mesh?
+		core::get_mesh_coordinates(landmarks, landmark_mapper, mesh, model_points, vertex_indices, image_points);
+
+		auto current_pose = fitting::estimate_orthographic_projection_linear(
+			image_points, model_points, true, frame_height
+		);
+
+		fitting::RenderingParameters rendering_params(current_pose, frame_width, frame_height);
+		fitting::FittingResult fitting_result;
+		fitting_result.rendering_parameters = rendering_params;
+		fitting_result.landmarks = landmarks;
+
+//		std::cout << face_roi << " ("<< frame_width << "," << frame_height << ")" << std::endl;
+//		for (auto &&lm : landmarks) {
+//			cv::rectangle(
+//					frame, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
+//					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
+//			);
+//		}
+//
+//		cv::imshow("frame", frame);
+//		cv::waitKey(0);
+		cv::Rect face_roi = core::get_face_roi(image_points, frame_width, frame_height);
+
+		float frame_laplacian_score = static_cast<float>(variance_of_laplacian(frame(face_roi)));
+		return Keyframe(frame_laplacian_score, frame, fitting_result, total_frames);
+	}
+
+	/**
+	 * Try to add a new key frame. Look at the laplacian score and the yaw_angle. The yaw_angle will
+	 * determine the bin in which it will be added. The score will be compared to the other frames in the
+	 * same bin, if a bitter score is found, the frames are sorted on the score and cut-off on max frames.
+	 *
+	 * Todo: make sure this this cutting-off is done correct, we don't want faulty frames.
+	 *
+	 * @param keyframe
+	 * @return
+	 */
+	bool try_add(Keyframe &keyframe) {
+		// Determine whether to add or not:
+		auto yaw_angle = glm::degrees(glm::yaw(keyframe.fitting_result.rendering_parameters.get_rotation()));
+		auto idx = angle_to_index(yaw_angle);
+		bool add_frame = false;
+
+		keyframe.yaw_angle = yaw_angle;
+
+		// Score is 0 for total black frames, we don't want those:
+		if (keyframe.score == 0) {
+			return false;
+		}
+
+		// always add when we don't have enough frames
+		if (bins[idx].size() < frames_per_bin) {
+			add_frame = true; // definitely adding - we wouldn't have to go through the for-loop on the next line.
+		}
+
+		for (auto&& f : bins[idx]) {
+			if (keyframe.score > f.score) {
+				add_frame = true;
+			}
+		}
+
+		if (!add_frame) {
+			return false;
+		}
+
+		// Add the keyframe:
+		bins[idx].push_back(keyframe);
+		if (bins[idx].size() > frames_per_bin) {
+			n_frames--;
+			// need to remove the lowest one:
+			std::sort(std::begin(bins[idx]), std::end(bins[idx]),
+					  [](const auto& lhs, const auto& rhs) { return lhs.score > rhs.score; });
+			bins[idx].resize(frames_per_bin);
+		}
+
+		return true;
 	}
+
+	/**
+	 * Builds string from current state of the buffer as informative as possible.
+	 * TODO: string representation can be better, maybe add a max precision for floats.
+	 *
+	 * @return
+	 */
+	std::string to_string() const {
+		std::string output = std::to_string(n_frames) + ": ";
+
+		for (auto&& b : bins) {
+			output.append("[");
+
+			for (auto&& f : b) {
+				std::size_t idx = angle_to_index(f.yaw_angle);
+				output.append("(" + std::to_string(f.score));
+				output.append("," + std::to_string(f.yaw_angle));
+				output.append("), ");
+			}
+
+			output.append("],");
+		}
+
+		return output;
+	}
+
+	/**
+	 * Get keyframes from all bins as a flattened list of keyframes.
+	 *
+	 * @return std::vector<Keyframe>
+	 */
+	std::vector<Keyframe> get_keyframes() const {
+		std::vector<Keyframe> keyframes;
+		for (auto&& b : bins) {
+			for (auto&& f : b) {
+				keyframes.push_back(f);
+			}
+		}
+
+		return keyframes;
+	}
+
+	/**
+	 * Start filling the buffer with frames and start a worker thread to keep doing so.
+	 *
+	 * @return
+	 */
+	std::vector<Keyframe> start() {
+		// blocking calling next for the first time, stop if empty frame.
+		if (!next()) {
+			__stop();
+		}
+
+		// start a thread to keep getting new frames into the buffer from video:
+		frame_buffer_worker = std::make_unique<std::thread>(&BufferedVideoIterator::video_iterator, this);
+		frame_buffer_worker.get()->detach();
+
+		return get_keyframes();
+	}
+
+	/**
+	 * Fill the buffer by iterating through the video until the very last frame.
+	 */
+	void video_iterator() {
+		// Go and fill the buffer with more frames while we are reconstructing.
+		while (next()) { }
+
+		// stop the video
+		__stop();
+
+		std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames <<  std::endl;
+	}
+
+	 /**
+	  *
+	  * Set next frame and return frame_buffer. Returns true or false if the next frame is empty.
+	  * Empty frames mean that we reached the end of the video stream (i.e., file or camera).
+	  *
+	  * @return bool if
+	  */
+	bool next() {
+		Mat frame = __get_new_frame();
+
+		if (frame.empty()) {
+			return false;
+		}
+
+		// keep the last frame here, so we can play a video subsequently:
+		last_frame_number = total_frames;
+		last_frame = frame;
+
+		// TODO: only calculate lapscore within the bounding box of the face.
+		auto keyframe = generate_new_keyframe(frame);
+		bool frame_added = try_add(keyframe);
+
+		if(frame_added) {
+			n_frames++;
+
+			// Setting that the buffer has changed:
+			frame_buffer_changed = true;
+			std::cout << "frame added(" << n_frames << "): " << keyframe.score << ", " << keyframe.yaw_angle << std::endl;
+		}
+
+		total_frames++;
+
+		// fill up the buffer until we hit the minimum frames we want in the buffer.
+		if(n_frames < min_frames && total_frames < 30) {
+			std::cout << "not enough frames yet: " << n_frames << "/" << min_frames << std::endl;
+			return next();
+		}
+
+		return true;
+	}
+
+	/**
+	 * Update pca shape coeff. Probably we need to make something with a mutex, for updating and reading
+	 * the pca_shape_coeff in the generate_new_keyframe functionality.
+	 *
+	 * @param pca_shape_coefficients
+	 */
+	void update_reconstruction_coeff(std::vector<float> pca_shape_coefficients) {
+		std::cout << "-- Update pca coeff -- " << std::endl;
+		this->pca_shape_coefficients = pca_shape_coefficients;
+	}
+
+	// Converts a given yaw angle to an index in the internal bins vector.
+	// Assumes 9 bins and 20� intervals.
+	static std::size_t angle_to_index(float yaw_angle)
+	{
+		if (yaw_angle <= -70.f)
+			return 0;
+		if (yaw_angle <= -50.f)
+			return 1;
+		if (yaw_angle <= -30.f)
+			return 2;
+		if (yaw_angle <= -10.f)
+			return 3;
+		if (yaw_angle <= 10.f)
+			return 4;
+		if (yaw_angle <= 30.f)
+			return 5;
+		if (yaw_angle <= 50.f)
+			return 6;
+		if (yaw_angle <= 70.f)
+			return 7;
+		return 8;
+	};
+
+	/**
+	 * Only by calling this function, we return if the buffer has changed since last check.
+	 *
+	 * @return
+	 */
+	bool has_changed() {
+		bool has_changed = frame_buffer_changed;
+
+		// record the event if the buffer is changed since last time:
+		frame_buffer_changed = false;
+
+		return has_changed;
+	}
+
+	/**
+	 * Get a new frame from the video source.
+	 *
+	 * @return cv::Mat
+	 */
+	Mat __get_new_frame() {
+		// Get a new frame from camera.
+		Mat frame;
+		cap >> frame;
+
+		return frame;
+	};
+
+
+	// todo: move to private if possible.
+	void __stop() {
+		cap.release();
+	};
+
+private:
+	int num_yaw_bins = 9;
+	unsigned int frames_per_bin;
+	bool frame_buffer_changed = false;
+
+	cv::VideoCapture cap;
+	std::deque<Keyframe> frame_buffer;
+	eos::fitting::ReconstructionData reconstruction_data;
+
+	using BinContent = std::vector<Keyframe>;
+	std::vector<BinContent> bins;
+
+	// latest pca_shape_coefficients
+	std::vector<float> pca_shape_coefficients;
+
+	// TODO: make set-able
+	// total frames in processed, not persee in buffer (skipped frames)
+	int total_frames;
+
+	// total frames in buffer
+	int n_frames;
+
+	// number of frames to skip at before starting
+	int skip_frames = 0;
+
+	// minimum amount of frames to keep in buffer.
+	int min_frames = 5;
+
+	// Note: these settings are for future use
+	int drop_frames = 0;
+
+	unsigned int num_shape_coefficients_to_fit = 0;
+
+	// laplacian threshold
+//	double laplacian_threshold = 10000000;
+};
+
+}
 }
 
 #endif

include/eos/video/Keyframe.hpp

@@ -24,6 +24,8 @@
 
 #include "eos/fitting/FittingResult.hpp"
 #include "eos/fitting/RenderingParameters.hpp"
+#include "eos/fitting/ReconstructionData.hpp"
+#include "eos/fitting/fitting.hpp"
 #include "eos/morphablemodel/Blendshape.hpp"
 #include "eos/morphablemodel/MorphableModel.hpp"
 #include "eos/render/texture_extraction.hpp"
@@ -33,6 +35,7 @@
 #include "opencv2/core/core.hpp"
 
 #include <deque>
+#include <cstdlib>
 
 namespace eos {
 namespace video {
@@ -72,108 +75,26 @@ public:
 	 * @param frame
 	 * @param fitting_result
 	 */
-	Keyframe(float score, cv::Mat frame, fitting::FittingResult fitting_result) {
+	Keyframe(float score, cv::Mat frame, fitting::FittingResult fitting_result, int frame_number) {
 		this->frame = frame;
 		this->score = score;
 		this->fitting_result = fitting_result;
+		this->frame_number = frame_number;
 	}
 
+	// The original frame
 	cv::Mat frame;
-	int frame_number;
-	float score = 0.0f;
 
-	fitting::FittingResult fitting_result;
-};
+	// Frame number such that order between frames can be maintained.
+	int frame_number;
 
-/**
- * @brief A keyframe selection that selects keyframes according to yaw pose and score.
- *
- * Separates the +-90� yaw pose range into 20� intervals (i.e. 90 to 70, ..., -10 to 10, ...), and puts frames
- * into each bin, until full. Replaces keyframes with better frames if the score is higher than that of
- * current keyframes.
- *
- * The yaw pose bins are currently hard-coded (9 bins, 20� intervals).
- */
-struct PoseBinningKeyframeSelector
-{
-public:
-    PoseBinningKeyframeSelector(int frames_per_bin = 2) : frames_per_bin(frames_per_bin)
-    {
-        bins.resize(num_yaw_bins);
-    };
+	// laplacian score for example
+	float score = 0.0f;
 
-    bool try_add(float frame_score, cv::Mat image, const fitting::FittingResult& fitting_result)
-    {
-        // Determine whether to add or not:
-        auto yaw_angle = glm::degrees(glm::yaw(fitting_result.rendering_parameters.get_rotation()));
-        auto idx = angle_to_index(yaw_angle);
-        bool add_frame = false;
-        if (bins[idx].size() < frames_per_bin) // always add when we don't have enough frames
-            add_frame =
-                true; // definitely adding - we wouldn't have to go through the for-loop on the next line.
-        for (auto&& f : bins[idx])
-        {
-            if (frame_score > f.score)
-                add_frame = true;
-        }
-        if (!add_frame)
-        {
-            return false;
-        }
-        // Add the keyframe:
-        bins[idx].push_back(video::Keyframe{frame_score, image, fitting_result});
-        if (bins[idx].size() > frames_per_bin)
-        {
-            // need to remove the lowest one:
-            std::sort(std::begin(bins[idx]), std::end(bins[idx]),
-                      [](const auto& lhs, const auto& rhs) { return lhs.score > rhs.score; });
-            bins[idx].resize(frames_per_bin);
-        }
-        return true;
-    };
-
-    // Returns the keyframes as a vector.
-    std::vector<Keyframe> get_keyframes() const
-    {
-        std::vector<Keyframe> keyframes;
-        for (auto&& b : bins)
-        {
-            for (auto&& f : b)
-            {
-                keyframes.push_back(f);
-            }
-        }
-        return keyframes;
-    };
+	// rotation from in fitting results
+	float yaw_angle = 0.0f;
 
-private:
-    using BinContent = std::vector<Keyframe>;
-    std::vector<BinContent> bins;
-    const int num_yaw_bins = 9;
-    int frames_per_bin;
-
-    // Converts a given yaw angle to an index in the internal bins vector.
-    // Assumes 9 bins and 20� intervals.
-    static std::size_t angle_to_index(float yaw_angle)
-    {
-        if (yaw_angle <= -70.f)
-            return 0;
-        if (yaw_angle <= -50.f)
-            return 1;
-        if (yaw_angle <= -30.f)
-            return 2;
-        if (yaw_angle <= -10.f)
-            return 3;
-        if (yaw_angle <= 10.f)
-            return 4;
-        if (yaw_angle <= 30.f)
-            return 5;
-        if (yaw_angle <= 50.f)
-            return 6;
-        if (yaw_angle <= 70.f)
-            return 7;
-        return 8;
-    };
+	fitting::FittingResult fitting_result;
 };
 
 /**
@@ -270,156 +191,6 @@ cv::Mat merge_weighted_mean(const std::vector<Keyframe>& keyframes,
     return merged_isomap;
 };
 
-/**
- * @brief Computes the variance of laplacian of the given image or patch.
- *
- * This should compute the variance of the laplacian of a given image or patch, according to the 'LAPV'
- * algorithm of Pech 2000.
- * It is used as a focus or blurriness measure, i.e. to assess the quality of the given patch.
- *
- * @param[in] image Input image or patch.
- * @return The computed variance of laplacian score.
- */
-double variance_of_laplacian(const cv::Mat& image)
-{
-    cv::Mat laplacian;
-    cv::Laplacian(image, laplacian, CV_64F);
-
-    cv::Scalar mu, sigma;
-    cv::meanStdDev(laplacian, mu, sigma);
-
-    double focus_measure = sigma.val[0] * sigma.val[0];
-    return focus_measure;
-};
-
-/**
- * BufferedVideo Iterator will keep a buffer of the last seen n_frames. By calling .next() it will load a new
- * frame from the given video and you will get a pointer to the front of the buffer (which has n_frames).
- *
- * Just imagine a sliding window accross the video, this is what we aim to implement here.
- *
- * Example:
- *    vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
- *
- *    std::deque<cv::mat> frames = vid_iterator.next();
- *    while(!(frames.empty())) {
- *        for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
- *            std::cout << ' ' << *it;
- *        }
- *
- *         frames = vid_iterator.next();
- *    }
- *
- * @tparam T
- */
-// Note for this template: later we can use other templates for easy testing (not using cv:Mat but <int> for example).
-class BufferedVideoIterator {
-public:
-	int width;
-	int height;
-
-	BufferedVideoIterator() {};
-
-	// TODO: build support for setting the amount of max_frames in the buffer.
-	BufferedVideoIterator(std::string videoFilePath, boost::property_tree::ptree settings) {
-		std::ifstream file(videoFilePath);
-		std::cout << "video file path: " << videoFilePath << std::endl;
-		if (!file.is_open()) {
-			throw std::runtime_error("Error opening given file: " + videoFilePath);
-		}
-
-		cv::VideoCapture tmp_cap(videoFilePath); // open video file
-
-		if (!tmp_cap.isOpened()) { // check if we succeeded
-			throw std::runtime_error("Could not play video");
-		}
-
-		this->cap = tmp_cap;
-		this->max_frames = settings.get<int>("video.max_frames", 5);
-		this->min_frames = settings.get<int>("video.min_frames", 5);
-		this->drop_frames = settings.get<int>("video.drop_frames", 0);
-		this->laplacian_threshold = settings.get<int>("video.blur_threshold", 1000);
-
-		// TODO: Implement this.
-		this->skip_frames = settings.get<int>("video.skip_frames", 0);
-	}
-
-	/**
-	 * Set next frame and return frame_buffer.
-	 *
-	 * @return dequeue<Mat> frame buffer.
-	 *
-	 * TODO: build support for returning landmarks AND frames.
-	 */
-	std::deque <Keyframe> next() {
-		long frame_buffer_length = frame_buffer.size();
-
-		// Get a new frame from camera.
-		cv::Mat frame;
-		cap >> frame;
-
-		// Pop if we exceeded max_frames.
-		if (n_frames > max_frames) {
-			frame_buffer.pop_front();
-			n_frames--;
-		}
-
-		float frame_laplacian_score = static_cast<float>(variance_of_laplacian(frame));
-
-		if (frame_laplacian_score < laplacian_threshold && frame_laplacian_score > 0) {
-			frame_buffer.push_back(Keyframe(frame_laplacian_score, frame, total_frames_processed));
-			n_frames++;
-			std::cout << total_frames_processed << ": laplacian score " << frame_laplacian_score << std::endl;
-		} else {
-			std::cout << total_frames_processed << ": skipping frame(";
-			if (frame_laplacian_score == 0) {
-				std::cout << "total black): " << frame_laplacian_score << std::endl;
-			} else {
-				std::cout << "too blurry): " << frame_laplacian_score << std::endl;
-			}
-
-		}
-
-		total_frames_processed++;
-
-		// fill up the buffer until we hit the minimum frames we want in the buffer.
-		if(n_frames < min_frames) {
-			frame_buffer = next();
-		}
-
-		return frame_buffer;
-	}
-
-	std::deque <Keyframe> get_frame_buffer() {
-		return frame_buffer;
-	}
-
-private:
-	cv::VideoCapture cap;
-	std::deque<Keyframe> frame_buffer;
-
-	// TODO: make set-able
-	// total frames in processed, not persee in buffer (skipped frames)
-	int total_frames_processed = 0;
-
-	// total frames in buffer
-	int n_frames = 0;
-
-	// number of frames to skip at before starting
-	int skip_frames = 0;
-
-	// minimum amount of frames to keep in buffer.
-	int min_frames = 5;
-
-	// keep max_frames into the buffer.
-	int max_frames = 5;
-
-	// Note: these settings are for future use
-	int drop_frames = 0;
-
-	// laplacian threshold
-	double laplacian_threshold = 10000000;
-};
 
 /**
 * @brief Merges isomaps from a live video with a weighted averaging, based
@@ -487,6 +258,18 @@ public:
 		return merged_isomap_uchar;
 	};
 
+	cv::Mat sharpen(const cv::Mat& isomap) {
+		cv::Mat output;
+		cv::Mat kernel = (cv::Mat_<float>(5, 5) << -0.125, -0.125, -0.125, -0.125, -0.125,
+											  -0.125,  0.25 ,  0.25 ,  0.25 , -0.125,
+											  -0.125,  0.25 ,  1.   ,  0.25 , -0.125,
+											  -0.125,  0.25 ,  0.25 ,  0.25 , -0.125,
+											  -0.125, -0.125, -0.125, -0.125, -0.125);
+		cv::filter2D(isomap, output, isomap.depth(), kernel);
+
+		return output;
+	}
+
 private:
 	cv::Mat visibility_counter;
 	cv::Mat merged_isomap;

utils/accuracy-evaluation.cpp

@@ -25,6 +25,7 @@
 #include "eos/render/texture_extraction.hpp"
 #include "eos/video/Keyframe.hpp"
 #include "eos/fitting/ReconstructionData.hpp"
+#include "eos/video/BufferedVideoIterator.hpp"
 
 #include "opencv2/core/core.hpp"
 #include "opencv2/opencv.hpp"
@@ -166,28 +167,28 @@ void evaluate_accuracy(
  * @param rendering_paramss
  * @param landmark_list
  * @param morphable_model
- * @param blend_shapes
+ * @param blendshapes
  * @param meshs
  * @param pca_shape_coefficients
- * @param blend_shape_coefficients
+ * @param blendshape_coefficients
  * @param fitted_image_points
  * @param annotations
  * @param landmark_mapper
  * @param settings
  * @param n_iter
  */
-void output_render(
-		std::deque<eos::video::Keyframe> key_frames,
+void render_output(
+		std::vector<eos::video::Keyframe> key_frames,
 		std::vector<fitting::RenderingParameters> rendering_paramss,
 		std::vector<core::LandmarkCollection<cv::Vec2f>> landmark_list,
-		morphablemodel::MorphableModel morphable_model,
-		vector<morphablemodel::Blendshape> blend_shapes,
 		vector<core::Mesh> meshs,
 		std::vector<float> pca_shape_coefficients,
-		std::vector<std::vector<float>> blend_shape_coefficients,
+		std::vector<std::vector<float>> blendshape_coefficients,
 		std::vector<std::vector<cv::Vec2f>> fitted_image_points,
 		std::vector<std::string> annotations,
-		core::LandmarkMapper landmark_mapper,
+		fitting::ReconstructionData reconstruction_data,
+		Mat last_frame,
+		int last_frame_number,
 		boost::property_tree::ptree settings,
 		int n_iter) {
 
@@ -195,6 +196,11 @@ void output_render(
 	WeightedIsomapAveraging isomap_averaging(60.f); // merge all triangles that are facing <60° towards the camera
 	eos::video::PcaCoefficientMerging pca_shape_merging;
 
+	auto landmark_mapper = reconstruction_data.landmark_mapper;
+	auto blendshapes = reconstruction_data.blendshapes;
+	auto morphable_model = reconstruction_data.morphable_model;
+	auto landmarks = reconstruction_data.landmark_list[last_frame_number];
+
 	auto outputfilebase = annotations[0];
 
 	for (uint i = 0; i < key_frames.size(); ++i) {
@@ -202,8 +208,11 @@ void output_render(
 
 		auto unmodified_frame = frame.clone();
 		int frame_number = key_frames[i].frame_number;
+		float yaw_angle = glm::degrees(glm::yaw(rendering_paramss[i].get_rotation()));
+
+		cv::imwrite("/tmp/eos/" +
+			std::to_string(frame_number) + "_" + std::to_string(yaw_angle) + ".png", frame);
 
-//		float yaw_angle = glm::degrees(glm::yaw(rendering_paramss[i].get_rotation()));
 		int frame_width = frame.cols;
 		int frame_height = frame.rows;
 
@@ -215,29 +224,47 @@ void output_render(
 				unmodified_frame,
 				true,
 				render::TextureInterpolation::NearestNeighbour,
-				512
-		);
+				512);
 
 		// Merge the isomaps - add the current one to the already merged ones:
 		merged_isomap = isomap_averaging.add_and_merge(isomap);
 	}
 
 	// Get last frame (that's the newest one).
-	Mat frame = key_frames.back().frame;
-	auto last_rendering_params = rendering_paramss.back();
-	auto last_blend_coefficients = blend_shape_coefficients.back();
-
+	Mat frame = last_frame.clone();
 	int frame_width = frame.cols;
 	int frame_height = frame.rows;
 
+	vector<cv::Vec4f> model_points;
+	vector<int> vertex_indices;
+	vector<cv::Vec2f> image_points;
+
+	// make a new one
+	std::vector<float> blend_shape_coefficients;
+
+	auto mesh = fitting::generate_new_mesh(
+		morphable_model,
+		blendshapes,
+		pca_shape_coefficients, // current pca_coeff will be the mean for the first iterations.
+		blend_shape_coefficients);
+
+	// Will yield model_points, vertex_indices and image_points
+	// todo: should this function not come from mesh?
+	core::get_mesh_coordinates(landmarks, landmark_mapper, mesh, model_points, vertex_indices, image_points);
+
+	auto current_pose = fitting::estimate_orthographic_projection_linear(
+		image_points, model_points, true, frame_height);
+
+	fitting::RenderingParameters rendering_params(current_pose, frame_width, frame_height);
+
 	// Same for the shape:
 	pca_shape_coefficients = pca_shape_merging.add_and_merge(pca_shape_coefficients);
-	auto blendshapes_as_basis = morphablemodel::to_matrix(blend_shapes);
+
+	auto blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
 	auto merged_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients) +
 						blendshapes_as_basis *
-						Eigen::Map<const Eigen::VectorXf>(last_blend_coefficients.data(),
-														  last_blend_coefficients.size()
-						);
+						Eigen::Map<const Eigen::VectorXf>(blend_shape_coefficients.data(),
+														  blend_shape_coefficients.size());
 
 	auto merged_mesh = morphablemodel::sample_to_mesh(
 			merged_shape,
@@ -250,24 +277,33 @@ void output_render(
 
 	// Render the model in a separate window using the estimated pose, shape and merged texture:
 	Mat rendering;
+
 	// render needs 4 channels 8 bits image, needs a two step conversion.
 	cvtColor(frame, rendering, CV_BGR2BGRA);
 
 	// make sure the image is CV_8UC4, maybe do check first?
 	rendering.convertTo(rendering, CV_8UC4);
 
+	Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+	Mat isomap = render::extract_texture(
+		merged_mesh,
+		affine_cam,
+		last_frame,
+		true,
+		render::TextureInterpolation::NearestNeighbour,
+		512);
+
 	std::tie(rendering, std::ignore) = render::render(
 			merged_mesh,
-			last_rendering_params.get_modelview(),
-			last_rendering_params.get_projection(),
+			rendering_params.get_modelview(),
+			rendering_params.get_projection(),
 			frame_width,
 			frame_height,
-			render::create_mipmapped_texture(merged_isomap),
+			render::create_mipmapped_texture(isomap),
 			true,
 			false,
 			false,
-			rendering
-	);
+			rendering);
 
 	cv::imshow("render", rendering);
 	cv::waitKey(10);
@@ -279,10 +315,10 @@ void output_render(
  * @param rendering_paramss
  * @param landmark_list
  * @param morphable_model
- * @param blend_shapes
+ * @param blendshapes
  * @param meshs
  * @param pca_shape_coefficients
- * @param blend_shape_coefficients
+ * @param blendshape_coefficients
  * @param fitted_image_points
  * @param annotations
  * @param landmark_mapper
@@ -290,111 +326,141 @@ void output_render(
  * @param n_iter
  */
 void evaluate_results(
-		std::deque<eos::video::Keyframe> key_frames,
+		std::vector<eos::video::Keyframe> key_frames,
 		std::vector<fitting::RenderingParameters> rendering_paramss,
-		std::vector<core::LandmarkCollection<cv::Vec2f>> landmark_list,
-		morphablemodel::MorphableModel morphable_model,
-		vector<morphablemodel::Blendshape> blend_shapes,
 		vector<core::Mesh> meshs,
 		std::vector<float> pca_shape_coefficients,
-		std::vector<std::vector<float>> blend_shape_coefficients,
+		std::vector<std::vector<float>> blendshape_coefficients,
 		std::vector<std::vector<cv::Vec2f>> fitted_image_points,
 		std::vector<std::string> annotations,
-		core::LandmarkMapper landmark_mapper,
+		fitting::ReconstructionData reconstruction_data,
 		boost::property_tree::ptree settings,
 		int n_iter) {
 
-	WeightedIsomapAveraging isomap_averaging(60.f); // merge all triangles that are facing <60° towards the camera
+	WeightedIsomapAveraging isomap_averaging(10.f); // merge all triangles that are facing <60° towards the camera
 
 	Mat merged_isomap;
 	fs::path outputfilebase = annotations[0];
 
+	auto landmark_mapper = reconstruction_data.landmark_mapper;
+	auto blendshapes = reconstruction_data.blendshapes;
+	auto morphable_model = reconstruction_data.morphable_model;
+
+
 	// The 3D head pose can be recovered as follows:
 	for (uint i = 0; i < key_frames.size(); ++i) {
-		float yaw_angle = glm::degrees(glm::yaw(rendering_paramss[i].get_rotation()));
 		int frame_number = key_frames[i].frame_number;
+		auto landmarks = key_frames[i].fitting_result.landmarks;
+		auto rendering_params = key_frames[i].fitting_result.rendering_parameters;
+		float yaw_angle = key_frames[i].yaw_angle;
+
 		Mat frame = key_frames[i].frame;
 
 		int frame_width = frame.cols;
 		int frame_height = frame.rows;
 
+		Mat outimg = frame.clone();
+
 		// and similarly for pitch and roll.
 		// Extract the texture from the image using given mesh and camera parameters:
 		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
-				rendering_paramss[i], frame_width, frame_height
-		);
+			rendering_params, frame_width, frame_height);
+
+//		for (auto &&lm : landmarks) {
+//			cv::rectangle(
+//					outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
+//					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
+//			);
+//		}
+//		// Draw the fitted mesh as wireframe, and save the image:
+//		draw_wireframe(
+//				outimg,
+//				meshs[i],
+//				rendering_params.get_modelview(),
+//				rendering_params.get_projection(),
+//				fitting::get_opencv_viewport(frame_width, frame_height));
+//
+//		cv::imshow("Img", outimg);
+//		cv::waitKey(1);
 
 		// Draw the loaded landmarks:
 		Mat isomap = render::extract_texture(meshs[i], affine_from_ortho, frame);
-		Mat outimg = frame.clone();
-
-		for (auto &&lm : landmark_list[i]) {
-			cv::rectangle(
-					outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
-					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
-			);
-		}
-
-		// Draw the fitted mesh as wireframe, and save the image:
-		draw_wireframe(
-				outimg,
-				meshs[i],
-				rendering_paramss[i].get_modelview(),
-				rendering_paramss[i].get_projection(),
-				fitting::get_opencv_viewport(frame_width, frame_height)
-		);
-
-		fs::path path = (fs::path(annotations[frame_number]).parent_path() / "eval");
-		std::string outputfile = (path / fs::path(annotations[frame_number]).replace_extension("").filename()).string();
-		std::string iter = "_" + std::to_string(n_iter) + "_" + std::to_string(i);
-		cv::imwrite(outputfile + iter + ".annotated.png", outimg);
-
-		// save frontal rendering with texture:
-		Mat frontal_rendering;
-		glm::mat4 modelview_frontal = glm::mat4( 1.0 );
-		core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
-				morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
-				morphable_model.get_color_model().get_mean(),
-				morphable_model.get_shape_model().get_triangle_list(),
-				morphable_model.get_color_model().get_triangle_list(),
-				morphable_model.get_texture_coordinates()
-		);
-
-		outimg = frame.clone();
-		cv::imwrite(outputfile + iter + ".isomap.png", isomap);
 
 		// merge the isomaps:
 		merged_isomap = isomap_averaging.add_and_merge(isomap);
 
-		std::tie(frontal_rendering, std::ignore) = eos::render::render(
-				neutral_expression,
-				rendering_paramss[i].get_modelview(),
-				rendering_paramss[i].get_projection(),
-				frame_width,
-				frame_height,
-				render::create_mipmapped_texture(merged_isomap),
-				true,
-				false,
-				false,
-				outimg
-		);
-
-		cv::imwrite(outputfile + iter + ".frontal.png", frontal_rendering);
-		evaluate_accuracy(
-			landmark_list[i],
-			landmark_mapper,
-			meshs[i],
-			affine_from_ortho,
-			settings
-		);
+//		cv::imshow("isomap", isomap);
+//		cv::imshow("merged_isomap", merged_isomap);
+//
+//		Mat outimg = frame.clone();
+//
+//		for (auto &&lm : landmarks) {
+//			cv::rectangle(
+//					outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
+//					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
+//			);
+//		}
+//		// Draw the fitted mesh as wireframe, and save the image:
+//		draw_wireframe(
+//				outimg,
+//				meshs[i],
+//				rendering_params.get_modelview(),
+//				rendering_params.get_projection(),
+//				fitting::get_opencv_viewport(frame_width, frame_height));
+//
+//		cv::imshow("Img", outimg);
+//		cv::waitKey(0);
+
+//		fs::path path = (fs::path(annotations[frame_number]).parent_path() / "eval");
+//		std::string outputfile = (path / fs::path(annotations[frame_number]).replace_extension("").filename()).string();
+//		std::string iter = "_" + std::to_string(n_iter) + "_" + std::to_string(i);
+//		cv::imwrite(outputfile + iter + ".annotated.png", outimg);
+//
+//		// save frontal rendering with texture:
+//		glm::mat4 modelview_frontal = glm::mat4( 1.0 );
+//		core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
+//				morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
+//				morphable_model.get_color_model().get_mean(),
+//				morphable_model.get_shape_model().get_triangle_list(),
+//				morphable_model.get_color_model().get_triangle_list(),
+//				morphable_model.get_texture_coordinates()
+//		);
+
+//		cv::imwrite(outputfile + iter + ".isomap.png", isomap);
+
+//		Mat frontal_rendering;
+//		std::tie(frontal_rendering, std::ignore) = eos::render::render(
+//				neutral_expression,
+//				rendering_params.get_modelview(),
+//				rendering_params.get_projection(),
+//				frame_width,
+//				frame_height,
+//				render::create_mipmapped_texture(merged_isomap),
+//				true,
+//				false,
+//				false
+//		);
+
+//		cv::imwrite(outputfile + iter + ".frontal.png", frontal_rendering);
+//		evaluate_accuracy(
+//			landmarks,
+//			landmark_mapper,
+//			meshs[i],
+//			affine_from_ortho,
+//			settings
+//		);
 	}
 
 	// save the merged isomap:
 	std::string iter = "_" + std::to_string(n_iter);
 	fs::path path = (fs::path(annotations[n_iter]).parent_path() / "eval");
+	fs::create_directory(path);
 	std::string outputfile = (path / fs::path(annotations[n_iter]).replace_extension("").filename()).string();
 
-	cv::imwrite(outputfile + iter + ".isomap.png", merged_isomap);
+	// sharpen isomap
+	Mat sharpen = isomap_averaging.sharpen(merged_isomap);
+
+	cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
 
 	// save the frontal rendering with merged isomap:
 	Mat frontal_rendering;
@@ -407,23 +473,20 @@ void evaluate_results(
 			morphable_model.get_texture_coordinates()
 	);
 
-	std::tie(frontal_rendering, std::ignore) = render::render(
-			neutral_expression,
-			modelview_frontal,
-			glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
-			512, 512,
-			render::create_mipmapped_texture(merged_isomap),
-			true,
-			false,
-			false
-	);
-
-	cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
+//	std::tie(frontal_rendering, std::ignore) = render::render(
+//			neutral_expression,
+//			modelview_frontal,
+//			glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
+//			512, 512,
+//			render::create_mipmapped_texture(merged_isomap),
+//			true,
+//			false,
+//			false
+//	);
+//	cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
 
 	// Save the mesh as textured obj:
-	core::write_textured_obj(morphable_model.draw_sample(
-			pca_shape_coefficients, std::vector<float>()), outputfile + iter + ".obj");
-
+	core::write_textured_obj(neutral_expression, outputfile + iter + ".obj");
 	std::cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile + iter + ".obj" << std::endl;
 }
 
@@ -450,7 +513,7 @@ boost::property_tree::ptree get_reconstruction_config(std::string filename) {
  * @return
  */
 vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
-		std::deque<eos::video::Keyframe> key_frames, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
+		std::vector<eos::video::Keyframe> key_frames, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
 	vector<core::LandmarkCollection<cv::Vec2f>> sublist;
 
 	for (auto& f : key_frames) {
@@ -523,8 +586,8 @@ int main(int argc, char *argv[]) {
 	morphablemodel::MorphableModel morphable_model;
 	std::vector<float> pca_shape_coefficients;
 
-	// List of blend_shapes coefficients:
-	std::vector<std::vector<float>> blend_shape_coefficients;
+	// List of blendshapes coefficients:
+	std::vector<std::vector<float>> blendshape_coefficients;
 	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
 
 	// These will be the final 2D and 3D points used for the fitting:
@@ -558,7 +621,7 @@ int main(int argc, char *argv[]) {
 	}
 
 	// The expression blendshapes:
-	auto blend_shapes = morphablemodel::load_blendshapes(blendshapesfile.string());
+	auto blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string());
 
 	// These two are used to fit the front-facing contour to the ibug contour landmarks:
 	auto model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
@@ -568,35 +631,39 @@ int main(int argc, char *argv[]) {
 
 	// Read .ini file to accommodate.
 	auto settings = get_reconstruction_config(reconstruction_config.string());
+	auto reconstruction_data = eos::fitting::ReconstructionData{
+		morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
 
 	// Start with the video play and get video file:
 	BufferedVideoIterator vid_iterator;
 
 	try {
-		vid_iterator = BufferedVideoIterator(videofile.string(), settings);
+		vid_iterator = BufferedVideoIterator(videofile.string(), reconstruction_data, settings);
 	} catch(std::runtime_error &e) {
 		std::cout << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
 
 	// Start getting video frames:
-	std::deque<eos::video::Keyframe> key_frames = vid_iterator.next();
-	eos::fitting::ReconstructionData data;
+	vid_iterator.start();
 
 	// Count the amount of iterations:
 	int n_iter = 0;
-	while(!(key_frames.empty())) {
-//		if (n_iter == 100) {
-//			break;
-//		}
+	while(vid_iterator.is_playing()) {
+		auto key_frames = vid_iterator.get_keyframes();
+
 		// Generate a sublist of the total available landmark list:
 		auto landmark_sublist = sample_landmarks(key_frames, landmark_list);
+//		std::cout << vid_iterator.to_string() << std::endl;
 
-		// Fit shape and pose:
-		std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_2(
+		// it makes no sense to update pca_coeff if nothing in the buffer has changed:
+		if (vid_iterator.has_changed()) {
+			std::cout << "Going to reconstruct with " << key_frames.size() << " images."<< std::endl;
+			// Fit shape and pose:
+			std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_2(
 				morphable_model,
-				blend_shapes,
-				landmark_sublist,
+				blendshapes,
+				key_frames,
 				landmark_mapper,
 				key_frames[0].frame.cols,
 				key_frames[0].frame.rows,
@@ -609,33 +676,68 @@ int main(int argc, char *argv[]) {
 				30.0f,
 				boost::none,
 				pca_shape_coefficients,
-				blend_shape_coefficients,
+				blendshape_coefficients,
 				fitted_image_points
-		);
+			);
 
-//		vid_iterator.setReconstructionResults(reconstructionData);
+			vid_iterator.update_reconstruction_coeff(pca_shape_coefficients);
 
-		// Render output:
-		output_render(
+			evaluate_results(
 				key_frames,
 				rendering_paramss,
-				landmark_sublist,
-				morphable_model,
-				blend_shapes,
 				meshs,
 				pca_shape_coefficients,
-				blend_shape_coefficients,
+				blendshape_coefficients,
 				fitted_image_points,
 				annotations,
-				landmark_mapper,
+				reconstruction_data,
 				settings,
 				n_iter
-		);
+			);
+
+		} else {
+			std::cout << "No reconstruction - buffer did not change" << std::endl;
+		}
+
+//		// Render output:
+//		render_output(
+//				key_frames,
+//				rendering_paramss,
+//				landmark_sublist,
+//				meshs,
+//				pca_shape_coefficients,
+//				blendshape_coefficients,
+//				fitted_image_points,
+//				annotations,
+//				reconstruction_data,
+//				vid_iterator.last_frame,
+//				vid_iterator.last_frame_number,
+//				settings,
+//				n_iter
+//		);
+
 
 		// Get new frames:
-		key_frames = vid_iterator.next();
 		n_iter++;
 	}
 
+	auto key_frames = vid_iterator.get_keyframes();
+
+	std::cout << "Going to reconstruct with " << key_frames.size() << " images."<< std::endl;
+	evaluate_results(
+		key_frames,
+		rendering_paramss,
+		meshs,
+		pca_shape_coefficients,
+		blendshape_coefficients,
+		fitted_image_points,
+		annotations,
+		reconstruction_data,
+		settings,
+		n_iter
+	);
+
+	//todo: we could build our final obj here?
+
 	return EXIT_SUCCESS;
 }