Backup progress

examples/fit-model-multi-frame.cpp

 /*
  * eos - A 3D Morphable Model fitting library written in modern C++11/14.
  *
- * File: examples/fit-model-simple.cpp
- *
- * Copyright 2015 Patrik Huber
+ * File: utils/json-to-cereal-binary.cpp
  *
+ * Copyright 2016 Patrik Huber *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -17,211 +16,359 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "eos/core/Landmark.hpp"
-#include "eos/core/LandmarkMapper.hpp"
-#include "eos/fitting/orthographic_camera_estimation_linear.hpp"
-#include "eos/fitting/RenderingParameters.hpp"
-#include "eos/fitting/linear_shape_fitting.hpp"
+#include "eos/core/landmark_utils.hpp"
+#include "eos/render/render.hpp"
+#include "eos/morphablemodel/MorphableModel.hpp"
+#include "eos/morphablemodel/Blendshape.hpp"
+#include "eos/fitting/fitting.hpp"
 #include "eos/render/utils.hpp"
 #include "eos/render/texture_extraction.hpp"
+#include "eos/video/Keyframe.hpp"
+#include "eos/fitting/ReconstructionData.hpp"
+#include "eos/video/BufferedVideoIterator.hpp"
 
 #include "opencv2/core/core.hpp"
+#include "opencv2/opencv.hpp"
 #include "opencv2/highgui/highgui.hpp"
 
 #include "boost/program_options.hpp"
 #include "boost/filesystem.hpp"
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/ini_parser.hpp>
+#include "glm/gtx/string_cast.hpp"
 
 #include <vector>
 #include <iostream>
 #include <fstream>
 
 using namespace eos;
-using namespace fitting;
 namespace po = boost::program_options;
 namespace fs = boost::filesystem;
 using eos::core::Landmark;
 using eos::core::LandmarkCollection;
+using eos::video::BufferedVideoIterator;
+using eos::video::WeightedIsomapAveraging;
+using eos::video::ReconstructionVideoWriter;
+using eos::video::Keyframe;
 using cv::Mat;
 using cv::Vec2f;
 using cv::Vec3f;
 using cv::Vec4f;
-using std::cout;
-using std::endl;
 using std::vector;
 using std::string;
 
+using namespace cv;
+
+typedef unsigned int uint;
+
 /**
- * Reads an ibug .pts landmark file and returns an ordered vector with
- * the 68 2D landmark coordinates.
+ * Draws the given mesh as wireframe into the image.
  *
- * @param[in] filename Path to a .pts file.
- * @return An ordered vector with the 68 ibug landmarks.
+ * It does backface culling, i.e. draws only vertices in CCW order.
+ *
+ * @param[in] image An image to draw into.
+ * @param[in] mesh The mesh to draw.
+ * @param[in] modelview Model-view matrix to draw the mesh.
+ * @param[in] projection Projection matrix to draw the mesh.
+ * @param[in] viewport Viewport to draw the mesh.
+ * @param[in] colour Colour of the mesh to be drawn.
  */
-LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename)
+void draw_wireframe(cv::Mat image, const eos::core::Mesh& mesh, glm::mat4x4 modelview, glm::mat4x4 projection, glm::vec4 viewport, cv::Scalar colour = cv::Scalar(0, 255, 0, 255))
 {
-	using std::getline;
-	using cv::Vec2f;
-	using std::string;
-	LandmarkCollection<Vec2f> landmarks;
-	landmarks.reserve(68);
-
-	std::ifstream file(filename);
-	if (!file.is_open()) {
-		throw std::runtime_error(string("Could not open landmark file: " + filename));
+	for (const auto& triangle : mesh.tvi)
+	{
+		const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport);
+		const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport);
+		const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport);
+		if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3))) {
+			cv::line(image, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour);
+			cv::line(image, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour);
+			cv::line(image, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour);
 		}
+	}
+};
 
-	string line;
-	// Skip the first 3 lines, they're header lines:
-	getline(file, line); // 'version:1'
-	getline(file, line); // 'n_points : 68'
-	getline(file, line); // '{'
+/**
+ *
+ * @param keyframes
+ * @param rendering_paramss
+ * @param landmark_list
+ * @param morphable_model
+ * @param blendshapes
+ * @param meshs
+ * @param pca_shape_coefficients
+ * @param blendshape_coefficients
+ * @param fitted_image_points
+ * @param annotations
+ * @param landmark_mapper
+ * @param settings
+ * @param n_iter
+ */
+void output_obj(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
+					  std::vector<float> pca_shape_coefficients,
+					  fitting::ReconstructionData reconstruction_data,
+					  boost::property_tree::ptree settings) {
 
-	int ibugId = 1;
-	while (getline(file, line))
-	{
-		if (line == "}") { // end of the file
-			break;
-		}
-		std::stringstream lineStream(line);
+	Mat isomap;
+	Mat merged_isomap;
+	float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
+	WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
+
+	auto landmark_mapper = reconstruction_data.landmark_mapper;
+	auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
+	auto morphable_model = reconstruction_data.morphable_model;
+
+	bool show_video = settings.get<bool>("output.show_video", false);
+
+	// The 3D head pose can be recovered as follows:
+	for (uint i = 0; i < keyframes.size(); ++i) {
+		Mat frame = keyframes[i].get()->frame;
+
+		int frame_width = frame.cols;
+		int frame_height = frame.rows;
+
+		int frame_number = keyframes[i].get()->frame_number;
+		auto landmarks = keyframes[i].get()->fitting_result.landmarks;
+		auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
+		auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+
+		// and similarly for pitch and roll.
+		// Extract the texture from the image using given mesh and camera parameters:
+		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
 
-		Landmark<Vec2f> landmark;
-		landmark.name = std::to_string(ibugId);
-		if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
-			throw std::runtime_error(string("Landmark format error while parsing the line: " + line));
+		// Draw the loaded landmarks:
+		isomap = render::extract_texture(keyframes[i].get()->fitting_result.mesh, affine_from_ortho, frame);
+
+		// merge the isomaps:
+		merged_isomap = isomap_averaging.add_and_merge(isomap);
 	}
-		// From the iBug website:
-		// "Please note that the re-annotated data for this challenge are saved in the Matlab convention of 1 being
-		// the first index, i.e. the coordinates of the top left pixel in an image are x=1, y=1."
-		// ==> So we shift every point by 1:
-		landmark.coordinates[0] -= 1.0f;
-		landmark.coordinates[1] -= 1.0f;
-		landmarks.emplace_back(landmark);
-		++ibugId;
+
+	// save an obj + current merged isomap to the disk:
+	core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
+		morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
+		morphable_model.get_color_model().get_mean(),
+		morphable_model.get_shape_model().get_triangle_list(),
+		morphable_model.get_color_model().get_triangle_list(),
+		morphable_model.get_texture_coordinates());
+
+	core::write_textured_obj(neutral_expression, "current_merged.obj");
+	cv::imwrite("current_merged.isomap.png", merged_isomap);
+	std::cout << "written current_merged.obj and current_merged.isomap" << std::endl;
+}
+
+/**
+ * Parse config file
+ * @param filename
+ */
+boost::property_tree::ptree get_reconstruction_config(std::string filename) {
+	boost::property_tree::ptree pt;
+	boost::property_tree::ini_parser::read_ini(filename, pt);
+
+	return pt;
+}
+
+/**
+ *
+ * Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
+ * are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
+ * causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
+ * annotations with the based on a given keyframe list.
+ *
+ * @param keyframes
+ * @param landmarks
+ * @return
+ */
+vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
+	std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
+
+	vector<core::LandmarkCollection<cv::Vec2f>> sublist;
+
+	for (auto& f : keyframes) {
+		sublist.push_back(landmarks[f.get()->frame_number]);
 	}
-	return landmarks;
-};
+
+	return sublist;
+}
 
 /**
  * This app demonstrates estimation of the camera and fitting of the shape
  * model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
+ * In addition to fit-model-simple, this example uses blendshapes, contour-
+ * fitting, and can iterate the fitting.
  *
- * First, the 68 ibug landmarks are loaded from the .pts file and converted
- * to vertex indices using the LandmarkMapper. Then, an orthographic camera
- * is estimated, and then, using this camera matrix, the shape is fitted
- * to the landmarks.
+ * 68 ibug landmarks are loaded from the .pts file and converted
+ * to vertex indices using the LandmarkMapper.
  */
 int main(int argc, char *argv[]) {
-	fs::path modelfile, isomapfile, imagefile, landmarksfile, mappingsfile, outputfile;
+	fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile, reconstruction_config;
+	std::vector<std::string> annotations;
+
+	// get annotations from one file
+	bool get_annotations = false;
+
 	try {
 		po::options_description desc("Allowed options");
 		desc.add_options()
 			("help,h",
 			 "display the help message")
-				("model,m", po::value<fs::path>(&modelfile)->required()->default_value("/data/share/sfm_shape_3448.bin"),
+			("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
 			 "a Morphable Model stored as cereal BinaryArchive")
-				("image,i", po::value<fs::path>(&imagefile)->required()->default_value("/data/image_0010.png"),
+			("video,v", po::value<fs::path>(&videofile)->required(),
 			 "an input image")
-				("landmarks,l", po::value<fs::path>(&landmarksfile)->required()->default_value("/data/image_0010.pts"),
+			("config,f", po::value<fs::path>(&reconstruction_config)->default_value("../share/default_reconstruction_config.ini"),
+			 "configuration file for the reconstruction")
+			("get_annotations,g", po::bool_switch(&get_annotations)->default_value(false),
+			 "read .pts annotation file locations from one file, put one file path on each line")
+			("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
+			 ".pts annotation files per frame of video")
+			("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
 			 "2D landmarks for the image, in ibug .pts format")
-				("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("/data/share/ibug2did.txt"),
-				 "landmark identifier to model vertex number mapping")
-				("output,o", po::value<fs::path>(&outputfile)->required()->default_value("/data/out"),
-				 "basename for the output rendering and obj files")
-				;
+			("model-contour,c",
+			 po::value<fs::path>(&contourfile)->required()->default_value("../share/model_contours.json"),
+			 "file with model contour indices")
+			("edge-topology,e", po::value<fs::path>(&edgetopologyfile)->required()->default_value(
+				"../share/sfm_3448_edge_topology.json"),
+			 "file with model's precomputed edge topology")
+			("blendshapes,b", po::value<fs::path>(&blendshapesfile)->required()->default_value(
+				"../share/expression_blendshapes_3448.bin"),
+			 "file with blendshapes")
+			("output,o", po::value<fs::path>(&outputfile)->required()->default_value("out"),
+			 "basename for the output rendering and obj files");
 		po::variables_map vm;
 		po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
 		if (vm.count("help")) {
-			cout << "Usage: fit-model-simple [options]" << endl;
-			cout << desc;
+			std::cout << "Usage: fit-model [options]" << std::endl;
+			std::cout << desc;
 			return EXIT_SUCCESS;
 		}
 		po::notify(vm);
 	}
-	catch (const po::error& e) {
-		cout << "Error while parsing command-line arguments: " << e.what() << endl;
-		cout << "Use --help to display a list of options." << endl;
+	catch (const po::error &e) {
+		std::cout << "Error while parsing command-line arguments: " << e.what() << std::endl;
+		std::cout << "Use --help to display a list of options." << std::endl;
 		return EXIT_SUCCESS;
 	}
 
-	// Load the image, landmarks, LandmarkMapper and the Morphable Model:
-	Mat image = cv::imread(imagefile.string());
-	LandmarkCollection<cv::Vec2f> landmarks;
-	try {
-		landmarks = read_pts_landmarks(landmarksfile.string());
-	}
-	catch (const std::runtime_error& e) {
-		cout << "Error reading the landmarks: " << e.what() << endl;
-		return EXIT_FAILURE;
-	}
+	// Start loading prerequisites:
 	morphablemodel::MorphableModel morphable_model;
+	std::vector<float> pca_shape_coefficients;
+
+	// List of blendshapes coefficients:
+	std::vector<std::vector<float>> blendshape_coefficients;
+	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
+
+	// These will be the final 2D and 3D points used for the fitting:
+	vector<cv::Vec3f> model_points; // the points in the 3D shape model
+	vector<int> vertex_indices; // their vertex indices
+	vector<core::Mesh> meshs;
+	vector<fitting::RenderingParameters> rendering_paramss;
+
+	// Load landmarks, LandmarkMapper and the Morphable Model:
+	auto landmark_mapper = core::LandmarkMapper(mappingsfile);
+
+	// Load all annotation files into lists of landmarks:
+	vector<core::LandmarkCollection<cv::Vec2f>> landmark_list;
 	try {
 		morphable_model = morphablemodel::load_model(modelfile.string());
-	}
-	catch (const std::runtime_error& e) {
-		cout << "Error loading the Morphable Model: " << e.what() << endl;
+	} catch (const std::runtime_error &e) {
+		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
 		return EXIT_FAILURE;
 	}
-	core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
 
-	// Draw the loaded landmarks:
-	Mat outimg = image.clone();
-	for (auto&& lm : landmarks) {
-		cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 });
+	// Load all annotation using input parameters:
+	try {
+		std::tie(landmark_list, annotations) = eos::core::load_annotations<cv::Vec2f>(
+			annotations,
+			landmark_mapper,
+			morphable_model,
+			get_annotations);
+	} catch(const std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
+		return EXIT_FAILURE;
 	}
 
-	// These will be the final 2D and 3D points used for the fitting:
-	vector<Vec4f> model_points; // the points in the 3D shape model
-	vector<int> vertex_indices; // their vertex indices
-	vector<Vec2f> image_points; // the corresponding 2D landmark points
+	// The expression blendshapes:
+	auto blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string());
 
-	// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
-	for (int i = 0; i < landmarks.size(); ++i) {
-		auto converted_name = landmark_mapper.convert(landmarks[i].name);
-		if (!converted_name) { // no mapping defined for the current landmark
-			continue;
-		}
-		int vertex_idx = std::stoi(converted_name.get());
-		auto vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
-		model_points.emplace_back(Vec4f(vertex.x(), vertex.y(), vertex.z(), 1.0f));
-		vertex_indices.emplace_back(vertex_idx);
-		image_points.emplace_back(landmarks[i].coordinates);
+	// These two are used to fit the front-facing contour to the ibug contour landmarks:
+	auto model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
+	auto ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string());
+	// The edge topology is used to speed up computation of the occluding face contour fitting:
+	auto edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string());
+
+	// Read .ini file to accommodate.
+	auto settings = get_reconstruction_config(reconstruction_config.string());
+	auto reconstruction_data = eos::fitting::ReconstructionData{
+		morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
+
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+	bool evaluate = settings.get<bool>("eval.evaluate", true);
+	int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
+	int max_frames = settings.get<int>("eval.max_frames", 0);
+	int drop_frames = settings.get<int>("frames.drop_frames", 0);
+
+	// Start with the video play and get video file:
+	BufferedVideoIterator vid_iterator;
+
+	try {
+		vid_iterator = BufferedVideoIterator(videofile.string(), reconstruction_data, settings);
+	} catch(std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
+		return EXIT_FAILURE;
 	}
 
-	// Estimate the camera (pose) from the 2D - 3D point correspondences
-	fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
-	fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
+	// start a thread to search for the best frames
+	vid_iterator.start();
 
-	// The 3D head pose can be recovered as follows:
-	float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+	// Waiting for the iterator to find the best keyframes, we will reconstruct when done, note that this is
+	// best for short videos. Although the vid iterator has a maximum capacity therefore memory leaks are
+	// not a problem, still short videos will demonstrate its usefulness best.
+	while(vid_iterator.is_playing()) { }
 
-	// and similarly for pitch and roll.
-	// Estimate the shape coefficients by fitting the shape to the landmarks:
-	Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
-		rendering_params, image.cols, image.rows
-	);
+	auto keyframes = vid_iterator.get_keyframes();
 
-	vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(
+	// Generate a sublist of the total available landmark list:
+	std::cout << "Going to reconstruct with " << keyframes.size() << " images, "<< num_iterations << " iterations:"<< std::endl;
+
+	// Fit shape and pose:
+	auto t1 = std::chrono::high_resolution_clock::now();
+	std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
 		morphable_model,
-		affine_from_ortho,
-		image_points,
-		vertex_indices
+		blendshapes,
+		keyframes,
+		landmark_mapper,
+		keyframes[0].get()->frame.cols,
+		keyframes[0].get()->frame.rows,
+		static_cast<int>(keyframes.size()),
+		edge_topology,
+		ibug_contour,
+		model_contour,
+		num_iterations,
+		boost::none,
+		30.0f,
+		boost::none,
+		pca_shape_coefficients,
+		blendshape_coefficients,
+		fitted_image_points,
+		settings
 	);
 
-	// Obtain the full mesh with the estimated coefficients:
-	eos::core::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
+	std::cout << pca_shape_coefficients.size() << std::endl;
+	std::cout << vid_iterator.to_string() << std::endl;
 
-	// Extract the texture from the image using given mesh and camera parameters:
-	Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
+	auto t2 = std::chrono::high_resolution_clock::now();
+	std::cout << "Reconstruction took "
+			  << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count()
+			  << "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() / (keyframes.size() * num_iterations)
+			  << "ms)" << std::endl;
 
-	// Save the mesh as textured obj:
-	outputfile += fs::path(".obj");
-	eos::core::write_textured_obj(mesh, outputfile.string());
-
-	// And save the isomap:
-	outputfile.replace_extension("isomap.png");
-	cv::imwrite(outputfile.string(), isomap);
-
-	cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
+	output_obj(
+		keyframes,
+		pca_shape_coefficients,
+		reconstruction_data,
+		settings
+	);
 
 	return EXIT_SUCCESS;
 }
+

include/eos/core/LandmarkMapper.hpp

@@ -119,6 +119,31 @@ public:
 		}
 	};
 
+	/**
+	 * @brief Converts the given landmark name to the mapped name.
+	 *
+	 * @param[in] landmark_name A landmark name to convert.
+	 * @return The mapped landmark name if a mapping exists, an empty optional otherwise.
+	 * @throws out_of_range exception if there is no mapping
+	 *         for the given landmarkName.
+	 */
+	std::string get_vertex_id(std::string landmark_name) const {
+		if (landmark_mappings.empty()) {
+			// perform identity mapping, i.e. return the input
+			return landmark_name;
+		}
+		else {
+			auto&& converted_landmark = landmark_mappings.find(landmark_name);
+			if (converted_landmark != std::end(landmark_mappings)) {
+				// landmark mapping found, return it
+				return converted_landmark->second;
+			}
+			else { // landmark_name does not match the key of any element in the map
+				return "";
+			}
+		}
+	};
+
 	/**
 	 * @brief Returns the number of loaded landmark mappings.
 	 *

include/eos/core/landmark_utils.hpp

@@ -193,13 +193,20 @@ namespace eos {
 	 * @param landmarks
 	 * @return
 	 */
-		cv::Rect get_face_roi(vector<Vec2f>image_points, int image_width, int image_height) {
+		cv::Rect get_face_roi(vector<cv::Vec2f>image_points, int image_width, int image_height) {
 			cv::Rect bbox = cv::boundingRect(image_points);
 
+			// sometimes the bbox starts outside the image boundaries due to wrong image points.
+			bbox.x = bbox.x < 0 ? 0 : bbox.x;
+			bbox.y = bbox.y < 0 ? 0 : bbox.y;
+
 			// cap on the image width and height.
 			bbox.width = bbox.x + bbox.width < image_width ? bbox.width: image_width - bbox.x - 1;
 			bbox.height = bbox.y + bbox.height < image_height ? bbox.height: image_height - bbox.y - 1;
 
+			assert(bbox.x + bbox.width < image_width);
+			assert(bbox.y + bbox.height < image_height);
+
 			return bbox;
 		}
 		/**

include/eos/fitting/FittingResult.hpp

@@ -46,6 +46,9 @@ struct FittingResult
 	std::vector<float> blendshape_coefficients;
 	core::LandmarkCollection<cv::Vec2f> landmarks;
 	core::Mesh mesh;
+	std::vector<Vec4f> model_points; // the points in the 3D shape model
+	std::vector<int> vertex_indices; // their vertex indices
+	std::vector<Vec2f> image_points; // the corresponding 2D landmark points
 };
 
 	} /* namespace fitting */

include/eos/fitting/contour_correspondence.hpp

@@ -72,7 +72,7 @@ struct ModelContour
 
 	// We store r/l separately because we currently only fit to the contour facing the camera.
 	// Also if we were to fit to the whole contour: Be careful not to just fit to the closest. The 
-	// "invisible" ones behind might be closer on an e.g 90° angle. Store CNT for left/right side separately?
+	// "invisible" ones behind might be closer on an e.g 90� angle. Store CNT for left/right side separately?
 	
 	/**
 	 * Helper method to load a ModelContour from
@@ -226,7 +226,7 @@ inline std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<in
  * have different size. 
  * Correspondence can be established using get_nearest_contour_correspondences().
  *
- * If the yaw angle is between +-7.5°, both contours will be selected.
+ * If the yaw angle is between +-7.5�, both contours will be selected.
  *
  * Note: Maybe rename to find_nearest_contour_points, to highlight that there is (potentially a lot) computational cost involved?
  *
@@ -251,7 +251,7 @@ std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_a
 		model_contour_indices.insert(end(model_contour_indices), begin(model_contour.left_contour), end(model_contour.left_contour));
 		contour_landmark_identifiers.insert(end(contour_landmark_identifiers), begin(contour_landmarks.left_contour), end(contour_landmarks.left_contour));
 	}
-	// Note there's an overlap between the angles - if a subject is between +- 7.5°, both contours get added.
+	// Note there's an overlap between the angles - if a subject is between +- 7.5�, both contours get added.
 	return std::make_pair(contour_landmark_identifiers, model_contour_indices);
 };
 

include/eos/fitting/fitting.hpp

@@ -880,255 +880,11 @@ inline eos::core::Mesh generate_new_mesh(
 	return mesh;
 }
 
-inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_2(
-	const morphablemodel::MorphableModel& morphable_model,
-	const std::vector<morphablemodel::Blendshape>& blendshapes,
-	std::vector<eos::video::Keyframe>& keyframes,
-	const core::LandmarkMapper& landmark_mapper,
-	int image_width,
-	int image_height,
-	int num_images,
-	const morphablemodel::EdgeTopology& edge_topology,
-	const fitting::ContourLandmarks& contour_landmarks,
-	const fitting::ModelContour& model_contour,
-	int num_iterations,
-	boost::optional<int> num_shape_coefficients_to_fit,
-	float lambda,
-	boost::optional<fitting::RenderingParameters> initial_rendering_params,
-	std::vector<float>& pca_shape_coefficients,
-	std::vector<std::vector<float>>& blendshape_coefficients,
-	std::vector<std::vector<cv::Vec2f>>& fitted_image_points) {
-
-	assert(blendshapes.size() > 0);
-	assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit?
-	assert(pca_shape_coefficients.size() <= morphable_model.get_shape_model().get_num_principal_components());
-
-	using std::vector;
-	using cv::Vec2f;
-	using cv::Vec4f;
-	using cv::Mat;
-	using Eigen::VectorXf;
-	using Eigen::MatrixXf;
-
-	if (!num_shape_coefficients_to_fit) {
-		num_shape_coefficients_to_fit = morphable_model.get_shape_model().get_num_principal_components();
-	}
-
-	if (pca_shape_coefficients.empty()) {
-		pca_shape_coefficients.resize(num_shape_coefficients_to_fit.get());
-	}
-
-	// TODO: This leaves the following case open: num_coeffs given is empty or defined, but the
-	// pca_shape_coefficients given is != num_coeffs or the model's max-coeffs. What to do then? Handle & document!
-	if (blendshape_coefficients.size() < num_images) {
-		for (int j = 0; j < num_images; ++j) {
-			std::vector<float> current_blendshape_coefficients;
-			current_blendshape_coefficients.resize(blendshapes.size());
-			blendshape_coefficients.push_back(current_blendshape_coefficients);
-		}
-	}
-
-	MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
-
-	// Current mesh - either from the given coefficients, or the mean:
-	VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
-	vector<VectorXf> current_combined_shapes;
-	vector<eos::core::Mesh> current_meshs;
-
-	// The 2D and 3D point correspondences used for the fitting:
-	vector<vector<Vec4f>> model_points; // the points in the 3D shape model of all frames.
-	vector<vector<int>> vertex_indices; // their vertex indices of all frames.
-	vector<vector<Vec2f>> image_points; // the corresponding 2D landmark points of all frames.
-
-	vector<fitting::RenderingParameters> rendering_params; // list of rendering params for all frames.
-
-	// Generate meshes from current shapes.
-	for (int j = 0; j < num_images; ++j) {
-		VectorXf current_combined_shape = current_pca_shape +
-			blendshapes_as_basis *
-				Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
-
-		eos::core::Mesh current_mesh = morphablemodel::sample_to_mesh(
-			current_combined_shape,
-			morphable_model.get_color_model().get_mean(),
-			morphable_model.get_shape_model().get_triangle_list(),
-			morphable_model.get_color_model().get_triangle_list(),
-			morphable_model.get_texture_coordinates());
-
-		// Get the locations of the model locations of the meshes, vertex_indices and image points
-		// (equal to landmark coordinates), for every image / mesh.
-
-//		std::tie(curr_model_points, curr_vertex_indices, curr_image_points) = eos::core::get_landmark_coordinates<Vec2f, Vec4f>(
-//			keyframes[j].fitting_result.landmarks, landmark_mapper, current_mesh);
-		eos::core::get_landmark_coordinates_inline<Vec2f, Vec4f>(
-			keyframes[j].fitting_result.landmarks,
-			landmark_mapper,
-			current_mesh,
-			// output parameters
-			model_points,
-			vertex_indices,
-			image_points);
-
-		// Start constructing a list of rendering parameters needed for reconstruction.
-		// Get the current points from the last added image points and model points
-		auto current_pose = fitting::estimate_orthographic_projection_linear(
-			image_points.back(), model_points.back(), true, image_height);
-
-		fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height);
-		rendering_params.push_back(current_rendering_params);
-
-		// update key frame rendering params
-		keyframes[j].fitting_result.rendering_parameters = current_rendering_params;
-
-		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
-
-		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
-			blendshapes, current_pca_shape, affine_from_ortho, image_points.back(), vertex_indices.back()
-		);
-
-		// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
-		current_combined_shape = current_pca_shape +
-			morphablemodel::to_matrix(blendshapes) *
-				Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size()
-				);
-
-		current_combined_shapes.push_back(current_combined_shape);
-
-		current_mesh = morphablemodel::sample_to_mesh(
-			current_combined_shape, morphable_model.get_color_model().get_mean(),
-			morphable_model.get_shape_model().get_triangle_list(),
-			morphable_model.get_color_model().get_triangle_list(),
-			morphable_model.get_texture_coordinates()
-		);
-
-		current_meshs.push_back(current_mesh);
-	}
-
-	// The static (fixed) landmark correspondences which will stay the same throughout
-	// the fitting (the inner face landmarks):
-	vector<vector<int>> fixed_vertex_indices (vertex_indices);
-	vector<vector<Vec2f>> fixed_image_points (image_points);
-
-	std::vector<cv::Mat> affine_from_orthos;
-	std::vector<VectorXf> mean_plus_blendshapes;
-
-	image_points = fixed_image_points;
-	vertex_indices = fixed_vertex_indices;
-
-	for (int j = 0; j < num_images; ++j) {
-		// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
-		vector<Vec2f> image_points_contour;
-		vector<int> vertex_indices_contour;
-
-		auto curr_keyframe = keyframes[j];
-		auto landmarks = curr_keyframe.fitting_result.landmarks;
-		auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
-
-		// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
-		std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(
-			landmarks,
-			contour_landmarks,
-			model_contour,
-			yaw_angle,
-			current_meshs[j],
-			rendering_params[j].get_modelview(),
-			rendering_params[j].get_projection(),
-			fitting::get_opencv_viewport(image_width, image_height)
-		);
-
-		// Add the contour correspondences to the set of landmarks that we use for the fitting:
-		vertex_indices[j] = fitting::concat(vertex_indices[j], vertex_indices_contour);
-		image_points[j] = fitting::concat(image_points[j], image_points_contour);
-
-		// Fit the occluding (away-facing) contour using the detected contour LMs:
-		vector<Eigen::Vector2f> occluding_contour_landmarks;
-
-		// positive yaw = subject looking to the left
-		if (yaw_angle >= 0.0f) {
-			// the left contour is the occluding one we want to use ("away-facing")
-			auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
-			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
-				occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
-			});
-		} else {
-			auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
-			std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
-				occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
-			});
-		}
-
-		auto edge_correspondences = fitting::find_occluding_edge_correspondences(
-			current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f
-		);
-
-		image_points[j] = fitting::concat(image_points[j], edge_correspondences.first);
-		vertex_indices[j] = fitting::concat(vertex_indices[j], edge_correspondences.second);
-
-		// Get the model points of the current mesh, for all correspondences that we've got:
-		model_points[j].clear();
-		for (const auto& v : vertex_indices[j]) {
-			model_points[j].push_back(
-				{
-					current_meshs[j].vertices[v][0],
-					current_meshs[j].vertices[v][1],
-					current_meshs[j].vertices[v][2],
-					current_meshs[j].vertices[v][3]
-				});
-		}
-
-		// Re-estimate the pose, using all correspondences:
-		auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
-		rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
-		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
-
-		affine_from_orthos.push_back(affine_from_ortho);
-
-		// Estimate the PCA shape coefficients with the current blendshape coefficients:
-		VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
-			blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
-		mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
-	}
-
-	pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel(
-		morphable_model,
-		affine_from_orthos,
-		image_points,
-		vertex_indices,
-		mean_plus_blendshapes,
-		lambda,
-		num_shape_coefficients_to_fit
-	);
-
-	// Estimate the blendshape coefficients with the current PCA model estimate:
-	current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
-
-	for (int j = 0; j < num_images; ++j) {
-		blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
-			blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
-		);
-		current_combined_shapes[j] = current_pca_shape +
-			blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
-
-		current_meshs[j] = morphablemodel::sample_to_mesh(
-			current_combined_shapes[j],
-			morphable_model.get_color_model().get_mean(),
-			morphable_model.get_shape_model().get_triangle_list(),
-			morphable_model.get_color_model().get_triangle_list(),
-			morphable_model.get_texture_coordinates()
-		);
-	}
-
-	fitted_image_points = image_points;
-
-	return { current_meshs, rendering_params }; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices).
-};
-
 #ifdef _OPENMP
 inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_parallel(
 		const morphablemodel::MorphableModel& morphable_model,
 		const std::vector<morphablemodel::Blendshape>& blendshapes,
 		std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
-//	std::vector<eos::video::Keyframe>& keyframes,
 		const core::LandmarkMapper& landmark_mapper,
 		int image_width,
 		int image_height,
@@ -1143,8 +899,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 		std::vector<float>& pca_shape_coefficients,
 		std::vector<std::vector<float>>& blendshape_coefficients,
 		std::vector<std::vector<cv::Vec2f>>& fitted_image_points,
-	boost::property_tree::ptree settings)
-{
+		boost::property_tree::ptree settings) {
 
 	assert(blendshapes.size() > 0);
 	assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit?
@@ -1173,10 +928,8 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 
 	// TODO: This leaves the following case open: num_coeffs given is empty or defined, but the
 	// pca_shape_coefficients given is != num_coeffs or the model's max-coeffs. What to do then? Handle & document!
-	if (blendshape_coefficients.size() < num_images)
-	{
-		for (int j = 0; j < num_images; ++j)
-		{
+	if (blendshape_coefficients.size() < num_images) {
+		for (int j = 0; j < num_images; ++j) {
 			std::vector<float> current_blendshape_coefficients;
 			current_blendshape_coefficients.resize(blendshapes.size());
 			blendshape_coefficients.push_back(current_blendshape_coefficients);
@@ -1236,9 +989,6 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 
 			Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
 
-			// if no contour
-//			affine_from_orthos[j] = affine_from_ortho;
-
 			blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
 				blendshapes, current_pca_shape, affine_from_ortho, curr_image_points, curr_vertex_indices);
 
@@ -1246,8 +996,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 			current_combined_shapes[j] = current_pca_shape +
 				morphablemodel::to_matrix(blendshapes) *
 					Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
-													  blendshape_coefficients[j].size()
-					);
+													  blendshape_coefficients[j].size());
 
 			current_meshs[j] = morphablemodel::sample_to_mesh(
 				current_combined_shape, morphable_model.get_color_model().get_mean(),
@@ -1259,21 +1008,14 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 			model_points[j] = curr_model_points;
 			vertex_indices[j] = curr_vertex_indices;
 			image_points[j] = curr_image_points;
-
-			// Estimate the PCA shape coefficients with the current blendshape coefficients:
-//			VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
-//				blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
-//																		 blendshape_coefficients[j].size());
-//			mean_plus_blendshapes[j] = current_mean_plus_blendshapes;
 		}
 	}
+
 	// The static (fixed) landmark correspondences which will stay the same throughout
 	// the fitting (the inner face landmarks):
 	vector<vector<int>> fixed_vertex_indices(vertex_indices);
 	vector<vector<Vec2f>> fixed_image_points(image_points);
 
-	if (use_contours)
-	{
 	for (int i = 0; i < num_iterations; ++i) {
 		std::vector<cv::Mat> affine_from_orthos;
 		std::vector<VectorXf> mean_plus_blendshapes;
@@ -1346,8 +1088,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 		#pragma omp parallel num_threads(NUM_THREADS)
 				{
 		#pragma omp for
-			for (int j = 0; j < num_images; ++j)
-			{
+		for (int j = 0; j < num_images; ++j) {
 			blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
 				blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
 			);
@@ -1367,11 +1108,13 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
 			// we could make it optional
 			keyframes[j].get()->fitting_result.mesh = current_meshs[j];
 			keyframes[j].get()->fitting_result.rendering_parameters = rendering_params[j];
+			keyframes[j].get()->fitting_result.image_points = image_points[j];
+			keyframes[j].get()->fitting_result.vertex_indices = vertex_indices[j];
+			keyframes[j].get()->fitting_result.model_points = model_points[j];
 		}
 
 		}
 	}
-		}
 
 	fitted_image_points = image_points;
 

include/eos/video/BufferedVideoIterator.hpp

@@ -460,6 +460,7 @@ public:
 	int width;
 	int height;
 	int last_frame_number;
+	bool add_random;
 
 	Keyframe last_keyframe;
 	std::unique_ptr<std::thread> frame_buffer_worker;
@@ -488,6 +489,8 @@ public:
 		skip_frames = settings.get<int>("frames.skip_frames", 0);
 		frames_per_bin = settings.get<unsigned int>("frames.frames_per_bin", 2);
 
+		add_random = settings.get<bool>("frames.add_random", false);
+
 		this->reconstruction_data = reconstruction_data;
 		unsigned int num_shape_coeff = reconstruction_data.
 			morphable_model.get_shape_model().get_num_principal_components();
@@ -501,6 +504,17 @@ public:
 		// reset frame count (to be sure)
 		n_frames = 0;
 		total_frames = 0;
+		frames_dropped = 0;
+
+		Mat frame;
+
+		// Get the first frame to see, sometimes frames are empty at the start, keep reading until we hit one.
+		while(frame.empty() || total_frames < skip_frames) {
+			cap.read(frame);
+			total_frames++;
+		}
+
+		std::cout << "Starting at video at frame:" << total_frames << std::endl;
 
 //		std::cout << "Settings: " << std::endl <<
 //			"min_frames: " << min_frames << std::endl <<
@@ -572,11 +586,108 @@ public:
 		fitting_result.rendering_parameters = rendering_params;
 
 		cv::Rect face_roi = core::get_face_roi(image_points, frame_width, frame_height);
-
 		float frame_laplacian_score = static_cast<float>(variance_of_laplacian(frame(face_roi)));
+
 		return Keyframe(frame_laplacian_score, frame, fitting_result, total_frames);
 	}
 
+
+	/**
+	 * Fill the buffer by iterating through the video until the very last frame.
+	 */
+	void video_iterator() {
+		// Go and fill the buffer with more frames while we are reconstructing.
+		while (next()) { }
+
+		// stop the video
+		__stop();
+
+		std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames <<  std::endl;
+	};
+
+	Keyframe get_last_keyframe() {
+		return last_keyframe;
+	};
+
+	bool try_add_random(Keyframe keyframe) {
+		// Determine whether to add or not:
+		float yaw_angle = glm::degrees(glm::yaw(keyframe.fitting_result.rendering_parameters.get_rotation()));
+		auto idx = angle_to_index(yaw_angle);
+		bool add_frame = false;
+
+		keyframe.yaw_angle = yaw_angle;
+
+		// Score is 0 for total black frames, we don't want those:
+		if (keyframe.score == 0) {
+			return false;
+		}
+
+		// only add when the look at the camera
+		if (idx == 3 || idx == 4) {
+			add_frame = true;
+		}
+
+		if (!add_frame) {
+			return false;
+		}
+
+		// Add the keyframe:
+		bins[idx].push_back(std::make_shared<Keyframe>(keyframe));
+		if (bins[idx].size() > frames_per_bin) {
+			n_frames--;
+			// need to remove the lowest one:
+			std::sort(std::begin(bins[idx]), std::end(bins[idx]),
+					  [](const auto& lhs, const auto& rhs) { return lhs->score > rhs->score; });
+			bins[idx].resize(frames_per_bin);
+		}
+
+		return true;
+	}
+
+	Keyframe next_key_frame() {
+		Mat frame = __get_new_frame();
+		auto keyframe = generate_new_keyframe(frame);
+
+		return keyframe;
+	}
+
+	/**
+	 *
+	 * Set next frame and return frame_buffer. Returns true or false if the next frame is empty.
+	 * Empty frames mean that we reached the end of the video stream (i.e., file or camera).
+	 *
+	 * @return bool if
+	 */
+	bool next() {
+		Mat frame = __get_new_frame();
+
+		if (frame.empty()) {
+			return false;
+		}
+
+		// TODO: only calculate lapscore within the bounding box of the face.
+		auto keyframe = generate_new_keyframe(frame);
+		bool frame_added = try_add(keyframe);
+
+		// Added or not, we put this as the last keyframe
+		last_keyframe = keyframe;
+
+		if(frame_added) {
+			n_frames++;
+			// Setting that the buffer has changed:
+			frame_buffer_changed = true;
+		}
+
+		total_frames++;
+
+		// fill up the buffer until we hit the minimum frames we want in the buffer.
+		if(n_frames < min_frames) {
+			return next();
+		}
+
+		return true;
+	}
+
 	/**
 	 * Try to add a new key frame. Look at the laplacian score and the yaw_angle. The yaw_angle will
 	 * determine the bin in which it will be added. The score will be compared to the other frames in the
@@ -600,8 +711,6 @@ public:
 			return false;
 		}
 
-		std::cout << "idx: " << idx << std::endl;
-
 		// always add when we don't have enough frames
 		if (bins[idx].size() < frames_per_bin) {
 			add_frame = true; // definitely adding - we wouldn't have to go through the for-loop on the next line.
@@ -689,62 +798,6 @@ public:
 		return get_keyframes();
 	}
 
-	/**
-	 * Fill the buffer by iterating through the video until the very last frame.
-	 */
-	void video_iterator() {
-		// Go and fill the buffer with more frames while we are reconstructing.
-		while (next()) { }
-
-		// stop the video
-		__stop();
-
-		std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames <<  std::endl;
-	};
-
-	Keyframe get_last_keyframe() {
-		return last_keyframe;
-	};
-
-	 /**
-	  *
-	  * Set next frame and return frame_buffer. Returns true or false if the next frame is empty.
-	  * Empty frames mean that we reached the end of the video stream (i.e., file or camera).
-	  *
-	  * @return bool if
-	  */
-	bool next() {
-		Mat frame = __get_new_frame();
-
-		if (frame.empty()) {
-			return false;
-		}
-
-		// keep the last frame here, so we can play a video subsequently:
-		last_frame_number = total_frames;
-
-		// TODO: only calculate lapscore within the bounding box of the face.
-		auto keyframe = generate_new_keyframe(frame);
-		bool frame_added = try_add(keyframe);
-
-		// Added or not, we put this as the last keyframe
-		last_keyframe = keyframe;
-
-		if(frame_added) {
-			n_frames++;
-			// Setting that the buffer has changed:
-			frame_buffer_changed = true;
-		}
-
-		total_frames++;
-
-		// fill up the buffer until we hit the minimum frames we want in the buffer.
-		if(n_frames < min_frames) {
-			return next();
-		}
-
-		return true;
-	}
 
 	/**
 	 * Update pca shape coeff. Probably we need to make something with a mutex, for updating and reading
@@ -856,6 +909,8 @@ private:
 	// Note: these settings are for future use
 	int drop_frames = 0;
 
+	int frames_dropped = 0;
+
 	unsigned int num_shape_coefficients_to_fit = 0;
 };
 

include/eos/video/Keyframe.hpp

@@ -263,9 +263,6 @@ public:
 	cv::Mat add_and_merge(const cv::Mat& isomap)
 	{
 		// Merge isomaps, add the current to the already merged, pixel by pixel:
-#pragma omp target
-		{
-#pragma omp parallel for
 		for (int r = 0; r < isomap.rows; ++r)
 		{
 			for (int c = 0; c < isomap.cols; ++c)
@@ -276,26 +273,19 @@ public:
 				}
 				// we're sure to have a visible pixel, merge it:
 				// merged_pixel = (old_average * visible_count + new_pixel) / (visible_count + 1)
-					merged_isomap.at<cv::Vec4f>(r, c)[0] =
-						(merged_isomap.at<cv::Vec4f>(r, c)[0] * visibility_counter.at<int>(r, c)
-							+ isomap.at<cv::Vec4b>(r, c)[0]) / (visibility_counter.at<int>(r, c) + 1);
-					merged_isomap.at<cv::Vec4f>(r, c)[1] =
-						(merged_isomap.at<cv::Vec4f>(r, c)[1] * visibility_counter.at<int>(r, c)
-							+ isomap.at<cv::Vec4b>(r, c)[1]) / (visibility_counter.at<int>(r, c) + 1);
-					merged_isomap.at<cv::Vec4f>(r, c)[2] =
-						(merged_isomap.at<cv::Vec4f>(r, c)[2] * visibility_counter.at<int>(r, c)
-							+ isomap.at<cv::Vec4b>(r, c)[2]) / (visibility_counter.at<int>(r, c) + 1);
-					merged_isomap.at<cv::Vec4f>(r, c)[3] =
-						255; // as soon as we've seen the pixel visible once, we set it to visible.
+				merged_isomap.at<cv::Vec4f>(r, c)[0] = (merged_isomap.at<cv::Vec4f>(r, c)[0] * visibility_counter.at<int>(r, c) + isomap.at<cv::Vec4b>(r, c)[0]) / (visibility_counter.at<int>(r, c) + 1);
+				merged_isomap.at<cv::Vec4f>(r, c)[1] = (merged_isomap.at<cv::Vec4f>(r, c)[1] * visibility_counter.at<int>(r, c) + isomap.at<cv::Vec4b>(r, c)[1]) / (visibility_counter.at<int>(r, c) + 1);
+				merged_isomap.at<cv::Vec4f>(r, c)[2] = (merged_isomap.at<cv::Vec4f>(r, c)[2] * visibility_counter.at<int>(r, c) + isomap.at<cv::Vec4b>(r, c)[2]) / (visibility_counter.at<int>(r, c) + 1);
+				merged_isomap.at<cv::Vec4f>(r, c)[3] = 255; // as soon as we've seen the pixel visible once, we set it to visible.
 				++visibility_counter.at<int>(r, c);
 			}
 		}
-		}
 		cv::Mat merged_isomap_uchar;
 		merged_isomap.convertTo(merged_isomap_uchar, CV_8UC4);
 		return merged_isomap_uchar;
 	};
 
+
 	cv::Mat sharpen(const cv::Mat& isomap) {
 		cv::Mat output;
 		cv::Mat kernel = (cv::Mat_<float>(5, 5) << -0.125, -0.125, -0.125, -0.125, -0.125,

python/generate-python-bindings.cpp

@@ -65,6 +65,7 @@ PYBIND11_PLUGIN(eos) {
 				new (&instance) core::LandmarkMapper(filename);
 			}, "Constructs a new landmark mapper from a file containing mappings from one set of landmark identifiers to another.", py::arg("filename"))
 		// We can't expose the convert member function yet - need std::optional (or some trick with self/this and a lambda)
+		.def("get_vertex_id", &core::LandmarkMapper::get_vertex_id, "Returns the vertex id given a landmark name")
 		;
 
 	py::class_<core::Mesh>(core_module, "Mesh", "This class represents a 3D mesh consisting of vertices, vertex colour information and texture coordinates.")
@@ -187,12 +188,21 @@ PYBIND11_PLUGIN(eos) {
 
 	py::class_<fitting::ContourLandmarks>(fitting_module, "ContourLandmarks", "Defines which 2D landmarks comprise the right and left face contour.")
 		.def_static("load", &fitting::ContourLandmarks::load, "Helper method to load contour landmarks from a text file with landmark mappings, like ibug_to_sfm.txt.", py::arg("filename"))
+//		.def("get_contour_landmarks", [](const fitting::RenderingParameters& rendering_params, const core::LandmarkCollection<cv::Vec2f>& landmarks, const fitting::ContourLandmarks& contour_landmarks, const fitting::ModelContour& model_contour, float yaw_angle, const core::Mesh& mesh, const glm::mat4x4& view_model, const glm::vec4& viewport) {
+//			cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, width, heightG;
+//		}, "Get contour landmarks", py::arg("morphable_model"), py::arg("landmarks"), py::arg("contour_landmarks"), py::arg("model_contour"), py::arg("yaw_angle"), py::arg("mesh"), py::arg("view_model"), py::arg("ortho_projection"), py::arg("contour_landmarks"), py::arg("model_contour"), py::arg("num_iterations") = 5, py::arg("num_shape_coefficients_to_fit") = -1, py::arg("lambda") = 30.0f);
 		;
 
 	py::class_<fitting::ModelContour>(fitting_module, "ModelContour", "Definition of the vertex indices that define the right and left model contour.")
 		.def_static("load", &fitting::ModelContour::load, "Helper method to load a ModelContour from a json file from the hard drive.", py::arg("filename"))
 		;
 
+//	fitting_module.def("get_contour_landmarks", [](
+//		const core::LandmarkCollection<cv::Vec2f>& landmarks, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour, float yaw_angle, const core::Mesh& mesh, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport) {
+//
+//	}, "Get contour landmarks");
+
+
 	fitting_module.def("fit_shape_and_pose", [](const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<glm::vec2>& landmarks, const std::vector<std::string>& landmark_ids, const core::LandmarkMapper& landmark_mapper, int image_width, int image_height, const morphablemodel::EdgeTopology& edge_topology, const fitting::ContourLandmarks& contour_landmarks, const fitting::ModelContour& model_contour, int num_iterations, int num_shape_coefficients_to_fit, float lambda) {
 			assert(landmarks.size() == landmark_ids.size());
 			std::vector<float> pca_coeffs;

utils/CMakeLists.txt

@@ -39,6 +39,10 @@ add_executable(accuracy-evaluation accuracy-evaluation.cpp)
 target_link_libraries(accuracy-evaluation eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
 target_link_libraries(accuracy-evaluation "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
 
+add_executable(accuracy-evaluation-random accuracy-evaluation-random.cpp)
+target_link_libraries(accuracy-evaluation-random eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
+target_link_libraries(accuracy-evaluation-random "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
+
 add_executable(openmp-test openmp-test.cpp)
 target_link_libraries(openmp-test eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
 target_link_libraries(openmp-test "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
@@ -48,4 +52,5 @@ install(TARGETS scm-to-cereal DESTINATION bin)
 install(TARGETS bfm-binary-to-cereal DESTINATION bin)
 install(TARGETS edgestruct-csv-to-json DESTINATION bin)
 install(TARGETS accuracy-evaluation DESTINATION bin)
+install(TARGETS accuracy-evaluation-random DESTINATION bin)
 install(TARGETS openmp-test DESTINATION bin)

utils/accuracy-evaluation-random.cpp

+/*
+ * eos - A 3D Morphable Model fitting library written in modern C++11/14.
+ *
+ * File: utils/json-to-cereal-binary.cpp
+ *
+ * Copyright 2016 Patrik Huber *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "eos/core/landmark_utils.hpp"
+#include "eos/render/render.hpp"
+#include "eos/morphablemodel/MorphableModel.hpp"
+#include "eos/morphablemodel/Blendshape.hpp"
+#include "eos/fitting/fitting.hpp"
+#include "eos/render/utils.hpp"
+#include "eos/render/texture_extraction.hpp"
+#include "eos/video/Keyframe.hpp"
+#include "eos/fitting/ReconstructionData.hpp"
+#include "eos/video/BufferedVideoIterator.hpp"
+
+#include "opencv2/core/core.hpp"
+#include "opencv2/opencv.hpp"
+#include "opencv2/highgui/highgui.hpp"
+
+#include "boost/program_options.hpp"
+#include "boost/filesystem.hpp"
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/ini_parser.hpp>
+#include "glm/gtx/string_cast.hpp"
+
+#include <vector>
+#include <iostream>
+#include <fstream>
+
+using namespace eos;
+namespace po = boost::program_options;
+namespace fs = boost::filesystem;
+using eos::core::Landmark;
+using eos::core::LandmarkCollection;
+using eos::video::BufferedVideoIterator;
+using eos::video::WeightedIsomapAveraging;
+using eos::video::ReconstructionVideoWriter;
+using eos::video::Keyframe;
+using cv::Mat;
+using cv::Vec2f;
+using cv::Vec3f;
+using cv::Vec4f;
+using std::vector;
+using std::string;
+
+using namespace cv;
+
+typedef unsigned int uint;
+
+std::mt19937 gen(1);
+
+/**
+ * Draws the given mesh as wireframe into the image.
+ *
+ * It does backface culling, i.e. draws only vertices in CCW order.
+ *
+ * @param[in] image An image to draw into.
+ * @param[in] mesh The mesh to draw.
+ * @param[in] modelview Model-view matrix to draw the mesh.
+ * @param[in] projection Projection matrix to draw the mesh.
+ * @param[in] viewport Viewport to draw the mesh.
+ * @param[in] colour Colour of the mesh to be drawn.
+ */
+void draw_wireframe(cv::Mat image, const eos::core::Mesh& mesh, glm::mat4x4 modelview, glm::mat4x4 projection, glm::vec4 viewport, cv::Scalar colour = cv::Scalar(0, 255, 0, 255))
+{
+	for (const auto& triangle : mesh.tvi)
+	{
+		const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport);
+		const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport);
+		const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport);
+		if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3))) {
+			cv::line(image, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour);
+			cv::line(image, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour);
+			cv::line(image, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour);
+		}
+	}
+};
+
+/**
+ * Helper function to calculate the euclidean distance between the landmark and a projected
+ * point. Nothing more than Pythagoras.
+ *
+ * @param landmark
+ * @param vertex_screen_coords
+ * @return
+ */
+inline float euclidean_distance(cv::Vec2f landmark, cv::Mat vertex_screen_coords) {
+	float screen_x = vertex_screen_coords.at<float>(0, 0);
+	float screen_y = vertex_screen_coords.at<float>(1, 0);
+
+	// Calc squared differences, ready for use in Pythagoras.
+	float landmark_diff_x_sq = std::fabs(landmark[0] - screen_x) * std::fabs(landmark[0] - screen_x);
+	float landmark_diff_y_sq = std::fabs(landmark[1] - screen_y) * std::fabs(landmark[0] - screen_x);
+
+	return std::sqrt(landmark_diff_x_sq + landmark_diff_y_sq);
+}
+
+/**
+ *
+ * @param landmarks
+ * @param landmark_mapper
+ * @param mesh
+ * @param affine_from_ortho
+ */
+
+std::pair<float, float> calc_reprojection_error(
+	core::LandmarkCollection<cv::Vec2f> landmarks,
+	core::LandmarkMapper landmark_mapper,
+	eos::core::Mesh& mesh,
+	Mat affine_from_ortho,
+	boost::property_tree::ptree settings) {
+
+	vector<float> total_error;
+
+	// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
+	for (uint i = 0; i < landmarks.size(); ++i) {
+		auto converted_name = landmark_mapper.convert(landmarks[i].name);
+
+		// no mapping defined for the current landmark
+		if (!converted_name) {
+			continue;
+		}
+
+		int vertex_idx = std::stoi(converted_name.get());
+
+		// The vertex_idx should be around the value of the original coordinates after we have
+		// projected it with the affine_from_ortho that is obtained earlier.
+		Mat vertex_screen_coords = affine_from_ortho *
+			Mat(cv::Vec4f(
+				mesh.vertices[vertex_idx].x,
+				mesh.vertices[vertex_idx].y,
+				mesh.vertices[vertex_idx].z,
+				mesh.vertices[vertex_idx].w
+				)
+			);
+
+		// using euclidean distance here, but should look at other ways too.
+		float dist = euclidean_distance(landmarks[i].coordinates, vertex_screen_coords);
+		total_error.push_back(dist);
+	}
+
+	// Calculate mean error and stddev.
+	float accum = 0.0;
+	float mean_error = static_cast<float>(std::accumulate(total_error.begin(), total_error.end(), 0)) / landmarks.size();
+
+	for(auto &d: total_error) {
+		accum += (d - mean_error) * (d - mean_error);
+	}
+
+	float stddev = std::sqrt(accum / (total_error.size() - 1));
+
+	return std::make_pair(mean_error, stddev);
+
+}
+
+inline std::string array_to_string(std::vector<std::string> input) {
+	std::string output;
+
+	for(int i = 0; i < input.size(); i++) {
+		output += input[i] + ((i < input.size() - 1) ? "," : "");
+	}
+
+	return output;
+}
+
+/**
+ *
+ *
+ * @param keyframes
+ * @param rendering_paramss
+ * @param landmark_list
+ * @param morphable_model
+ * @param blendshapes
+ * @param meshs
+ * @param pca_shape_coefficients
+ * @param blendshape_coefficients
+ * @param fitted_image_points
+ * @param annotations
+ * @param landmark_mapper
+ * @param settings
+ * @param n_iter
+ */
+void eval(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
+					  vector<core::Mesh> meshs,
+					  std::vector<fitting::RenderingParameters> rendering_paramss,
+					  std::vector<float> pca_shape_coefficients,
+					  std::vector<std::vector<cv::Vec2f>> fitted_image_points,
+					  std::vector<std::string> annotations,
+					  fitting::ReconstructionData reconstruction_data,
+					  boost::property_tree::ptree settings,
+					  long process_time,
+					  std::string videofile,
+					  std::string reconstruction_config,
+					  int n_iter) {
+
+	std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
+	float merge_isomap_face_angle = 90.0; //settings.get<float>("output.merge_isomap_face_angle", 60.f);
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+	bool show_video = settings.get<bool>("output.show_video", false);
+
+	Mat merged_isomap;
+	fs::path outputfilebase = reconstruction_config;
+	WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
+
+	auto landmark_mapper = reconstruction_data.landmark_mapper;
+	auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
+	auto morphable_model = reconstruction_data.morphable_model;
+
+	// The 3D head pose can be recovered as follows:
+	for (uint i = 0; i < keyframes.size(); ++i) {
+		Mat frame = keyframes[i].get()->frame;
+
+		int frame_width = frame.cols;
+		int frame_height = frame.rows;
+
+		auto landmarks = keyframes[i].get()->fitting_result.landmarks;
+		auto rendering_params = rendering_paramss[i];
+		auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+
+		// and similarly for pitch and roll.
+		// Extract the texture from the image using given mesh and camera parameters:
+		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+		Mat isomap = render::extract_texture(
+			meshs[i],
+			affine_from_ortho,
+			frame,
+			true,
+			render::TextureInterpolation::NearestNeighbour,
+			512);
+
+		// Merge the isomaps - add the current one to the already merged ones:
+		merged_isomap = isomap_averaging.add_and_merge(isomap);
+	}
+
+	// gather csv output here
+	std::vector<std::string> output_csv;
+
+	size_t last_frame_index = keyframes.size() - 1;
+	auto keyframe = keyframes[last_frame_index].get();
+	Mat frame = keyframe->frame;
+
+	int frame_width = frame.cols;
+	int frame_height = frame.rows;
+
+	int frame_number = keyframe->frame_number;
+	auto landmarks = keyframe->fitting_result.landmarks;
+	auto rendering_params = rendering_paramss[last_frame_index];
+	auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+	auto mesh = meshs[last_frame_index];
+
+	Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+	float reprojection_error;
+	float stddev;
+	std::tie(reprojection_error, stddev) = calc_reprojection_error(
+		landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
+
+	output_csv.push_back(std::to_string(keyframe->frame_number));
+	output_csv.push_back(std::to_string(keyframes.size()));
+	output_csv.push_back(std::to_string(reprojection_error));
+	output_csv.push_back(std::to_string(stddev));
+	output_csv.push_back(std::to_string(yaw_angle));
+	output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
+	output_csv.push_back(use_pose_binning ? "true" : "false");
+	output_csv.push_back("true");
+	output_csv.push_back(std::to_string(process_time));
+	output_csv.push_back(std::to_string(frame_width));
+	output_csv.push_back(std::to_string(frame_height));
+	output_csv.push_back(videofile);
+	output_csv.push_back(reconstruction_config);
+
+	if (show_video) {
+		Mat outimg = frame.clone();
+
+		std::cout << "fitted points size: " << fitted_image_points.size() << " -- " << landmarks.size() << std::endl;
+		// Draw the fitted mesh as wireframe, and save the image:
+		if (settings.get<bool>("output.landmarks", true))
+		{
+			for (auto &&f: fitted_image_points[last_frame_index]) {
+				cv::rectangle(
+					outimg, cv::Point2f(f[0] - 2.0f, f[1] - 2.0f),
+					cv::Point2f(f[0], f[1] + 2.0f), {255, 0, 0}
+				);
+
+			}
+		}
+
+		if (settings.get<bool>("output.wireframe", true)) {
+			draw_wireframe(
+				outimg,
+				mesh,
+				rendering_params.get_modelview(),
+				rendering_params.get_projection(),
+				fitting::get_opencv_viewport(frame_width, frame_height));
+		}
+
+		cv::imshow("Reconstruction", outimg);
+		cv::waitKey(20);
+	}
+
+	if (settings.get<bool>("output.build_obj", true)) {
+		// save the merged isomap:
+		std::string iter = "_" + std::to_string(n_iter);
+		fs::path path = (fs::path(reconstruction_config).parent_path()
+			/ fs::path(reconstruction_config).replace_extension("").filename()).string();
+		fs::create_directory(path);
+		std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
+
+		outputfile += use_pose_binning ? "_true" : "_false";
+
+		// sharpen isomap
+		Mat sharpen = isomap_averaging.sharpen(merged_isomap);
+		cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
+
+		// save the frontal rendering with merged isomap:
+		Mat frontal_rendering;
+		glm::mat4 modelview_frontal = glm::mat4(1.0);
+
+		std::tie(frontal_rendering, std::ignore) = render::render(
+			mesh,
+			modelview_frontal,
+			glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
+			512, 512,
+			render::create_mipmapped_texture(merged_isomap),
+			true,
+			false,
+			false
+		);
+
+		cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
+
+		output_csv.push_back(outputfile + iter + ".obj");
+
+		// Save the mesh as textured obj:
+		core::write_textured_obj(mesh, outputfile + iter + ".obj");
+		std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
+				  << outputfile + iter + ".obj" << std::endl;
+	}
+
+	// add all alpha's
+	for (auto &alpha: pca_shape_coefficients) {
+		output_csv.push_back(std::to_string(alpha));
+	}
+
+	std::cerr << array_to_string(output_csv) << std::endl;
+}
+
+/**
+ *
+ * @param keyframes
+ * @param rendering_paramss
+ * @param landmark_list
+ * @param morphable_model
+ * @param blendshapes
+ * @param meshs
+ * @param pca_shape_coefficients
+ * @param blendshape_coefficients
+ * @param fitted_image_points
+ * @param annotations
+ * @param landmark_mapper
+ * @param settings
+ * @param n_iter
+ */
+void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
+		  std::vector<float> pca_shape_coefficients,
+		  std::vector<std::vector<cv::Vec2f>> fitted_image_points,
+		  std::vector<std::string> annotations,
+		  fitting::ReconstructionData reconstruction_data,
+		  boost::property_tree::ptree settings,
+		  long process_time,
+		  std::string videofile,
+		  std::string reconstruction_config,
+		  int n_iter) {
+
+	std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
+	float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+
+	Mat merged_isomap;
+	Mat isomap;
+	fs::path outputfilebase = reconstruction_config;
+	WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
+
+	auto landmark_mapper = reconstruction_data.landmark_mapper;
+	auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
+	auto morphable_model = reconstruction_data.morphable_model;
+
+	bool show_video = settings.get<bool>("output.show_video", false);
+
+	// The 3D head pose can be recovered as follows:
+	for (uint i = 0; i < keyframes.size(); ++i) {
+		Mat frame = keyframes[i].get()->frame;
+
+		int frame_width = frame.cols;
+		int frame_height = frame.rows;
+
+		int frame_number = keyframes[i].get()->frame_number;
+		auto landmarks = keyframes[i].get()->fitting_result.landmarks;
+		auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
+		auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+
+		// and similarly for pitch and roll.
+		// Extract the texture from the image using given mesh and camera parameters:
+		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+		// Draw the loaded landmarks:
+		isomap = render::extract_texture(keyframes[i].get()->fitting_result.mesh, affine_from_ortho, frame);
+
+		// merge the isomaps:
+		merged_isomap = isomap_averaging.add_and_merge(isomap);
+	}
+
+	// gather csv output here
+	std::vector<std::string> output_csv;
+
+	size_t last_frame_index = keyframes.size() - 1;
+	auto keyframe = keyframes[last_frame_index].get();
+	Mat frame = keyframe->frame;
+
+	int frame_width = frame.cols;
+	int frame_height = frame.rows;
+
+	int frame_number = keyframe->frame_number;
+	auto landmarks = keyframe->fitting_result.landmarks;
+	auto rendering_params = keyframe->fitting_result.rendering_parameters;
+	auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+	auto mesh = keyframes[last_frame_index].get()->fitting_result.mesh;
+
+	Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+	float reprojection_error;
+	float stddev;
+//	std::tie(reprojection_error, stddev) = calc_reprojection_error(
+//		landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
+
+//	output_csv.push_back(std::to_string(keyframe->frame_number));
+//	output_csv.push_back(std::to_string(keyframes.size()));
+//	output_csv.push_back(std::to_string(reprojection_error));
+//	output_csv.push_back(std::to_string(stddev));
+//	output_csv.push_back(std::to_string(yaw_angle));
+//	output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
+//	output_csv.push_back(use_pose_binning ? "true": "false");
+//	output_csv.push_back(std::to_string(process_time));
+//	output_csv.push_back(std::to_string(frame_width));
+//	output_csv.push_back(std::to_string(frame_height));
+//	output_csv.push_back(videofile);
+//	output_csv.push_back(reconstruction_config);
+
+//	for (auto &alpha: pca_shape_coefficients) {
+//		output_csv.push_back(std::to_string(alpha));
+//	}
+//
+//	std::cerr << array_to_string(output_csv) << std::endl;
+//
+//	if (show_video) {
+//		Mat outimg = frame.clone();
+//
+//		// Draw the fitted mesh as wireframe, and save the image:
+//		if (settings.get<bool>("output.landmarks", true)) {
+//			for (auto &&lm : landmarks) {
+//				cv::rectangle(
+//					outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
+//					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
+//				);
+//			}
+//		}
+//
+//		if (settings.get<bool>("output.wireframe", true)) {
+//			draw_wireframe(
+//				outimg,
+//				mesh,
+//				rendering_params.get_modelview(),
+//				rendering_params.get_projection(),
+//				fitting::get_opencv_viewport(frame_width, frame_height));
+//		}
+//
+//		cv::imshow("Reconstruction", outimg);
+//		cv::waitKey(20);
+//	}
+//
+//	if (settings.get<bool>("output.build_obj", true)) {
+//		// save the merged isomap:
+//		std::string iter = "_" + std::to_string(n_iter);
+//		fs::path path = (fs::path(reconstruction_config).parent_path() / fs::path(reconstruction_config).replace_extension("").filename()).string();
+//		fs::create_directory(path);
+//		std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
+//
+//		outputfile += use_pose_binning ? "_true" : "_false";
+//
+//		// sharpen isomap
+//		Mat sharpen = isomap_averaging.sharpen(merged_isomap);
+//		cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
+//
+//		// save the frontal rendering with merged isomap:
+//		Mat frontal_rendering;
+//		glm::mat4 modelview_frontal = glm::mat4(1.0);
+//
+//		std::tie(frontal_rendering, std::ignore) = render::render(
+//			mesh,
+//			modelview_frontal,
+//			glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
+//			512, 512,
+//			render::create_mipmapped_texture(merged_isomap),
+//			true,
+//			false,
+//			false
+//		);
+//
+//		cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
+//
+//		// Save the mesh as textured obj:
+//		core::write_textured_obj(mesh, outputfile + iter + ".obj");
+//		std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
+//				  << outputfile + iter + ".obj" << std::endl;
+//	}
+}
+
+/**
+ * Parse config file
+ * @param filename
+ */
+boost::property_tree::ptree get_reconstruction_config(std::string filename) {
+	boost::property_tree::ptree pt;
+	boost::property_tree::ini_parser::read_ini(filename, pt);
+
+	return pt;
+}
+
+/**
+ *
+ * Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
+ * are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
+ * causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
+ * annotations with the based on a given keyframe list.
+ *
+ * @param keyframes
+ * @param landmarks
+ * @return
+ */
+core::LandmarkCollection<cv::Vec2f> add_noise(core::LandmarkCollection<cv::Vec2f> landmarks, int noise_x, int noise_y) {
+	std::uniform_real_distribution<> dis_x(0, noise_x);
+	std::uniform_real_distribution<> dis_y(0, noise_y);
+
+	core::LandmarkCollection<cv::Vec2f> sublist;
+	vector<cv::Vec2f> noise;
+
+	for (auto &lm : landmarks) {
+		cv::Vec2f offset(static_cast<float>(dis_x(gen)), dis_y(gen));
+		lm.coordinates += offset;
+
+		noise.push_back(offset);
+		sublist.push_back(lm);
+	}
+
+	return sublist; } /**
+ *
+ * Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
+ * are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
+ * causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
+ * annotations with the based on a given keyframe list.
+ *
+ * @param keyframes
+ * @param landmarks
+ * @return
+ */
+vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
+	std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks, int noise_x, int noise_y) {
+	vector<core::LandmarkCollection<cv::Vec2f>> sublist;
+
+	for (auto& f : keyframes) {
+		auto orig_landmarks = landmarks[f.get()->frame_number];
+		auto noisy_landmarks = add_noise(orig_landmarks, noise_x, noise_y);
+
+		sublist.push_back(noisy_landmarks);
+	}
+
+	return sublist;
+}
+
+/**
+ * This app demonstrates estimation of the camera and fitting of the shape
+ * model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
+ * In addition to fit-model-simple, this example uses blendshapes, contour-
+ * fitting, and can iterate the fitting.
+ *
+ * 68 ibug landmarks are loaded from the .pts file and converted
+ * to vertex indices using the LandmarkMapper.
+ */
+int main(int argc, char *argv[])
+{
+	fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile,
+		blendshapesfile, outputfile, reconstruction_config;
+	std::vector<std::string> annotations;
+
+	// get annotations from one file
+	bool get_annotations = false;
+
+	try
+	{
+		po::options_description desc("Allowed options");
+		desc.add_options()
+			("help,h",
+			 "display the help message")
+			("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
+			 "a Morphable Model stored as cereal BinaryArchive")
+			("video,v", po::value<fs::path>(&videofile)->required(),
+			 "an input image")
+			("config,f",
+			 po::value<fs::path>(&reconstruction_config)->default_value("../share/default_reconstruction_config.ini"),
+			 "configuration file for the reconstruction")
+			("get_annotations,g", po::bool_switch(&get_annotations)->default_value(false),
+			 "read .pts annotation file locations from one file, put one file path on each line")
+			("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
+			 ".pts annotation files per frame of video")
+			("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
+			 "2D landmarks for the image, in ibug .pts format")
+			("model-contour,c",
+			 po::value<fs::path>(&contourfile)->required()->default_value("../share/model_contours.json"),
+			 "file with model contour indices")
+			("edge-topology,e", po::value<fs::path>(&edgetopologyfile)->required()->default_value(
+				"../share/sfm_3448_edge_topology.json"),
+			 "file with model's precomputed edge topology")
+			("blendshapes,b", po::value<fs::path>(&blendshapesfile)->required()->default_value(
+				"../share/expression_blendshapes_3448.bin"),
+			 "file with blendshapes")
+			("output,o", po::value<fs::path>(&outputfile)->required()->default_value("out"),
+			 "basename for the output rendering and obj files");
+		po::variables_map vm;
+		po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
+		if (vm.count("help"))
+		{
+			std::cout << "Usage: fit-model [options]" << std::endl;
+			std::cout << desc;
+			return EXIT_SUCCESS;
+		}
+		po::notify(vm);
+	}
+	catch (const po::error &e)
+	{
+		std::cout << "Error while parsing command-line arguments: " << e.what() << std::endl;
+		std::cout << "Use --help to display a list of options." << std::endl;
+		return EXIT_SUCCESS;
+	}
+
+	// Start loading prerequisites:
+	morphablemodel::MorphableModel morphable_model;
+	std::vector<float> pca_shape_coefficients;
+
+	// List of blendshapes coefficients:
+	std::vector<std::vector<float>> blendshape_coefficients;
+	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
+
+	// These will be the final 2D and 3D points used for the fitting:
+	vector<cv::Vec3f> model_points; // the points in the 3D shape model
+	vector<int> vertex_indices; // their vertex indices
+	vector<core::Mesh> meshs;
+	vector<fitting::RenderingParameters> rendering_paramss;
+
+	// Load landmarks, LandmarkMapper and the Morphable Model:
+	auto landmark_mapper = core::LandmarkMapper(mappingsfile);
+
+	// Load all annotation files into lists of landmarks:
+	vector<core::LandmarkCollection<cv::Vec2f>> landmark_list;
+	try {
+		morphable_model = morphablemodel::load_model(modelfile.string());
+	}
+	catch (const std::runtime_error &e) {
+		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	// Load all annotation using input parameters:
+	try {
+		std::tie(landmark_list, annotations) = eos::core::load_annotations<cv::Vec2f>(
+			annotations,
+			landmark_mapper,
+			morphable_model,
+			get_annotations);
+	}
+	catch (const std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	// The expression blendshapes:
+	auto blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string());
+
+	// These two are used to fit the front-facing contour to the ibug contour landmarks:
+	auto model_contour =
+		contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
+	auto ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string());
+	// The edge topology is used to speed up computation of the occluding face contour fitting:
+	auto edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string());
+
+	// Read .ini file to accommodate.
+	auto settings = get_reconstruction_config(reconstruction_config.string());
+	auto reconstruction_data = eos::fitting::ReconstructionData{
+		morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
+
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+	float threshold = settings.get<float>("frames.threshold", 0.1);
+	bool evaluate = settings.get<bool>("eval.evaluate", true);
+	int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
+	int max_frames = settings.get<int>("eval.max_frames", 0);
+	int drop_frames = settings.get<int>("frames.drop_frames", 0);
+	int noise_x = settings.get<int>("reconstruction.noise_x", 0);
+	int noise_y = settings.get<int>("reconstruction.noise_y", 0);
+
+	// Start with the video play and get video file:
+	BufferedVideoIterator vid_iterator;
+
+	try {
+		vid_iterator = BufferedVideoIterator(videofile.string(), reconstruction_data, settings);
+	} catch (std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	// TODO: determine if this following one should only be used in other programs:
+	// Count the amount of iterations:
+	int n_iter = 0;
+
+	int frames_dropped = 0;
+
+	std::vector<std::shared_ptr<Keyframe>> keyframes;
+	std::uniform_real_distribution<> keyframe_add_random(0, 1);
+
+	// global random generator with given seed
+	std::random_device rd;
+	std::mt19937 local_gen(rd());
+
+	// Get frames randomly until video is done playing OR we've reached max_frames
+	while (vid_iterator.is_playing()) {
+		bool new_frame = vid_iterator.next();
+		if (new_frame == false) {
+			break;
+		}
+
+		double add_frame = keyframe_add_random(local_gen);
+
+		if (add_frame < threshold) {
+			keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
+			if (keyframes.size() >= max_frames) {
+				break;
+			}
+		}
+
+		n_iter++;
+	}
+
+	vid_iterator.__stop();
+
+	// Generate a sublist of the total available landmark list:
+	auto landmark_sublist = sample_landmarks(keyframes, landmark_list, noise_x, noise_y);
+	std::cout << "Going to reconstruct with " << keyframes.size() <<
+				   " images, " << num_iterations << " iterations: " << std::endl;
+
+	// Fit shape and pose:
+	auto t1 = std::chrono::high_resolution_clock::now();
+	std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
+		morphable_model,
+		blendshapes,
+		landmark_sublist,
+		landmark_mapper,
+		keyframes[0].get()->frame.cols,
+		keyframes[0].get()->frame.rows,
+		static_cast<int>(keyframes.size()),
+		edge_topology,
+		ibug_contour,
+		model_contour,
+		num_iterations,
+		boost::none,
+		30.0f,
+		boost::none,
+		pca_shape_coefficients,
+		blendshape_coefficients,
+		fitted_image_points
+	);
+
+	std::cout << vid_iterator.to_string() << std::endl;
+	auto t2 = std::chrono::high_resolution_clock::now();
+
+	std::cout << "Reconstruction took "
+			  << std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count()
+			  << "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count()
+				  / (keyframes.size() * num_iterations)
+			  << "ms)" << std::endl;
+
+	if (settings.get<bool>("eval.evaluate", false)) {
+		eval(
+			keyframes,
+			meshs,
+			rendering_paramss,
+			pca_shape_coefficients,
+			fitted_image_points,
+			annotations,
+			reconstruction_data,
+			settings,
+			std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count(),
+			videofile.string(),
+			reconstruction_config.string(),
+			n_iter
+		);
+	}
+
+	return EXIT_SUCCESS;
+}

utils/accuracy-evaluation.cpp

@@ -61,6 +61,9 @@ using namespace cv;
 
 typedef unsigned int uint;
 
+// global random generator with given seed
+std::mt19937 gen(1);
+
 /**
  * Draws the given mesh as wireframe into the image.
  *
@@ -107,62 +110,6 @@ inline float euclidean_distance(cv::Vec2f landmark, cv::Mat vertex_screen_coords
 	return std::sqrt(landmark_diff_x_sq + landmark_diff_y_sq);
 }
 
-/**
- *
- * @param landmarks
- * @param landmark_mapper
- * @param mesh
- * @param affine_from_ortho
- */
-void evaluate_accuracy(
-		core::LandmarkCollection<cv::Vec2f> landmarks,
-		core::LandmarkMapper landmark_mapper,
-		const eos::core::Mesh& mesh,
-		Mat affine_from_ortho,
-		boost::property_tree::ptree settings) {
-
-	vector<float> total_error;
-
-	// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
-	for (uint i = 0; i < landmarks.size(); ++i) {
-		auto converted_name = landmark_mapper.convert(landmarks[i].name);
-
-		// no mapping defined for the current landmark
-		if (!converted_name) {
-			continue;
-		}
-
-		int vertex_idx = std::stoi(converted_name.get());
-
-		// The vertex_idx should be around the value of the original coordinates after we have
-		// projected it with the affine_from_ortho that is obtained earlier.
-		Mat vertex_screen_coords = affine_from_ortho *
-				Mat(cv::Vec4f(
-						mesh.vertices[vertex_idx].x,
-						mesh.vertices[vertex_idx].y,
-						mesh.vertices[vertex_idx].z,
-						mesh.vertices[vertex_idx].w
-					)
-				);
-
-		// using euclidean distance here, but should look at other ways too.
-		float dist = euclidean_distance(landmarks[i].coordinates, vertex_screen_coords);
-		total_error.push_back(dist);
-	}
-
-	// Calculate mean error and stddev.
-	float accum = 0.0;
-	float mean_error = std::accumulate(total_error.begin(), total_error.end(), 0) / landmarks.size();
-
-	for(auto &d: total_error) {
-		accum += (d - mean_error) * (d - mean_error);
-	}
-
-	float stddev = std::sqrt(accum / (total_error.size() - 1));
-
-	std::cout << "stddev/mean_error: " << stddev << " " <<  mean_error << std::endl;
-}
-
 /**
  *
  * @param keyframes
@@ -325,6 +272,75 @@ void render_output(
 }
 
 /**
+ *
+ * @param landmarks
+ * @param landmark_mapper
+ * @param mesh
+ * @param affine_from_ortho
+ */
+
+std::pair<float, float> calc_reprojection_error(
+	core::LandmarkCollection<cv::Vec2f> landmarks,
+	core::LandmarkMapper landmark_mapper,
+	eos::core::Mesh& mesh,
+	Mat affine_from_ortho,
+	boost::property_tree::ptree settings) {
+
+	vector<float> total_error;
+
+	// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
+	for (uint i = 0; i < landmarks.size(); ++i) {
+		auto converted_name = landmark_mapper.convert(landmarks[i].name);
+
+		// no mapping defined for the current landmark
+		if (!converted_name) {
+			continue;
+		}
+
+		int vertex_idx = std::stoi(converted_name.get());
+
+		// The vertex_idx should be around the value of the original coordinates after we have
+		// projected it with the affine_from_ortho that is obtained earlier.
+		Mat vertex_screen_coords = affine_from_ortho *
+			Mat(cv::Vec4f(
+				mesh.vertices[vertex_idx].x,
+				mesh.vertices[vertex_idx].y,
+				mesh.vertices[vertex_idx].z,
+				mesh.vertices[vertex_idx].w
+				)
+			);
+
+		// using euclidean distance here, but should look at other ways too.
+		float dist = euclidean_distance(landmarks[i].coordinates, vertex_screen_coords);
+		total_error.push_back(dist);
+	}
+
+	// Calculate mean error and stddev.
+	float accum = 0.0;
+	float mean_error = static_cast<float>(std::accumulate(total_error.begin(), total_error.end(), 0)) / landmarks.size();
+
+	for(auto &d: total_error) {
+		accum += (d - mean_error) * (d - mean_error);
+	}
+
+	float stddev = std::sqrt(accum / (total_error.size() - 1));
+
+	return std::make_pair(mean_error, stddev);
+
+}
+
+inline std::string array_to_string(std::vector<std::string> input) {
+	std::string output;
+
+	for(int i = 0; i < input.size(); i++) {
+		output += input[i] + ((i < input.size() - 1) ? "," : "");
+	}
+
+	return output;
+}
+
+/**
+ *
  *
  * @param keyframes
  * @param rendering_paramss
@@ -340,27 +356,30 @@ void render_output(
  * @param settings
  * @param n_iter
  */
-void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
-		std::vector<fitting::RenderingParameters> rendering_paramss,
+void eval(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
 					  vector<core::Mesh> meshs,
+					  std::vector<fitting::RenderingParameters> rendering_paramss,
 					  std::vector<float> pca_shape_coefficients,
-		std::vector<std::vector<float>> blendshape_coefficients,
 					  std::vector<std::vector<cv::Vec2f>> fitted_image_points,
 					  std::vector<std::string> annotations,
 					  fitting::ReconstructionData reconstruction_data,
 					  boost::property_tree::ptree settings,
+					  long process_time,
+					  std::string videofile,
+					  std::string reconstruction_config,
 					  int n_iter) {
 
-
 	std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
-	float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
+	float merge_isomap_face_angle = 90.0; //settings.get<float>("output.merge_isomap_face_angle", 60.f);
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+	bool show_video = settings.get<bool>("output.show_video", false);
 
-	WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
 	Mat merged_isomap;
-	fs::path outputfilebase = annotations[0];
+	fs::path outputfilebase = reconstruction_config;
+	WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
 
 	auto landmark_mapper = reconstruction_data.landmark_mapper;
-	auto blendshapes = reconstruction_data.blendshapes;
+	auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
 	auto morphable_model = reconstruction_data.morphable_model;
 
 	// The 3D head pose can be recovered as follows:
@@ -370,76 +389,312 @@ void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyfram
 		int frame_width = frame.cols;
 		int frame_height = frame.rows;
 
-		int frame_number = keyframes[i].get()->frame_number;
 		auto landmarks = keyframes[i].get()->fitting_result.landmarks;
-		auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
+		auto rendering_params = rendering_paramss[i];
 		auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
 
-		Mat outimg = frame.clone();
-
 		// and similarly for pitch and roll.
 		// Extract the texture from the image using given mesh and camera parameters:
-		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
-			rendering_params, frame_width, frame_height);
+		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+		Mat isomap = render::extract_texture(
+			meshs[i],
+			affine_from_ortho,
+			frame,
+			true,
+			render::TextureInterpolation::NearestNeighbour,
+			512);
+
+		Mat outimg = frame.clone();
+
+		// Merge the isomaps - add the current one to the already merged ones:
+		merged_isomap = isomap_averaging.add_and_merge(isomap);
 
-		for (auto &&lm : landmarks) {
+		for (auto &&f: fitted_image_points[i]) {
 			cv::rectangle(
-				outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
-				cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
+				outimg, cv::Point2f(f[0] - 2.0f, f[1] - 2.0f),
+				cv::Point2f(f[0], f[1] + 2.0f), {255, 0, 0}
 			);
 		}
 
+		draw_wireframe(
+			outimg,
+			meshs[i],
+			rendering_params.get_modelview(),
+			rendering_params.get_projection(),
+			fitting::get_opencv_viewport(frame_width, frame_height));
+
+		std::string iter = std::to_string(n_iter) + "_" + std::to_string(yaw_angle);
+
+		cv::imwrite("/tmp/video/" + iter + ".png", outimg);
+	}
+
+	// gather csv output here
+	std::vector<std::string> output_csv;
+
+	size_t last_frame_index = keyframes.size() - 1;
+	auto keyframe = keyframes[last_frame_index].get();
+	Mat frame = keyframe->frame;
+
+	int frame_width = frame.cols;
+	int frame_height = frame.rows;
+
+	int frame_number = keyframe->frame_number;
+	auto landmarks = keyframe->fitting_result.landmarks;
+	auto rendering_params = rendering_paramss[last_frame_index];
+	auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+	auto mesh = meshs[last_frame_index];
+
+	Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+	float reprojection_error;
+	float stddev;
+	std::tie(reprojection_error, stddev) = calc_reprojection_error(
+		landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
+
+	output_csv.push_back(std::to_string(keyframe->frame_number));
+	output_csv.push_back(std::to_string(keyframes.size()));
+	output_csv.push_back(std::to_string(reprojection_error));
+	output_csv.push_back(std::to_string(stddev));
+	output_csv.push_back(std::to_string(yaw_angle));
+	output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
+	output_csv.push_back(use_pose_binning ? "true" : "false");
+	output_csv.push_back("false");
+	output_csv.push_back(std::to_string(process_time));
+	output_csv.push_back(std::to_string(frame_width));
+	output_csv.push_back(std::to_string(frame_height));
+	output_csv.push_back(videofile);
+	output_csv.push_back(reconstruction_config);
+
+	if (show_video) {
+		Mat outimg = frame.clone();
+
+		std::cout << "fitted points size: " << fitted_image_points.size() << " -- " << landmarks.size() << std::endl;
+
 		// Draw the fitted mesh as wireframe, and save the image:
+		if (settings.get<bool>("output.landmarks", true))
+		{
+			for (auto &&f: fitted_image_points[last_frame_index]) {
+				cv::rectangle(
+					outimg, cv::Point2f(f[0] - 2.0f, f[1] - 2.0f),
+					cv::Point2f(f[0], f[1] + 2.0f), {255, 0, 0}
+				);
+
+			}
+		}
+
+		if (settings.get<bool>("output.wireframe", true)) {
 			draw_wireframe(
 				outimg,
-				keyframes[i].get()->fitting_result.mesh,
+				mesh,
 				rendering_params.get_modelview(),
 				rendering_params.get_projection(),
 				fitting::get_opencv_viewport(frame_width, frame_height));
+		}
+
+		cv::imshow("Reconstruction", outimg);
+		cv::waitKey(0);
+	}
+
+	if (settings.get<bool>("output.build_obj", true)) {
+		// save the merged isomap:
+		std::string iter = "_" + std::to_string(n_iter);
+		fs::path path = (fs::path(reconstruction_config).parent_path()
+			/ fs::path(reconstruction_config).replace_extension("").filename()).string();
+		fs::create_directory(path);
+		std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
+
+		outputfile += use_pose_binning ? "_true" : "_false";
+
+		// sharpen isomap
+		Mat sharpen = isomap_averaging.sharpen(merged_isomap);
+		cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
+
+		// save the frontal rendering with merged isomap:
+		Mat frontal_rendering;
+		glm::mat4 modelview_frontal = glm::mat4(1.0);
+
+		std::tie(frontal_rendering, std::ignore) = render::render(
+			mesh,
+			modelview_frontal,
+			glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
+			512, 512,
+			render::create_mipmapped_texture(merged_isomap),
+			true,
+			false,
+			false
+		);
+
+		cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
+
+		output_csv.push_back(outputfile + iter + ".obj");
+
+		// Save the mesh as textured obj:
+		core::write_textured_obj(mesh, outputfile + iter + ".obj");
+		std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
+				  << outputfile + iter + ".obj" << std::endl;
+	}
+
+	// add all alpha's
+	for (auto &alpha: pca_shape_coefficients) {
+		output_csv.push_back(std::to_string(alpha));
+	}
+
+	std::cerr << array_to_string(output_csv) << std::endl;
+}
+
+/**
+ *
+ * @param keyframes
+ * @param rendering_paramss
+ * @param landmark_list
+ * @param morphable_model
+ * @param blendshapes
+ * @param meshs
+ * @param pca_shape_coefficients
+ * @param blendshape_coefficients
+ * @param fitted_image_points
+ * @param annotations
+ * @param landmark_mapper
+ * @param settings
+ * @param n_iter
+ */
+void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
+		  std::vector<float> pca_shape_coefficients,
+		  std::vector<std::vector<cv::Vec2f>> fitted_image_points,
+		  std::vector<std::string> annotations,
+		  fitting::ReconstructionData reconstruction_data,
+		  boost::property_tree::ptree settings,
+		  long process_time,
+		  std::string videofile,
+		  std::string reconstruction_config,
+		  int n_iter) {
+
+	std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
+	float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+
+	Mat merged_isomap;
+	Mat isomap;
+	fs::path outputfilebase = reconstruction_config;
+	WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
+
+	auto landmark_mapper = reconstruction_data.landmark_mapper;
+	auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
+	auto morphable_model = reconstruction_data.morphable_model;
+
+	bool show_video = settings.get<bool>("output.show_video", false);
+
+	// The 3D head pose can be recovered as follows:
+	for (uint i = 0; i < keyframes.size(); ++i) {
+		Mat frame = keyframes[i].get()->frame;
+
+		int frame_width = frame.cols;
+		int frame_height = frame.rows;
 
+		int frame_number = keyframes[i].get()->frame_number;
+		auto landmarks = keyframes[i].get()->fitting_result.landmarks;
+		auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
+		auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
 
-		cv::imshow("Img", outimg);
-		cv::waitKey(20);
+		// and similarly for pitch and roll.
+		// Extract the texture from the image using given mesh and camera parameters:
+		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
 
 		// Draw the loaded landmarks:
-		Mat isomap = render::extract_texture(meshs[i], affine_from_ortho, frame);
+		isomap = render::extract_texture(keyframes[i].get()->fitting_result.mesh, affine_from_ortho, frame);
 
 		// merge the isomaps:
 		merged_isomap = isomap_averaging.add_and_merge(isomap);
-
-		evaluate_accuracy(
-			landmarks,
-			landmark_mapper,
-			meshs[i],
-			affine_from_ortho,
-			settings
-		);
 	}
 
-	// save the merged isomap:
+	// gather csv output here
+	std::vector<std::string> output_csv;
+
+	size_t last_frame_index = keyframes.size() - 1;
+	auto keyframe = keyframes[last_frame_index].get();
+	Mat frame = keyframe->frame;
+
+	int frame_width = frame.cols;
+	int frame_height = frame.rows;
+
+	int frame_number = keyframe->frame_number;
+	auto landmarks = keyframe->fitting_result.landmarks;
+	auto rendering_params = keyframe->fitting_result.rendering_parameters;
+	auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+	auto mesh = keyframes[last_frame_index].get()->fitting_result.mesh;
+
+	Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
+
+	float reprojection_error;
+	float stddev;
+//	std::tie(reprojection_error, stddev) = calc_reprojection_error(
+//		landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
+
+//	output_csv.push_back(std::to_string(keyframe->frame_number));
+//	output_csv.push_back(std::to_string(keyframes.size()));
+//	output_csv.push_back(std::to_string(reprojection_error));
+//	output_csv.push_back(std::to_string(stddev));
+//	output_csv.push_back(std::to_string(yaw_angle));
+//	output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
+//	output_csv.push_back(use_pose_binning ? "true": "false");
+//	output_csv.push_back(std::to_string(process_time));
+//	output_csv.push_back(std::to_string(frame_width));
+//	output_csv.push_back(std::to_string(frame_height));
+//	output_csv.push_back(videofile);
+//	output_csv.push_back(reconstruction_config);
+
+//	for (auto &alpha: pca_shape_coefficients) {
+//		output_csv.push_back(std::to_string(alpha));
+//	}
+//
+//	std::cerr << array_to_string(output_csv) << std::endl;
+//
+//	if (show_video) {
+//		Mat outimg = frame.clone();
+//
+//		// Draw the fitted mesh as wireframe, and save the image:
+//		if (settings.get<bool>("output.landmarks", true)) {
+//			for (auto &&lm : landmarks) {
+//				cv::rectangle(
+//					outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
+//					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
+//				);
+//			}
+//		}
+//
+//		if (settings.get<bool>("output.wireframe", true)) {
+//			draw_wireframe(
+//				outimg,
+//				mesh,
+//				rendering_params.get_modelview(),
+//				rendering_params.get_projection(),
+//				fitting::get_opencv_viewport(frame_width, frame_height));
+//		}
+//
+//		cv::imshow("Reconstruction", outimg);
+//		cv::waitKey(20);
+//	}
+//
+//	if (settings.get<bool>("output.build_obj", true)) {
+//		// save the merged isomap:
 //		std::string iter = "_" + std::to_string(n_iter);
-//	fs::path path = (fs::path(annotations[n_iter]).parent_path() / "eval");
+//		fs::path path = (fs::path(reconstruction_config).parent_path() / fs::path(reconstruction_config).replace_extension("").filename()).string();
 //		fs::create_directory(path);
-//	std::string outputfile = (path / fs::path(annotations[n_iter]).replace_extension("").filename()).string();
+//		std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
+//
+//		outputfile += use_pose_binning ? "_true" : "_false";
 //
 //		// sharpen isomap
 //		Mat sharpen = isomap_averaging.sharpen(merged_isomap);
-//
 //		cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
 //
 //		// save the frontal rendering with merged isomap:
 //		Mat frontal_rendering;
-//	glm::mat4 modelview_frontal = glm::mat4( 1.0 );
-//	core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
-//			morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
-//			morphable_model.get_color_model().get_mean(),
-//			morphable_model.get_shape_model().get_triangle_list(),
-//			morphable_model.get_color_model().get_triangle_list(),
-//			morphable_model.get_texture_coordinates()
-//	);
-
+//		glm::mat4 modelview_frontal = glm::mat4(1.0);
+//
 //		std::tie(frontal_rendering, std::ignore) = render::render(
-//			neutral_expression,
+//			mesh,
 //			modelview_frontal,
 //			glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
 //			512, 512,
@@ -448,11 +703,14 @@ void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyfram
 //			false,
 //			false
 //		);
+//
 //		cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
-
-	// Save the mesh as textured obj:
-//	core::write_textured_obj(neutral_expression, outputfile + iter + ".obj");
-//	std::cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile + iter + ".obj" << std::endl;
+//
+//		// Save the mesh as textured obj:
+//		core::write_textured_obj(mesh, outputfile + iter + ".obj");
+//		std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
+//				  << outputfile + iter + ".obj" << std::endl;
+//	}
 }
 
 /**
@@ -477,13 +735,41 @@ boost::property_tree::ptree get_reconstruction_config(std::string filename) {
  * @param landmarks
  * @return
  */
-vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
-	std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
+core::LandmarkCollection<cv::Vec2f> add_noise(core::LandmarkCollection<cv::Vec2f> landmarks, int noise_x, int noise_y) {
+	std::uniform_real_distribution<> dis_x(0, noise_x);
+	std::uniform_real_distribution<> dis_y(0, noise_y);
+
+	core::LandmarkCollection<cv::Vec2f> sublist;
+	vector<cv::Vec2f> noise;
+
+	for (auto &lm : landmarks) {
+		cv::Vec2f offset(static_cast<float>(dis_x(gen)), dis_y(gen));
+		lm.coordinates += offset;
 
+		noise.push_back(offset);
+		sublist.push_back(lm);
+	}
+
+	return sublist; } /**
+ *
+ * Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
+ * are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
+ * causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
+ * annotations with the based on a given keyframe list.
+ *
+ * @param keyframes
+ * @param landmarks
+ * @return
+ */
+vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
+	std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks, int noise_x, int noise_y) {
 	vector<core::LandmarkCollection<cv::Vec2f>> sublist;
 
 	for (auto& f : keyframes) {
-		sublist.push_back(landmarks[f.get()->frame_number]);
+		auto orig_landmarks = landmarks[f.get()->frame_number];
+		auto noisy_landmarks = add_noise(orig_landmarks, noise_x, noise_y);
+
+		sublist.push_back(noisy_landmarks);
 	}
 
 	return sublist;
@@ -600,6 +886,14 @@ int main(int argc, char *argv[]) {
 	auto reconstruction_data = eos::fitting::ReconstructionData{
 		morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
 
+	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
+	bool evaluate = settings.get<bool>("eval.evaluate", true);
+	int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
+	int max_frames = settings.get<int>("eval.max_frames", 0);
+	int drop_frames = settings.get<int>("frames.drop_frames", 0);
+	int min_frames = settings.get<int>("frames.min_frames", 0);
+	int noise_x = settings.get<int>("reconstruction.noise_x", 0);
+	int noise_y = settings.get<int>("reconstruction.noise_y", 0);
 
 	// Start with the video play and get video file:
 	BufferedVideoIterator vid_iterator;
@@ -611,52 +905,75 @@ int main(int argc, char *argv[]) {
 		return EXIT_FAILURE;
 	}
 
+	// TODO: determine if this following one should only be used in other programs:
+	//	vid_iterator.start();
+
+	//ReconstructionVideoWriter vid_writer;
+	//try {
+	//	vid_writer = ReconstructionVideoWriter(videofile.string(), reconstruction_data, settings);
+	//} catch(std::runtime_error &e) {
+	//	std::cout << e.what() << std::endl;
+	//	return EXIT_FAILURE;
+	//}
+	// vid_writer.start();
 	// Start getting video frames:
-	bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
-
-	if (use_pose_binning) {
-		vid_iterator.start();
-	}
-
-
-	ReconstructionVideoWriter vid_writer;
-	try {
-		vid_writer = ReconstructionVideoWriter(videofile.string(), reconstruction_data, settings);
-	} catch(std::runtime_error &e) {
-		std::cout << e.what() << std::endl;
-		return EXIT_FAILURE;
-	}
-
-	int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
-//	vid_writer.start();
 
 	// Count the amount of iterations:
 	int n_iter = 0;
 
-	while(vid_iterator.is_playing() || n_iter < 20) {
+//	if (use_pose_binning == false && (max_frames > 0 && n_iter > max_frames)) {
+//		break;
+//
+	int frames_dropped = 0;
+
+	while(vid_iterator.is_playing()) {
+		std::cout << vid_iterator.to_string() << std::endl;
 		std::vector<std::shared_ptr<Keyframe>> keyframes;
+		std::cout << "Try to reconstruct: " << vid_iterator.get_last_keyframe().frame_number << "/" << max_frames << std::endl;
+
+		if (vid_iterator.get_last_keyframe().frame_number > max_frames) {
+			vid_iterator.__stop();
+			break;
+		}
 
 		if (use_pose_binning) {
+			bool new_frame = vid_iterator.next();
+			if (new_frame == false) {
+				vid_iterator.__stop();
+			}
 			keyframes = vid_iterator.get_keyframes();
+			std::cout << "pose binning size:" << keyframes.size() << std::endl;
+//			keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
 		} else {
-			vid_iterator.next();
+			bool new_frame = vid_iterator.next();
+			if (new_frame == false) {
+				vid_iterator.__stop();
+			}
+
 			keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
 		}
 
+		if (drop_frames > 0 && (frames_dropped < drop_frames) && n_iter > 20) {
+			std::cout << "frames_dropped: " << frames_dropped << ", " << drop_frames << std::endl;
+			frames_dropped++;
+			continue;
+		}
 
-		// Generate a sublist of the total available landmark list:
-		auto landmark_sublist = sample_landmarks(keyframes, landmark_list);
+		frames_dropped = 0;
 
 		// it makes no sense to update pca_coeff if nothing in the buffer has changed:
-		if (vid_iterator.has_changed() || use_pose_binning == false) {
-			std::cout << "Going to reconstruct with " << keyframes.size() << " images."<< num_iterations << std::endl;
+		if (vid_iterator.has_changed()) {
+			// Generate a sublist of the total available landmark list:
+			auto landmark_sublist = sample_landmarks(keyframes, landmark_list, noise_x, noise_y);
+			std::cout << "Going to reconstruct with " << keyframes.size() << " images, "<< num_iterations << " iterations:"<< std::endl;
 
 			// Fit shape and pose:
 			auto t1 = std::chrono::high_resolution_clock::now();
-			std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
+
+			std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
 				morphable_model,
 				blendshapes,
-				keyframes,
+				landmark_sublist,
 				landmark_mapper,
 				keyframes[0].get()->frame.cols,
 				keyframes[0].get()->frame.rows,
@@ -670,9 +987,28 @@ int main(int argc, char *argv[]) {
 				boost::none,
 				pca_shape_coefficients,
 				blendshape_coefficients,
-				fitted_image_points,
-				settings
+				fitted_image_points
 			);
+//			std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
+//				morphable_model,
+//				blendshapes,
+//				keyframes,
+//				landmark_mapper,
+//				keyframes[0].get()->frame.cols,
+//				keyframes[0].get()->frame.rows,
+//				static_cast<int>(keyframes.size()),
+//				edge_topology,
+//				ibug_contour,
+//				model_contour,
+//				num_iterations,
+//				boost::none,
+//				30.0f,
+//				boost::none,
+//				pca_shape_coefficients,
+//				blendshape_coefficients,
+//				fitted_image_points,
+//				settings
+//			);
 
 			std::cout << vid_iterator.to_string() << std::endl;
 
@@ -681,71 +1017,95 @@ int main(int argc, char *argv[]) {
 					  << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count()
 					  << "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() / (keyframes.size() * num_iterations)
 					  << "ms)" << std::endl;
-//
-//			if(settings.get<bool>("output.make_video", false)) {
-//				vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
-//				vid_writer.next();
-//			}
 
-			evaluate_results(
+			//if(settings.get<bool>("output.make_video", false)) {
+			//	vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
+			//	vid_writer.next();
+			//}
+
+			if(settings.get<bool>("eval.evaluate", false)) {
+				eval(
 					keyframes,
-				rendering_paramss,
 					meshs,
+					rendering_paramss,
 					pca_shape_coefficients,
-				blendshape_coefficients,
 					fitted_image_points,
 					annotations,
 					reconstruction_data,
 					settings,
+					std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count(),
+					videofile.string(),
+					reconstruction_config.string(),
 					n_iter
 				);
 
+//				evaluate_results(
+//					keyframes,
+//					pca_shape_coefficients,
+//					fitted_image_points,
+//					annotations,
+//					reconstruction_data,
+//					settings,
+//					std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count(),
+//					videofile.string(),
+//					reconstruction_config.string(),
+//					n_iter
+//				);
+			}
+
 		} else {
 			std::cout << vid_iterator.to_string() << std::endl;
 			std::cout << "No reconstruction - buffer did not change" << std::endl;
 		}
 
-		std::cout << n_iter << std::endl;
-		// Get new frames:
+		// only count after a reconstruction
 		n_iter++;
 	}
 
-	std::cout << "stopping?" << std::endl;
 
-	vid_writer.__stop();
-//	if(settings.get<bool>("output.make_video", false)) {
-//		// Render output:
-//		std::cout << "Waiting for video to be completed..." << std::endl;
-//		vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
-//
-//		int count = 0;
-//		while (count < 40) {
-//			auto t1 = std::chrono::high_resolution_clock::now();
-//			vid_writer.next();
-//			auto t2 = std::chrono::high_resolution_clock::now();
-//			printf("Frame %d/%d (%d)\n", vid_writer.get_frame_number(), vid_iterator.get_frame_number(), count);
-//
-//			std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() << "ms" << std::endl;
-//			count++;
-//		}
+	//if(settings.get<bool>("output.make_video", false)) {
+	//	// Render output:
+	//	std::cout << "Waiting for video to be completed..." << std::endl;
+	//	vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
+
+	//	int count = 0;
+	//	while (count < 40) {
+	//		auto t1 = std::chrono::high_resolution_clock::now();
+	//		vid_writer.next();
+	//		auto t2 = std::chrono::high_resolution_clock::now();
+	//		printf("Frame %d/%d (%d)\n", vid_writer.get_frame_number(), vid_iterator.get_frame_number(), count);
+
+	//		std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() << "ms" << std::endl;
+	//		count++;
+	//	}
 //
 //		vid_writer.__stop();
 //	}
 
-//	auto keyframes = vid_iterator.get_keyframes();
+//	if(settings.get<bool>("eval.evaluate", false)) {
+//		std::vector<std::shared_ptr<Keyframe>> keyframes;
+//
+//		if (use_pose_binning) {
+//			keyframes = vid_iterator.get_keyframes();
+//			keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
+//		} else {
+//			vid_iterator.next();
+//			keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
+//		}
 //
 //		evaluate_results(
 //			keyframes,
-//		rendering_paramss,
-//		meshs,
 //			pca_shape_coefficients,
-//		blendshape_coefficients,
 //			fitted_image_points,
 //			annotations,
 //			reconstruction_data,
 //			settings,
+//			0,
+//			videofile.string(),
+//			reconstruction_config.string(),
 //			n_iter
 //		);
+//	}
 
 	//todo: we could build our final obj here?