Add capability to read a settings file (.ini format) and fix bug with...

Add capability to read a settings file (.ini format) and fix bug with unaligned landmarks and frames

include/eos/video/Keyframe.hpp

@@ -28,6 +28,8 @@
 #include "eos/morphablemodel/MorphableModel.hpp"
 #include "eos/render/texture_extraction.hpp"
 
+#include <boost/property_tree/ptree.hpp>
+
 #include "opencv2/core/core.hpp"
 
 #include <deque>
@@ -40,7 +42,7 @@ namespace video {
  *
  * Contains the original frame, all necessary fitting parameters, and a score.
  */
-struct Keyframe {
+class Keyframe {
 public:
 	Keyframe() {}
 	/**
@@ -76,10 +78,11 @@ public:
 		this->fitting_result = fitting_result;
 	}
 
+	int frame_number;
 	float score = 0.0f;
+
 	cv::Mat frame;
 	fitting::FittingResult fitting_result;
-	int frame_number;
 };
 
 /**
@@ -318,8 +321,7 @@ public:
 	BufferedVideoIterator() {};
 
 	// TODO: build support for setting the amount of max_frames in the buffer.
-	BufferedVideoIterator(std::string videoFilePath,
-						  uint max_frames = 5, uint min_frames = 4) {
+	BufferedVideoIterator(std::string videoFilePath, boost::property_tree::ptree settings) {
 		std::ifstream file(videoFilePath);
 
 		std::cout << "video file path: " << videoFilePath << std::endl;
@@ -334,8 +336,13 @@ public:
 		}
 
 		this->cap = tmp_cap;
-		this->max_frames = max_frames;
-		this->min_frames = min_frames;
+		this->max_frames = settings.get<int>("video.max_frames", 5);
+		this->min_frames = settings.get<int>("video.min_frames", 5);
+		this->drop_frames = settings.get<int>("video.drop_frames", 0);
+		this->laplacian_threshold = settings.get<int>("video.blur_threshold", 1000);
+
+		// TODO: Implement this.
+		this->skip_frames = settings.get<int>("video.skip_frames", 0);
 	}
 
 	/**
@@ -352,6 +359,7 @@ public:
 		cv::Mat frame;
 		cap >> frame;
 
+
 		// Pop if we exceeded max_frames.
 		if (n_frames > max_frames) {
 			frame_buffer.pop_front();
@@ -362,13 +370,20 @@ public:
 
 		if (frame_laplacian_score < laplacian_threshold && frame_laplacian_score > 0) {
 			frame_buffer.push_back(Keyframe(frame_laplacian_score, frame, total_frames_processed));
-			total_frames_processed++;
 			n_frames++;
 			std::cout << total_frames_processed << ": laplacian score " << frame_laplacian_score << std::endl;
 		} else {
-			std::cout << "skipping frame, too blurry or total black" << std::endl;
+			std::cout << total_frames_processed << ": skipping frame " << std::endl;
+			if (frame_laplacian_score == 0) {
+				std::cout << "total black" << std::endl;
+			} else {
+				std::cout << "too blurry" << std::endl;
+			}
+
 		}
 
+		total_frames_processed++;
+
 		// fill up the buffer until we hit the minimum frames we want in the buffer.
 		if(n_frames < min_frames) {
 			frame_buffer = next();
@@ -386,21 +401,23 @@ private:
 	std::deque<Keyframe> frame_buffer;
 
 	// TODO: make set-able
-	uint total_frames_processed = 0;
-
-	uint skip_frames = 0;
+	// total frames in processed, not persee in buffer (skipped frames)
+	int total_frames_processed = 0;
 
 	// total frames in buffer
-	uint n_frames = 0;
+	int n_frames = 0;
+
+	// number of frames to skip at before starting
+	int skip_frames = 0;
 
-	// min frames to load at the start.
-	uint min_frames = 5;
+	// minimum amount of frames to keep in buffer.
+	int min_frames = 5;
 
 	// keep max_frames into the buffer.
-	uint max_frames = 5;
+	int max_frames = 5;
 
 	// Note: these settings are for future use
-	uint drop_frames = 0;
+	int drop_frames = 0;
 
 	// laplacian threshold
 	double laplacian_threshold = 10000000;

utils/accuracy-evaluation.cpp

@@ -31,6 +31,8 @@
 
 #include "boost/program_options.hpp"
 #include "boost/filesystem.hpp"
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/ini_parser.hpp>
 #include "glm/gtx/string_cast.hpp"
 
 #include <vector>
@@ -185,17 +187,17 @@ void evaluate_results(
 		// and similarly for pitch and roll.
 		// Extract the texture from the image using given mesh and camera parameters:
 		Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
-			rendering_paramss[i], frame_width, frame_height
+				rendering_paramss[i], frame_width, frame_height
 		);
 
 		// Draw the loaded landmarks:
 		Mat isomap = render::extract_texture(meshs[i], affine_from_ortho, frame);
 		Mat outimg = frame.clone();
 
-		for (auto&& lm : landmark_list[frame_number]) {
+		for (auto &&lm : landmark_list[i]) {
 			cv::rectangle(
 					outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
-					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), { 255, 0, 0 }
+					cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
 			);
 		}
 
@@ -208,7 +210,8 @@ void evaluate_results(
 				fitting::get_opencv_viewport(frame_width, frame_height)
 		);
 
-		std::string outputfile = fs::path(annotations[frame_number]).replace_extension("").string();
+		fs::path path = (fs::path(annotations[frame_number]).parent_path() / "eval");
+		std::string outputfile = (path / fs::path(annotations[frame_number]).replace_extension("").filename()).string();
 		std::string iter = "_" + std::to_string(n_iter) + "_" + std::to_string(i);
 		cv::imwrite(outputfile + iter + ".annotated.png", outimg);
 
@@ -234,16 +237,17 @@ void evaluate_results(
 		merged_isomap = isomap_averaging.add_and_merge(isomap);
 
 		evaluate_accuracy(
-				landmark_list[frame_number],
-				landmark_mapper,
-				meshs[i],
-				affine_from_ortho
+			landmark_list[i],
+			landmark_mapper,
+			meshs[i],
+			affine_from_ortho
 		);
 	}
 
 	// save the merged isomap:
 	std::string iter = "_" + std::to_string(n_iter);
-	std::string outputfile = fs::path(annotations[n_iter]).replace_extension("").string();
+	fs::path path = (fs::path(annotations[n_iter]).parent_path() / "eval");
+	std::string outputfile = (path / fs::path(annotations[n_iter]).replace_extension("").filename()).string();
 
 	cv::imwrite(outputfile + iter + ".isomap.png", merged_isomap);
 
@@ -264,6 +268,44 @@ void evaluate_results(
 	std::cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile + iter + ".obj" << std::endl;
 }
 
+/**
+ * Parse config file
+ * @param filename
+ */
+boost::property_tree::ptree get_reconstruction_config(std::string filename) {
+	boost::property_tree::ptree pt;
+	boost::property_tree::ini_parser::read_ini(filename, pt);
+
+	std::cout << pt.get<std::string>("video.max_frames") << std::endl;
+	std::cout << pt.get<std::string>("video.drop_frames") << std::endl;
+	std::cout << pt.get<std::string>("video.min_frames") << std::endl;
+	std::cout << pt.get<std::string>("video.skip_frames") << std::endl;
+	std::cout << pt.get<std::string>("video.blur_threshold") << std::endl;
+
+	return pt;
+}
+
+/**
+ *
+ * Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
+ * are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
+ * causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
+ * annotations with the based on a given keyframe list.
+ *
+ * @param key_frames
+ * @param landmarks
+ * @return
+ */
+vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(std::deque<eos::video::Keyframe> key_frames, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
+		vector<core::LandmarkCollection<cv::Vec2f>> sublist;
+
+	for (auto& f : key_frames) {
+		sublist.push_back(landmarks[f.frame_number]);
+	}
+
+	return sublist;
+}
+
 /**
  * This app demonstrates estimation of the camera and fitting of the shape
  * model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
@@ -274,7 +316,7 @@ void evaluate_results(
  * to vertex indices using the LandmarkMapper.
  */
 int main(int argc, char *argv[]) {
-	fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile;
+	fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile, reconstruction_config;
 	std::vector<std::string> annotations;
 
 	// get annotaitions from one file
@@ -287,10 +329,12 @@ int main(int argc, char *argv[]) {
 				 "display the help message")
 				("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
 				 "a Morphable Model stored as cereal BinaryArchive")
-				("video,i", po::value<fs::path>(&videofile)->required(),
+				("video,v", po::value<fs::path>(&videofile)->required(),
 				 "an input image")
+				("config,c", po::value<fs::path>(&reconstruction_config)->default_value("../share/default_reconstruction_config.ini"),
+				 "configuration file for the reconstruction")
 				("get_annotations,g", po::bool_switch(&get_annotations)->default_value(false),
-				 "read .pts annotation file locations from one file, one file path per line")
+				 "read .pts annotation file locations from one file, put one file path on each line")
 				("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
 					 ".pts annotation files per frame of video")
 				("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
@@ -323,6 +367,7 @@ int main(int argc, char *argv[]) {
 
 	// start loading prerequisites
 	morphablemodel::MorphableModel morphable_model;
+
 	try {
 		morphable_model = morphablemodel::load_model(modelfile.string());
 	} catch (const std::runtime_error &e) {
@@ -359,9 +404,10 @@ int main(int argc, char *argv[]) {
 	vector<fitting::RenderingParameters> rendering_paramss;
 
 	BufferedVideoIterator vid_iterator;
+	boost::property_tree::ptree settings = get_reconstruction_config(reconstruction_config.string());
 
 	try {
-		vid_iterator = BufferedVideoIterator(videofile.string(), 5, 5);
+		vid_iterator = BufferedVideoIterator(videofile.string(), settings);
 	} catch(std::runtime_error &e) {
 		std::cout << e.what() << std::endl;
 		return EXIT_FAILURE;
@@ -378,17 +424,16 @@ int main(int argc, char *argv[]) {
 	std::vector<std::vector<float>> blend_shape_coefficients;
 	std::vector<std::vector<cv::Vec2f>> fitted_image_points;
 
-
 	int n_iter = 0;
 
-
 	while(!(key_frames.empty())) {
 		if (n_iter == 10) {
 			break;
 		}
 
-		// load all annotation files into lists of landmarks
-		vector<core::LandmarkCollection<cv::Vec2f>> landmark_sublist(landmark_list.begin() + n_iter, landmark_list.end());
+		vector<core::LandmarkCollection<cv::Vec2f>> landmark_sublist = sample_landmarks(
+			key_frames, landmark_list
+		);
 
 		std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
 				morphable_model,
@@ -413,7 +458,7 @@ int main(int argc, char *argv[]) {
 		evaluate_results(
 				key_frames,
 				rendering_paramss,
-				landmark_list,
+				landmark_sublist,
 				morphable_model,
 				meshs,
 				pca_shape_coefficients,