Add test files for future tests, made some minor changes to BufferedVideoIter and use it in utils

CMakeLists.txt

@@ -56,7 +56,7 @@ option(BUILD_EXAMPLES "Build the example applications." ON)
 message(STATUS "BUILD_EXAMPLES: ${BUILD_EXAMPLES}")
 option(BUILD_CERES_EXAMPLE "Build the fit-model-ceres example (requires Ceres)." OFF)
 message(STATUS "BUILD_CERES_EXAMPLE: ${BUILD_CERES_EXAMPLE}")
-option(BUILD_UTILS "Build utility applications." OFF)
+option(BUILD_UTILS "Build utility applications." ON)
 message(STATUS "BUILD_UTILS: ${BUILD_UTILS}")
 option(BUILD_DOCUMENTATION "Build the library documentation." OFF)
 message(STATUS "BUILD_DOCUMENTATION: ${BUILD_DOCUMENTATION}")
@@ -109,6 +109,7 @@ set(HEADERS
 	include/eos/core/Landmark.hpp
 	include/eos/core/LandmarkMapper.hpp
 	include/eos/core/BufferedVideoIterator.hpp
+	include/eos/core/landmark_utils.hpp
 	include/eos/morphablemodel/PcaModel.hpp
 	include/eos/morphablemodel/MorphableModel.hpp
 	include/eos/morphablemodel/Blendshape.hpp

examples/fit-model-multi-frame.cpp

@@ -74,7 +74,7 @@ LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename)
 
 	string line;
 	// Skip the first 3 lines, they're header lines:
-	getline(file, line); // 'version: 1'
+	getline(file, line); // 'version:1'
 	getline(file, line); // 'n_points : 68'
 	getline(file, line); // '{'
 

include/eos/core/BufferedVideoIterator.hpp

@@ -29,8 +29,6 @@
 #include <vector>
 #include <string>
 
-using eos::core::Landmark;
-using eos::core::LandmarkCollection;
 using cv::Mat;
 using cv::Vec2f;
 using cv::Vec3f;
@@ -38,76 +36,104 @@ using cv::Vec4f;
 using std::vector;
 using std::string;
 
+namespace fs = boost::filesystem;
 using namespace cv;
+using namespace std;
 
 namespace eos {
 	namespace core {
-
+		/**
+		 * BufferedVideo Iterator will keep a buffer of the last seen n_frames. By calling .next() it will load a new
+		 * frame from the given video and you will get a pointer to the front of the buffer (which has n_frames).
+		 *
+		 * Just imagine a sliding window accross the video, this is what we aim to implement here.
+		 *
+		 * Example:
+		 *		vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
+		 *
+		 *		std::deque<cv::mat> frames = vid_iterator.next();
+		 *		while(!(frames.empty())) {
+		 *			for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+		 *				std::cout << ' ' << *it;
+		 *			}
+		 *
+		 *			frames = vid_iterator.next();
+		 *		}
+		 *
+		 * @tparam T
+		 */
+		// Note for this template: later we can use other templates for easy testing (not using cv:Mat but <int> for example).
 		template<typename T>
 		class BufferedVideoIterator {
 		public:
 			BufferedVideoIterator() {};
 
-			BufferedVideoIterator(std::string videoFilePath) {
-				std::cout << "capturing:" << videoFilePath << std::endl;;
+			// TODO: build support for setting the amount of max_frames in the buffer.
+			BufferedVideoIterator(std::string videoFilePath, std::vector<std::vector<cv::Vec2f>> landmarks) {
+				std::ifstream file(videoFilePath);
 
-				std::ifstream file(videoFilePath, std::ios::binary);
-				if (file.fail()) {
+				if (!file.is_open()) {
 					throw std::runtime_error("Error opening given file: " + videoFilePath);
 				}
 
-				cv::VideoCapture tmp_cap(videoFilePath); // open the default camera
+				cv::VideoCapture tmp_cap(videoFilePath); // open video file
 
 				if (!tmp_cap.isOpened()) { // check if we succeeded
 					throw std::runtime_error("Could not play video");
 				}
 
 				cap = tmp_cap;
+				this->landmarks = landmarks;
 			}
 
-			std::deque <T> next() {
-				int frame_buffer_length = frame_buffer.size();
-				int landmark_landmark_length = landmark_buffer.size();
+			/**
+			 * Set next frame and return frame_buffer.
+			 *
+			 * @return dequeue<Mat> frame buffer.
+			 *
+			 * TODO: build support for returning landmarks AND frames.
+			 */
+			std::deque <Mat> next() {
+				long frame_buffer_length = frame_buffer.size();
+				long landmark_landmark_length = landmark_buffer.size();
 
 				// Get a new frame from camera.
 				Mat frame;
 				cap >> frame;
 
-				if (n_frames >= max_frames) {
+				// Pop if we exceeded max_frames.
+				if (n_frames > max_frames) {
 					frame_buffer.pop_front();
-					landmark_buffer.pop_front();
 				}
 
-				std::cout << "frame_buff" << frame.empty() << std::endl;
 				if (frame.empty() == 0) {
-					std::cout << "derpio" << std::endl;
-					frame_buffer.push_back(n_frames);
-					landmark_buffer.push_back(n_frames);
+					frame_buffer.push_back(frame);
 				}
 
-				std::cout << "frame_buff" << frame_buffer.empty() << std::endl;
-
 				n_frames++;
 
-
 				return frame_buffer;
 			}
 
-			std::deque <T> get_frame_buffer() {
+			std::deque <Mat> get_frame_buffer() {
 				return frame_buffer;
 			}
 
-			std::deque <T> get_landmark_buffer() {
+			std::deque <Mat> get_landmark_buffer() {
 				return landmark_buffer;
 			}
 
 		private:
 			cv::VideoCapture cap;
-			std::deque <T> frame_buffer;
-			std::deque <T> landmark_buffer;
-
-			int n_frames = 0;
-			int max_frames = 5;
+			std::deque <Mat> frame_buffer;
+			std::deque <Mat> landmark_buffer;
+			std::vector<std::vector<cv::Vec2f>> landmarks;
+
+			// TODO: make set-able
+			// load n_frames at once into the buffer.
+			long n_frames = 1;
+			// keep max_frames into the buffer.
+			long max_frames = 5;
 		};
 	}
 }

include/eos/core/Landmark.hpp

@@ -22,9 +22,23 @@
 #ifndef EOS_LANDMARK_HPP_
 #define EOS_LANDMARK_HPP_
 
+#include "opencv2/core/core.hpp"
+#include "opencv2/opencv.hpp"
+
+#include <vector>
 #include <string>
 #include <vector>
 #include <algorithm>
+#include <iostream>
+#include <fstream>
+
+using cv::Mat;
+using cv::Vec2f;
+using cv::Vec3f;
+using cv::Vec4f;
+using std::vector;
+using std::string;
+
 
 namespace eos {
 	namespace core {

include/eos/core/landmark_utils.hpp

+//
+// Created by RA Torenvliet on 07/02/2017.
+//
+
+#ifndef EOS_LANDMARK_UTILS_H
+#define EOS_LANDMARK_UTILS_H
+
+#include "opencv2/core/core.hpp"
+#include "opencv2/opencv.hpp"
+
+#include "eos/morphablemodel/morphablemodel.hpp"
+#include "eos/core/LandmarkMapper.hpp"
+
+#include "boost/filesystem.hpp"
+
+#include <vector>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+
+using cv::Mat;
+using cv::Vec2f;
+using cv::Vec3f;
+using cv::Vec4f;
+using std::vector;
+using std::string;
+
+namespace fs = boost::filesystem;
+
+/**
+ * Reads an ibug .pts landmark file and returns an ordered vector with
+ * the 68 2D landmark coordinates.
+ *
+ * @param[in] filename Path to a .pts file.
+ * @return An ordered vector with the 68 ibug landmarks.
+ */
+namespace eos {
+	namespace core {
+		eos::core::LandmarkCollection <cv::Vec2f> read_pts_landmarks(std::string filename) {
+			using std::getline;
+
+			LandmarkCollection <Vec2f> landmarks;
+			landmarks.reserve(68);
+
+			std::ifstream file(filename);
+			if (!file.is_open()) {
+				throw std::runtime_error(string("Could not open landmark file: " + filename));
+			}
+
+			string line;
+			// Skip the first 3 lines, they're header lines:
+			getline(file, line); // 'version: 1'
+			getline(file, line); // 'n_points : 68'
+			getline(file, line); // '{'
+
+			int ibugId = 1;
+			while (getline(file, line)) {
+				if (line[0] == '}') { // end of the file
+					break;
+				}
+
+				std::stringstream lineStream(line);
+				Landmark <Vec2f> landmark;
+				landmark.name = std::to_string(ibugId);
+
+				if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
+					throw std::runtime_error(string("Landmark format error while parsing the line: " + line));
+				}
+
+				// From the iBug website:
+				// "Please note that the re-annotated data for this challenge are saved in the Matlab convention of 1 being
+				// the first index, i.e. the coordinates of the top left pixel in an image are x=1, y=1."
+				// ==> So we shift every point by 1:
+				landmark.coordinates[0] -= 1.0f;
+				landmark.coordinates[1] -= 1.0f;
+				landmarks.emplace_back(landmark);
+				++ibugId;
+			}
+
+			return landmarks;
+		}
+
+		/**
+		 * Helper function, gathers matching model_points with given landmarks and LandmarkMapper.
+		 *
+		 * @param landmarks
+		 * @param morphable_model
+		 * @param landmark_mapper
+		 * @return std::pair<std::vector<Vec4f>, std::vector<int> model_points and vertex_indices.
+		 */
+		std::pair <std::vector<Vec4f>, std::vector<int>>
+		load_model_data(eos::core::LandmarkCollection<cv::Vec2f> landmarks,
+		                morphablemodel::MorphableModel morphable_model, eos::core::LandmarkMapper landmark_mapper) {
+			std::vector <cv::Vec4f> model_points;
+			std::vector<int> vertex_indices;
+
+			// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
+			for (int i = 0; i < landmarks.size(); ++i) {
+				auto converted_name = landmark_mapper.convert(landmarks[i].name);
+				if (!converted_name) { // no mapping defined for the current landmark
+					continue;
+				}
+				int vertex_idx = std::stoi(converted_name.get());
+				Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
+				model_points.emplace_back(vertex);
+				vertex_indices.emplace_back(vertex_idx);
+			}
+
+			return std::make_pair(model_points, vertex_indices);
+		}
+
+		/**
+		* Load annotations, return all annotations as image points (vectors of Vec2f).
+		 *
+		 * @param annotations
+		 * @param mappingsfile
+		 * @throws std::runtime_error in case of faulty annotation file
+		 * @return std::vector<std::vector<cv::Vec2f>> image_points in a vector of OpenCV float pairs (Vec2f).
+		 */
+		std::vector <std::vector<cv::Vec2f>> load_annotations(std::vector <std::string> annotations, fs::path mappingsfile) {
+			std::vector <std::vector<cv::Vec2f>> image_points; // the corresponding 2D landmark points of all annotation files.
+			eos::core::LandmarkMapper landmark_mapper = eos::core::LandmarkMapper(mappingsfile);
+
+			// These will be the final 2D and 3D points used for the fitting:
+			std::vector <Vec4f> model_points; // the points in the 3D shape model
+			std::vector<int> vertex_indices; // their vertex indices
+
+			for (int i = 0; i < annotations.size(); i++) {
+				eos::core::LandmarkCollection <cv::Vec2f> landmarks;
+
+				try {
+					landmarks = read_pts_landmarks(annotations[i]);
+				}
+				catch (const std::runtime_error &e) {
+					std::cout << e.what() << std::endl;
+					throw std::runtime_error("Error reading the landmarks:  " + annotations[i]);
+				}
+
+				std::vector <cv::Vec2f> image_points_tmp;
+
+				// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
+				for (int j = 0; j < landmarks.size(); j++) {
+					image_points_tmp.emplace_back(landmarks[i].coordinates);
+				}
+			}
+
+			return image_points;
+		}
+	}
+}
+
+#endif //EOS_LANDMARK_UTILS_H

tests/CMakeLists.txt

@@ -41,17 +41,19 @@ set(CATCH_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/Catch/include CACHE INTERNAL
 
 add_library(Catch INTERFACE)
 target_include_directories(Catch INTERFACE ${CATCH_INCLUDE_DIR})
+include_directories(${CMAKE_SOURCE_DIR}/tests/include)
 
-set(${EOS_TESTS_DIRS} ${CMAKE_SOURCE_DIR}/tests/fitting "Paths to test include directories")
+set(EOS_TESTS_DIRS ${CMAKE_SOURCE_DIR}/tests/fitting ${CMAKE_SOURCE_DIR}/tests/core ${CMAKE_SOURCE_DIR}/tests/include "Paths to test include directories")
 
 add_executable (main-tests main_tests.cpp)
-target_link_libraries(main-tests Catch ${EOS_TESTS_DIRS}  ${OpenCV_LIBS} ${Boost_LIBRARIES})
+target_link_libraries(main-tests Catch ${OpenCV_LIBS} ${Boost_LIBRARIES})
 
 install(TARGETS main-tests DESTINATION tests)
 install(DIRECTORY ${CMAKE_SOURCE_DIR}/tests/data DESTINATION tests)
+install(DIRECTORY ${CMAKE_SOURCE_DIR}/tests/include DESTINATION tests)
 
 # add tests
 enable_testing()
 add_test(NAME fitting-test COMMAND fitting-test)
 add_test(NAME linear-shape-fitting-test COMMAND linear-shape-fitting-test)
-
+add_test(NAME landmark-test COMMAND landmark-test)
\ No newline at end of file

tests/core/landmark_test.hpp

+// Example file, show to use Catch with multiple files through one main_tests file.
+#pragma once
+
+#ifndef EOS_LANDMARK_TEST_HPP_
+#define EOS_LANDMARK_TEST_HPP_
+
+#include "catch.hpp"
+
+
+TEST_CASE("Test 1 == 1", "[landmarks]") {
+	REQUIRE(1 == 1);
+}
+
+#endif //EOS_LANDMARK_TEST_H

tests/fitting/fitting_test.hpp

 #include "catch.hpp"
+#include "test_helper.hpp"
 
 #include "glm/ext.hpp"
 #include "glm/gtc/matrix_transform.hpp"
@@ -14,6 +15,7 @@
 #include "eos/render/render.hpp"
 #include "eos/render/detail/render_detail.hpp"
 
+
 #include "opencv2/core/core.hpp"
 #include "opencv2/highgui/highgui.hpp"
 
@@ -94,10 +96,6 @@ LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename)
 	return landmarks;
 };
 
-morphablemodel::MorphableModel loadTestModel() {
-	return morphablemodel::load_model("../share/sfm_shape_3448.bin");
-}
-
 /**
  * Loads test data. Returns
  * @param landmarks
@@ -171,11 +169,6 @@ TEST_CASE("Test ortographic projection", "[projection]" ){
 		image_points, model_points, true, image.rows
 	);
 
-	std::cout << glm::to_string(pose.R) << endl;
-	std::cout << pose.s << endl;
-	std::cout << pose.tx << endl;
-	std::cout << pose.ty << endl;
-
 	fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
 
 	// Estimate the shape coefficients by fitting the shape to the landmarks:

tests/include/test_helper.hpp

+#ifndef EOS_TEST_HELPER_HPP
+#define EOS_TEST_HELPER_HPP
+
+#include "eos/morphablemodel/MorphableModel.hpp"
+
+eos::morphablemodel::MorphableModel loadTestModel() {
+	return eos::morphablemodel::load_model("../share/sfm_shape_3448.bin");
+}
+
+#endif //EOS_TEST_HELPER_HPP

tests/main_tests.cpp

@@ -3,5 +3,7 @@
 
 // Include all tests here and don't forget to alter the CMakeLists.txt in the tests folder,
 // that means use add_test to add the a new test file.
+#include "core/landmark_test.hpp"
 #include "fitting/fitting_test.hpp"
 #include "fitting/linear_shape_fitting_test.hpp"
+

utils/accuracy-evaluation.cpp

@@ -18,9 +18,10 @@
  */
 #include "eos/core/Landmark.hpp"
 #include "eos/core/LandmarkMapper.hpp"
+#include "eos/core/landmark_utils.hpp"
 #include "eos/core/BufferedVideoIterator.hpp"
-#include "eos/morphablemodel/MorphableModel.hpp"
-#include "eos/morphablemodel/Blendshape.hpp"
+#include "eos/morphablemodel/morphablemodel.hpp"
+#include "eos/morphablemodel/blendshape.hpp"
 #include "eos/fitting/fitting.hpp"
 #include "eos/render/utils.hpp"
 #include "eos/render/texture_extraction.hpp"
@@ -46,66 +47,12 @@ using cv::Mat;
 using cv::Vec2f;
 using cv::Vec3f;
 using cv::Vec4f;
-using std::cout;
-using std::endl;
 using std::vector;
 using std::string;
 
 
 using namespace cv;
 
-
-/**
- * Reads an ibug .pts landmark file and returns an ordered vector with
- * the 68 2D landmark coordinates.
- *
- * @param[in] filename Path to a .pts file.
- * @return An ordered vector with the 68 ibug landmarks.
- */
-LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename)
-{
-	using std::getline;
-	using cv::Vec2f;
-	using std::string;
-	LandmarkCollection<Vec2f> landmarks;
-	landmarks.reserve(68);
-
-	std::ifstream file(filename);
-	if (!file.is_open()) {
-		throw std::runtime_error(string("Could not open landmark file: " + filename));
-	}
-
-	string line;
-	// Skip the first 3 lines, they're header lines:
-	getline(file, line); // 'version: 1'
-	getline(file, line); // 'n_points : 68'
-	getline(file, line); // '{'
-
-	int ibugId = 1;
-	while (getline(file, line))
-	{
-		if (line == "}") { // end of the file
-			break;
-		}
-		std::stringstream lineStream(line);
-
-		Landmark<Vec2f> landmark;
-		landmark.name = std::to_string(ibugId);
-		if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
-			throw std::runtime_error(string("Landmark format error while parsing the line: " + line));
-		}
-		// From the iBug website:
-		// "Please note that the re-annotated data for this challenge are saved in the Matlab convention of 1 being
-		// the first index, i.e. the coordinates of the top left pixel in an image are x=1, y=1."
-		// ==> So we shift every point by 1:
-		landmark.coordinates[0] -= 1.0f;
-		landmark.coordinates[1] -= 1.0f;
-		landmarks.emplace_back(landmark);
-		++ibugId;
-	}
-	return landmarks;
-};
-
 /**
  * Draws the given mesh as wireframe into the image.
  *
@@ -145,6 +92,8 @@ void draw_wireframe(cv::Mat image, const eos::render::Mesh& mesh, glm::mat4x4 mo
  */
 int main(int argc, char *argv[]) {
 	fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile;
+	std::vector<std::string> annotations;
+
 	try {
 		po::options_description desc("Allowed options");
 		desc.add_options()
@@ -154,10 +103,10 @@ int main(int argc, char *argv[]) {
 				 "a Morphable Model stored as cereal BinaryArchive")
 				("video,i", po::value<fs::path>(&videofile)->required(),
 				 "an input image")
-				("landmarks,l", po::value<fs::path>(&landmarksfile)->required()->default_value("data/image_0010.pts"),
-				 "2D landmarks for the image, in ibug .pts format")
+				("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
+					 ".pts annotation files per frame of video")
 				("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
-				 "landmark identifier to model vertex number mapping")
+					"2D landmarks for the image, in ibug .pts format")
 				("model-contour,c",
 				 po::value<fs::path>(&contourfile)->required()->default_value("../share/model_contours.json"),
 				 "file with model contour indices")
@@ -184,42 +133,71 @@ int main(int argc, char *argv[]) {
 		return EXIT_SUCCESS;
 	}
 
-	BufferedVideoIterator<int> vid_iterator;
 	try {
-		vid_iterator = BufferedVideoIterator<int>(videofile.string());
+		vector <vector<Vec2f>> multi_frame_points = eos::core::load_annotations(annotations, mappingsfile);
+	} catch(const std::runtime_error &e) {
+		std::cout << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	// Load landmarks, LandmarkMapper and the Morphable Model:
+	LandmarkCollection <cv::Vec2f> landmarks;
+	core::LandmarkMapper landmark_mapper = core::LandmarkMapper(mappingsfile);
+
+	try {
+		landmarks = eos::core::read_pts_landmarks(annotations[0]);
+	}
+	catch (const std::runtime_error &e) {
+		cout << "Error reading the landmarks: " << e.what() << endl;
+		return EXIT_FAILURE;
+	}
+
+	morphablemodel::MorphableModel morphable_model;
+
+	try {
+		morphable_model = morphablemodel::load_model(modelfile.string());
+	} catch (const std::runtime_error &e) {
+		std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
+			return EXIT_FAILURE;
+	}
+
+	// These will be the final 2D and 3D points used for the fitting:
+	vector<Vec4f> model_points; // the points in the 3D shape model
+	vector<int> vertex_indices; // their vertex indices
+	std::tie(model_points, vertex_indices) = eos::core::load_model_data(landmarks, morphable_model, landmark_mapper);
+
+	BufferedVideoIterator<cv::Mat> vid_iterator;
+	std::vector <std::vector<cv::Vec2f>> landmark_annotation_list = eos::core::load_annotations(annotations, mappingsfile);
+
+	try {
+		vid_iterator = bufferedvideoiterator<cv::mat>(videofile.string(), landmark_annotation_list);
 	} catch(std::runtime_error &e) {
 		cout << e.what() << endl;
-		return EXIT_FAILURE;
+		return exit_failure;
 	}
 
-	std::deque<int> frames = vid_iterator.next();
 
+	// todo: expand this to really perform some reconstruction, and move this to a test file.
+	// test with loading 10 frames subsequently.
+	// vid_iterator.next() will return a number of frames, depending on
+	std::deque<cv::mat> frames = vid_iterator.next();
+	int count = 0;
 	while(!(frames.empty())) {
-		for (std::deque<int>::iterator it = frames.begin(); it!=frames.end(); ++it) {
-			std::cout << ' ' << *it;
+		if (count == 10) {
+			break;
+		}
+		int frame_count = 0;
+		for (std::deque<cv::mat>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+			//std::cout << ' ' << *it;
+			std::cout << frame_count << " ";
+			frame_count++;
 		}
 
+		std::cout << std::endl << "frames processed: " << count * frame_count << std::endl;
+
 		frames = vid_iterator.next();
-		usleep(500);
+		usleep(10);
+		count++;
 	}
-
-	// Load the image, landmarks, LandmarkMapper and the Morphable Model:
-//	Mat image = cv::imread(imagefile.string());
-//	LandmarkCollection <cv::Vec2f> landmarks;
-//	try {
-//		landmarks = read_pts_landmarks(landmarksfile.string());
-//	}
-//	catch (const std::runtime_error &e) {
-//		cout << "Error reading the landmarks: " << e.what() << endl;
-//		return EXIT_FAILURE;
-//	}
-//	morphablemodel::MorphableModel morphable_model;
-//	try {
-//		morphable_model = morphablemodel::load_model(modelfile.string());
-//	}
-//	catch (const std::runtime_error &e) {
-//		cout << "Error loading the Morphable Model: " << e.what() << endl;
-//		return EXIT_FAILURE;
-//	}
 }