Add a first version of buffered video iterator, needed later for multi-frame tests

CMakeLists.txt

@@ -108,6 +108,7 @@ set(eigen3_nnls_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/eigen3-nnls/src")
 set(HEADERS
 	include/eos/core/Landmark.hpp
 	include/eos/core/LandmarkMapper.hpp
+	include/eos/core/BufferedVideoIterator.hpp
 	include/eos/morphablemodel/PcaModel.hpp
 	include/eos/morphablemodel/MorphableModel.hpp
 	include/eos/morphablemodel/Blendshape.hpp

include/eos/core/BufferedVideoIterator.hpp

+/*
+ * eos - A 3D Morphable Model fitting library written in modern C++11/14.
+ *
+ * File: include/eos/core/BufferedVideoIterator.hpp
+ *
+ * Copyright 2014, 2015 Patrik Huber
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#ifndef BUFFERED_VIDEO_ITERATOR_HPP_
+#define BUFFERED_VIDEO_ITERATOR_HPP_
+
+#include "opencv2/core/core.hpp"
+#include "opencv2/opencv.hpp"
+#include "opencv2/highgui/highgui.hpp"
+
+#include <vector>
+#include <string>
+
+using eos::core::Landmark;
+using eos::core::LandmarkCollection;
+using cv::Mat;
+using cv::Vec2f;
+using cv::Vec3f;
+using cv::Vec4f;
+using std::vector;
+using std::string;
+
+using namespace cv;
+
+namespace eos {
+	namespace core {
+
+		template<typename T>
+		class BufferedVideoIterator {
+		public:
+			BufferedVideoIterator() {};
+
+			BufferedVideoIterator(std::string videoFilePath) {
+				std::cout << "capturing:" << videoFilePath << std::endl;;
+
+				std::ifstream file(videoFilePath, std::ios::binary);
+				if (file.fail()) {
+					throw std::runtime_error("Error opening given file: " + videoFilePath);
+				}
+
+				cv::VideoCapture tmp_cap(videoFilePath); // open the default camera
+
+				if (!tmp_cap.isOpened()) { // check if we succeeded
+					throw std::runtime_error("Could not play video");
+				}
+
+				cap = tmp_cap;
+			}
+
+			std::deque <T> next() {
+				int frame_buffer_length = frame_buffer.size();
+				int landmark_landmark_length = landmark_buffer.size();
+
+				// Get a new frame from camera.
+				Mat frame;
+				cap >> frame;
+
+				if (n_frames >= max_frames) {
+					frame_buffer.pop_front();
+					landmark_buffer.pop_front();
+				}
+
+				std::cout << "frame_buff" << frame.empty() << std::endl;
+				if (frame.empty() == 0) {
+					std::cout << "derpio" << std::endl;
+					frame_buffer.push_back(n_frames);
+					landmark_buffer.push_back(n_frames);
+				}
+
+				std::cout << "frame_buff" << frame_buffer.empty() << std::endl;
+
+				n_frames++;
+
+
+				return frame_buffer;
+			}
+
+			std::deque <T> get_frame_buffer() {
+				return frame_buffer;
+			}
+
+			std::deque <T> get_landmark_buffer() {
+				return landmark_buffer;
+			}
+
+		private:
+			cv::VideoCapture cap;
+			std::deque <T> frame_buffer;
+			std::deque <T> landmark_buffer;
+
+			int n_frames = 0;
+			int max_frames = 5;
+		};
+	}
+}
+
+#endif

utils/accuracy-evaluation.cpp

+/*
+ * eos - A 3D Morphable Model fitting library written in modern C++11/14.
+ *
+ * File: utils/json-to-cereal-binary.cpp
+ *
+ * Copyright 2016 Patrik Huber *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "eos/core/Landmark.hpp"
+#include "eos/core/LandmarkMapper.hpp"
+#include "eos/core/BufferedVideoIterator.hpp"
+#include "eos/morphablemodel/MorphableModel.hpp"
+#include "eos/morphablemodel/Blendshape.hpp"
+#include "eos/fitting/fitting.hpp"
+#include "eos/render/utils.hpp"
+#include "eos/render/texture_extraction.hpp"
+
+#include "opencv2/core/core.hpp"
+#include "opencv2/opencv.hpp"
+#include "opencv2/highgui/highgui.hpp"
+
+#include "boost/program_options.hpp"
+#include "boost/filesystem.hpp"
+
+#include <vector>
+#include <iostream>
+#include <fstream>
+
+using namespace eos;
+namespace po = boost::program_options;
+namespace fs = boost::filesystem;
+using eos::core::Landmark;
+using eos::core::LandmarkCollection;
+using eos::core::BufferedVideoIterator;
+using cv::Mat;
+using cv::Vec2f;
+using cv::Vec3f;
+using cv::Vec4f;
+using std::cout;
+using std::endl;
+using std::vector;
+using std::string;
+
+
+using namespace cv;
+
+
+/**
+ * Reads an ibug .pts landmark file and returns an ordered vector with
+ * the 68 2D landmark coordinates.
+ *
+ * @param[in] filename Path to a .pts file.
+ * @return An ordered vector with the 68 ibug landmarks.
+ */
+LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename)
+{
+	using std::getline;
+	using cv::Vec2f;
+	using std::string;
+	LandmarkCollection<Vec2f> landmarks;
+	landmarks.reserve(68);
+
+	std::ifstream file(filename);
+	if (!file.is_open()) {
+		throw std::runtime_error(string("Could not open landmark file: " + filename));
+	}
+
+	string line;
+	// Skip the first 3 lines, they're header lines:
+	getline(file, line); // 'version: 1'
+	getline(file, line); // 'n_points : 68'
+	getline(file, line); // '{'
+
+	int ibugId = 1;
+	while (getline(file, line))
+	{
+		if (line == "}") { // end of the file
+			break;
+		}
+		std::stringstream lineStream(line);
+
+		Landmark<Vec2f> landmark;
+		landmark.name = std::to_string(ibugId);
+		if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
+			throw std::runtime_error(string("Landmark format error while parsing the line: " + line));
+		}
+		// From the iBug website:
+		// "Please note that the re-annotated data for this challenge are saved in the Matlab convention of 1 being
+		// the first index, i.e. the coordinates of the top left pixel in an image are x=1, y=1."
+		// ==> So we shift every point by 1:
+		landmark.coordinates[0] -= 1.0f;
+		landmark.coordinates[1] -= 1.0f;
+		landmarks.emplace_back(landmark);
+		++ibugId;
+	}
+	return landmarks;
+};
+
+/**
+ * Draws the given mesh as wireframe into the image.
+ *
+ * It does backface culling, i.e. draws only vertices in CCW order.
+ *
+ * @param[in] image An image to draw into.
+ * @param[in] mesh The mesh to draw.
+ * @param[in] modelview Model-view matrix to draw the mesh.
+ * @param[in] projection Projection matrix to draw the mesh.
+ * @param[in] viewport Viewport to draw the mesh.
+ * @param[in] colour Colour of the mesh to be drawn.
+ */
+void draw_wireframe(cv::Mat image, const eos::render::Mesh& mesh, glm::mat4x4 modelview, glm::mat4x4 projection, glm::vec4 viewport, cv::Scalar colour = cv::Scalar(0, 255, 0, 255))
+{
+	for (const auto& triangle : mesh.tvi)
+	{
+		const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport);
+		const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport);
+		const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport);
+		if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3)))
+		{
+			cv::line(image, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour);
+			cv::line(image, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour);
+			cv::line(image, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour);
+		}
+	}
+};
+
+/**
+ * This app demonstrates estimation of the camera and fitting of the shape
+ * model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
+ * In addition to fit-model-simple, this example uses blendshapes, contour-
+ * fitting, and can iterate the fitting.
+ *
+ * 68 ibug landmarks are loaded from the .pts file and converted
+ * to vertex indices using the LandmarkMapper.
+ */
+int main(int argc, char *argv[]) {
+	fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile;
+	try {
+		po::options_description desc("Allowed options");
+		desc.add_options()
+				("help,h",
+				 "display the help message")
+				("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
+				 "a Morphable Model stored as cereal BinaryArchive")
+				("video,i", po::value<fs::path>(&videofile)->required(),
+				 "an input image")
+				("landmarks,l", po::value<fs::path>(&landmarksfile)->required()->default_value("data/image_0010.pts"),
+				 "2D landmarks for the image, in ibug .pts format")
+				("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
+				 "landmark identifier to model vertex number mapping")
+				("model-contour,c",
+				 po::value<fs::path>(&contourfile)->required()->default_value("../share/model_contours.json"),
+				 "file with model contour indices")
+				("edge-topology,e", po::value<fs::path>(&edgetopologyfile)->required()->default_value(
+						"../share/sfm_3448_edge_topology.json"),
+				 "file with model's precomputed edge topology")
+				("blendshapes,b", po::value<fs::path>(&blendshapesfile)->required()->default_value(
+						"../share/expression_blendshapes_3448.bin"),
+				 "file with blendshapes")
+				("output,o", po::value<fs::path>(&outputfile)->required()->default_value("out"),
+				 "basename for the output rendering and obj files");
+		po::variables_map vm;
+		po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
+		if (vm.count("help")) {
+			cout << "Usage: fit-model [options]" << endl;
+			cout << desc;
+			return EXIT_SUCCESS;
+		}
+		po::notify(vm);
+	}
+	catch (const po::error &e) {
+		cout << "Error while parsing command-line arguments: " << e.what() << endl;
+		cout << "Use --help to display a list of options." << endl;
+		return EXIT_SUCCESS;
+	}
+
+	BufferedVideoIterator<int> vid_iterator;
+	try {
+		vid_iterator = BufferedVideoIterator<int>(videofile.string());
+	} catch(std::runtime_error &e) {
+		cout << e.what() << endl;
+		return EXIT_FAILURE;
+	}
+
+	std::deque<int> frames = vid_iterator.next();
+
+	while(!(frames.empty())) {
+		for (std::deque<int>::iterator it = frames.begin(); it!=frames.end(); ++it) {
+			std::cout << ' ' << *it;
+		}
+
+		frames = vid_iterator.next();
+		usleep(500);
+	}
+
+	// Load the image, landmarks, LandmarkMapper and the Morphable Model:
+//	Mat image = cv::imread(imagefile.string());
+//	LandmarkCollection <cv::Vec2f> landmarks;
+//	try {
+//		landmarks = read_pts_landmarks(landmarksfile.string());
+//	}
+//	catch (const std::runtime_error &e) {
+//		cout << "Error reading the landmarks: " << e.what() << endl;
+//		return EXIT_FAILURE;
+//	}
+//	morphablemodel::MorphableModel morphable_model;
+//	try {
+//		morphable_model = morphablemodel::load_model(modelfile.string());
+//	}
+//	catch (const std::runtime_error &e) {
+//		cout << "Error loading the Morphable Model: " << e.what() << endl;
+//		return EXIT_FAILURE;
+//	}
+}
+