Skip to content

E
eos
Commit fcae2025 authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Backup progress

parent 574ccbce
Hide whitespace changes
Inline Side-by-side
Showing with 1957 additions and 790 deletions
examples/fit-model-multi-frame.cpp
View file @ fcae2025
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: examples/fit-model-simple.cpp
*
* Copyright 2015 Patrik Huber
* File: utils/json-to-cereal-binary.cpp
*
* Copyright 2016 Patrik Huber *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
......@@ -17,211 +16,359 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "eos/core/Landmark.hpp"
#include "eos/core/LandmarkMapper.hpp"
#include "eos/fitting/orthographic_camera_estimation_linear.hpp"
#include "eos/fitting/RenderingParameters.hpp"
#include "eos/fitting/linear_shape_fitting.hpp"
#include "eos/core/landmark_utils.hpp"
#include "eos/render/render.hpp"
#include "eos/morphablemodel/MorphableModel.hpp"
#include "eos/morphablemodel/Blendshape.hpp"
#include "eos/fitting/fitting.hpp"
#include "eos/render/utils.hpp"
#include "eos/render/texture_extraction.hpp"
#include "eos/video/Keyframe.hpp"
#include "eos/fitting/ReconstructionData.hpp"
#include "eos/video/BufferedVideoIterator.hpp"
#include "opencv2/core/core.hpp"
#include "opencv2/opencv.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "boost/program_options.hpp"
#include "boost/filesystem.hpp"
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include "glm/gtx/string_cast.hpp"
#include <vector>
#include <iostream>
#include <fstream>
using namespace eos;
using namespace fitting;
namespace po = boost::program_options;
namespace fs = boost::filesystem;
using eos::core::Landmark;
using eos::core::LandmarkCollection;
using eos::video::BufferedVideoIterator;
using eos::video::WeightedIsomapAveraging;
using eos::video::ReconstructionVideoWriter;
using eos::video::Keyframe;
using cv::Mat;
using cv::Vec2f;
using cv::Vec3f;
using cv::Vec4f;
using std::cout;
using std::endl;
using std::vector;
using std::string;
using namespace cv;
typedef unsigned int uint;
/**
* Reads an ibug .pts landmark file and returns an ordered vector with
* the 68 2D landmark coordinates.
* Draws the given mesh as wireframe into the image.
*
* @param[in] filename Path to a .pts file.
* @return An ordered vector with the 68 ibug landmarks.
* It does backface culling, i.e. draws only vertices in CCW order.
*
* @param[in] image An image to draw into.
* @param[in] mesh The mesh to draw.
* @param[in] modelview Model-view matrix to draw the mesh.
* @param[in] projection Projection matrix to draw the mesh.
* @param[in] viewport Viewport to draw the mesh.
* @param[in] colour Colour of the mesh to be drawn.
*/
LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename)
void draw_wireframe(cv::Mat image, const eos::core::Mesh& mesh, glm::mat4x4 modelview, glm::mat4x4 projection, glm::vec4 viewport, cv::Scalar colour = cv::Scalar(0, 255, 0, 255))
{
using std::getline;
using cv::Vec2f;
using std::string;
LandmarkCollection<Vec2f> landmarks;
landmarks.reserve(68);
std::ifstream file(filename);
if (!file.is_open()) {
throw std::runtime_error(string("Could not open landmark file: " + filename));
for (const auto& triangle : mesh.tvi)
{
const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport);
const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport);
const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport);
if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3))) {
cv::line(image, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour);
cv::line(image, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour);
cv::line(image, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour);
}
}
};
/**
*
* @param keyframes
* @param rendering_paramss
* @param landmark_list
* @param morphable_model
* @param blendshapes
* @param meshs
* @param pca_shape_coefficients
* @param blendshape_coefficients
* @param fitted_image_points
* @param annotations
* @param landmark_mapper
* @param settings
* @param n_iter
*/
void output_obj(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
std::vector<float> pca_shape_coefficients,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings) {
string line;
// Skip the first 3 lines, they're header lines:
getline(file, line); // 'version:1'
getline(file, line); // 'n_points : 68'
getline(file, line); // '{'
Mat isomap;
Mat merged_isomap;
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
int ibugId = 1;
while (getline(file, line))
{
if (line == "}") { // end of the file
break;
}
std::stringstream lineStream(line);
auto landmark_mapper = reconstruction_data.landmark_mapper;
auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
auto morphable_model = reconstruction_data.morphable_model;
Landmark<Vec2f> landmark;
landmark.name = std::to_string(ibugId);
if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) {
throw std::runtime_error(string("Landmark format error while parsing the line: " + line));
}
// From the iBug website:
// "Please note that the re-annotated data for this challenge are saved in the Matlab convention of 1 being
// the first index, i.e. the coordinates of the top left pixel in an image are x=1, y=1."
// ==> So we shift every point by 1:
landmark.coordinates[0] -= 1.0f;
landmark.coordinates[1] -= 1.0f;
landmarks.emplace_back(landmark);
++ibugId;
bool show_video = settings.get<bool>("output.show_video", false);
// The 3D head pose can be recovered as follows:
for (uint i = 0; i < keyframes.size(); ++i) {
Mat frame = keyframes[i].get()->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframes[i].get()->frame_number;
auto landmarks = keyframes[i].get()->fitting_result.landmarks;
auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// and similarly for pitch and roll.
// Extract the texture from the image using given mesh and camera parameters:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
// Draw the loaded landmarks:
isomap = render::extract_texture(keyframes[i].get()->fitting_result.mesh, affine_from_ortho, frame);
// merge the isomaps:
merged_isomap = isomap_averaging.add_and_merge(isomap);
}
return landmarks;
};
// save an obj + current merged isomap to the disk:
core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates());
core::write_textured_obj(neutral_expression, "current_merged.obj");
cv::imwrite("current_merged.isomap.png", merged_isomap);
std::cout << "written current_merged.obj and current_merged.isomap" << std::endl;
}
/**
* Parse config file
* @param filename
*/
boost::property_tree::ptree get_reconstruction_config(std::string filename) {
boost::property_tree::ptree pt;
boost::property_tree::ini_parser::read_ini(filename, pt);
return pt;
}
/**
*
* Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
* are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
* causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
* annotations with the based on a given keyframe list.
*
* @param keyframes
* @param landmarks
* @return
*/
vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
vector<core::LandmarkCollection<cv::Vec2f>> sublist;
for (auto& f : keyframes) {
sublist.push_back(landmarks[f.get()->frame_number]);
}
return sublist;
}
/**
* This app demonstrates estimation of the camera and fitting of the shape
* model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
* In addition to fit-model-simple, this example uses blendshapes, contour-
* fitting, and can iterate the fitting.
*
* First, the 68 ibug landmarks are loaded from the .pts file and converted
* to vertex indices using the LandmarkMapper. Then, an orthographic camera
* is estimated, and then, using this camera matrix, the shape is fitted
* to the landmarks.
* 68 ibug landmarks are loaded from the .pts file and converted
* to vertex indices using the LandmarkMapper.
*/
int main(int argc, char *argv[]) {
fs::path modelfile, isomapfile, imagefile, landmarksfile, mappingsfile, outputfile;
fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile, reconstruction_config;
std::vector<std::string> annotations;
// get annotations from one file
bool get_annotations = false;
try {
po::options_description desc("Allowed options");
desc.add_options()
("help,h",
"display the help message")
("model,m", po::value<fs::path>(&modelfile)->required()->default_value("/data/share/sfm_shape_3448.bin"),
"a Morphable Model stored as cereal BinaryArchive")
("image,i", po::value<fs::path>(&imagefile)->required()->default_value("/data/image_0010.png"),
"an input image")
("landmarks,l", po::value<fs::path>(&landmarksfile)->required()->default_value("/data/image_0010.pts"),
"2D landmarks for the image, in ibug .pts format")
("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("/data/share/ibug2did.txt"),
"landmark identifier to model vertex number mapping")
("output,o", po::value<fs::path>(&outputfile)->required()->default_value("/data/out"),
"basename for the output rendering and obj files")
;
("help,h",
"display the help message")
("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
"a Morphable Model stored as cereal BinaryArchive")
("video,v", po::value<fs::path>(&videofile)->required(),
"an input image")
("config,f", po::value<fs::path>(&reconstruction_config)->default_value("../share/default_reconstruction_config.ini"),
"configuration file for the reconstruction")
("get_annotations,g", po::bool_switch(&get_annotations)->default_value(false),
"read .pts annotation file locations from one file, put one file path on each line")
("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
".pts annotation files per frame of video")
("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
"2D landmarks for the image, in ibug .pts format")
("model-contour,c",
po::value<fs::path>(&contourfile)->required()->default_value("../share/model_contours.json"),
"file with model contour indices")
("edge-topology,e", po::value<fs::path>(&edgetopologyfile)->required()->default_value(
"../share/sfm_3448_edge_topology.json"),
"file with model's precomputed edge topology")
("blendshapes,b", po::value<fs::path>(&blendshapesfile)->required()->default_value(
"../share/expression_blendshapes_3448.bin"),
"file with blendshapes")
("output,o", po::value<fs::path>(&outputfile)->required()->default_value("out"),
"basename for the output rendering and obj files");
po::variables_map vm;
po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
if (vm.count("help")) {
cout << "Usage: fit-model-simple [options]" << endl;
cout << desc;
std::cout << "Usage: fit-model [options]" << std::endl;
std::cout << desc;
return EXIT_SUCCESS;
}
po::notify(vm);
}
catch (const po::error& e) {
cout << "Error while parsing command-line arguments: " << e.what() << endl;
cout << "Use --help to display a list of options." << endl;
catch (const po::error &e) {
std::cout << "Error while parsing command-line arguments: " << e.what() << std::endl;
std::cout << "Use --help to display a list of options." << std::endl;
return EXIT_SUCCESS;
}
// Load the image, landmarks, LandmarkMapper and the Morphable Model:
Mat image = cv::imread(imagefile.string());
LandmarkCollection<cv::Vec2f> landmarks;
try {
landmarks = read_pts_landmarks(landmarksfile.string());
}
catch (const std::runtime_error& e) {
cout << "Error reading the landmarks: " << e.what() << endl;
return EXIT_FAILURE;
}
// Start loading prerequisites:
morphablemodel::MorphableModel morphable_model;
std::vector<float> pca_shape_coefficients;
// List of blendshapes coefficients:
std::vector<std::vector<float>> blendshape_coefficients;
std::vector<std::vector<cv::Vec2f>> fitted_image_points;
// These will be the final 2D and 3D points used for the fitting:
vector<cv::Vec3f> model_points; // the points in the 3D shape model
vector<int> vertex_indices; // their vertex indices
vector<core::Mesh> meshs;
vector<fitting::RenderingParameters> rendering_paramss;
// Load landmarks, LandmarkMapper and the Morphable Model:
auto landmark_mapper = core::LandmarkMapper(mappingsfile);
// Load all annotation files into lists of landmarks:
vector<core::LandmarkCollection<cv::Vec2f>> landmark_list;
try {
morphable_model = morphablemodel::load_model(modelfile.string());
}
catch (const std::runtime_error& e) {
cout << "Error loading the Morphable Model: " << e.what() << endl;
} catch (const std::runtime_error &e) {
std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
return EXIT_FAILURE;
}
core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
// Draw the loaded landmarks:
Mat outimg = image.clone();
for (auto&& lm : landmarks) {
cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 });
// Load all annotation using input parameters:
try {
std::tie(landmark_list, annotations) = eos::core::load_annotations<cv::Vec2f>(
annotations,
landmark_mapper,
morphable_model,
get_annotations);
} catch(const std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
}
// These will be the final 2D and 3D points used for the fitting:
vector<Vec4f> model_points; // the points in the 3D shape model
vector<int> vertex_indices; // their vertex indices
vector<Vec2f> image_points; // the corresponding 2D landmark points
// The expression blendshapes:
auto blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string());
// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
for (int i = 0; i < landmarks.size(); ++i) {
auto converted_name = landmark_mapper.convert(landmarks[i].name);
if (!converted_name) { // no mapping defined for the current landmark
continue;
}
int vertex_idx = std::stoi(converted_name.get());
auto vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
model_points.emplace_back(Vec4f(vertex.x(), vertex.y(), vertex.z(), 1.0f));
vertex_indices.emplace_back(vertex_idx);
image_points.emplace_back(landmarks[i].coordinates);
// These two are used to fit the front-facing contour to the ibug contour landmarks:
auto model_contour = contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
auto ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string());
// The edge topology is used to speed up computation of the occluding face contour fitting:
auto edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string());
// Read .ini file to accommodate.
auto settings = get_reconstruction_config(reconstruction_config.string());
auto reconstruction_data = eos::fitting::ReconstructionData{
morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
bool evaluate = settings.get<bool>("eval.evaluate", true);
int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
int max_frames = settings.get<int>("eval.max_frames", 0);
int drop_frames = settings.get<int>("frames.drop_frames", 0);
// Start with the video play and get video file:
BufferedVideoIterator vid_iterator;
try {
vid_iterator = BufferedVideoIterator(videofile.string(), reconstruction_data, settings);
} catch(std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
}
// Estimate the camera (pose) from the 2D - 3D point correspondences
fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
// start a thread to search for the best frames
vid_iterator.start();
// The 3D head pose can be recovered as follows:
float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// Waiting for the iterator to find the best keyframes, we will reconstruct when done, note that this is
// best for short videos. Although the vid iterator has a maximum capacity therefore memory leaks are
// not a problem, still short videos will demonstrate its usefulness best.
while(vid_iterator.is_playing()) { }
// and similarly for pitch and roll.
// Estimate the shape coefficients by fitting the shape to the landmarks:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
rendering_params, image.cols, image.rows
);
auto keyframes = vid_iterator.get_keyframes();
// Generate a sublist of the total available landmark list:
std::cout << "Going to reconstruct with " << keyframes.size() << " images, "<< num_iterations << " iterations:"<< std::endl;
vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(
// Fit shape and pose:
auto t1 = std::chrono::high_resolution_clock::now();
std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
morphable_model,
affine_from_ortho,
image_points,
vertex_indices
blendshapes,
keyframes,
landmark_mapper,
keyframes[0].get()->frame.cols,
keyframes[0].get()->frame.rows,
static_cast<int>(keyframes.size()),
edge_topology,
ibug_contour,
model_contour,
num_iterations,
boost::none,
30.0f,
boost::none,
pca_shape_coefficients,
blendshape_coefficients,
fitted_image_points,
settings
);
// Obtain the full mesh with the estimated coefficients:
eos::core::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
std::cout << pca_shape_coefficients.size() << std::endl;
std::cout << vid_iterator.to_string() << std::endl;
// Extract the texture from the image using given mesh and camera parameters:
Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
auto t2 = std::chrono::high_resolution_clock::now();
std::cout << "Reconstruction took "
<< std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count()
<< "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() / (keyframes.size() * num_iterations)
<< "ms)" << std::endl;
// Save the mesh as textured obj:
outputfile += fs::path(".obj");
eos::core::write_textured_obj(mesh, outputfile.string());
// And save the isomap:
outputfile.replace_extension("isomap.png");
cv::imwrite(outputfile.string(), isomap);
cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
output_obj(
keyframes,
pca_shape_coefficients,
reconstruction_data,
settings
);
return EXIT_SUCCESS;
}
include/eos/core/LandmarkMapper.hpp
View file @ fcae2025
......@@ -119,6 +119,31 @@ public:
}
};
/**
* @brief Converts the given landmark name to the mapped name.
*
* @param[in] landmark_name A landmark name to convert.
* @return The mapped landmark name if a mapping exists, an empty optional otherwise.
* @throws out_of_range exception if there is no mapping
* for the given landmarkName.
*/
std::string get_vertex_id(std::string landmark_name) const {
if (landmark_mappings.empty()) {
// perform identity mapping, i.e. return the input
return landmark_name;
}
else {
auto&& converted_landmark = landmark_mappings.find(landmark_name);
if (converted_landmark != std::end(landmark_mappings)) {
// landmark mapping found, return it
return converted_landmark->second;
}
else { // landmark_name does not match the key of any element in the map
return "";
}
}
};
/**
* @brief Returns the number of loaded landmark mappings.
*
......
include/eos/core/landmark_utils.hpp
View file @ fcae2025
......@@ -193,13 +193,20 @@ namespace eos {
* @param landmarks
* @return
*/
cv::Rect get_face_roi(vector<Vec2f>image_points, int image_width, int image_height) {
cv::Rect get_face_roi(vector<cv::Vec2f>image_points, int image_width, int image_height) {
cv::Rect bbox = cv::boundingRect(image_points);
// sometimes the bbox starts outside the image boundaries due to wrong image points.
bbox.x = bbox.x < 0 ? 0 : bbox.x;
bbox.y = bbox.y < 0 ? 0 : bbox.y;
// cap on the image width and height.
bbox.width = bbox.x + bbox.width < image_width ? bbox.width: image_width - bbox.x - 1;
bbox.height = bbox.y + bbox.height < image_height ? bbox.height: image_height - bbox.y - 1;
assert(bbox.x + bbox.width < image_width);
assert(bbox.y + bbox.height < image_height);
return bbox;
}
/**
......
include/eos/fitting/FittingResult.hpp
View file @ fcae2025
......@@ -46,6 +46,9 @@ struct FittingResult
std::vector<float> blendshape_coefficients;
core::LandmarkCollection<cv::Vec2f> landmarks;
core::Mesh mesh;
std::vector<Vec4f> model_points; // the points in the 3D shape model
std::vector<int> vertex_indices; // their vertex indices
std::vector<Vec2f> image_points; // the corresponding 2D landmark points
};
} /* namespace fitting */
......
include/eos/fitting/contour_correspondence.hpp
View file @ fcae2025
......@@ -72,7 +72,7 @@ struct ModelContour
// We store r/l separately because we currently only fit to the contour facing the camera.
// Also if we were to fit to the whole contour: Be careful not to just fit to the closest. The
// "invisible" ones behind might be closer on an e.g 90° angle. Store CNT for left/right side separately?
// "invisible" ones behind might be closer on an e.g 90� angle. Store CNT for left/right side separately?
/**
* Helper method to load a ModelContour from
......@@ -226,7 +226,7 @@ inline std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<in
* have different size.
* Correspondence can be established using get_nearest_contour_correspondences().
*
* If the yaw angle is between +-7.5°, both contours will be selected.
* If the yaw angle is between +-7.5�, both contours will be selected.
*
* Note: Maybe rename to find_nearest_contour_points, to highlight that there is (potentially a lot) computational cost involved?
*
......@@ -251,7 +251,7 @@ std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_a
model_contour_indices.insert(end(model_contour_indices), begin(model_contour.left_contour), end(model_contour.left_contour));
contour_landmark_identifiers.insert(end(contour_landmark_identifiers), begin(contour_landmarks.left_contour), end(contour_landmarks.left_contour));
}
// Note there's an overlap between the angles - if a subject is between +- 7.5°, both contours get added.
// Note there's an overlap between the angles - if a subject is between +- 7.5�, both contours get added.
return std::make_pair(contour_landmark_identifiers, model_contour_indices);
};
......@@ -273,7 +273,7 @@ std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_a
* @param[in] viewport Current viewport to use.
* @return A tuple with the 2D contour landmark points, the corresponding points in the 3D shape model and their vertex indices.
*/
inline std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_nearest_contour_correspondences(const core::LandmarkCollection<cv::Vec2f>& landmarks, const std::vector<std::string>& landmark_contour_identifiers, const std::vector<int>& model_contour_indices, const core::Mesh& mesh, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport)
inline std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_nearest_contour_correspondences(const core::LandmarkCollection<cv::Vec2f>& landmarks, const std::vector<std::string>& landmark_contour_identifiers, const std::vector<int>& model_contour_indices, const core::Mesh& mesh, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport)
{
// These are the additional contour-correspondences we're going to find and then use!
std::vector<cv::Vec4f> model_points_cnt; // the points in the 3D shape model
......
include/eos/fitting/fitting.hpp
View file @ fcae2025
......@@ -880,271 +880,26 @@ inline eos::core::Mesh generate_new_mesh(
return mesh;
}
inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_2(
const morphablemodel::MorphableModel& morphable_model,
const std::vector<morphablemodel::Blendshape>& blendshapes,
std::vector<eos::video::Keyframe>& keyframes,
const core::LandmarkMapper& landmark_mapper,
int image_width,
int image_height,
int num_images,
const morphablemodel::EdgeTopology& edge_topology,
const fitting::ContourLandmarks& contour_landmarks,
const fitting::ModelContour& model_contour,
int num_iterations,
boost::optional<int> num_shape_coefficients_to_fit,
float lambda,
boost::optional<fitting::RenderingParameters> initial_rendering_params,
std::vector<float>& pca_shape_coefficients,
std::vector<std::vector<float>>& blendshape_coefficients,
std::vector<std::vector<cv::Vec2f>>& fitted_image_points) {
assert(blendshapes.size() > 0);
assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit?
assert(pca_shape_coefficients.size() <= morphable_model.get_shape_model().get_num_principal_components());
using std::vector;
using cv::Vec2f;
using cv::Vec4f;
using cv::Mat;
using Eigen::VectorXf;
using Eigen::MatrixXf;
if (!num_shape_coefficients_to_fit) {
num_shape_coefficients_to_fit = morphable_model.get_shape_model().get_num_principal_components();
}
if (pca_shape_coefficients.empty()) {
pca_shape_coefficients.resize(num_shape_coefficients_to_fit.get());
}
// TODO: This leaves the following case open: num_coeffs given is empty or defined, but the
// pca_shape_coefficients given is != num_coeffs or the model's max-coeffs. What to do then? Handle & document!
if (blendshape_coefficients.size() < num_images) {
for (int j = 0; j < num_images; ++j) {
std::vector<float> current_blendshape_coefficients;
current_blendshape_coefficients.resize(blendshapes.size());
blendshape_coefficients.push_back(current_blendshape_coefficients);
}
}
MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
// Current mesh - either from the given coefficients, or the mean:
VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
vector<VectorXf> current_combined_shapes;
vector<eos::core::Mesh> current_meshs;
// The 2D and 3D point correspondences used for the fitting:
vector<vector<Vec4f>> model_points; // the points in the 3D shape model of all frames.
vector<vector<int>> vertex_indices; // their vertex indices of all frames.
vector<vector<Vec2f>> image_points; // the corresponding 2D landmark points of all frames.
vector<fitting::RenderingParameters> rendering_params; // list of rendering params for all frames.
// Generate meshes from current shapes.
for (int j = 0; j < num_images; ++j) {
VectorXf current_combined_shape = current_pca_shape +
blendshapes_as_basis *
Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
eos::core::Mesh current_mesh = morphablemodel::sample_to_mesh(
current_combined_shape,
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates());
// Get the locations of the model locations of the meshes, vertex_indices and image points
// (equal to landmark coordinates), for every image / mesh.
// std::tie(curr_model_points, curr_vertex_indices, curr_image_points) = eos::core::get_landmark_coordinates<Vec2f, Vec4f>(
// keyframes[j].fitting_result.landmarks, landmark_mapper, current_mesh);
eos::core::get_landmark_coordinates_inline<Vec2f, Vec4f>(
keyframes[j].fitting_result.landmarks,
landmark_mapper,
current_mesh,
// output parameters
model_points,
vertex_indices,
image_points);
// Start constructing a list of rendering parameters needed for reconstruction.
// Get the current points from the last added image points and model points
auto current_pose = fitting::estimate_orthographic_projection_linear(
image_points.back(), model_points.back(), true, image_height);
fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height);
rendering_params.push_back(current_rendering_params);
// update key frame rendering params
keyframes[j].fitting_result.rendering_parameters = current_rendering_params;
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_ortho, image_points.back(), vertex_indices.back()
);
// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
current_combined_shape = current_pca_shape +
morphablemodel::to_matrix(blendshapes) *
Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size()
);
current_combined_shapes.push_back(current_combined_shape);
current_mesh = morphablemodel::sample_to_mesh(
current_combined_shape, morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()
);
current_meshs.push_back(current_mesh);
}
// The static (fixed) landmark correspondences which will stay the same throughout
// the fitting (the inner face landmarks):
vector<vector<int>> fixed_vertex_indices (vertex_indices);
vector<vector<Vec2f>> fixed_image_points (image_points);
std::vector<cv::Mat> affine_from_orthos;
std::vector<VectorXf> mean_plus_blendshapes;
image_points = fixed_image_points;
vertex_indices = fixed_vertex_indices;
for (int j = 0; j < num_images; ++j) {
// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
vector<Vec2f> image_points_contour;
vector<int> vertex_indices_contour;
auto curr_keyframe = keyframes[j];
auto landmarks = curr_keyframe.fitting_result.landmarks;
auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(
landmarks,
contour_landmarks,
model_contour,
yaw_angle,
current_meshs[j],
rendering_params[j].get_modelview(),
rendering_params[j].get_projection(),
fitting::get_opencv_viewport(image_width, image_height)
);
// Add the contour correspondences to the set of landmarks that we use for the fitting:
vertex_indices[j] = fitting::concat(vertex_indices[j], vertex_indices_contour);
image_points[j] = fitting::concat(image_points[j], image_points_contour);
// Fit the occluding (away-facing) contour using the detected contour LMs:
vector<Eigen::Vector2f> occluding_contour_landmarks;
// positive yaw = subject looking to the left
if (yaw_angle >= 0.0f) {
// the left contour is the occluding one we want to use ("away-facing")
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
});
} else {
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
});
}
auto edge_correspondences = fitting::find_occluding_edge_correspondences(
current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f
);
image_points[j] = fitting::concat(image_points[j], edge_correspondences.first);
vertex_indices[j] = fitting::concat(vertex_indices[j], edge_correspondences.second);
// Get the model points of the current mesh, for all correspondences that we've got:
model_points[j].clear();
for (const auto& v : vertex_indices[j]) {
model_points[j].push_back(
{
current_meshs[j].vertices[v][0],
current_meshs[j].vertices[v][1],
current_meshs[j].vertices[v][2],
current_meshs[j].vertices[v][3]
});
}
// Re-estimate the pose, using all correspondences:
auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
affine_from_orthos.push_back(affine_from_ortho);
// Estimate the PCA shape coefficients with the current blendshape coefficients:
VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
}
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel(
morphable_model,
affine_from_orthos,
image_points,
vertex_indices,
mean_plus_blendshapes,
lambda,
num_shape_coefficients_to_fit
);
// Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
for (int j = 0; j < num_images; ++j) {
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
);
current_combined_shapes[j] = current_pca_shape +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
current_meshs[j] = morphablemodel::sample_to_mesh(
current_combined_shapes[j],
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()
);
}
fitted_image_points = image_points;
return { current_meshs, rendering_params }; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices).
};
#ifdef _OPENMP
inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_parallel(
const morphablemodel::MorphableModel& morphable_model,
const std::vector<morphablemodel::Blendshape>& blendshapes,
std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
// std::vector<eos::video::Keyframe>& keyframes,
const core::LandmarkMapper& landmark_mapper,
int image_width,
int image_height,
int num_images,
const morphablemodel::EdgeTopology& edge_topology,
const fitting::ContourLandmarks& contour_landmarks,
const fitting::ModelContour& model_contour,
int num_iterations,
boost::optional<int> num_shape_coefficients_to_fit,
float lambda,
boost::optional<fitting::RenderingParameters> initial_rendering_params,
std::vector<float>& pca_shape_coefficients,
std::vector<std::vector<float>>& blendshape_coefficients,
std::vector<std::vector<cv::Vec2f>>& fitted_image_points,
boost::property_tree::ptree settings)
{
const morphablemodel::MorphableModel& morphable_model,
const std::vector<morphablemodel::Blendshape>& blendshapes,
std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
const core::LandmarkMapper& landmark_mapper,
int image_width,
int image_height,
int num_images,
const morphablemodel::EdgeTopology& edge_topology,
const fitting::ContourLandmarks& contour_landmarks,
const fitting::ModelContour& model_contour,
int num_iterations,
boost::optional<int> num_shape_coefficients_to_fit,
float lambda,
boost::optional<fitting::RenderingParameters> initial_rendering_params,
std::vector<float>& pca_shape_coefficients,
std::vector<std::vector<float>>& blendshape_coefficients,
std::vector<std::vector<cv::Vec2f>>& fitted_image_points,
boost::property_tree::ptree settings) {
assert(blendshapes.size() > 0);
assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit?
......@@ -1173,10 +928,8 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// TODO: This leaves the following case open: num_coeffs given is empty or defined, but the
// pca_shape_coefficients given is != num_coeffs or the model's max-coeffs. What to do then? Handle & document!
if (blendshape_coefficients.size() < num_images)
{
for (int j = 0; j < num_images; ++j)
{
if (blendshape_coefficients.size() < num_images) {
for (int j = 0; j < num_images; ++j) {
std::vector<float> current_blendshape_coefficients;
current_blendshape_coefficients.resize(blendshapes.size());
blendshape_coefficients.push_back(current_blendshape_coefficients);
......@@ -1236,9 +989,6 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
// if no contour
// affine_from_orthos[j] = affine_from_ortho;
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_ortho, curr_image_points, curr_vertex_indices);
......@@ -1246,8 +996,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
current_combined_shapes[j] = current_pca_shape +
morphablemodel::to_matrix(blendshapes) *
Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
blendshape_coefficients[j].size()
);
blendshape_coefficients[j].size());
current_meshs[j] = morphablemodel::sample_to_mesh(
current_combined_shape, morphable_model.get_color_model().get_mean(),
......@@ -1259,119 +1008,113 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
model_points[j] = curr_model_points;
vertex_indices[j] = curr_vertex_indices;
image_points[j] = curr_image_points;
// Estimate the PCA shape coefficients with the current blendshape coefficients:
// VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
// blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
// blendshape_coefficients[j].size());
// mean_plus_blendshapes[j] = current_mean_plus_blendshapes;
}
}
// The static (fixed) landmark correspondences which will stay the same throughout
// the fitting (the inner face landmarks):
vector<vector<int>> fixed_vertex_indices(vertex_indices);
vector<vector<Vec2f>> fixed_image_points(image_points);
if (use_contours)
{
for (int i = 0; i < num_iterations; ++i) {
std::vector<cv::Mat> affine_from_orthos;
std::vector<VectorXf> mean_plus_blendshapes;
image_points = fixed_image_points;
vertex_indices = fixed_vertex_indices;
for (int j = 0; j < num_images; ++j) {
// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
vector<Vec2f> image_points_contour;
vector<int> vertex_indices_contour;
auto curr_keyframe = keyframes[j].get();
auto landmarks = curr_keyframe->fitting_result.landmarks;
auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(landmarks, contour_landmarks, model_contour, yaw_angle, current_meshs[j], rendering_params[j].get_modelview(), rendering_params[j].get_projection(), fitting::get_opencv_viewport(image_width, image_height));
// Add the contour correspondences to the set of landmarks that we use for the fitting:
vertex_indices[j] = fitting::concat(vertex_indices[j], vertex_indices_contour);
image_points[j] = fitting::concat(image_points[j], image_points_contour);
// Fit the occluding (away-facing) contour using the detected contour LMs:
vector<Eigen::Vector2f> occluding_contour_landmarks;
if (yaw_angle >= 0.0f) // positive yaw = subject looking to the left
{ // the left contour is the occluding one we want to use ("away-facing")
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
}
else {
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
}
auto edge_correspondences = fitting::find_occluding_edge_correspondences_parallel(current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f);
image_points[j] = fitting::concat(image_points[j], edge_correspondences.first);
vertex_indices[j] = fitting::concat(vertex_indices[j], edge_correspondences.second);
// Get the model points of the current mesh, for all correspondences that we've got:
model_points[j].clear();
for (const auto& v : vertex_indices[j])
{
model_points[j].push_back({ current_meshs[j].vertices[v][0], current_meshs[j].vertices[v][1], current_meshs[j].vertices[v][2], current_meshs[j].vertices[v][3] });
}
for (int i = 0; i < num_iterations; ++i) {
std::vector<cv::Mat> affine_from_orthos;
std::vector<VectorXf> mean_plus_blendshapes;
// Re-estimate the pose, using all correspondences:
fitting::ScaledOrthoProjectionParameters current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
image_points = fixed_image_points;
vertex_indices = fixed_vertex_indices;
cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
affine_from_orthos.push_back(affine_from_ortho);
for (int j = 0; j < num_images; ++j) {
// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
vector<Vec2f> image_points_contour;
vector<int> vertex_indices_contour;
auto curr_keyframe = keyframes[j].get();
auto landmarks = curr_keyframe->fitting_result.landmarks;
auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(landmarks, contour_landmarks, model_contour, yaw_angle, current_meshs[j], rendering_params[j].get_modelview(), rendering_params[j].get_projection(), fitting::get_opencv_viewport(image_width, image_height));
// Add the contour correspondences to the set of landmarks that we use for the fitting:
vertex_indices[j] = fitting::concat(vertex_indices[j], vertex_indices_contour);
image_points[j] = fitting::concat(image_points[j], image_points_contour);
// Fit the occluding (away-facing) contour using the detected contour LMs:
vector<Eigen::Vector2f> occluding_contour_landmarks;
if (yaw_angle >= 0.0f) // positive yaw = subject looking to the left
{ // the left contour is the occluding one we want to use ("away-facing")
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
}
else {
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); });
}
auto edge_correspondences = fitting::find_occluding_edge_correspondences_parallel(current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f);
image_points[j] = fitting::concat(image_points[j], edge_correspondences.first);
vertex_indices[j] = fitting::concat(vertex_indices[j], edge_correspondences.second);
// Estimate the PCA shape coefficients with the current blendshape coefficients:
VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
// Get the model points of the current mesh, for all correspondences that we've got:
model_points[j].clear();
for (const auto& v : vertex_indices[j])
{
model_points[j].push_back({ current_meshs[j].vertices[v][0], current_meshs[j].vertices[v][1], current_meshs[j].vertices[v][2], current_meshs[j].vertices[v][3] });
}
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel(
morphable_model,
affine_from_orthos,
image_points,
vertex_indices,
mean_plus_blendshapes,
lambda,
num_shape_coefficients_to_fit
// Re-estimate the pose, using all correspondences:
fitting::ScaledOrthoProjectionParameters current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
affine_from_orthos.push_back(affine_from_ortho);
// Estimate the PCA shape coefficients with the current blendshape coefficients:
VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
}
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel(
morphable_model,
affine_from_orthos,
image_points,
vertex_indices,
mean_plus_blendshapes,
lambda,
num_shape_coefficients_to_fit
);
// Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
#pragma omp parallel num_threads(NUM_THREADS)
{
#pragma omp for
for (int j = 0; j < num_images; ++j) {
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
);
current_combined_shapes[j] = current_pca_shape +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
blendshape_coefficients[j].size());
// Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
current_meshs[j] = morphablemodel::sample_to_mesh(
current_combined_shapes[j],
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()
);
#pragma omp parallel num_threads(NUM_THREADS)
{
#pragma omp for
for (int j = 0; j < num_images; ++j)
{
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
);
current_combined_shapes[j] = current_pca_shape +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
blendshape_coefficients[j].size());
current_meshs[j] = morphablemodel::sample_to_mesh(
current_combined_shapes[j],
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()
);
// save it to the keyframe, we might need it for showing the reconstruction.
// we could make it optional
keyframes[j].get()->fitting_result.mesh = current_meshs[j];
keyframes[j].get()->fitting_result.rendering_parameters = rendering_params[j];
}
// save it to the keyframe, we might need it for showing the reconstruction.
// we could make it optional
keyframes[j].get()->fitting_result.mesh = current_meshs[j];
keyframes[j].get()->fitting_result.rendering_parameters = rendering_params[j];
keyframes[j].get()->fitting_result.image_points = image_points[j];
keyframes[j].get()->fitting_result.vertex_indices = vertex_indices[j];
keyframes[j].get()->fitting_result.model_points = model_points[j];
}
}
}
}
}
fitted_image_points = image_points;
......
include/eos/video/BufferedVideoIterator.hpp
View file @ fcae2025
......@@ -460,6 +460,7 @@ public:
int width;
int height;
int last_frame_number;
bool add_random;
Keyframe last_keyframe;
std::unique_ptr<std::thread> frame_buffer_worker;
......@@ -488,6 +489,8 @@ public:
skip_frames = settings.get<int>("frames.skip_frames", 0);
frames_per_bin = settings.get<unsigned int>("frames.frames_per_bin", 2);
add_random = settings.get<bool>("frames.add_random", false);
this->reconstruction_data = reconstruction_data;
unsigned int num_shape_coeff = reconstruction_data.
morphable_model.get_shape_model().get_num_principal_components();
......@@ -501,6 +504,17 @@ public:
// reset frame count (to be sure)
n_frames = 0;
total_frames = 0;
frames_dropped = 0;
Mat frame;
// Get the first frame to see, sometimes frames are empty at the start, keep reading until we hit one.
while(frame.empty() || total_frames < skip_frames) {
cap.read(frame);
total_frames++;
}
std::cout << "Starting at video at frame:" << total_frames << std::endl;
// std::cout << "Settings: " << std::endl <<
// "min_frames: " << min_frames << std::endl <<
......@@ -572,11 +586,108 @@ public:
fitting_result.rendering_parameters = rendering_params;
cv::Rect face_roi = core::get_face_roi(image_points, frame_width, frame_height);
float frame_laplacian_score = static_cast<float>(variance_of_laplacian(frame(face_roi)));
return Keyframe(frame_laplacian_score, frame, fitting_result, total_frames);
}
/**
* Fill the buffer by iterating through the video until the very last frame.
*/
void video_iterator() {
// Go and fill the buffer with more frames while we are reconstructing.
while (next()) { }
// stop the video
__stop();
std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames << std::endl;
};
Keyframe get_last_keyframe() {
return last_keyframe;
};
bool try_add_random(Keyframe keyframe) {
// Determine whether to add or not:
float yaw_angle = glm::degrees(glm::yaw(keyframe.fitting_result.rendering_parameters.get_rotation()));
auto idx = angle_to_index(yaw_angle);
bool add_frame = false;
keyframe.yaw_angle = yaw_angle;
// Score is 0 for total black frames, we don't want those:
if (keyframe.score == 0) {
return false;
}
// only add when the look at the camera
if (idx == 3 || idx == 4) {
add_frame = true;
}
if (!add_frame) {
return false;
}
// Add the keyframe:
bins[idx].push_back(std::make_shared<Keyframe>(keyframe));
if (bins[idx].size() > frames_per_bin) {
n_frames--;
// need to remove the lowest one:
std::sort(std::begin(bins[idx]), std::end(bins[idx]),
[](const auto& lhs, const auto& rhs) { return lhs->score > rhs->score; });
bins[idx].resize(frames_per_bin);
}
return true;
}
Keyframe next_key_frame() {
Mat frame = __get_new_frame();
auto keyframe = generate_new_keyframe(frame);
return keyframe;
}
/**
*
* Set next frame and return frame_buffer. Returns true or false if the next frame is empty.
* Empty frames mean that we reached the end of the video stream (i.e., file or camera).
*
* @return bool if
*/
bool next() {
Mat frame = __get_new_frame();
if (frame.empty()) {
return false;
}
// TODO: only calculate lapscore within the bounding box of the face.
auto keyframe = generate_new_keyframe(frame);
bool frame_added = try_add(keyframe);
// Added or not, we put this as the last keyframe
last_keyframe = keyframe;
if(frame_added) {
n_frames++;
// Setting that the buffer has changed:
frame_buffer_changed = true;
}
total_frames++;
// fill up the buffer until we hit the minimum frames we want in the buffer.
if(n_frames < min_frames) {
return next();
}
return true;
}
/**
* Try to add a new key frame. Look at the laplacian score and the yaw_angle. The yaw_angle will
* determine the bin in which it will be added. The score will be compared to the other frames in the
......@@ -600,8 +711,6 @@ public:
return false;
}
std::cout << "idx: " << idx << std::endl;
// always add when we don't have enough frames
if (bins[idx].size() < frames_per_bin) {
add_frame = true; // definitely adding - we wouldn't have to go through the for-loop on the next line.
......@@ -689,62 +798,6 @@ public:
return get_keyframes();
}
/**
* Fill the buffer by iterating through the video until the very last frame.
*/
void video_iterator() {
// Go and fill the buffer with more frames while we are reconstructing.
while (next()) { }
// stop the video
__stop();
std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames << std::endl;
};
Keyframe get_last_keyframe() {
return last_keyframe;
};
/**
*
* Set next frame and return frame_buffer. Returns true or false if the next frame is empty.
* Empty frames mean that we reached the end of the video stream (i.e., file or camera).
*
* @return bool if
*/
bool next() {
Mat frame = __get_new_frame();
if (frame.empty()) {
return false;
}
// keep the last frame here, so we can play a video subsequently:
last_frame_number = total_frames;
// TODO: only calculate lapscore within the bounding box of the face.
auto keyframe = generate_new_keyframe(frame);
bool frame_added = try_add(keyframe);
// Added or not, we put this as the last keyframe
last_keyframe = keyframe;
if(frame_added) {
n_frames++;
// Setting that the buffer has changed:
frame_buffer_changed = true;
}
total_frames++;
// fill up the buffer until we hit the minimum frames we want in the buffer.
if(n_frames < min_frames) {
return next();
}
return true;
}
/**
* Update pca shape coeff. Probably we need to make something with a mutex, for updating and reading
......@@ -856,6 +909,8 @@ private:
// Note: these settings are for future use
int drop_frames = 0;
int frames_dropped = 0;
unsigned int num_shape_coefficients_to_fit = 0;
};
......
include/eos/video/Keyframe.hpp
View file @ fcae2025
......@@ -263,32 +263,21 @@ public:
cv::Mat add_and_merge(const cv::Mat& isomap)
{
// Merge isomaps, add the current to the already merged, pixel by pixel:
#pragma omp target
for (int r = 0; r < isomap.rows; ++r)
{
#pragma omp parallel for
for (int r = 0; r < isomap.rows; ++r)
for (int c = 0; c < isomap.cols; ++c)
{
for (int c = 0; c < isomap.cols; ++c)
if (isomap.at<cv::Vec4b>(r, c)[3] <= threshold)
{
if (isomap.at<cv::Vec4b>(r, c)[3] <= threshold)
{
continue; // ignore this pixel, not visible in the extracted isomap of this current frame
}
// we're sure to have a visible pixel, merge it:
// merged_pixel = (old_average * visible_count + new_pixel) / (visible_count + 1)
merged_isomap.at<cv::Vec4f>(r, c)[0] =
(merged_isomap.at<cv::Vec4f>(r, c)[0] * visibility_counter.at<int>(r, c)
+ isomap.at<cv::Vec4b>(r, c)[0]) / (visibility_counter.at<int>(r, c) + 1);
merged_isomap.at<cv::Vec4f>(r, c)[1] =
(merged_isomap.at<cv::Vec4f>(r, c)[1] * visibility_counter.at<int>(r, c)
+ isomap.at<cv::Vec4b>(r, c)[1]) / (visibility_counter.at<int>(r, c) + 1);
merged_isomap.at<cv::Vec4f>(r, c)[2] =
(merged_isomap.at<cv::Vec4f>(r, c)[2] * visibility_counter.at<int>(r, c)
+ isomap.at<cv::Vec4b>(r, c)[2]) / (visibility_counter.at<int>(r, c) + 1);
merged_isomap.at<cv::Vec4f>(r, c)[3] =
255; // as soon as we've seen the pixel visible once, we set it to visible.
++visibility_counter.at<int>(r, c);
continue; // ignore this pixel, not visible in the extracted isomap of this current frame
}
// we're sure to have a visible pixel, merge it:
// merged_pixel = (old_average * visible_count + new_pixel) / (visible_count + 1)
merged_isomap.at<cv::Vec4f>(r, c)[0] = (merged_isomap.at<cv::Vec4f>(r, c)[0] * visibility_counter.at<int>(r, c) + isomap.at<cv::Vec4b>(r, c)[0]) / (visibility_counter.at<int>(r, c) + 1);
merged_isomap.at<cv::Vec4f>(r, c)[1] = (merged_isomap.at<cv::Vec4f>(r, c)[1] * visibility_counter.at<int>(r, c) + isomap.at<cv::Vec4b>(r, c)[1]) / (visibility_counter.at<int>(r, c) + 1);
merged_isomap.at<cv::Vec4f>(r, c)[2] = (merged_isomap.at<cv::Vec4f>(r, c)[2] * visibility_counter.at<int>(r, c) + isomap.at<cv::Vec4b>(r, c)[2]) / (visibility_counter.at<int>(r, c) + 1);
merged_isomap.at<cv::Vec4f>(r, c)[3] = 255; // as soon as we've seen the pixel visible once, we set it to visible.
++visibility_counter.at<int>(r, c);
}
}
cv::Mat merged_isomap_uchar;
......@@ -296,6 +285,7 @@ public:
return merged_isomap_uchar;
};
cv::Mat sharpen(const cv::Mat& isomap) {
cv::Mat output;
cv::Mat kernel = (cv::Mat_<float>(5, 5) << -0.125, -0.125, -0.125, -0.125, -0.125,
......
python/generate-python-bindings.cpp
View file @ fcae2025
......@@ -65,6 +65,7 @@ PYBIND11_PLUGIN(eos) {
new (&instance) core::LandmarkMapper(filename);
}, "Constructs a new landmark mapper from a file containing mappings from one set of landmark identifiers to another.", py::arg("filename"))
// We can't expose the convert member function yet - need std::optional (or some trick with self/this and a lambda)
.def("get_vertex_id", &core::LandmarkMapper::get_vertex_id, "Returns the vertex id given a landmark name")
;
py::class_<core::Mesh>(core_module, "Mesh", "This class represents a 3D mesh consisting of vertices, vertex colour information and texture coordinates.")
......@@ -187,12 +188,21 @@ PYBIND11_PLUGIN(eos) {
py::class_<fitting::ContourLandmarks>(fitting_module, "ContourLandmarks", "Defines which 2D landmarks comprise the right and left face contour.")
.def_static("load", &fitting::ContourLandmarks::load, "Helper method to load contour landmarks from a text file with landmark mappings, like ibug_to_sfm.txt.", py::arg("filename"))
// .def("get_contour_landmarks", [](const fitting::RenderingParameters& rendering_params, const core::LandmarkCollection<cv::Vec2f>& landmarks, const fitting::ContourLandmarks& contour_landmarks, const fitting::ModelContour& model_contour, float yaw_angle, const core::Mesh& mesh, const glm::mat4x4& view_model, const glm::vec4& viewport) {
// cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, width, heightG;
// }, "Get contour landmarks", py::arg("morphable_model"), py::arg("landmarks"), py::arg("contour_landmarks"), py::arg("model_contour"), py::arg("yaw_angle"), py::arg("mesh"), py::arg("view_model"), py::arg("ortho_projection"), py::arg("contour_landmarks"), py::arg("model_contour"), py::arg("num_iterations") = 5, py::arg("num_shape_coefficients_to_fit") = -1, py::arg("lambda") = 30.0f);
;
py::class_<fitting::ModelContour>(fitting_module, "ModelContour", "Definition of the vertex indices that define the right and left model contour.")
.def_static("load", &fitting::ModelContour::load, "Helper method to load a ModelContour from a json file from the hard drive.", py::arg("filename"))
;
// fitting_module.def("get_contour_landmarks", [](
// const core::LandmarkCollection<cv::Vec2f>& landmarks, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour, float yaw_angle, const core::Mesh& mesh, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport) {
//
// }, "Get contour landmarks");
fitting_module.def("fit_shape_and_pose", [](const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<glm::vec2>& landmarks, const std::vector<std::string>& landmark_ids, const core::LandmarkMapper& landmark_mapper, int image_width, int image_height, const morphablemodel::EdgeTopology& edge_topology, const fitting::ContourLandmarks& contour_landmarks, const fitting::ModelContour& model_contour, int num_iterations, int num_shape_coefficients_to_fit, float lambda) {
assert(landmarks.size() == landmark_ids.size());
std::vector<float> pca_coeffs;
......
share/model_contours.json
View file @ fcae2025
......@@ -39,4 +39,4 @@
465
]
}
}
\ No newline at end of file
}
utils/CMakeLists.txt
View file @ fcae2025
......@@ -39,6 +39,10 @@ add_executable(accuracy-evaluation accuracy-evaluation.cpp)
target_link_libraries(accuracy-evaluation eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
target_link_libraries(accuracy-evaluation "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
add_executable(accuracy-evaluation-random accuracy-evaluation-random.cpp)
target_link_libraries(accuracy-evaluation-random eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
target_link_libraries(accuracy-evaluation-random "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
add_executable(openmp-test openmp-test.cpp)
target_link_libraries(openmp-test eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
target_link_libraries(openmp-test "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
......@@ -48,4 +52,5 @@ install(TARGETS scm-to-cereal DESTINATION bin)
install(TARGETS bfm-binary-to-cereal DESTINATION bin)
install(TARGETS edgestruct-csv-to-json DESTINATION bin)
install(TARGETS accuracy-evaluation DESTINATION bin)
install(TARGETS accuracy-evaluation-random DESTINATION bin)
install(TARGETS openmp-test DESTINATION bin)
utils/accuracy-evaluation-random.cpp 0 → 100644
View file @ fcae2025
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: utils/json-to-cereal-binary.cpp
*
* Copyright 2016 Patrik Huber *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "eos/core/landmark_utils.hpp"
#include "eos/render/render.hpp"
#include "eos/morphablemodel/MorphableModel.hpp"
#include "eos/morphablemodel/Blendshape.hpp"
#include "eos/fitting/fitting.hpp"
#include "eos/render/utils.hpp"
#include "eos/render/texture_extraction.hpp"
#include "eos/video/Keyframe.hpp"
#include "eos/fitting/ReconstructionData.hpp"
#include "eos/video/BufferedVideoIterator.hpp"
#include "opencv2/core/core.hpp"
#include "opencv2/opencv.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "boost/program_options.hpp"
#include "boost/filesystem.hpp"
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include "glm/gtx/string_cast.hpp"
#include <vector>
#include <iostream>
#include <fstream>
using namespace eos;
namespace po = boost::program_options;
namespace fs = boost::filesystem;
using eos::core::Landmark;
using eos::core::LandmarkCollection;
using eos::video::BufferedVideoIterator;
using eos::video::WeightedIsomapAveraging;
using eos::video::ReconstructionVideoWriter;
using eos::video::Keyframe;
using cv::Mat;
using cv::Vec2f;
using cv::Vec3f;
using cv::Vec4f;
using std::vector;
using std::string;
using namespace cv;
typedef unsigned int uint;
std::mt19937 gen(1);
/**
* Draws the given mesh as wireframe into the image.
*
* It does backface culling, i.e. draws only vertices in CCW order.
*
* @param[in] image An image to draw into.
* @param[in] mesh The mesh to draw.
* @param[in] modelview Model-view matrix to draw the mesh.
* @param[in] projection Projection matrix to draw the mesh.
* @param[in] viewport Viewport to draw the mesh.
* @param[in] colour Colour of the mesh to be drawn.
*/
void draw_wireframe(cv::Mat image, const eos::core::Mesh& mesh, glm::mat4x4 modelview, glm::mat4x4 projection, glm::vec4 viewport, cv::Scalar colour = cv::Scalar(0, 255, 0, 255))
{
for (const auto& triangle : mesh.tvi)
{
const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport);
const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport);
const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport);
if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3))) {
cv::line(image, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour);
cv::line(image, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour);
cv::line(image, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour);
}
}
};
/**
* Helper function to calculate the euclidean distance between the landmark and a projected
* point. Nothing more than Pythagoras.
*
* @param landmark
* @param vertex_screen_coords
* @return
*/
inline float euclidean_distance(cv::Vec2f landmark, cv::Mat vertex_screen_coords) {
float screen_x = vertex_screen_coords.at<float>(0, 0);
float screen_y = vertex_screen_coords.at<float>(1, 0);
// Calc squared differences, ready for use in Pythagoras.
float landmark_diff_x_sq = std::fabs(landmark[0] - screen_x) * std::fabs(landmark[0] - screen_x);
float landmark_diff_y_sq = std::fabs(landmark[1] - screen_y) * std::fabs(landmark[0] - screen_x);
return std::sqrt(landmark_diff_x_sq + landmark_diff_y_sq);
}
/**
*
* @param landmarks
* @param landmark_mapper
* @param mesh
* @param affine_from_ortho
*/
std::pair<float, float> calc_reprojection_error(
core::LandmarkCollection<cv::Vec2f> landmarks,
core::LandmarkMapper landmark_mapper,
eos::core::Mesh& mesh,
Mat affine_from_ortho,
boost::property_tree::ptree settings) {
vector<float> total_error;
// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
for (uint i = 0; i < landmarks.size(); ++i) {
auto converted_name = landmark_mapper.convert(landmarks[i].name);
// no mapping defined for the current landmark
if (!converted_name) {
continue;
}
int vertex_idx = std::stoi(converted_name.get());
// The vertex_idx should be around the value of the original coordinates after we have
// projected it with the affine_from_ortho that is obtained earlier.
Mat vertex_screen_coords = affine_from_ortho *
Mat(cv::Vec4f(
mesh.vertices[vertex_idx].x,
mesh.vertices[vertex_idx].y,
mesh.vertices[vertex_idx].z,
mesh.vertices[vertex_idx].w
)
);
// using euclidean distance here, but should look at other ways too.
float dist = euclidean_distance(landmarks[i].coordinates, vertex_screen_coords);
total_error.push_back(dist);
}
// Calculate mean error and stddev.
float accum = 0.0;
float mean_error = static_cast<float>(std::accumulate(total_error.begin(), total_error.end(), 0)) / landmarks.size();
for(auto &d: total_error) {
accum += (d - mean_error) * (d - mean_error);
}
float stddev = std::sqrt(accum / (total_error.size() - 1));
return std::make_pair(mean_error, stddev);
}
inline std::string array_to_string(std::vector<std::string> input) {
std::string output;
for(int i = 0; i < input.size(); i++) {
output += input[i] + ((i < input.size() - 1) ? "," : "");
}
return output;
}
/**
*
*
* @param keyframes
* @param rendering_paramss
* @param landmark_list
* @param morphable_model
* @param blendshapes
* @param meshs
* @param pca_shape_coefficients
* @param blendshape_coefficients
* @param fitted_image_points
* @param annotations
* @param landmark_mapper
* @param settings
* @param n_iter
*/
void eval(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
vector<core::Mesh> meshs,
std::vector<fitting::RenderingParameters> rendering_paramss,
std::vector<float> pca_shape_coefficients,
std::vector<std::vector<cv::Vec2f>> fitted_image_points,
std::vector<std::string> annotations,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings,
long process_time,
std::string videofile,
std::string reconstruction_config,
int n_iter) {
std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
float merge_isomap_face_angle = 90.0; //settings.get<float>("output.merge_isomap_face_angle", 60.f);
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
bool show_video = settings.get<bool>("output.show_video", false);
Mat merged_isomap;
fs::path outputfilebase = reconstruction_config;
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
auto landmark_mapper = reconstruction_data.landmark_mapper;
auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
auto morphable_model = reconstruction_data.morphable_model;
// The 3D head pose can be recovered as follows:
for (uint i = 0; i < keyframes.size(); ++i) {
Mat frame = keyframes[i].get()->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
auto landmarks = keyframes[i].get()->fitting_result.landmarks;
auto rendering_params = rendering_paramss[i];
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// and similarly for pitch and roll.
// Extract the texture from the image using given mesh and camera parameters:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
Mat isomap = render::extract_texture(
meshs[i],
affine_from_ortho,
frame,
true,
render::TextureInterpolation::NearestNeighbour,
512);
// Merge the isomaps - add the current one to the already merged ones:
merged_isomap = isomap_averaging.add_and_merge(isomap);
}
// gather csv output here
std::vector<std::string> output_csv;
size_t last_frame_index = keyframes.size() - 1;
auto keyframe = keyframes[last_frame_index].get();
Mat frame = keyframe->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframe->frame_number;
auto landmarks = keyframe->fitting_result.landmarks;
auto rendering_params = rendering_paramss[last_frame_index];
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
auto mesh = meshs[last_frame_index];
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
float reprojection_error;
float stddev;
std::tie(reprojection_error, stddev) = calc_reprojection_error(
landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
output_csv.push_back(std::to_string(keyframe->frame_number));
output_csv.push_back(std::to_string(keyframes.size()));
output_csv.push_back(std::to_string(reprojection_error));
output_csv.push_back(std::to_string(stddev));
output_csv.push_back(std::to_string(yaw_angle));
output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
output_csv.push_back(use_pose_binning ? "true" : "false");
output_csv.push_back("true");
output_csv.push_back(std::to_string(process_time));
output_csv.push_back(std::to_string(frame_width));
output_csv.push_back(std::to_string(frame_height));
output_csv.push_back(videofile);
output_csv.push_back(reconstruction_config);
if (show_video) {
Mat outimg = frame.clone();
std::cout << "fitted points size: " << fitted_image_points.size() << " -- " << landmarks.size() << std::endl;
// Draw the fitted mesh as wireframe, and save the image:
if (settings.get<bool>("output.landmarks", true))
{
for (auto &&f: fitted_image_points[last_frame_index]) {
cv::rectangle(
outimg, cv::Point2f(f[0] - 2.0f, f[1] - 2.0f),
cv::Point2f(f[0], f[1] + 2.0f), {255, 0, 0}
);
}
}
if (settings.get<bool>("output.wireframe", true)) {
draw_wireframe(
outimg,
mesh,
rendering_params.get_modelview(),
rendering_params.get_projection(),
fitting::get_opencv_viewport(frame_width, frame_height));
}
cv::imshow("Reconstruction", outimg);
cv::waitKey(20);
}
if (settings.get<bool>("output.build_obj", true)) {
// save the merged isomap:
std::string iter = "_" + std::to_string(n_iter);
fs::path path = (fs::path(reconstruction_config).parent_path()
/ fs::path(reconstruction_config).replace_extension("").filename()).string();
fs::create_directory(path);
std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
outputfile += use_pose_binning ? "_true" : "_false";
// sharpen isomap
Mat sharpen = isomap_averaging.sharpen(merged_isomap);
cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
// save the frontal rendering with merged isomap:
Mat frontal_rendering;
glm::mat4 modelview_frontal = glm::mat4(1.0);
std::tie(frontal_rendering, std::ignore) = render::render(
mesh,
modelview_frontal,
glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
512, 512,
render::create_mipmapped_texture(merged_isomap),
true,
false,
false
);
cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
output_csv.push_back(outputfile + iter + ".obj");
// Save the mesh as textured obj:
core::write_textured_obj(mesh, outputfile + iter + ".obj");
std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
<< outputfile + iter + ".obj" << std::endl;
}
// add all alpha's
for (auto &alpha: pca_shape_coefficients) {
output_csv.push_back(std::to_string(alpha));
}
std::cerr << array_to_string(output_csv) << std::endl;
}
/**
*
* @param keyframes
* @param rendering_paramss
* @param landmark_list
* @param morphable_model
* @param blendshapes
* @param meshs
* @param pca_shape_coefficients
* @param blendshape_coefficients
* @param fitted_image_points
* @param annotations
* @param landmark_mapper
* @param settings
* @param n_iter
*/
void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
std::vector<float> pca_shape_coefficients,
std::vector<std::vector<cv::Vec2f>> fitted_image_points,
std::vector<std::string> annotations,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings,
long process_time,
std::string videofile,
std::string reconstruction_config,
int n_iter) {
std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
Mat merged_isomap;
Mat isomap;
fs::path outputfilebase = reconstruction_config;
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
auto landmark_mapper = reconstruction_data.landmark_mapper;
auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
auto morphable_model = reconstruction_data.morphable_model;
bool show_video = settings.get<bool>("output.show_video", false);
// The 3D head pose can be recovered as follows:
for (uint i = 0; i < keyframes.size(); ++i) {
Mat frame = keyframes[i].get()->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframes[i].get()->frame_number;
auto landmarks = keyframes[i].get()->fitting_result.landmarks;
auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// and similarly for pitch and roll.
// Extract the texture from the image using given mesh and camera parameters:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
// Draw the loaded landmarks:
isomap = render::extract_texture(keyframes[i].get()->fitting_result.mesh, affine_from_ortho, frame);
// merge the isomaps:
merged_isomap = isomap_averaging.add_and_merge(isomap);
}
// gather csv output here
std::vector<std::string> output_csv;
size_t last_frame_index = keyframes.size() - 1;
auto keyframe = keyframes[last_frame_index].get();
Mat frame = keyframe->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframe->frame_number;
auto landmarks = keyframe->fitting_result.landmarks;
auto rendering_params = keyframe->fitting_result.rendering_parameters;
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
auto mesh = keyframes[last_frame_index].get()->fitting_result.mesh;
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
float reprojection_error;
float stddev;
// std::tie(reprojection_error, stddev) = calc_reprojection_error(
// landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
// output_csv.push_back(std::to_string(keyframe->frame_number));
// output_csv.push_back(std::to_string(keyframes.size()));
// output_csv.push_back(std::to_string(reprojection_error));
// output_csv.push_back(std::to_string(stddev));
// output_csv.push_back(std::to_string(yaw_angle));
// output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
// output_csv.push_back(use_pose_binning ? "true": "false");
// output_csv.push_back(std::to_string(process_time));
// output_csv.push_back(std::to_string(frame_width));
// output_csv.push_back(std::to_string(frame_height));
// output_csv.push_back(videofile);
// output_csv.push_back(reconstruction_config);
// for (auto &alpha: pca_shape_coefficients) {
// output_csv.push_back(std::to_string(alpha));
// }
//
// std::cerr << array_to_string(output_csv) << std::endl;
//
// if (show_video) {
// Mat outimg = frame.clone();
//
// // Draw the fitted mesh as wireframe, and save the image:
// if (settings.get<bool>("output.landmarks", true)) {
// for (auto &&lm : landmarks) {
// cv::rectangle(
// outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
// cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
// );
// }
// }
//
// if (settings.get<bool>("output.wireframe", true)) {
// draw_wireframe(
// outimg,
// mesh,
// rendering_params.get_modelview(),
// rendering_params.get_projection(),
// fitting::get_opencv_viewport(frame_width, frame_height));
// }
//
// cv::imshow("Reconstruction", outimg);
// cv::waitKey(20);
// }
//
// if (settings.get<bool>("output.build_obj", true)) {
// // save the merged isomap:
// std::string iter = "_" + std::to_string(n_iter);
// fs::path path = (fs::path(reconstruction_config).parent_path() / fs::path(reconstruction_config).replace_extension("").filename()).string();
// fs::create_directory(path);
// std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
//
// outputfile += use_pose_binning ? "_true" : "_false";
//
// // sharpen isomap
// Mat sharpen = isomap_averaging.sharpen(merged_isomap);
// cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
//
// // save the frontal rendering with merged isomap:
// Mat frontal_rendering;
// glm::mat4 modelview_frontal = glm::mat4(1.0);
//
// std::tie(frontal_rendering, std::ignore) = render::render(
// mesh,
// modelview_frontal,
// glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
// 512, 512,
// render::create_mipmapped_texture(merged_isomap),
// true,
// false,
// false
// );
//
// cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
//
// // Save the mesh as textured obj:
// core::write_textured_obj(mesh, outputfile + iter + ".obj");
// std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
// << outputfile + iter + ".obj" << std::endl;
// }
}
/**
* Parse config file
* @param filename
*/
boost::property_tree::ptree get_reconstruction_config(std::string filename) {
boost::property_tree::ptree pt;
boost::property_tree::ini_parser::read_ini(filename, pt);
return pt;
}
/**
*
* Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
* are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
* causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
* annotations with the based on a given keyframe list.
*
* @param keyframes
* @param landmarks
* @return
*/
core::LandmarkCollection<cv::Vec2f> add_noise(core::LandmarkCollection<cv::Vec2f> landmarks, int noise_x, int noise_y) {
std::uniform_real_distribution<> dis_x(0, noise_x);
std::uniform_real_distribution<> dis_y(0, noise_y);
core::LandmarkCollection<cv::Vec2f> sublist;
vector<cv::Vec2f> noise;
for (auto &lm : landmarks) {
cv::Vec2f offset(static_cast<float>(dis_x(gen)), dis_y(gen));
lm.coordinates += offset;
noise.push_back(offset);
sublist.push_back(lm);
}
return sublist; } /**
*
* Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
* are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
* causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
* annotations with the based on a given keyframe list.
*
* @param keyframes
* @param landmarks
* @return
*/
vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks, int noise_x, int noise_y) {
vector<core::LandmarkCollection<cv::Vec2f>> sublist;
for (auto& f : keyframes) {
auto orig_landmarks = landmarks[f.get()->frame_number];
auto noisy_landmarks = add_noise(orig_landmarks, noise_x, noise_y);
sublist.push_back(noisy_landmarks);
}
return sublist;
}
/**
* This app demonstrates estimation of the camera and fitting of the shape
* model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
* In addition to fit-model-simple, this example uses blendshapes, contour-
* fitting, and can iterate the fitting.
*
* 68 ibug landmarks are loaded from the .pts file and converted
* to vertex indices using the LandmarkMapper.
*/
int main(int argc, char *argv[])
{
fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile,
blendshapesfile, outputfile, reconstruction_config;
std::vector<std::string> annotations;
// get annotations from one file
bool get_annotations = false;
try
{
po::options_description desc("Allowed options");
desc.add_options()
("help,h",
"display the help message")
("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
"a Morphable Model stored as cereal BinaryArchive")
("video,v", po::value<fs::path>(&videofile)->required(),
"an input image")
("config,f",
po::value<fs::path>(&reconstruction_config)->default_value("../share/default_reconstruction_config.ini"),
"configuration file for the reconstruction")
("get_annotations,g", po::bool_switch(&get_annotations)->default_value(false),
"read .pts annotation file locations from one file, put one file path on each line")
("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
".pts annotation files per frame of video")
("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
"2D landmarks for the image, in ibug .pts format")
("model-contour,c",
po::value<fs::path>(&contourfile)->required()->default_value("../share/model_contours.json"),
"file with model contour indices")
("edge-topology,e", po::value<fs::path>(&edgetopologyfile)->required()->default_value(
"../share/sfm_3448_edge_topology.json"),
"file with model's precomputed edge topology")
("blendshapes,b", po::value<fs::path>(&blendshapesfile)->required()->default_value(
"../share/expression_blendshapes_3448.bin"),
"file with blendshapes")
("output,o", po::value<fs::path>(&outputfile)->required()->default_value("out"),
"basename for the output rendering and obj files");
po::variables_map vm;
po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);
if (vm.count("help"))
{
std::cout << "Usage: fit-model [options]" << std::endl;
std::cout << desc;
return EXIT_SUCCESS;
}
po::notify(vm);
}
catch (const po::error &e)
{
std::cout << "Error while parsing command-line arguments: " << e.what() << std::endl;
std::cout << "Use --help to display a list of options." << std::endl;
return EXIT_SUCCESS;
}
// Start loading prerequisites:
morphablemodel::MorphableModel morphable_model;
std::vector<float> pca_shape_coefficients;
// List of blendshapes coefficients:
std::vector<std::vector<float>> blendshape_coefficients;
std::vector<std::vector<cv::Vec2f>> fitted_image_points;
// These will be the final 2D and 3D points used for the fitting:
vector<cv::Vec3f> model_points; // the points in the 3D shape model
vector<int> vertex_indices; // their vertex indices
vector<core::Mesh> meshs;
vector<fitting::RenderingParameters> rendering_paramss;
// Load landmarks, LandmarkMapper and the Morphable Model:
auto landmark_mapper = core::LandmarkMapper(mappingsfile);
// Load all annotation files into lists of landmarks:
vector<core::LandmarkCollection<cv::Vec2f>> landmark_list;
try {
morphable_model = morphablemodel::load_model(modelfile.string());
}
catch (const std::runtime_error &e) {
std::cout << "Error loading the Morphable Model: " << e.what() << std::endl;
return EXIT_FAILURE;
}
// Load all annotation using input parameters:
try {
std::tie(landmark_list, annotations) = eos::core::load_annotations<cv::Vec2f>(
annotations,
landmark_mapper,
morphable_model,
get_annotations);
}
catch (const std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
}
// The expression blendshapes:
auto blendshapes = morphablemodel::load_blendshapes(blendshapesfile.string());
// These two are used to fit the front-facing contour to the ibug contour landmarks:
auto model_contour =
contourfile.empty() ? fitting::ModelContour() : fitting::ModelContour::load(contourfile.string());
auto ibug_contour = fitting::ContourLandmarks::load(mappingsfile.string());
// The edge topology is used to speed up computation of the occluding face contour fitting:
auto edge_topology = morphablemodel::load_edge_topology(edgetopologyfile.string());
// Read .ini file to accommodate.
auto settings = get_reconstruction_config(reconstruction_config.string());
auto reconstruction_data = eos::fitting::ReconstructionData{
morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
float threshold = settings.get<float>("frames.threshold", 0.1);
bool evaluate = settings.get<bool>("eval.evaluate", true);
int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
int max_frames = settings.get<int>("eval.max_frames", 0);
int drop_frames = settings.get<int>("frames.drop_frames", 0);
int noise_x = settings.get<int>("reconstruction.noise_x", 0);
int noise_y = settings.get<int>("reconstruction.noise_y", 0);
// Start with the video play and get video file:
BufferedVideoIterator vid_iterator;
try {
vid_iterator = BufferedVideoIterator(videofile.string(), reconstruction_data, settings);
} catch (std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
}
// TODO: determine if this following one should only be used in other programs:
// Count the amount of iterations:
int n_iter = 0;
int frames_dropped = 0;
std::vector<std::shared_ptr<Keyframe>> keyframes;
std::uniform_real_distribution<> keyframe_add_random(0, 1);
// global random generator with given seed
std::random_device rd;
std::mt19937 local_gen(rd());
// Get frames randomly until video is done playing OR we've reached max_frames
while (vid_iterator.is_playing()) {
bool new_frame = vid_iterator.next();
if (new_frame == false) {
break;
}
double add_frame = keyframe_add_random(local_gen);
if (add_frame < threshold) {
keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
if (keyframes.size() >= max_frames) {
break;
}
}
n_iter++;
}
vid_iterator.__stop();
// Generate a sublist of the total available landmark list:
auto landmark_sublist = sample_landmarks(keyframes, landmark_list, noise_x, noise_y);
std::cout << "Going to reconstruct with " << keyframes.size() <<
" images, " << num_iterations << " iterations: " << std::endl;
// Fit shape and pose:
auto t1 = std::chrono::high_resolution_clock::now();
std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
morphable_model,
blendshapes,
landmark_sublist,
landmark_mapper,
keyframes[0].get()->frame.cols,
keyframes[0].get()->frame.rows,
static_cast<int>(keyframes.size()),
edge_topology,
ibug_contour,
model_contour,
num_iterations,
boost::none,
30.0f,
boost::none,
pca_shape_coefficients,
blendshape_coefficients,
fitted_image_points
);
std::cout << vid_iterator.to_string() << std::endl;
auto t2 = std::chrono::high_resolution_clock::now();
std::cout << "Reconstruction took "
<< std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count()
<< "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count()
/ (keyframes.size() * num_iterations)
<< "ms)" << std::endl;
if (settings.get<bool>("eval.evaluate", false)) {
eval(
keyframes,
meshs,
rendering_paramss,
pca_shape_coefficients,
fitted_image_points,
annotations,
reconstruction_data,
settings,
std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count(),
videofile.string(),
reconstruction_config.string(),
n_iter
);
}
return EXIT_SUCCESS;
}
utils/accuracy-evaluation.cpp
View file @ fcae2025
......@@ -61,6 +61,9 @@ using namespace cv;
typedef unsigned int uint;
// global random generator with given seed
std::mt19937 gen(1);
/**
* Draws the given mesh as wireframe into the image.
*
......@@ -107,62 +110,6 @@ inline float euclidean_distance(cv::Vec2f landmark, cv::Mat vertex_screen_coords
return std::sqrt(landmark_diff_x_sq + landmark_diff_y_sq);
}
/**
*
* @param landmarks
* @param landmark_mapper
* @param mesh
* @param affine_from_ortho
*/
void evaluate_accuracy(
core::LandmarkCollection<cv::Vec2f> landmarks,
core::LandmarkMapper landmark_mapper,
const eos::core::Mesh& mesh,
Mat affine_from_ortho,
boost::property_tree::ptree settings) {
vector<float> total_error;
// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
for (uint i = 0; i < landmarks.size(); ++i) {
auto converted_name = landmark_mapper.convert(landmarks[i].name);
// no mapping defined for the current landmark
if (!converted_name) {
continue;
}
int vertex_idx = std::stoi(converted_name.get());
// The vertex_idx should be around the value of the original coordinates after we have
// projected it with the affine_from_ortho that is obtained earlier.
Mat vertex_screen_coords = affine_from_ortho *
Mat(cv::Vec4f(
mesh.vertices[vertex_idx].x,
mesh.vertices[vertex_idx].y,
mesh.vertices[vertex_idx].z,
mesh.vertices[vertex_idx].w
)
);
// using euclidean distance here, but should look at other ways too.
float dist = euclidean_distance(landmarks[i].coordinates, vertex_screen_coords);
total_error.push_back(dist);
}
// Calculate mean error and stddev.
float accum = 0.0;
float mean_error = std::accumulate(total_error.begin(), total_error.end(), 0) / landmarks.size();
for(auto &d: total_error) {
accum += (d - mean_error) * (d - mean_error);
}
float stddev = std::sqrt(accum / (total_error.size() - 1));
std::cout << "stddev/mean_error: " << stddev << " " << mean_error << std::endl;
}
/**
*
* @param keyframes
......@@ -325,6 +272,75 @@ void render_output(
}
/**
*
* @param landmarks
* @param landmark_mapper
* @param mesh
* @param affine_from_ortho
*/
std::pair<float, float> calc_reprojection_error(
core::LandmarkCollection<cv::Vec2f> landmarks,
core::LandmarkMapper landmark_mapper,
eos::core::Mesh& mesh,
Mat affine_from_ortho,
boost::property_tree::ptree settings) {
vector<float> total_error;
// Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
for (uint i = 0; i < landmarks.size(); ++i) {
auto converted_name = landmark_mapper.convert(landmarks[i].name);
// no mapping defined for the current landmark
if (!converted_name) {
continue;
}
int vertex_idx = std::stoi(converted_name.get());
// The vertex_idx should be around the value of the original coordinates after we have
// projected it with the affine_from_ortho that is obtained earlier.
Mat vertex_screen_coords = affine_from_ortho *
Mat(cv::Vec4f(
mesh.vertices[vertex_idx].x,
mesh.vertices[vertex_idx].y,
mesh.vertices[vertex_idx].z,
mesh.vertices[vertex_idx].w
)
);
// using euclidean distance here, but should look at other ways too.
float dist = euclidean_distance(landmarks[i].coordinates, vertex_screen_coords);
total_error.push_back(dist);
}
// Calculate mean error and stddev.
float accum = 0.0;
float mean_error = static_cast<float>(std::accumulate(total_error.begin(), total_error.end(), 0)) / landmarks.size();
for(auto &d: total_error) {
accum += (d - mean_error) * (d - mean_error);
}
float stddev = std::sqrt(accum / (total_error.size() - 1));
return std::make_pair(mean_error, stddev);
}
inline std::string array_to_string(std::vector<std::string> input) {
std::string output;
for(int i = 0; i < input.size(); i++) {
output += input[i] + ((i < input.size() - 1) ? "," : "");
}
return output;
}
/**
*
*
* @param keyframes
* @param rendering_paramss
......@@ -340,27 +356,30 @@ void render_output(
* @param settings
* @param n_iter
*/
void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
std::vector<fitting::RenderingParameters> rendering_paramss,
vector<core::Mesh> meshs,
std::vector<float> pca_shape_coefficients,
std::vector<std::vector<float>> blendshape_coefficients,
std::vector<std::vector<cv::Vec2f>> fitted_image_points,
std::vector<std::string> annotations,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings,
int n_iter) {
void eval(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
vector<core::Mesh> meshs,
std::vector<fitting::RenderingParameters> rendering_paramss,
std::vector<float> pca_shape_coefficients,
std::vector<std::vector<cv::Vec2f>> fitted_image_points,
std::vector<std::string> annotations,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings,
long process_time,
std::string videofile,
std::string reconstruction_config,
int n_iter) {
std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
float merge_isomap_face_angle = 90.0; //settings.get<float>("output.merge_isomap_face_angle", 60.f);
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
bool show_video = settings.get<bool>("output.show_video", false);
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
Mat merged_isomap;
fs::path outputfilebase = annotations[0];
fs::path outputfilebase = reconstruction_config;
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
auto landmark_mapper = reconstruction_data.landmark_mapper;
auto blendshapes = reconstruction_data.blendshapes;
auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
auto morphable_model = reconstruction_data.morphable_model;
// The 3D head pose can be recovered as follows:
......@@ -370,76 +389,312 @@ void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyfram
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframes[i].get()->frame_number;
auto landmarks = keyframes[i].get()->fitting_result.landmarks;
auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
auto rendering_params = rendering_paramss[i];
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
Mat outimg = frame.clone();
// and similarly for pitch and roll.
// Extract the texture from the image using given mesh and camera parameters:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
rendering_params, frame_width, frame_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
Mat isomap = render::extract_texture(
meshs[i],
affine_from_ortho,
frame,
true,
render::TextureInterpolation::NearestNeighbour,
512);
Mat outimg = frame.clone();
for (auto &&lm : landmarks) {
// Merge the isomaps - add the current one to the already merged ones:
merged_isomap = isomap_averaging.add_and_merge(isomap);
for (auto &&f: fitted_image_points[i]) {
cv::rectangle(
outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
outimg, cv::Point2f(f[0] - 2.0f, f[1] - 2.0f),
cv::Point2f(f[0], f[1] + 2.0f), {255, 0, 0}
);
}
// Draw the fitted mesh as wireframe, and save the image:
draw_wireframe(
outimg,
meshs[i],
rendering_params.get_modelview(),
rendering_params.get_projection(),
fitting::get_opencv_viewport(frame_width, frame_height));
std::string iter = std::to_string(n_iter) + "_" + std::to_string(yaw_angle);
cv::imwrite("/tmp/video/" + iter + ".png", outimg);
}
// gather csv output here
std::vector<std::string> output_csv;
size_t last_frame_index = keyframes.size() - 1;
auto keyframe = keyframes[last_frame_index].get();
Mat frame = keyframe->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframe->frame_number;
auto landmarks = keyframe->fitting_result.landmarks;
auto rendering_params = rendering_paramss[last_frame_index];
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
auto mesh = meshs[last_frame_index];
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
float reprojection_error;
float stddev;
std::tie(reprojection_error, stddev) = calc_reprojection_error(
landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
output_csv.push_back(std::to_string(keyframe->frame_number));
output_csv.push_back(std::to_string(keyframes.size()));
output_csv.push_back(std::to_string(reprojection_error));
output_csv.push_back(std::to_string(stddev));
output_csv.push_back(std::to_string(yaw_angle));
output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
output_csv.push_back(use_pose_binning ? "true" : "false");
output_csv.push_back("false");
output_csv.push_back(std::to_string(process_time));
output_csv.push_back(std::to_string(frame_width));
output_csv.push_back(std::to_string(frame_height));
output_csv.push_back(videofile);
output_csv.push_back(reconstruction_config);
if (show_video) {
Mat outimg = frame.clone();
std::cout << "fitted points size: " << fitted_image_points.size() << " -- " << landmarks.size() << std::endl;
// Draw the fitted mesh as wireframe, and save the image:
if (settings.get<bool>("output.landmarks", true))
{
for (auto &&f: fitted_image_points[last_frame_index]) {
cv::rectangle(
outimg, cv::Point2f(f[0] - 2.0f, f[1] - 2.0f),
cv::Point2f(f[0], f[1] + 2.0f), {255, 0, 0}
);
}
}
if (settings.get<bool>("output.wireframe", true)) {
draw_wireframe(
outimg,
keyframes[i].get()->fitting_result.mesh,
mesh,
rendering_params.get_modelview(),
rendering_params.get_projection(),
fitting::get_opencv_viewport(frame_width, frame_height));
}
cv::imshow("Reconstruction", outimg);
cv::waitKey(0);
}
if (settings.get<bool>("output.build_obj", true)) {
// save the merged isomap:
std::string iter = "_" + std::to_string(n_iter);
fs::path path = (fs::path(reconstruction_config).parent_path()
/ fs::path(reconstruction_config).replace_extension("").filename()).string();
fs::create_directory(path);
std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
outputfile += use_pose_binning ? "_true" : "_false";
// sharpen isomap
Mat sharpen = isomap_averaging.sharpen(merged_isomap);
cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
// save the frontal rendering with merged isomap:
Mat frontal_rendering;
glm::mat4 modelview_frontal = glm::mat4(1.0);
std::tie(frontal_rendering, std::ignore) = render::render(
mesh,
modelview_frontal,
glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
512, 512,
render::create_mipmapped_texture(merged_isomap),
true,
false,
false
);
cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
output_csv.push_back(outputfile + iter + ".obj");
cv::imshow("Img", outimg);
cv::waitKey(20);
// Save the mesh as textured obj:
core::write_textured_obj(mesh, outputfile + iter + ".obj");
std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
<< outputfile + iter + ".obj" << std::endl;
}
// add all alpha's
for (auto &alpha: pca_shape_coefficients) {
output_csv.push_back(std::to_string(alpha));
}
std::cerr << array_to_string(output_csv) << std::endl;
}
/**
*
* @param keyframes
* @param rendering_paramss
* @param landmark_list
* @param morphable_model
* @param blendshapes
* @param meshs
* @param pca_shape_coefficients
* @param blendshape_coefficients
* @param fitted_image_points
* @param annotations
* @param landmark_mapper
* @param settings
* @param n_iter
*/
void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes,
std::vector<float> pca_shape_coefficients,
std::vector<std::vector<cv::Vec2f>> fitted_image_points,
std::vector<std::string> annotations,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings,
long process_time,
std::string videofile,
std::string reconstruction_config,
int n_iter) {
std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
Mat merged_isomap;
Mat isomap;
fs::path outputfilebase = reconstruction_config;
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
auto landmark_mapper = reconstruction_data.landmark_mapper;
auto blendshapes = morphablemodel::to_matrix(reconstruction_data.blendshapes);
auto morphable_model = reconstruction_data.morphable_model;
bool show_video = settings.get<bool>("output.show_video", false);
// The 3D head pose can be recovered as follows:
for (uint i = 0; i < keyframes.size(); ++i) {
Mat frame = keyframes[i].get()->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframes[i].get()->frame_number;
auto landmarks = keyframes[i].get()->fitting_result.landmarks;
auto rendering_params = keyframes[i].get()->fitting_result.rendering_parameters;
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// and similarly for pitch and roll.
// Extract the texture from the image using given mesh and camera parameters:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
// Draw the loaded landmarks:
Mat isomap = render::extract_texture(meshs[i], affine_from_ortho, frame);
isomap = render::extract_texture(keyframes[i].get()->fitting_result.mesh, affine_from_ortho, frame);
// merge the isomaps:
merged_isomap = isomap_averaging.add_and_merge(isomap);
evaluate_accuracy(
landmarks,
landmark_mapper,
meshs[i],
affine_from_ortho,
settings
);
}
// save the merged isomap:
// std::string iter = "_" + std::to_string(n_iter);
// fs::path path = (fs::path(annotations[n_iter]).parent_path() / "eval");
// fs::create_directory(path);
// std::string outputfile = (path / fs::path(annotations[n_iter]).replace_extension("").filename()).string();
// gather csv output here
std::vector<std::string> output_csv;
size_t last_frame_index = keyframes.size() - 1;
auto keyframe = keyframes[last_frame_index].get();
Mat frame = keyframe->frame;
int frame_width = frame.cols;
int frame_height = frame.rows;
int frame_number = keyframe->frame_number;
auto landmarks = keyframe->fitting_result.landmarks;
auto rendering_params = keyframe->fitting_result.rendering_parameters;
auto yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
auto mesh = keyframes[last_frame_index].get()->fitting_result.mesh;
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
float reprojection_error;
float stddev;
// std::tie(reprojection_error, stddev) = calc_reprojection_error(
// landmarks, landmark_mapper, mesh, affine_from_ortho, settings);
// output_csv.push_back(std::to_string(keyframe->frame_number));
// output_csv.push_back(std::to_string(keyframes.size()));
// output_csv.push_back(std::to_string(reprojection_error));
// output_csv.push_back(std::to_string(stddev));
// output_csv.push_back(std::to_string(yaw_angle));
// output_csv.push_back(std::to_string(settings.get<int>("frames.frames_per_bin")));
// output_csv.push_back(use_pose_binning ? "true": "false");
// output_csv.push_back(std::to_string(process_time));
// output_csv.push_back(std::to_string(frame_width));
// output_csv.push_back(std::to_string(frame_height));
// output_csv.push_back(videofile);
// output_csv.push_back(reconstruction_config);
// for (auto &alpha: pca_shape_coefficients) {
// output_csv.push_back(std::to_string(alpha));
// }
//
// // sharpen isomap
// Mat sharpen = isomap_averaging.sharpen(merged_isomap);
// std::cerr << array_to_string(output_csv) << std::endl;
//
// cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
// if (show_video) {
// Mat outimg = frame.clone();
//
// // save the frontal rendering with merged isomap:
// Mat frontal_rendering;
// glm::mat4 modelview_frontal = glm::mat4( 1.0 );
// core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
// morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
// morphable_model.get_color_model().get_mean(),
// morphable_model.get_shape_model().get_triangle_list(),
// morphable_model.get_color_model().get_triangle_list(),
// morphable_model.get_texture_coordinates()
// );
// std::tie(frontal_rendering, std::ignore) = render::render(
// neutral_expression,
// // Draw the fitted mesh as wireframe, and save the image:
// if (settings.get<bool>("output.landmarks", true)) {
// for (auto &&lm : landmarks) {
// cv::rectangle(
// outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
// cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
// );
// }
// }
//
// if (settings.get<bool>("output.wireframe", true)) {
// draw_wireframe(
// outimg,
// mesh,
// rendering_params.get_modelview(),
// rendering_params.get_projection(),
// fitting::get_opencv_viewport(frame_width, frame_height));
// }
//
// cv::imshow("Reconstruction", outimg);
// cv::waitKey(20);
// }
//
// if (settings.get<bool>("output.build_obj", true)) {
// // save the merged isomap:
// std::string iter = "_" + std::to_string(n_iter);
// fs::path path = (fs::path(reconstruction_config).parent_path() / fs::path(reconstruction_config).replace_extension("").filename()).string();
// fs::create_directory(path);
// std::string outputfile = (path / fs::path(reconstruction_config).replace_extension("").filename()).string();
//
// outputfile += use_pose_binning ? "_true" : "_false";
//
// // sharpen isomap
// Mat sharpen = isomap_averaging.sharpen(merged_isomap);
// cv::imwrite(outputfile + iter + ".isomap.png", sharpen);
//
// // save the frontal rendering with merged isomap:
// Mat frontal_rendering;
// glm::mat4 modelview_frontal = glm::mat4(1.0);
//
// std::tie(frontal_rendering, std::ignore) = render::render(
// mesh,
// modelview_frontal,
// glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
// 512, 512,
......@@ -447,12 +702,15 @@ void evaluate_results(std::vector<std::shared_ptr<eos::video::Keyframe>> keyfram
// true,
// false,
// false
// );
// cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
// Save the mesh as textured obj:
// core::write_textured_obj(neutral_expression, outputfile + iter + ".obj");
// std::cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile + iter + ".obj" << std::endl;
// );
//
// cv::imwrite(outputfile + iter + ".merged_frontal.png", frontal_rendering);
//
// // Save the mesh as textured obj:
// core::write_textured_obj(mesh, outputfile + iter + ".obj");
// std::cout << "Finished fitting and wrote result mesh and isomap to files with basename "
// << outputfile + iter + ".obj" << std::endl;
// }
}
/**
......@@ -477,13 +735,41 @@ boost::property_tree::ptree get_reconstruction_config(std::string filename) {
* @param landmarks
* @return
*/
vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
core::LandmarkCollection<cv::Vec2f> add_noise(core::LandmarkCollection<cv::Vec2f> landmarks, int noise_x, int noise_y) {
std::uniform_real_distribution<> dis_x(0, noise_x);
std::uniform_real_distribution<> dis_y(0, noise_y);
core::LandmarkCollection<cv::Vec2f> sublist;
vector<cv::Vec2f> noise;
for (auto &lm : landmarks) {
cv::Vec2f offset(static_cast<float>(dis_x(gen)), dis_y(gen));
lm.coordinates += offset;
noise.push_back(offset);
sublist.push_back(lm);
}
return sublist; } /**
*
* Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
* are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
* causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
* annotations with the based on a given keyframe list.
*
* @param keyframes
* @param landmarks
* @return
*/
vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(
std::vector<std::shared_ptr<eos::video::Keyframe>> keyframes, vector<core::LandmarkCollection<cv::Vec2f>> landmarks, int noise_x, int noise_y) {
vector<core::LandmarkCollection<cv::Vec2f>> sublist;
for (auto& f : keyframes) {
sublist.push_back(landmarks[f.get()->frame_number]);
auto orig_landmarks = landmarks[f.get()->frame_number];
auto noisy_landmarks = add_noise(orig_landmarks, noise_x, noise_y);
sublist.push_back(noisy_landmarks);
}
return sublist;
......@@ -600,6 +886,14 @@ int main(int argc, char *argv[]) {
auto reconstruction_data = eos::fitting::ReconstructionData{
morphable_model, blendshapes, landmark_mapper, landmark_list, model_contour, ibug_contour, edge_topology};
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
bool evaluate = settings.get<bool>("eval.evaluate", true);
int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
int max_frames = settings.get<int>("eval.max_frames", 0);
int drop_frames = settings.get<int>("frames.drop_frames", 0);
int min_frames = settings.get<int>("frames.min_frames", 0);
int noise_x = settings.get<int>("reconstruction.noise_x", 0);
int noise_y = settings.get<int>("reconstruction.noise_y", 0);
// Start with the video play and get video file:
BufferedVideoIterator vid_iterator;
......@@ -611,52 +905,75 @@ int main(int argc, char *argv[]) {
return EXIT_FAILURE;
}
// TODO: determine if this following one should only be used in other programs:
// vid_iterator.start();
//ReconstructionVideoWriter vid_writer;
//try {
// vid_writer = ReconstructionVideoWriter(videofile.string(), reconstruction_data, settings);
//} catch(std::runtime_error &e) {
// std::cout << e.what() << std::endl;
// return EXIT_FAILURE;
//}
// vid_writer.start();
// Start getting video frames:
bool use_pose_binning = settings.get<bool>("frames.use_pose_binning", true);
if (use_pose_binning) {
vid_iterator.start();
}
ReconstructionVideoWriter vid_writer;
try {
vid_writer = ReconstructionVideoWriter(videofile.string(), reconstruction_data, settings);
} catch(std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
}
int num_iterations = settings.get<int>("reconstruction.num_iterations", 10);
// vid_writer.start();
// Count the amount of iterations:
int n_iter = 0;
while(vid_iterator.is_playing() || n_iter < 20) {
// if (use_pose_binning == false && (max_frames > 0 && n_iter > max_frames)) {
// break;
//
int frames_dropped = 0;
while(vid_iterator.is_playing()) {
std::cout << vid_iterator.to_string() << std::endl;
std::vector<std::shared_ptr<Keyframe>> keyframes;
std::cout << "Try to reconstruct: " << vid_iterator.get_last_keyframe().frame_number << "/" << max_frames << std::endl;
if (vid_iterator.get_last_keyframe().frame_number > max_frames) {
vid_iterator.__stop();
break;
}
if (use_pose_binning) {
bool new_frame = vid_iterator.next();
if (new_frame == false) {
vid_iterator.__stop();
}
keyframes = vid_iterator.get_keyframes();
std::cout << "pose binning size:" << keyframes.size() << std::endl;
// keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
} else {
vid_iterator.next();
bool new_frame = vid_iterator.next();
if (new_frame == false) {
vid_iterator.__stop();
}
keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
}
if (drop_frames > 0 && (frames_dropped < drop_frames) && n_iter > 20) {
std::cout << "frames_dropped: " << frames_dropped << ", " << drop_frames << std::endl;
frames_dropped++;
continue;
}
// Generate a sublist of the total available landmark list:
auto landmark_sublist = sample_landmarks(keyframes, landmark_list);
frames_dropped = 0;
// it makes no sense to update pca_coeff if nothing in the buffer has changed:
if (vid_iterator.has_changed() || use_pose_binning == false) {
std::cout << "Going to reconstruct with " << keyframes.size() << " images."<< num_iterations << std::endl;
if (vid_iterator.has_changed()) {
// Generate a sublist of the total available landmark list:
auto landmark_sublist = sample_landmarks(keyframes, landmark_list, noise_x, noise_y);
std::cout << "Going to reconstruct with " << keyframes.size() << " images, "<< num_iterations << " iterations:"<< std::endl;
// Fit shape and pose:
auto t1 = std::chrono::high_resolution_clock::now();
std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
morphable_model,
blendshapes,
keyframes,
landmark_sublist,
landmark_mapper,
keyframes[0].get()->frame.cols,
keyframes[0].get()->frame.rows,
......@@ -670,9 +987,28 @@ int main(int argc, char *argv[]) {
boost::none,
pca_shape_coefficients,
blendshape_coefficients,
fitted_image_points,
settings
fitted_image_points
);
// std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
// morphable_model,
// blendshapes,
// keyframes,
// landmark_mapper,
// keyframes[0].get()->frame.cols,
// keyframes[0].get()->frame.rows,
// static_cast<int>(keyframes.size()),
// edge_topology,
// ibug_contour,
// model_contour,
// num_iterations,
// boost::none,
// 30.0f,
// boost::none,
// pca_shape_coefficients,
// blendshape_coefficients,
// fitted_image_points,
// settings
// );
std::cout << vid_iterator.to_string() << std::endl;
......@@ -681,71 +1017,95 @@ int main(int argc, char *argv[]) {
<< std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count()
<< "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() / (keyframes.size() * num_iterations)
<< "ms)" << std::endl;
//
// if(settings.get<bool>("output.make_video", false)) {
// vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
// vid_writer.next();
// }
evaluate_results(
keyframes,
rendering_paramss,
meshs,
pca_shape_coefficients,
blendshape_coefficients,
fitted_image_points,
annotations,
reconstruction_data,
settings,
n_iter
);
//if(settings.get<bool>("output.make_video", false)) {
// vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
// vid_writer.next();
//}
if(settings.get<bool>("eval.evaluate", false)) {
eval(
keyframes,
meshs,
rendering_paramss,
pca_shape_coefficients,
fitted_image_points,
annotations,
reconstruction_data,
settings,
std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count(),
videofile.string(),
reconstruction_config.string(),
n_iter
);
// evaluate_results(
// keyframes,
// pca_shape_coefficients,
// fitted_image_points,
// annotations,
// reconstruction_data,
// settings,
// std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count(),
// videofile.string(),
// reconstruction_config.string(),
// n_iter
// );
}
} else {
std::cout << vid_iterator.to_string() << std::endl;
std::cout << "No reconstruction - buffer did not change" << std::endl;
}
std::cout << n_iter << std::endl;
// Get new frames:
// only count after a reconstruction
n_iter++;
}
std::cout << "stopping?" << std::endl;
vid_writer.__stop();
// if(settings.get<bool>("output.make_video", false)) {
// // Render output:
// std::cout << "Waiting for video to be completed..." << std::endl;
// vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
//
// int count = 0;
// while (count < 40) {
// auto t1 = std::chrono::high_resolution_clock::now();
// vid_writer.next();
// auto t2 = std::chrono::high_resolution_clock::now();
// printf("Frame %d/%d (%d)\n", vid_writer.get_frame_number(), vid_iterator.get_frame_number(), count);
//
// std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() << "ms" << std::endl;
// count++;
// }
//if(settings.get<bool>("output.make_video", false)) {
// // Render output:
// std::cout << "Waiting for video to be completed..." << std::endl;
// vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
// int count = 0;
// while (count < 40) {
// auto t1 = std::chrono::high_resolution_clock::now();
// vid_writer.next();
// auto t2 = std::chrono::high_resolution_clock::now();
// printf("Frame %d/%d (%d)\n", vid_writer.get_frame_number(), vid_iterator.get_frame_number(), count);
// std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() << "ms" << std::endl;
// count++;
// }
//
// vid_writer.__stop();
// }
// auto keyframes = vid_iterator.get_keyframes();
// if(settings.get<bool>("eval.evaluate", false)) {
// std::vector<std::shared_ptr<Keyframe>> keyframes;
//
// if (use_pose_binning) {
// keyframes = vid_iterator.get_keyframes();
// keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
// } else {
// vid_iterator.next();
// keyframes.push_back(std::make_shared<Keyframe>(vid_iterator.get_last_keyframe()));
// }
//
// evaluate_results(
// keyframes,
// rendering_paramss,
// meshs,
// pca_shape_coefficients,
// blendshape_coefficients,
// fitted_image_points,
// annotations,
// reconstruction_data,
// settings,
// n_iter
// );
// evaluate_results(
// keyframes,
// pca_shape_coefficients,
// fitted_image_points,
// annotations,
// reconstruction_data,
// settings,
// 0,
// videofile.string(),
// reconstruction_config.string(),
// n_iter
// );
// }
//todo: we could build our final obj here?
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment