Commit c5628500 authored by Richard Torenvliet's avatar Richard Torenvliet

Test with OpenMP GPU - added for occluding boundary vertices

parent e861a303
...@@ -113,6 +113,7 @@ set(HEADERS ...@@ -113,6 +113,7 @@ set(HEADERS
${CMAKE_CURRENT_SOURCE_DIR}/include/eos/video/Keyframe.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/eos/video/Keyframe.hpp
) )
# Set OpenMP compiler flags if possible
find_package(OpenMP) find_package(OpenMP)
if (OPENMP_FOUND) if (OPENMP_FOUND)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
......
...@@ -42,9 +42,10 @@ namespace eos { ...@@ -42,9 +42,10 @@ namespace eos {
struct FittingResult struct FittingResult
{ {
RenderingParameters rendering_parameters; RenderingParameters rendering_parameters;
core::LandmarkCollection<cv::Vec2f> landmarks;
std::vector<float> pca_shape_coefficients; std::vector<float> pca_shape_coefficients;
std::vector<float> blendshape_coefficients; std::vector<float> blendshape_coefficients;
core::LandmarkCollection<cv::Vec2f> landmarks;
core::Mesh mesh;
}; };
} /* namespace fitting */ } /* namespace fitting */
......
...@@ -45,6 +45,7 @@ ...@@ -45,6 +45,7 @@
#include <utility> #include <utility>
#include <cstddef> #include <cstddef>
namespace eos { namespace eos {
namespace fitting { namespace fitting {
...@@ -391,7 +392,11 @@ inline std::pair<std::vector<cv::Vec2f>, std::vector<int>> find_occluding_edge_c ...@@ -391,7 +392,11 @@ inline std::pair<std::vector<cv::Vec2f>, std::vector<int>> find_occluding_edge_c
} }
return { image_points, vertex_indices }; return { image_points, vertex_indices };
}; };
#ifdef _OPENMP
/** /**
* OpenMP version
*
* @brief Computes the vertices that lie on occluding boundaries, given a particular pose. * @brief Computes the vertices that lie on occluding boundaries, given a particular pose.
* *
* This algorithm computes the edges that lie on occluding boundaries of the mesh. * This algorithm computes the edges that lie on occluding boundaries of the mesh.
...@@ -404,7 +409,7 @@ inline std::pair<std::vector<cv::Vec2f>, std::vector<int>> find_occluding_edge_c ...@@ -404,7 +409,7 @@ inline std::pair<std::vector<cv::Vec2f>, std::vector<int>> find_occluding_edge_c
* @param[in] R The rotation (pose) under which the occluding boundaries should be computed. * @param[in] R The rotation (pose) under which the occluding boundaries should be computed.
* @return A vector with unique vertex id's making up the edges. * @return A vector with unique vertex id's making up the edges.
*/ */
inline std::vector<int> occluding_boundary_vertices_parallel(const core::Mesh& mesh, const morphablemodel::EdgeTopology& edge_topology, glm::mat4x4 R) std::vector<int> occluding_boundary_vertices_parallel(const core::Mesh& mesh, const morphablemodel::EdgeTopology& edge_topology, glm::mat4x4 R)
{ {
// Rotate the mesh: // Rotate the mesh:
std::vector<glm::vec4> rotated_vertices; std::vector<glm::vec4> rotated_vertices;
...@@ -423,7 +428,8 @@ inline std::vector<int> occluding_boundary_vertices_parallel(const core::Mesh& m ...@@ -423,7 +428,8 @@ inline std::vector<int> occluding_boundary_vertices_parallel(const core::Mesh& m
// Find occluding edges: // Find occluding edges:
std::vector<int> occluding_edges_indices; std::vector<int> occluding_edges_indices;
for (int edge_idx = 0; edge_idx < edge_topology.adjacent_faces.size(); ++edge_idx) // For each edge... Ef contains the indices of the two adjacent faces for (int edge_idx = 0; edge_idx < edge_topology.adjacent_faces.size();
++edge_idx) // For each edge... Ef contains the indices of the two adjacent faces
{ {
const auto &edge = edge_topology.adjacent_faces[edge_idx]; const auto &edge = edge_topology.adjacent_faces[edge_idx];
if (edge[0] == 0) // ==> NOTE/Todo Need to change this if we use 0-based indexing! if (edge[0] == 0) // ==> NOTE/Todo Need to change this if we use 0-based indexing!
...@@ -455,65 +461,81 @@ inline std::vector<int> occluding_boundary_vertices_parallel(const core::Mesh& m ...@@ -455,65 +461,81 @@ inline std::vector<int> occluding_boundary_vertices_parallel(const core::Mesh& m
// Remove duplicate vertex id's (std::unique works only on sorted sequences): // Remove duplicate vertex id's (std::unique works only on sorted sequences):
std::sort(begin(occluding_vertices), end(occluding_vertices)); std::sort(begin(occluding_vertices), end(occluding_vertices));
occluding_vertices.erase(std::unique(begin(occluding_vertices), end(occluding_vertices)), end(occluding_vertices)); occluding_vertices.erase(std::unique(begin(occluding_vertices), end(occluding_vertices)), end(occluding_vertices));
// Perform ray-casting to find out which vertices are not visible (i.e. self-occluded): // Perform ray-casting to find out which vertices are not visible (i.e. self-occluded):
std::vector<bool> visibility; // Use map to allow parallelism, all vertex_ids are unique
std::vector<int> vertex_id_visible_map(occluding_vertices.size());
int NUM_THREADS = 4;
auto t1 = std::chrono::high_resolution_clock::now(); auto t1 = std::chrono::high_resolution_clock::now();
// #pragma omp target map(to: occluding_vertices, mesh, rotated_vertices)
// we shoot the ray from the vertex towards the camera // #pragma omp parallel for
glm::vec3 ray_direction(0.0f, 0.0f, 1.0f); #pragma omp target map(alloc:vertex_id_visible_map) map(from:occluding_vertices, rotated_vertices, mesh)
{
#pragma omp parallel for schedule num_threads(NUM_THREADS) #pragma omp parallel for
for(int i = 0; i < occluding_vertices.size(); i++) { for (int i = 0; i < occluding_vertices.size(); i++) {
const auto vertex_idx = occluding_vertices[i]; const auto vertex_idx = occluding_vertices[i];
bool visible = true; int visible = 1;
glm::vec3 ray_origin(rotated_vertices[vertex_idx]); glm::vec3 ray_origin(rotated_vertices[vertex_idx]);
// we shoot the ray from the vertex towards the camera
glm::vec3 ray_direction(0.0f, 0.0f, 1.0f);
// For every tri of the rotated mesh: // For every tri of the rotated mesh:
for(int j = 0; j < mesh.tvi.size(); j++) { for (int j = 0; j < mesh.tvi.size(); j++) {
auto tri = mesh.tvi[j]; auto tri = mesh.tvi[j];
auto &v0 = rotated_vertices[tri[0]]; auto &v0 = rotated_vertices[tri[0]];
auto &v1 = rotated_vertices[tri[1]]; auto &v1 = rotated_vertices[tri[1]];
auto &v2 = rotated_vertices[tri[2]]; auto &v2 = rotated_vertices[tri[2]];
auto intersect = ray_triangle_intersect(ray_origin, ray_direction, glm::vec3(v0), glm::vec3(v1), glm::vec3(v2), false); auto intersect = ray_triangle_intersect(ray_origin,
ray_direction,
glm::vec3(v0),
glm::vec3(v1),
glm::vec3(v2),
false);
// first is bool intersect, second is the distance t // first is bool intersect, second is the distance t
if (intersect.first == true) { if (intersect.first == true) {
// We've hit a triangle. Ray hit its own triangle. If it's behind the ray origin, ignore the intersection: // We've hit a triangle. Ray hit its own triangle. If it's behind the ray origin, ignore the intersection:
// Check if in front or behind? // Check if in front or behind?
if (intersect.second.get() <= 1e-4) if (intersect.second.get() <= 1e-4) {
{
continue; // the intersection is behind the vertex, we don't care about it continue; // the intersection is behind the vertex, we don't care about it
} }
// Otherwise, we've hit a genuine triangle, and the vertex is not visible: // Otherwise, we've hit a genuine triangle, and the vertex is not visible:
visible = false; visible = 0;
break; break;
} }
} }
visibility.push_back(visible); vertex_id_visible_map[i] = visible;
}
} }
auto t2 = std::chrono::high_resolution_clock::now(); auto t2 = std::chrono::high_resolution_clock::now();
std::cout << "test derp 1: " << std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count() << "ms" << std::endl;
// Remove vertices from occluding boundary list that are not visible: // copy the results to final vertex ids
std::vector<int> final_vertex_ids; std::vector<int> final_vertex_ids;
for (int i = 0; i < occluding_vertices.size(); ++i) for (int i = 0; i < occluding_vertices.size(); ++i) {
{ const auto vertex_idx = occluding_vertices[i];
if (visibility[i] == true) int visible = vertex_id_visible_map[i];
{ if (visible == 1) {
final_vertex_ids.push_back(occluding_vertices[i]); final_vertex_ids.push_back(vertex_idx);
} }
} }
auto final_timing = std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count();
printf("S %lu %lld ms (mean: %f)\n",
final_vertex_ids.size(),
final_timing,
static_cast<float>(final_timing) / final_vertex_ids.size()
);
return final_vertex_ids; return final_vertex_ids;
}; };
/** /**
*
* OpenMP version
*
* @brief For a given list of 2D edge points, find corresponding 3D vertex IDs. * @brief For a given list of 2D edge points, find corresponding 3D vertex IDs.
* *
* This algorithm first computes the 3D mesh's occluding boundary vertices under * This algorithm first computes the 3D mesh's occluding boundary vertices under
...@@ -593,6 +615,7 @@ inline std::pair<std::vector<cv::Vec2f>, std::vector<int>> find_occluding_edge_c ...@@ -593,6 +615,7 @@ inline std::pair<std::vector<cv::Vec2f>, std::vector<int>> find_occluding_edge_c
} }
return { image_points, vertex_indices }; return { image_points, vertex_indices };
}; };
#endif // _OPENMP
......
...@@ -43,7 +43,9 @@ ...@@ -43,7 +43,9 @@
#include <vector> #include <vector>
#include <algorithm> #include <algorithm>
#ifdef _OPENMP
#include <omp.h> #include <omp.h>
#endif
namespace eos { namespace eos {
namespace fitting { namespace fitting {
...@@ -842,32 +844,6 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -842,32 +844,6 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
); );
}; };
//void calculate_rendering_params() {
// // Need to do an initial pose fit to do the contour fitting inside the loop.
// // We'll do an expression fit too, since face shapes vary quite a lot, depending on expressions.
// vector<fitting::RenderingParameters> rendering_params;
// for (int j = 0; j < num_images; ++j) {
// fitting::ScaledOrthoProjectionParameters current_pose = fitting::estimate_orthographic_projection_linear (image_points[j], model_points[j], true, image_height);
// fitting::RenderingParameters current_rendering_params (current_pose, image_width, image_height);
// rendering_params.push_back (current_rendering_params);
//
// cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix (current_rendering_params, image_width, image_height);
// blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls (blendshapes, current_pca_shape, affine_from_ortho, image_points[j], vertex_indices[j]);
//
// // Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
// current_combined_shapes[j] = current_pca_shape + morphablemodel::to_matrix (blendshapes)
// * Eigen::Map<const Eigen::VectorXf> (blendshape_coefficients[j]
// .data (), blendshape_coefficients[j]
// .size ());
// current_meshs[j] = morphablemodel::sample_to_mesh (current_combined_shapes[j], morphable_model
// .get_color_model ().get_mean (), morphable_model.get_shape_model ().get_triangle_list (), morphable_model
// .get_color_model ()
// .get_triangle_list (), morphable_model
// .get_texture_coordinates ());
// }
//}}
/** /**
* Generate a new mesh given pca_coefficients and blend_shape_coefficients. * Generate a new mesh given pca_coefficients and blend_shape_coefficients.
* @param pca_shape_coefficients * @param pca_shape_coefficients
...@@ -905,10 +881,10 @@ inline eos::core::Mesh generate_new_mesh( ...@@ -905,10 +881,10 @@ inline eos::core::Mesh generate_new_mesh(
} }
inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_parallel( inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_2(
const morphablemodel::MorphableModel& morphable_model, const morphablemodel::MorphableModel& morphable_model,
const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<morphablemodel::Blendshape>& blendshapes,
std::vector<eos::video::Keyframe>& key_frames, std::vector<eos::video::Keyframe>& keyframes,
const core::LandmarkMapper& landmark_mapper, const core::LandmarkMapper& landmark_mapper,
int image_width, int image_width,
int image_height, int image_height,
...@@ -957,21 +933,17 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -957,21 +933,17 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Current mesh - either from the given coefficients, or the mean: // Current mesh - either from the given coefficients, or the mean:
VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients); VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
vector<VectorXf> current_combined_shapes(num_images); vector<VectorXf> current_combined_shapes;
vector<eos::core::Mesh> current_meshs(num_images); vector<eos::core::Mesh> current_meshs;
// The 2D and 3D point correspondences used for the fitting: // The 2D and 3D point correspondences used for the fitting:
vector<vector<Vec4f>> model_points(num_images); // the points in the 3D shape model of all frames. vector<vector<Vec4f>> model_points; // the points in the 3D shape model of all frames.
vector<vector<int>> vertex_indices(num_images); // their vertex indices of all frames. vector<vector<int>> vertex_indices; // their vertex indices of all frames.
vector<vector<Vec2f>> image_points(num_images); // the corresponding 2D landmark points of all frames. vector<vector<Vec2f>> image_points; // the corresponding 2D landmark points of all frames.
int NUM_THREADS = 4;
vector<fitting::RenderingParameters> rendering_params(num_images); // list of rendering params for all frames. vector<fitting::RenderingParameters> rendering_params; // list of rendering params for all frames.
#pragma omp parallel num_threads(NUM_THREADS) // Generate meshes from current shapes.
{
#pragma omp for
for (int j = 0; j < num_images; ++j) { for (int j = 0; j < num_images; ++j) {
VectorXf current_combined_shape = current_pca_shape + VectorXf current_combined_shape = current_pca_shape +
blendshapes_as_basis * blendshapes_as_basis *
...@@ -984,30 +956,36 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -984,30 +956,36 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
morphable_model.get_color_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()); morphable_model.get_texture_coordinates());
vector<Vec4f> curr_model_points;
vector<int> curr_vertex_indices;
vector<Vec2f> curr_image_points;
// Get the locations of the model locations of the meshes, vertex_indices and image points // Get the locations of the model locations of the meshes, vertex_indices and image points
// (equal to landmark coordinates), for every image / mesh. // (equal to landmark coordinates), for every image / mesh.
std::tie(curr_model_points, curr_vertex_indices, curr_image_points) = eos::core::get_landmark_coordinates<Vec2f, Vec4f>(
key_frames[j].fitting_result.landmarks, landmark_mapper, current_mesh); // std::tie(curr_model_points, curr_vertex_indices, curr_image_points) = eos::core::get_landmark_coordinates<Vec2f, Vec4f>(
// keyframes[j].fitting_result.landmarks, landmark_mapper, current_mesh);
eos::core::get_landmark_coordinates_inline<Vec2f, Vec4f>(
keyframes[j].fitting_result.landmarks,
landmark_mapper,
current_mesh,
// output parameters
model_points,
vertex_indices,
image_points);
// Start constructing a list of rendering parameters needed for reconstruction. // Start constructing a list of rendering parameters needed for reconstruction.
// Get the current points from the last added image points and model points // Get the current points from the last added image points and model points
auto current_pose = fitting::estimate_orthographic_projection_linear( auto current_pose = fitting::estimate_orthographic_projection_linear(
curr_image_points, curr_model_points, true, image_height); image_points.back(), model_points.back(), true, image_height);
fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height); fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height);
rendering_params[j] = current_rendering_params; rendering_params.push_back(current_rendering_params);
// update key frame rendering params // update key frame rendering params
key_frames[j].fitting_result.rendering_parameters = current_rendering_params; keyframes[j].fitting_result.rendering_parameters = current_rendering_params;
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height); Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls( blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_ortho, curr_image_points, curr_vertex_indices); blendshapes, current_pca_shape, affine_from_ortho, image_points.back(), vertex_indices.back()
);
// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given): // Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
current_combined_shape = current_pca_shape + current_combined_shape = current_pca_shape +
...@@ -1015,7 +993,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1015,7 +993,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size() Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size()
); );
current_combined_shapes[j] = current_combined_shape; current_combined_shapes.push_back(current_combined_shape);
current_mesh = morphablemodel::sample_to_mesh( current_mesh = morphablemodel::sample_to_mesh(
current_combined_shape, morphable_model.get_color_model().get_mean(), current_combined_shape, morphable_model.get_color_model().get_mean(),
...@@ -1024,36 +1002,28 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1024,36 +1002,28 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
morphable_model.get_texture_coordinates() morphable_model.get_texture_coordinates()
); );
current_meshs[j] = current_mesh; current_meshs.push_back(current_mesh);
model_points[j] = curr_model_points;
vertex_indices[j] = curr_vertex_indices;
image_points[j] = curr_image_points;
} }
}
// The static (fixed) landmark correspondences which will stay the same throughout // The static (fixed) landmark correspondences which will stay the same throughout
// the fitting (the inner face landmarks): // the fitting (the inner face landmarks):
vector<vector<int>> fixed_vertex_indices (vertex_indices); vector<vector<int>> fixed_vertex_indices (vertex_indices);
vector<vector<Vec2f>> fixed_image_points (image_points); vector<vector<Vec2f>> fixed_image_points (image_points);
std::vector<cv::Mat> affine_from_orthos(num_images); std::vector<cv::Mat> affine_from_orthos;
std::vector<VectorXf> mean_plus_blendshapes(num_images); std::vector<VectorXf> mean_plus_blendshapes;
image_points = fixed_image_points; image_points = fixed_image_points;
vertex_indices = fixed_vertex_indices; vertex_indices = fixed_vertex_indices;
#pragma omp parallel num_threads(NUM_THREADS)
{
#pragma omp for
for (int j = 0; j < num_images; ++j) { for (int j = 0; j < num_images; ++j) {
// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour: // Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
vector<Vec2f> image_points_contour; vector<Vec2f> image_points_contour;
vector<int> vertex_indices_contour; vector<int> vertex_indices_contour;
auto curr_keyframe = key_frames[j]; auto curr_keyframe = keyframes[j];
auto landmarks = curr_keyframe.fitting_result.landmarks; auto landmarks = curr_keyframe.fitting_result.landmarks;
float yaw_angle = curr_keyframe.yaw_angle; auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
// auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id: // For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences( std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences(
...@@ -1077,27 +1047,18 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1077,27 +1047,18 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// positive yaw = subject looking to the left // positive yaw = subject looking to the left
if (yaw_angle >= 0.0f) { if (yaw_angle >= 0.0f) {
// the left contour is the occluding one we want to use ("away-facing") // the left contour is the occluding one we want to use ("away-facing")
auto contour_landmarks_ = auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
std::for_each(begin(contour_landmarks_), occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
end(contour_landmarks_),
[&occluding_contour_landmarks](auto &&lm)
{
occluding_contour_landmarks.push_back({lm.coordinates[0], lm.coordinates[1]});
}); });
} } else {
else
{
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour); auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
std::for_each(begin(contour_landmarks_), std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) {
end(contour_landmarks_), occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] });
[&occluding_contour_landmarks](auto &&lm)
{
occluding_contour_landmarks.push_back({lm.coordinates[0], lm.coordinates[1]});
}); });
} }
auto edge_correspondences = fitting::find_occluding_edge_correspondences_parallel( auto edge_correspondences = fitting::find_occluding_edge_correspondences(
current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f
); );
...@@ -1106,9 +1067,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1106,9 +1067,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Get the model points of the current mesh, for all correspondences that we've got: // Get the model points of the current mesh, for all correspondences that we've got:
model_points[j].clear(); model_points[j].clear();
for (const auto& v : vertex_indices[j]) {
for (const auto &v : vertex_indices[j])
{
model_points[j].push_back( model_points[j].push_back(
{ {
current_meshs[j].vertices[v][0], current_meshs[j].vertices[v][0],
...@@ -1121,17 +1080,14 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1121,17 +1080,14 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Re-estimate the pose, using all correspondences: // Re-estimate the pose, using all correspondences:
auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height); auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height);
rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height); rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height); Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
affine_from_orthos[j] = affine_from_ortho; affine_from_orthos.push_back(affine_from_ortho);
// Estimate the PCA shape coefficients with the current blendshape coefficients: // Estimate the PCA shape coefficients with the current blendshape coefficients:
VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size());
blendshape_coefficients[j].size()); mean_plus_blendshapes.push_back(current_mean_plus_blendshapes);
mean_plus_blendshapes[j] = current_mean_plus_blendshapes;
}
} }
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel( pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel(
...@@ -1147,17 +1103,12 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1147,17 +1103,12 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Estimate the blendshape coefficients with the current PCA model estimate: // Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients); current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
#pragma omp parallel num_threads(NUM_THREADS) for (int j = 0; j < num_images; ++j) {
{
#pragma omp for
for (int j = 0; j < num_images; ++j)
{
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls( blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j] blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
); );
current_combined_shapes[j] = current_pca_shape + current_combined_shapes[j] = current_pca_shape +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size());
blendshape_coefficients[j].size());
current_meshs[j] = morphablemodel::sample_to_mesh( current_meshs[j] = morphablemodel::sample_to_mesh(
current_combined_shapes[j], current_combined_shapes[j],
...@@ -1168,16 +1119,16 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1168,16 +1119,16 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
); );
} }
}
fitted_image_points = image_points; fitted_image_points = image_points;
return {current_meshs, rendering_params}; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices). return { current_meshs, rendering_params }; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices).
}; };
inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_2( #ifdef _OPENMP
inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParameters>> fit_shape_and_pose_multi_parallel(
const morphablemodel::MorphableModel& morphable_model, const morphablemodel::MorphableModel& morphable_model,
const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<morphablemodel::Blendshape>& blendshapes,
std::vector<eos::video::Keyframe>& key_frames, std::vector<eos::video::Keyframe>& keyframes,
const core::LandmarkMapper& landmark_mapper, const core::LandmarkMapper& landmark_mapper,
int image_width, int image_width,
int image_height, int image_height,
...@@ -1191,7 +1142,8 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1191,7 +1142,8 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
boost::optional<fitting::RenderingParameters> initial_rendering_params, boost::optional<fitting::RenderingParameters> initial_rendering_params,
std::vector<float>& pca_shape_coefficients, std::vector<float>& pca_shape_coefficients,
std::vector<std::vector<float>>& blendshape_coefficients, std::vector<std::vector<float>>& blendshape_coefficients,
std::vector<std::vector<cv::Vec2f>>& fitted_image_points) { std::vector<std::vector<cv::Vec2f>>& fitted_image_points,
boost::property_tree::ptree settings) {
assert(blendshapes.size() > 0); assert(blendshapes.size() > 0);
assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit? assert(num_iterations > 0); // Can we allow 0, for only the initial pose-fit?
...@@ -1204,6 +1156,10 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1204,6 +1156,10 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
using Eigen::VectorXf; using Eigen::VectorXf;
using Eigen::MatrixXf; using Eigen::MatrixXf;
// read settings and trade-offs:
int NUM_THREADS = settings.get<int>("reconstruction.num_threads", 1);
bool use_contours = settings.get<bool>("reconstruction.use_contours", true);
if (!num_shape_coefficients_to_fit) { if (!num_shape_coefficients_to_fit) {
num_shape_coefficients_to_fit = morphable_model.get_shape_model().get_num_principal_components(); num_shape_coefficients_to_fit = morphable_model.get_shape_model().get_num_principal_components();
} }
...@@ -1226,17 +1182,21 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1226,17 +1182,21 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Current mesh - either from the given coefficients, or the mean: // Current mesh - either from the given coefficients, or the mean:
VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients); VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
vector<VectorXf> current_combined_shapes; vector<VectorXf> current_combined_shapes(num_images);
vector<eos::core::Mesh> current_meshs; vector<eos::core::Mesh> current_meshs(num_images);
// The 2D and 3D point correspondences used for the fitting: // The 2D and 3D point correspondences used for the fitting:
vector<vector<Vec4f>> model_points; // the points in the 3D shape model of all frames. vector<vector<Vec4f>> model_points(num_images); // the points in the 3D shape model of all frames.
vector<vector<int>> vertex_indices; // their vertex indices of all frames. vector<vector<int>> vertex_indices(num_images); // their vertex indices of all frames.
vector<vector<Vec2f>> image_points; // the corresponding 2D landmark points of all frames. vector<vector<Vec2f>> image_points(num_images); // the corresponding 2D landmark points of all frames.
vector<fitting::RenderingParameters> rendering_params; // list of rendering params for all frames. vector<fitting::RenderingParameters> rendering_params(num_images); // list of rendering params for all frames.
std::vector<cv::Mat> affine_from_orthos(num_images);
std::vector<VectorXf> mean_plus_blendshapes(num_images);
// Generate meshes from current shapes. #pragma omp parallel num_threads(NUM_THREADS)
{
#pragma omp for
for (int j = 0; j < num_images; ++j) { for (int j = 0; j < num_images; ++j) {
VectorXf current_combined_shape = current_pca_shape + VectorXf current_combined_shape = current_pca_shape +
blendshapes_as_basis * blendshapes_as_basis *
...@@ -1249,36 +1209,33 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1249,36 +1209,33 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
morphable_model.get_color_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()); morphable_model.get_texture_coordinates());
vector<Vec4f> curr_model_points;
vector<int> curr_vertex_indices;
vector<Vec2f> curr_image_points;
// Get the locations of the model locations of the meshes, vertex_indices and image points // Get the locations of the model locations of the meshes, vertex_indices and image points
// (equal to landmark coordinates), for every image / mesh. // (equal to landmark coordinates), for every image / mesh.
std::tie(curr_model_points, curr_vertex_indices, curr_image_points) = eos::core::get_landmark_coordinates<Vec2f, Vec4f>(
// std::tie(curr_model_points, curr_vertex_indices, curr_image_points) = eos::core::get_landmark_coordinates<Vec2f, Vec4f>( keyframes[j].fitting_result.landmarks, landmark_mapper, current_mesh);
// key_frames[j].fitting_result.landmarks, landmark_mapper, current_mesh);
eos::core::get_landmark_coordinates_inline<Vec2f, Vec4f>(
key_frames[j].fitting_result.landmarks,
landmark_mapper,
current_mesh,
// output parameters
model_points,
vertex_indices,
image_points);
// Start constructing a list of rendering parameters needed for reconstruction. // Start constructing a list of rendering parameters needed for reconstruction.
// Get the current points from the last added image points and model points // Get the current points from the last added image points and model points
auto current_pose = fitting::estimate_orthographic_projection_linear( auto current_pose = fitting::estimate_orthographic_projection_linear(
image_points.back(), model_points.back(), true, image_height); curr_image_points, curr_model_points, true, image_height);
fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height); fitting::RenderingParameters current_rendering_params(current_pose, image_width, image_height);
rendering_params.push_back(current_rendering_params); rendering_params[j] = current_rendering_params;
// update key frame rendering params // update key frame rendering params
key_frames[j].fitting_result.rendering_parameters = current_rendering_params; keyframes[j].fitting_result.rendering_parameters = current_rendering_params;
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height); Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(current_rendering_params, image_width, image_height);
// if no contour
affine_from_orthos[j] = affine_from_ortho;
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls( blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_ortho, image_points.back(), vertex_indices.back() blendshapes, current_pca_shape, affine_from_ortho, curr_image_points, curr_vertex_indices);
);
// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given): // Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
current_combined_shape = current_pca_shape + current_combined_shape = current_pca_shape +
...@@ -1286,7 +1243,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1286,7 +1243,7 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size() Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size()
); );
current_combined_shapes.push_back(current_combined_shape); current_combined_shapes[j] = current_combined_shape;
current_mesh = morphablemodel::sample_to_mesh( current_mesh = morphablemodel::sample_to_mesh(
current_combined_shape, morphable_model.get_color_model().get_mean(), current_combined_shape, morphable_model.get_color_model().get_mean(),
...@@ -1295,31 +1252,41 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1295,31 +1252,41 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
morphable_model.get_texture_coordinates() morphable_model.get_texture_coordinates()
); );
current_meshs.push_back(current_mesh); current_meshs[j] = current_mesh;
} model_points[j] = curr_model_points;
vertex_indices[j] = curr_vertex_indices;
image_points[j] = curr_image_points;
// Estimate the PCA shape coefficients with the current blendshape coefficients:
VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
blendshape_coefficients[j].size());
mean_plus_blendshapes[j] = current_mean_plus_blendshapes;
}
}
// The static (fixed) landmark correspondences which will stay the same throughout // The static (fixed) landmark correspondences which will stay the same throughout
// the fitting (the inner face landmarks): // the fitting (the inner face landmarks):
vector<vector<int>> fixed_vertex_indices (vertex_indices); vector<vector<int>> fixed_vertex_indices (vertex_indices);
vector<vector<Vec2f>> fixed_image_points (image_points); vector<vector<Vec2f>> fixed_image_points (image_points);
std::vector<cv::Mat> affine_from_orthos;
std::vector<VectorXf> mean_plus_blendshapes;
image_points = fixed_image_points; image_points = fixed_image_points;
vertex_indices = fixed_vertex_indices; vertex_indices = fixed_vertex_indices;
for (int j = 0; j < num_images; ++j) { if (use_contours) {
for (int j = 0; j < num_images; ++j)
{
// Given the current pose, find 2D-3D contour correspondences of the front-facing face contour: // Given the current pose, find 2D-3D contour correspondences of the front-facing face contour:
vector<Vec2f> image_points_contour; vector<Vec2f> image_points_contour;
vector<int> vertex_indices_contour; vector<int> vertex_indices_contour;
auto curr_keyframe = key_frames[j]; auto curr_keyframe = keyframes[j];
auto landmarks = curr_keyframe.fitting_result.landmarks; auto landmarks = curr_keyframe.fitting_result.landmarks;
auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]); auto yaw_angle = glm::degrees(glm::eulerAngles(rendering_params[j].get_rotation())[1]);
// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id: // For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
std::tie(image_points_contour, std::ignore, vertex_indices_contour) = fitting::get_contour_correspondences( std::tie(image_points_contour, std::ignore, vertex_indices_contour) =
fitting::get_contour_correspondences(
landmarks, landmarks,
contour_landmarks, contour_landmarks,
model_contour, model_contour,
...@@ -1338,20 +1305,30 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1338,20 +1305,30 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
vector<Eigen::Vector2f> occluding_contour_landmarks; vector<Eigen::Vector2f> occluding_contour_landmarks;
// positive yaw = subject looking to the left // positive yaw = subject looking to the left
if (yaw_angle >= 0.0f) { if (yaw_angle >= 0.0f)
{
// the left contour is the occluding one we want to use ("away-facing") // the left contour is the occluding one we want to use ("away-facing")
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop auto contour_landmarks_ =
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { core::filter(landmarks, contour_landmarks.left_contour); // Can do this outside of the loop
occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); std::for_each(begin(contour_landmarks_),
end(contour_landmarks_),
[&occluding_contour_landmarks](auto &&lm)
{
occluding_contour_landmarks.push_back({lm.coordinates[0], lm.coordinates[1]});
}); });
} else { }
else
{
auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour); auto contour_landmarks_ = core::filter(landmarks, contour_landmarks.right_contour);
std::for_each(begin(contour_landmarks_), end(contour_landmarks_), [&occluding_contour_landmarks](auto&& lm) { std::for_each(begin(contour_landmarks_),
occluding_contour_landmarks.push_back({ lm.coordinates[0], lm.coordinates[1] }); end(contour_landmarks_),
[&occluding_contour_landmarks](auto &&lm)
{
occluding_contour_landmarks.push_back({lm.coordinates[0], lm.coordinates[1]});
}); });
} }
auto edge_correspondences = fitting::find_occluding_edge_correspondences( auto edge_correspondences = fitting::find_occluding_edge_correspondences_parallel(
current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f current_meshs[j], edge_topology, rendering_params[j], occluding_contour_landmarks, 180.0f
); );
...@@ -1360,7 +1337,9 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1360,7 +1337,9 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Get the model points of the current mesh, for all correspondences that we've got: // Get the model points of the current mesh, for all correspondences that we've got:
model_points[j].clear(); model_points[j].clear();
for (const auto& v : vertex_indices[j]) {
for (const auto &v : vertex_indices[j])
{
model_points[j].push_back( model_points[j].push_back(
{ {
current_meshs[j].vertices[v][0], current_meshs[j].vertices[v][0],
...@@ -1371,16 +1350,22 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1371,16 +1350,22 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
} }
// Re-estimate the pose, using all correspondences: // Re-estimate the pose, using all correspondences:
auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j], model_points[j], true, image_height); auto current_pose = fitting::estimate_orthographic_projection_linear(image_points[j],
model_points[j],
true,
image_height);
rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height); rendering_params[j] = fitting::RenderingParameters(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
affine_from_orthos.push_back(affine_from_ortho); Mat affine_from_ortho =
fitting::get_3x4_affine_camera_matrix(rendering_params[j], image_width, image_height);
affine_from_orthos[j] = affine_from_ortho;
// Estimate the PCA shape coefficients with the current blendshape coefficients: // Estimate the PCA shape coefficients with the current blendshape coefficients:
VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + VectorXf current_mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),blendshape_coefficients[j].size()); blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
mean_plus_blendshapes.push_back(current_mean_plus_blendshapes); blendshape_coefficients[j].size());
mean_plus_blendshapes[j] = current_mean_plus_blendshapes;
}
} }
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel( pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear_multi_parallel(
...@@ -1396,12 +1381,16 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1396,12 +1381,16 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
// Estimate the blendshape coefficients with the current PCA model estimate: // Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients); current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
#pragma omp parallel num_threads(NUM_THREADS)
{
#pragma omp for
for (int j = 0; j < num_images; ++j) { for (int j = 0; j < num_images; ++j) {
blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls( blendshape_coefficients[j] = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j] blendshapes, current_pca_shape, affine_from_orthos[j], image_points[j], vertex_indices[j]
); );
current_combined_shapes[j] = current_pca_shape + current_combined_shapes[j] = current_pca_shape +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(), blendshape_coefficients[j].size()); blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients[j].data(),
blendshape_coefficients[j].size());
current_meshs[j] = morphablemodel::sample_to_mesh( current_meshs[j] = morphablemodel::sample_to_mesh(
current_combined_shapes[j], current_combined_shapes[j],
...@@ -1410,12 +1399,19 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete ...@@ -1410,12 +1399,19 @@ inline std::pair<std::vector<core::Mesh>, std::vector<fitting::RenderingParamete
morphable_model.get_color_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates() morphable_model.get_texture_coordinates()
); );
// save it to the keyframe, we might need it for showing the reconstruction.
// we could make it optional
keyframes[j].fitting_result.mesh = current_meshs[j];
keyframes[j].fitting_result.rendering_parameters = rendering_params[j];
} }
}
fitted_image_points = image_points; fitted_image_points = image_points;
return { current_meshs, rendering_params }; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices). return {current_meshs, rendering_params}; // I think we could also work with a Mat face_instance in this function instead of a Mesh, but it would convolute the code more (i.e. more complicated to access vertices).
}; };
#endif // if OpenMP
} /* namespace fitting */ } /* namespace fitting */
} /* namespace eos */ } /* namespace eos */
......
...@@ -315,8 +315,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel( ...@@ -315,8 +315,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel(
#pragma omp parallel num_threads(num_threads) #pragma omp parallel num_threads(num_threads)
{ {
#pragma omp for #pragma omp for
for (int i = 0; i < num_landmarks; ++i) for (int i = 0; i < num_landmarks; ++i) {
{ // calculate index from landmark offset (not always the same landmarks amount).
int index = (landmark_offset_index[k] * 4) + 4 * i; int index = (landmark_offset_index[k] * 4) + 4 * i;
int offset = landmark_offset_index[k] + i; int offset = landmark_offset_index[k] + i;
auto vertex_id = vertex_ids[k][i]; auto vertex_id = vertex_ids[k][i];
...@@ -328,16 +328,14 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel( ...@@ -328,16 +328,14 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel(
basis_rows_.block(0, 0, basis_rows_.rows(), num_coeffs_to_fit); basis_rows_.block(0, 0, basis_rows_.rows(), num_coeffs_to_fit);
// Set y with landmark coordinates (will be used in linear lsq later) // Set y with landmark coordinates (will be used in linear lsq later)
#pragma omp simd #pragma omp simd
for (int j = 0; j < 2; j++) for (int j = 0; j < 2; j++) {
{
y(3 * offset + j) = landmarks[k][i][j]; y(3 * offset + j) = landmarks[k][i][j];
} }
// Set v_bar with face mesh (will be used in linear lsq later) // Set v_bar with face mesh (will be used in linear lsq later)
#pragma omp simd #pragma omp simd
for (int j = 0; j < 3; j++) for (int j = 0; j < 3; j++) {
{
v_bar(4 * offset + j) = base_face[k](vertex_id * 3 + j); v_bar(4 * offset + j) = base_face[k](vertex_id * 3 + j);
} }
} }
...@@ -368,13 +366,13 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel( ...@@ -368,13 +366,13 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel(
auto t2 = std::chrono::high_resolution_clock::now(); auto t2 = std::chrono::high_resolution_clock::now();
t1 = std::chrono::high_resolution_clock::now();
// fill P with coefficients // fill P with coefficients
P.setFromTriplets(P_coefficients.begin(), P_coefficients.end()); P.setFromTriplets(P_coefficients.begin(), P_coefficients.end());
MatrixXf one; MatrixXf one;
MatrixXf two; MatrixXf two;
MatrixXf A; MatrixXf A;
MatrixXf b; MatrixXf b;
MatrixXf rhs;
#pragma omp parallel sections num_threads(2) #pragma omp parallel sections num_threads(2)
{ {
...@@ -382,6 +380,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel( ...@@ -382,6 +380,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel(
{ {
A = P * V_hat_h; // camera matrix times the basis A = P * V_hat_h; // camera matrix times the basis
one = A.transpose() * Omega.asDiagonal(); one = A.transpose() * Omega.asDiagonal();
rhs = -one * b; // It's -A^t*Omega^t*b, but we don't need to transpose Omega, since it's a diagonal matrix, and Omega^t = Omega.
} }
#pragma omp section #pragma omp section
...@@ -392,12 +391,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel( ...@@ -392,12 +391,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear_multi_parallel(
} }
const MatrixXf AtOmegaAReg = one * A + two; const MatrixXf AtOmegaAReg = one * A + two;
const MatrixXf rhs = -one * b; // It's -A^t*Omega^t*b, but we don't need to transpose Omega, since it's a diagonal matrix, and Omega^t = Omega.
const VectorXf c_s = AtOmegaAReg.colPivHouseholderQr().solve(rhs); const VectorXf c_s = AtOmegaAReg.colPivHouseholderQr().solve(rhs);
t2 = std::chrono::high_resolution_clock::now();
std::cout << "matrix xf " << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() << "ms" << std::endl;
return std::vector<float>(c_s.data(), c_s.data() + c_s.size()); return std::vector<float>(c_s.data(), c_s.data() + c_s.size());
}; };
#endif #endif
......
...@@ -141,6 +141,7 @@ inline std::pair<cv::Mat, cv::Mat> render( ...@@ -141,6 +141,7 @@ inline std::pair<cv::Mat, cv::Mat> render(
// bool enable_texturing = false; Maybe re-add later, not sure // bool enable_texturing = false; Maybe re-add later, not sure
// take a cv::Mat texture instead and convert to Texture internally? no, we don't want to recreate mipmap levels on each render() call. // take a cv::Mat texture instead and convert to Texture internally? no, we don't want to recreate mipmap levels on each render() call.
auto t1 = std::chrono::high_resolution_clock::now();
assert(mesh.vertices.size() == mesh.colors.size() || mesh.colors.empty()); // The number of vertices has to be equal for both shape and colour, or, alternatively, it has to be a shape-only model. assert(mesh.vertices.size() == mesh.colors.size() || mesh.colors.empty()); // The number of vertices has to be equal for both shape and colour, or, alternatively, it has to be a shape-only model.
assert(mesh.vertices.size() == mesh.texcoords.size() || mesh.texcoords.empty()); // same for the texcoords assert(mesh.vertices.size() == mesh.texcoords.size() || mesh.texcoords.empty()); // same for the texcoords
// another assert: If cv::Mat texture != empty, then we need texcoords? // another assert: If cv::Mat texture != empty, then we need texcoords?
...@@ -177,7 +178,11 @@ inline std::pair<cv::Mat, cv::Mat> render( ...@@ -177,7 +178,11 @@ inline std::pair<cv::Mat, cv::Mat> render(
// Prepare the rasterisation stage. // Prepare the rasterisation stage.
// For every vertex/tri: // For every vertex/tri:
vector<detail::TriangleToRasterize> triangles_to_raster; vector<detail::TriangleToRasterize> triangles_to_raster;
for (const auto& tri_indices : mesh.tvi) { //#pragma omp target
// {
//#pragma omp parallel for
for(int i = 0; i < mesh.tvi.size(); i++) {
const auto tri_indices = mesh.tvi[i];
// Todo: Split this whole stuff up. Make a "clip" function, ... rename "processProspective..".. what is "process"... get rid of "continue;"-stuff by moving stuff inside process... // Todo: Split this whole stuff up. Make a "clip" function, ... rename "processProspective..".. what is "process"... get rid of "continue;"-stuff by moving stuff inside process...
// classify vertices visibility with respect to the planes of the view frustum // classify vertices visibility with respect to the planes of the view frustum
// we're in clip-coords (NDC), so just check if outside [-1, 1] x ... // we're in clip-coords (NDC), so just check if outside [-1, 1] x ...
...@@ -186,8 +191,7 @@ inline std::pair<cv::Mat, cv::Mat> render( ...@@ -186,8 +191,7 @@ inline std::pair<cv::Mat, cv::Mat> render(
// However, when comparing against w_c below, we might run into the trouble of the sign again in the affine case. // However, when comparing against w_c below, we might run into the trouble of the sign again in the affine case.
// 'w' is always positive, as it is -z_camspace, and all z_camspace are negative. // 'w' is always positive, as it is -z_camspace, and all z_camspace are negative.
unsigned char visibility_bits[3]; unsigned char visibility_bits[3];
for (unsigned char k = 0; k < 3; k++) for (unsigned char k = 0; k < 3; k++) {
{
visibility_bits[k] = 0; visibility_bits[k] = 0;
float x_cc = clipspace_vertices[tri_indices[k]].position[0]; float x_cc = clipspace_vertices[tri_indices[k]].position[0];
float y_cc = clipspace_vertices[tri_indices[k]].position[1]; float y_cc = clipspace_vertices[tri_indices[k]].position[1];
...@@ -214,8 +218,15 @@ inline std::pair<cv::Mat, cv::Mat> render( ...@@ -214,8 +218,15 @@ inline std::pair<cv::Mat, cv::Mat> render(
// all vertices are visible - pass the whole triangle to the rasterizer. = All bits of all 3 triangles are 0. // all vertices are visible - pass the whole triangle to the rasterizer. = All bits of all 3 triangles are 0.
if ((visibility_bits[0] | visibility_bits[1] | visibility_bits[2]) == 0) if ((visibility_bits[0] | visibility_bits[1] | visibility_bits[2]) == 0)
{ {
boost::optional<detail::TriangleToRasterize> t = detail::process_prospective_tri(clipspace_vertices[tri_indices[0]], clipspace_vertices[tri_indices[1]], clipspace_vertices[tri_indices[2]], viewport_width, viewport_height, enable_backface_culling); boost::optional<detail::TriangleToRasterize> t =
if (t) { detail::process_prospective_tri(clipspace_vertices[tri_indices[0]],
clipspace_vertices[tri_indices[1]],
clipspace_vertices[tri_indices[2]],
viewport_width,
viewport_height,
enable_backface_culling);
if (t)
{
triangles_to_raster.push_back(*t); triangles_to_raster.push_back(*t);
} }
continue; continue;
...@@ -228,7 +239,11 @@ inline std::pair<cv::Mat, cv::Mat> render( ...@@ -228,7 +239,11 @@ inline std::pair<cv::Mat, cv::Mat> render(
// split the triangle if it intersects the near plane: // split the triangle if it intersects the near plane:
if (enable_near_clipping) if (enable_near_clipping)
{ {
vertices = detail::clip_polygon_to_plane_in_4d(vertices, glm::tvec4<float>(0.0f, 0.0f, -1.0f, -1.0f)); // "Normal" (or "4D hyperplane") of the near-plane. I tested it and it works like this but I'm a little bit unsure because Songho says the normal of the near-plane is (0,0,-1,1) (maybe I have to switch around the < 0 checks in the function?) vertices = detail::clip_polygon_to_plane_in_4d(vertices,
glm::tvec4<float>(0.0f,
0.0f,
-1.0f,
-1.0f)); // "Normal" (or "4D hyperplane") of the near-plane. I tested it and it works like this but I'm a little bit unsure because Songho says the normal of the near-plane is (0,0,-1,1) (maybe I have to switch around the < 0 checks in the function?)
} }
// triangulation of the polygon formed of vertices array // triangulation of the polygon formed of vertices array
...@@ -236,18 +251,29 @@ inline std::pair<cv::Mat, cv::Mat> render( ...@@ -236,18 +251,29 @@ inline std::pair<cv::Mat, cv::Mat> render(
{ {
for (unsigned char k = 0; k < vertices.size() - 2; k++) for (unsigned char k = 0; k < vertices.size() - 2; k++)
{ {
boost::optional<detail::TriangleToRasterize> t = detail::process_prospective_tri(vertices[0], vertices[1 + k], vertices[2 + k], viewport_width, viewport_height, enable_backface_culling); boost::optional<detail::TriangleToRasterize> t = detail::process_prospective_tri(vertices[0],
if (t) { vertices[1 + k],
vertices[2 + k],
viewport_width,
viewport_height,
enable_backface_culling);
if (t)
{
triangles_to_raster.push_back(*t); triangles_to_raster.push_back(*t);
} }
} }
} }
} }
// }
// Fragment/pixel shader: Colour the pixel values // Fragment/pixel shader: Colour the pixel values
for (const auto& tri : triangles_to_raster) { for (const auto& tri : triangles_to_raster) {
detail::raster_triangle(tri, colourbuffer, depthbuffer, texture, enable_far_clipping); detail::raster_triangle(tri, colourbuffer, depthbuffer, texture, enable_far_clipping);
} }
auto t2 = std::chrono::high_resolution_clock::now();
auto final_timing = std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count();
printf("Tri %lu %lld ms\n", triangles_to_raster.size(), final_timing);
return std::make_pair(colourbuffer, depthbuffer); return std::make_pair(colourbuffer, depthbuffer);
}; };
......
...@@ -35,6 +35,8 @@ ...@@ -35,6 +35,8 @@
#include <atomic> #include <atomic>
#include <unistd.h> #include <unistd.h>
#include <glm/gtx/rotate_vector.hpp>
using cv::Mat; using cv::Mat;
using cv::Vec2f; using cv::Vec2f;
using cv::Vec3f; using cv::Vec3f;
...@@ -49,6 +51,8 @@ using namespace std; ...@@ -49,6 +51,8 @@ using namespace std;
namespace eos { namespace eos {
namespace video { namespace video {
// TODO: maybe move video iterator here.. or remove fil. // TODO: maybe move video iterator here.. or remove fil.
/** /**
* @brief Computes the variance of laplacian of the given image or patch. * @brief Computes the variance of laplacian of the given image or patch.
* *
...@@ -71,6 +75,374 @@ double variance_of_laplacian(const cv::Mat& image) ...@@ -71,6 +75,374 @@ double variance_of_laplacian(const cv::Mat& image)
return focus_measure; return focus_measure;
}; };
class ReconstructionVideoWriter
{
public:
std::unique_ptr<std::thread> frame_buffer_worker;
boost::property_tree::ptree settings;
ReconstructionVideoWriter() {};
ReconstructionVideoWriter(std::string video_file,
fitting::ReconstructionData reconstruction_data,
boost::property_tree::ptree settings) {
// make available in settings, would be a bit difficult to parse, but not impossible ofc.
int codec = CV_FOURCC('a','v','c','1');
fs::path output_file_path = settings.get<fs::path>("output.file_path", "/tmp/video.mp4");
// Remove output video file if exists, OpenCV does not overwrite by default so will stop if we don't do this.
if(fs::exists(output_file_path)) {
fs::remove(output_file_path);
}
fps = settings.get<int>("output.fps", 30);
show_video = settings.get<bool>("output.show_video", false);
wireframe = settings.get<bool>("output.wireframe", false);
landmarks = settings.get<bool>("output.landmarks", false);
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle);
unsigned int num_shape_coeff = reconstruction_data.
morphable_model.get_shape_model().get_num_principal_components();
this->num_shape_coefficients_to_fit = settings.get<unsigned int>(
"frames.num_shape_coefficients_to_fit", num_shape_coeff);
// Open source video file.
std::ifstream file(video_file);
std::cout << "Opening video: " << video_file << std::endl;
if (!file.is_open()) {
throw std::runtime_error("Error opening given file: " + video_file);
}
cv::VideoCapture tmp_cap(video_file); // open video file
if (!tmp_cap.isOpened()) {
throw std::runtime_error("Could not play video");
}
cap = tmp_cap;
Mat frame;
// reset total frames, just to make sure.
total_frames = 0;
// Get the first frame to see, sometimes frames are empty at the start, keep reading until we hit one.
while(frame.empty()) {
cap.read(frame);
total_frames++;
}
std::cout << frame.size() << std::endl;
std::cout << frame.cols << std::endl;
std::cout << frame.rows << std::endl;
// Take over the size of the original video or take width / height from the settings.
int frame_width = settings.get<int>("frames.width", frame.cols);
int frame_height = settings.get<int>("frames.height", frame.rows);
Size frame_size = Size(frame_width, frame_height);
// Initialize writer with given output file
VideoWriter tmp_writer(output_file_path.string(), codec, fps, frame_size);
if (!tmp_writer.isOpened()) {
cout << "Could not open the output video for write: " << output_file_path << endl;
throw std::runtime_error("Could open output video");
}
writer = tmp_writer;
this->reconstruction_data = reconstruction_data;
};
/**
* Generate a new keyframe containing information about pose and landmarks
* These are needed to determine if we want the image in the first place.
*
* @param frame
* @return Keyframe
*/
Keyframe generate_new_keyframe(cv::Mat frame) {
int frame_height = frame.rows;
int frame_width = frame.cols;
// Get the necessary information for reconstruction.
auto landmarks = reconstruction_data.landmark_list[total_frames];
auto landmark_mapper = reconstruction_data.landmark_mapper;
auto blendshapes = reconstruction_data.blendshapes;
auto morphable_model = reconstruction_data.morphable_model;
// Reached the end of the landmarks (only applicable for annotated videos):
if (reconstruction_data.landmark_list.size() <= total_frames) {
std::cout << "Reached end of landmarks(" <<
reconstruction_data.landmark_list.size() <<
"/" << total_frames << ")" << std::endl;
return Keyframe();
}
if (pca_shape_coefficients.empty()) {
pca_shape_coefficients.resize(num_shape_coefficients_to_fit);
}
// make a new one
std::vector<float> blend_shape_coefficients;
vector<cv::Vec4f> model_points;
vector<int> vertex_indices;
vector<cv::Vec2f> image_points;
auto mesh = fitting::generate_new_mesh(
morphable_model,
blendshapes,
pca_shape_coefficients, // current pca_coeff will be the mean for the first iterations.
blend_shape_coefficients);
// Will yield model_points, vertex_indices and image_points
// todo: should this function not come from mesh?
core::get_mesh_coordinates(landmarks, landmark_mapper, mesh, model_points, vertex_indices, image_points);
auto current_pose = fitting::estimate_orthographic_projection_linear(
image_points, model_points, true, frame_height);
fitting::RenderingParameters rendering_params(current_pose, frame_width, frame_height);
Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_params, frame_width, frame_height);
auto blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
auto current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
blend_shape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(
blendshapes, current_pca_shape, affine_cam, image_points, vertex_indices);
auto merged_shape = current_pca_shape +
blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blend_shape_coefficients.data(),
blend_shape_coefficients.size());
auto merged_mesh = morphablemodel::sample_to_mesh(
merged_shape,
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()
);
auto R = rendering_params.get_rotation();
// Render the model in a separate window using the estimated pose, shape and merged texture:
Mat rendering;
// render needs 4 channels 8 bits image, needs a two step conversion.
cvtColor(frame, rendering, CV_BGR2BGRA);
// make sure the image is CV_8UC4, maybe do check first?
rendering.convertTo(rendering, CV_8UC4);
Mat isomap = render::extract_texture(merged_mesh, affine_cam, frame);
Mat merged_isomap = isomap_averaging.add_and_merge(isomap);
Mat frontal_rendering;
glm::mat4 modelview_frontal = glm::rotate(glm::mat4(1.0f), angle, glm::vec3(0.0f, 1.0f, 0.0f));
std::cout << angle << std::endl;
core::Mesh neutral_expression = morphablemodel::sample_to_mesh(
morphable_model.get_shape_model().draw_sample(pca_shape_coefficients),
morphable_model.get_color_model().get_mean(),
morphable_model.get_shape_model().get_triangle_list(),
morphable_model.get_color_model().get_triangle_list(),
morphable_model.get_texture_coordinates()
);
// angle -= 10.0;
std::tie(frontal_rendering, std::ignore) = render::render(
neutral_expression,
modelview_frontal,
glm::ortho(-130.0f, 130.0f, -130.0f, 130.0f),
512, 512,
render::create_mipmapped_texture(merged_isomap),
true,
false,
false
);
// cv::imshow("rendering", frontal_rendering);
// cv::waitKey(0);
fitting::FittingResult fitting_result;
fitting_result.rendering_parameters = rendering_params;
fitting_result.landmarks = landmarks;
fitting_result.mesh = mesh;
// output this?
cv::Rect face_roi = core::get_face_roi(image_points, frame_width, frame_height);
float frame_laplacian_score = static_cast<float>(variance_of_laplacian(frame(face_roi)));
return Keyframe(frame_laplacian_score, rendering, fitting_result, total_frames);
}
/**
* Checks if the writer is still open. The writer thread will close if the last frame is processed.
*
* @return bool
*/
bool is_writing() {
return writer.isOpened();
}
/**
* Get a new frame from the video source.
*
* @return cv::Mat
*/
Mat __get_new_frame() {
// Get a new frame from camera.
Mat frame;
cap.read(frame);
return frame;
};
/**
* Stop by releasing the VideoCapture.
*/
void __stop() {
writer.release();
cap.release();
};
/**
* Start filling the buffer with frames and start a worker thread to keep doing so.
*
* @return
*/
void start() {
// start a thread to keep getting new frames into the buffer from video:
frame_buffer_worker = std::make_unique<std::thread>(&ReconstructionVideoWriter::video_iterator, this);
frame_buffer_worker.get()->detach();
}
/**
* Fill the buffer by iterating through the video until the very last frame.
*/
void video_iterator() {
// Go and fill the buffer with more frames while we are reconstructing.
while (next()) { }
// stop the video
__stop();
std::cout << "Video stopped at: " << total_frames;
};
/**
*
* @param keyframe
*/
bool next() {
Mat frame = __get_new_frame();
if (frame.empty()) {
return false;
}
// makes a copy of the frame
Keyframe keyframe = generate_new_keyframe(frame);
if(wireframe) {
draw_wireframe(keyframe.frame, keyframe);
}
if(landmarks) {
draw_landmarks(keyframe.frame, keyframe);
}
writer.write(keyframe.frame);
total_frames++;
// if (show_video) {
// std::cout << "show video" << std::endl;
// cv::imshow("video", frame);
// cv::waitKey(static_cast<int>((1.0 / fps) * 1000.0));
// }
return true;
}
/**
*
* @param pca_shape_coefficients
*/
void update_reconstruction_coeff(std::vector<float> pca_shape_coefficients) {
this->pca_shape_coefficients = pca_shape_coefficients;
}
/**
*
* @param image
* @param landmarks
*/
void draw_landmarks(Mat &frame, Keyframe keyframe) {
auto landmarks = keyframe.fitting_result.landmarks;
for (auto &&lm : landmarks) {
cv::rectangle(frame, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
);
}
}
/**
* Draws the given mesh as wireframe into the image.
*
* It does backface culling, i.e. draws only vertices in CCW order.
*
* @param[in] image An image to draw into.
* @param[in] mesh The mesh to draw.
* @param[in] modelview Model-view matrix to draw the mesh.
* @param[in] projection Projection matrix to draw the mesh.
* @param[in] viewport Viewport to draw the mesh.
* @param[in] colour Colour of the mesh to be drawn.
*/
void draw_wireframe(Mat &frame, Keyframe keyframe, cv::Scalar colour = cv::Scalar(0, 255, 0, 255)) {
// make a copy, we dont want to alter the original
auto mesh = keyframe.fitting_result.mesh;
auto modelview = keyframe.fitting_result.rendering_parameters.get_modelview();
auto projection = keyframe.fitting_result.rendering_parameters.get_projection();
auto viewport = fitting::get_opencv_viewport(frame.cols, frame.rows);
for (const auto& triangle : mesh.tvi)
{
const auto p1 = glm::project({ mesh.vertices[triangle[0]][0], mesh.vertices[triangle[0]][1], mesh.vertices[triangle[0]][2] }, modelview, projection, viewport);
const auto p2 = glm::project({ mesh.vertices[triangle[1]][0], mesh.vertices[triangle[1]][1], mesh.vertices[triangle[1]][2] }, modelview, projection, viewport);
const auto p3 = glm::project({ mesh.vertices[triangle[2]][0], mesh.vertices[triangle[2]][1], mesh.vertices[triangle[2]][2] }, modelview, projection, viewport);
if (eos::render::detail::are_vertices_ccw_in_screen_space(glm::vec2(p1), glm::vec2(p2), glm::vec2(p3))) {
cv::line(frame, cv::Point(p1.x, p1.y), cv::Point(p2.x, p2.y), colour);
cv::line(frame, cv::Point(p2.x, p2.y), cv::Point(p3.x, p3.y), colour);
cv::line(frame, cv::Point(p3.x, p3.y), cv::Point(p1.x, p1.y), colour);
}
}
};
int get_frame_number() {
return total_frames;
}
private:
float angle = -45.0;
int total_frames = 0;
int num_shape_coefficients_to_fit = 0;
// merge all triangles that are facing <60° towards the camera
WeightedIsomapAveraging isomap_averaging;
// video options
bool show_video = false;
bool wireframe = false;
bool landmarks = false;
int fps;
cv::VideoCapture cap;
cv::VideoWriter writer;
// latest pca_shape_coefficients
std::vector<float> pca_shape_coefficients;
eos::fitting::ReconstructionData reconstruction_data;
};
/** /**
* *
* *
...@@ -81,58 +453,53 @@ class BufferedVideoIterator ...@@ -81,58 +453,53 @@ class BufferedVideoIterator
public: public:
int width; int width;
int height; int height;
Mat last_frame;
int last_frame_number; int last_frame_number;
Keyframe last_keyframe;
bool reached_eof;
std::unique_ptr<std::thread> frame_buffer_worker; std::unique_ptr<std::thread> frame_buffer_worker;
BufferedVideoIterator() {}; BufferedVideoIterator() {};
// TODO: build support for setting the amount of max_frames in the buffer. // TODO: build support for setting the amount of max_frames in the buffer.
BufferedVideoIterator(std::string videoFilePath, fitting::ReconstructionData reconstruction_data, boost::property_tree::ptree settings) { BufferedVideoIterator(std::string video_file, fitting::ReconstructionData reconstruction_data, boost::property_tree::ptree settings) {
std::ifstream file(videoFilePath); std::ifstream file(video_file);
std::cout << "Opening video: " << videoFilePath << std::endl; std::cout << "Opening video: " << video_file << std::endl;
if (!file.is_open()) { if (!file.is_open()) {
throw std::runtime_error("Error opening given file: " + videoFilePath); throw std::runtime_error("Error opening given file: " + video_file);
} }
cv::VideoCapture tmp_cap(videoFilePath); // open video file cv::VideoCapture tmp_cap(video_file); // open video file
if (!tmp_cap.isOpened()) { if (!tmp_cap.isOpened()) {
throw std::runtime_error("Could not play video"); throw std::runtime_error("Could not play video");
} }
cap = tmp_cap; cap = tmp_cap;
this->reconstruction_data = reconstruction_data; // Initialize settings
min_frames = settings.get<int>("frames.min_frames", 5);
// copy settings from gathered from a .ini file drop_frames = settings.get<int>("frames.drop_frames", 0);
min_frames = settings.get<int>("video.min_frames", 5); skip_frames = settings.get<int>("frames.skip_frames", 0);
drop_frames = settings.get<int>("video.drop_frames", 0); frames_per_bin = settings.get<unsigned int>("frames.frames_per_bin", 2);
skip_frames = settings.get<int>("video.skip_frames", 0);
frames_per_bin = settings.get<unsigned int>("video.frames_per_bin", 2);
unsigned int num_shape_coeff = reconstruction_data.morphable_model.get_shape_model().get_num_principal_components(); this->reconstruction_data = reconstruction_data;
unsigned int num_shape_coeff = reconstruction_data.
morphable_model.get_shape_model().get_num_principal_components();
this->num_shape_coefficients_to_fit = settings.get<unsigned int>( this->num_shape_coefficients_to_fit = settings.get<unsigned int>(
"video.num_shape_coefficients_to_fit", num_shape_coeff); "frames.num_shape_coefficients_to_fit", num_shape_coeff);
// initialize bins // initialize bins
bins.resize(num_yaw_bins); bins.resize(num_yaw_bins);
// reset frame count // reset frame count (to be sure)
n_frames = 0; n_frames = 0;
total_frames = 0; total_frames = 0;
std::cout << "Settings: " << std::endl << // std::cout << "Settings: " << std::endl <<
"min_frames: " << min_frames << std::endl << // "min_frames: " << min_frames << std::endl <<
"drop_frames: " << drop_frames << std::endl << // "drop_frames: " << drop_frames << std::endl <<
"frames_per_bin: " << frames_per_bin << std::endl << // "frames_per_bin: " << frames_per_bin << std::endl <<
"num_shape_coefficients_to_fit: " << num_shape_coefficients_to_fit << std::endl; // "num_shape_coefficients_to_fit: " << num_shape_coefficients_to_fit << std::endl;
std::cout << "total frames in video: " << cap.get(CV_CAP_PROP_FRAME_COUNT) << std::endl;
} }
bool is_playing() { bool is_playing() {
...@@ -323,7 +690,11 @@ public: ...@@ -323,7 +690,11 @@ public:
__stop(); __stop();
std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames << std::endl; std::cout << "Video stopped at:" << total_frames << " frames - in buff " << n_frames << std::endl;
} };
Keyframe get_last_keyframe() {
return last_keyframe;
};
/** /**
* *
...@@ -341,25 +712,24 @@ public: ...@@ -341,25 +712,24 @@ public:
// keep the last frame here, so we can play a video subsequently: // keep the last frame here, so we can play a video subsequently:
last_frame_number = total_frames; last_frame_number = total_frames;
last_frame = frame;
// TODO: only calculate lapscore within the bounding box of the face. // TODO: only calculate lapscore within the bounding box of the face.
auto keyframe = generate_new_keyframe(frame); auto keyframe = generate_new_keyframe(frame);
bool frame_added = try_add(keyframe); bool frame_added = try_add(keyframe);
// Added or not, we put this as the last keyframe
last_keyframe = keyframe;
if(frame_added) { if(frame_added) {
n_frames++; n_frames++;
// Setting that the buffer has changed: // Setting that the buffer has changed:
frame_buffer_changed = true; frame_buffer_changed = true;
std::cout << "frame added(" << total_frames << "): " << keyframe.score << ", " << keyframe.yaw_angle << std::endl;
} }
total_frames++; total_frames++;
// fill up the buffer until we hit the minimum frames we want in the buffer. // fill up the buffer until we hit the minimum frames we want in the buffer.
if(n_frames < min_frames && total_frames < 30) { if(n_frames < min_frames) {
std::cout << "not enough frames yet: " << n_frames << "/" << min_frames << std::endl;
return next(); return next();
} }
...@@ -442,9 +812,12 @@ public: ...@@ -442,9 +812,12 @@ public:
cap.release(); cap.release();
}; };
int get_frame_number() {
return total_frames;
}
private: private:
cv::VideoCapture cap; cv::VideoCapture cap;
std::deque<Keyframe> frame_buffer;
eos::fitting::ReconstructionData reconstruction_data; eos::fitting::ReconstructionData reconstruction_data;
using BinContent = std::vector<Keyframe>; using BinContent = std::vector<Keyframe>;
...@@ -458,7 +831,7 @@ private: ...@@ -458,7 +831,7 @@ private:
unsigned int frames_per_bin; unsigned int frames_per_bin;
// TODO: make set-able // TODO: make set-able
// total frames in processed, not persee in buffer (skipped frames) // total frames in processed, not necessarily in buffer (skipped frames)
int total_frames; int total_frames;
// total frames in buffer // total frames in buffer
......
...@@ -39,8 +39,13 @@ add_executable(accuracy-evaluation accuracy-evaluation.cpp) ...@@ -39,8 +39,13 @@ add_executable(accuracy-evaluation accuracy-evaluation.cpp)
target_link_libraries(accuracy-evaluation eos ${OpenCV_LIBS} ${Boost_LIBRARIES}) target_link_libraries(accuracy-evaluation eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
target_link_libraries(accuracy-evaluation "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>") target_link_libraries(accuracy-evaluation "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
add_executable(openmp-test openmp-test.cpp)
target_link_libraries(openmp-test eos ${OpenCV_LIBS} ${Boost_LIBRARIES})
target_link_libraries(openmp-test "$<$<CXX_COMPILER_ID:GNU>:-pthread>$<$<CXX_COMPILER_ID:Clang>:-pthreads>")
# Install targets: # Install targets:
install(TARGETS scm-to-cereal DESTINATION bin) install(TARGETS scm-to-cereal DESTINATION bin)
install(TARGETS bfm-binary-to-cereal DESTINATION bin) install(TARGETS bfm-binary-to-cereal DESTINATION bin)
install(TARGETS edgestruct-csv-to-json DESTINATION bin) install(TARGETS edgestruct-csv-to-json DESTINATION bin)
install(TARGETS accuracy-evaluation DESTINATION bin) install(TARGETS accuracy-evaluation DESTINATION bin)
install(TARGETS openmp-test DESTINATION bin)
...@@ -48,6 +48,7 @@ using eos::core::Landmark; ...@@ -48,6 +48,7 @@ using eos::core::Landmark;
using eos::core::LandmarkCollection; using eos::core::LandmarkCollection;
using eos::video::BufferedVideoIterator; using eos::video::BufferedVideoIterator;
using eos::video::WeightedIsomapAveraging; using eos::video::WeightedIsomapAveraging;
using eos::video::ReconstructionVideoWriter;
using cv::Mat; using cv::Mat;
using cv::Vec2f; using cv::Vec2f;
using cv::Vec3f; using cv::Vec3f;
...@@ -192,8 +193,13 @@ void render_output( ...@@ -192,8 +193,13 @@ void render_output(
boost::property_tree::ptree settings, boost::property_tree::ptree settings,
int n_iter) { int n_iter) {
std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
bool make_video = settings.get<bool>("output.save_video", false);
bool show_video = settings.get<bool>("output.show_video", false);
Mat merged_isomap; Mat merged_isomap;
WeightedIsomapAveraging isomap_averaging(60.f); // merge all triangles that are facing <60° towards the camera WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
eos::video::PcaCoefficientMerging pca_shape_merging; eos::video::PcaCoefficientMerging pca_shape_merging;
auto landmark_mapper = reconstruction_data.landmark_mapper; auto landmark_mapper = reconstruction_data.landmark_mapper;
...@@ -208,10 +214,9 @@ void render_output( ...@@ -208,10 +214,9 @@ void render_output(
auto unmodified_frame = frame.clone(); auto unmodified_frame = frame.clone();
int frame_number = key_frames[i].frame_number; int frame_number = key_frames[i].frame_number;
float yaw_angle = glm::degrees(glm::yaw(rendering_paramss[i].get_rotation())); float yaw_angle = key_frames[i].yaw_angle;// glm::degrees(glm::yaw(rendering_paramss[i].get_rotation()));
cv::imwrite("/tmp/eos/" + cv::imwrite("/tmp/eos/" + std::to_string(frame_number) + "_" + std::to_string(yaw_angle) + ".png", frame);
std::to_string(frame_number) + "_" + std::to_string(yaw_angle) + ".png", frame);
// Extract the texture using the fitted mesh from this frame: // Extract the texture using the fitted mesh from this frame:
Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_paramss[i], frame.cols, frame.rows); Mat affine_cam = fitting::get_3x4_affine_camera_matrix(rendering_paramss[i], frame.cols, frame.rows);
...@@ -302,8 +307,20 @@ void render_output( ...@@ -302,8 +307,20 @@ void render_output(
false, false,
rendering); rendering);
cv::imshow("render", rendering); // if(save_video) {
cv::waitKey(10); // cv::imshow("render", rendering);
// cv::waitKey(1);
// }
//
// if(show_video) {
// cv::imshow("render", rendering);
// cv::waitKey(1);
// }
//
// eos::video::ReconstructionVideoWriter outputVideo;
// Size S = Size((int) inputVideo.get(CV_CAP_PROP_FRAME_WIDTH), //Acquire input size
// (int) inputVideo.get(CV_CAP_PROP_FRAME_HEIGHT));
// outputVideo.open(NAME , ex, inputVideo.get(CV_CAP_PROP_FPS),S, true);
} }
/** /**
...@@ -334,8 +351,10 @@ void evaluate_results( ...@@ -334,8 +351,10 @@ void evaluate_results(
boost::property_tree::ptree settings, boost::property_tree::ptree settings,
int n_iter) { int n_iter) {
WeightedIsomapAveraging isomap_averaging(10.f); // merge all triangles that are facing <60° towards the camera std::string output_path = settings.get<std::string>("output.output_path", "/tmp");
float merge_isomap_face_angle = settings.get<float>("output.merge_isomap_face_angle", 60.f);
WeightedIsomapAveraging isomap_averaging(merge_isomap_face_angle); // merge all triangles that are facing <60° towards the camera
Mat merged_isomap; Mat merged_isomap;
fs::path outputfilebase = annotations[0]; fs::path outputfilebase = annotations[0];
...@@ -377,11 +396,11 @@ void evaluate_results( ...@@ -377,11 +396,11 @@ void evaluate_results(
rendering_params.get_projection(), rendering_params.get_projection(),
fitting::get_opencv_viewport(frame_width, frame_height)); fitting::get_opencv_viewport(frame_width, frame_height));
bool eyes_open = isomap_averaging.has_eyes_open(frame, landmarks); // bool eyes_open = isomap_averaging.has_eyes_open(frame, landmarks);
//
if (!eyes_open) { // if (!eyes_open) {
continue; // continue;
} // }
// cv::imshow("Img", outimg); // cv::imshow("Img", outimg);
// cv::waitKey(0); // cv::waitKey(0);
...@@ -649,19 +668,21 @@ int main(int argc, char *argv[]) { ...@@ -649,19 +668,21 @@ int main(int argc, char *argv[]) {
// Start getting video frames: // Start getting video frames:
vid_iterator.start(); vid_iterator.start();
// vid_writer.start();
// Count the amount of iterations: // Count the amount of iterations:
int n_iter = 0; int n_iter = 0;
while(vid_iterator.is_playing()) { while(vid_iterator.is_playing()) {
auto key_frames = vid_iterator.get_keyframes(); auto key_frames = vid_iterator.get_keyframes();
// Generate a sublist of the total available landmark list: // Generate a sublist of the total available landmark list:
auto landmark_sublist = sample_landmarks(key_frames, landmark_list); auto landmark_sublist = sample_landmarks(key_frames, landmark_list);
// std::cout << vid_iterator.to_string() << std::endl;
// it makes no sense to update pca_coeff if nothing in the buffer has changed: // it makes no sense to update pca_coeff if nothing in the buffer has changed:
if (vid_iterator.has_changed()) { if (vid_iterator.has_changed()) {
std::cout << "Going to reconstruct with " << key_frames.size() << " images."<< std::endl; std::cout << "Going to reconstruct with " << key_frames.size() << " images."<< std::endl;
// Fit shape and pose: // Fit shape and pose:
auto t1 = std::chrono::high_resolution_clock::now(); auto t1 = std::chrono::high_resolution_clock::now();
std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel( std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi_parallel(
...@@ -681,7 +702,8 @@ int main(int argc, char *argv[]) { ...@@ -681,7 +702,8 @@ int main(int argc, char *argv[]) {
boost::none, boost::none,
pca_shape_coefficients, pca_shape_coefficients,
blendshape_coefficients, blendshape_coefficients,
fitted_image_points fitted_image_points,
settings
); );
auto t2 = std::chrono::high_resolution_clock::now(); auto t2 = std::chrono::high_resolution_clock::now();
...@@ -690,62 +712,62 @@ int main(int argc, char *argv[]) { ...@@ -690,62 +712,62 @@ int main(int argc, char *argv[]) {
<< "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() / key_frames.size() << "ms, mean(" << std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count() / key_frames.size()
<< "ms)" << std::endl; << "ms)" << std::endl;
vid_iterator.update_reconstruction_coeff(pca_shape_coefficients);
evaluate_results( // evaluate_results(
key_frames,
rendering_paramss,
meshs,
pca_shape_coefficients,
blendshape_coefficients,
fitted_image_points,
annotations,
reconstruction_data,
settings,
n_iter
);
} else {
std::cout << "No reconstruction - buffer did not change" << std::endl;
}
// // Render output:
// render_output(
// key_frames, // key_frames,
// rendering_paramss, // rendering_paramss,
// landmark_sublist,
// meshs, // meshs,
// pca_shape_coefficients, // pca_shape_coefficients,
// blendshape_coefficients, // blendshape_coefficients,
// fitted_image_points, // fitted_image_points,
// annotations, // annotations,
// reconstruction_data, // reconstruction_data,
// vid_iterator.last_frame,
// vid_iterator.last_frame_number,
// settings, // settings,
// n_iter // n_iter
// ); // );
} else {
// std::cout << "No reconstruction - buffer did not change" << std::endl;
}
// Get new frames: // Get new frames:
n_iter++; n_iter++;
} }
auto key_frames = vid_iterator.get_keyframes(); // vid_writer.__stop();
if(settings.get<bool>("output.make_video", false)) {
ReconstructionVideoWriter vid_writer;
try {
vid_writer = ReconstructionVideoWriter(videofile.string(), reconstruction_data, settings);
} catch(std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
}
// Render output:
std::cout << "Waiting for video to be completed..." << std::endl;
vid_writer.update_reconstruction_coeff(pca_shape_coefficients);
std::cout << "Going to reconstruct with " << key_frames.size() << " images."<< std::endl; while (vid_writer.next()) {
evaluate_results( printf("%d/%d\r", vid_writer.get_frame_number(), vid_iterator.get_frame_number());
key_frames, }
rendering_paramss,
meshs, }
pca_shape_coefficients,
blendshape_coefficients, // auto key_frames = vid_iterator.get_keyframes();
fitted_image_points, // std::cout << "Going to reconstruct with " << key_frames.size() << " images."<< std::endl;
annotations, //
reconstruction_data, // evaluate_results(
settings, // key_frames,
n_iter // rendering_paramss,
); // meshs,
// pca_shape_coefficients,
// blendshape_coefficients,
// fitted_image_points,
// annotations,
// reconstruction_data,
// settings,
// n_iter
// );
//todo: we could build our final obj here? //todo: we could build our final obj here?
......
#include "opencv2/core/core.hpp"
#include "opencv2/opencv.hpp"
#include "opencv2/highgui/highgui.hpp"
#include <complex>
#include <iostream>
#include <omp.h>
typedef std::complex<double> complex;
/**
* Calculates distance between two images, returns -1 if dims are not the same.
*
* @param one
* @param two
* @return
*/
float calculate_distance_images(cv::Mat one, cv::Mat two) {
int width = one.cols;
int height = one.rows;
if(width != two.cols || height != two.rows) {
return -1;
}
float distance = 0.0;
for (int y = 0; y < height - 1; y++) {
for (int x = 0; x < width -1; x++) {
distance += fabs(one.at<uchar>(y, x) - two.at<uchar>(y, x));
}
}
return distance;
}
void cpu(cv::Mat A, cv::Mat Anew) {
int n = A.rows;
int m = A.cols;
float error = 1000;
float tol = 30;
int iter = 0;
int iter_max = 100;
auto t1 = std::chrono::high_resolution_clock::now();
while (error > tol && iter < iter_max)
{
error = 0.0;
#pragma omp parallel num_threads(4)
{
#pragma omp for reduction(max:error)
for (int j = 1; j < n - 1; j++)
{
for (int i = 1; i < m - 1; i++)
{
Anew.at<uchar>(j, i) = 0.25 * (A.at<uchar>(j, i + 1) + A.at<uchar>(j, i - 1)
+ A.at<uchar>(j - 1, i) + A.at<uchar>(j + 1, i));
error = fmax(error, fabs(Anew.at<uchar>(j, i) - A.at<uchar>(j, i)));
}
}
}
iter++;
}
}
void gpu(cv::Mat A, cv::Mat Anew) {
int n = A.rows;
int m = A.cols;
float error = 1000;
float tol = 30;
int iter = 0;
int iter_max = 100;
auto t1 = std::chrono::high_resolution_clock::now();
while ( error > tol && iter < iter_max )
{
error = 0.0;
#pragma omp target
{
#pragma omp parallel for reduction(max:error)
for (int j = 1; j < n - 1; j++)
{
for (int i = 1; i < m - 1; i++)
{
Anew.at<uchar>(j, i) = 0.25 * (A.at<uchar>(j, i + 1) + A.at<uchar>(j, i - 1)
+ A.at<uchar>(j - 1, i) + A.at<uchar>(j + 1, i));
error = fmax(error, fabs(Anew.at<uchar>(j, i) - A.at<uchar>(j, i)));
}
}
iter++;
}
}
}
int main(int argc, char** argv)
{
cv::Mat original = cv::imread("/data/img.jpg");
cv::cvtColor(original, original, CV_BGR2GRAY);
cv::Mat gpuImage = original.clone();
cv::Mat gpuImageNew = original.clone();
auto t1 = std::chrono::high_resolution_clock::now();
gpu(gpuImage, gpuImageNew);
auto t2 = std::chrono::high_resolution_clock::now();
std::cout << "gpu time: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count()
<< "ms" << std::endl;
cv::Mat cpuImage = original.clone();
cv::Mat cpuImageNew = original.clone();
t1 = std::chrono::high_resolution_clock::now();
cpu(cpuImage, cpuImageNew);
t2 = std::chrono::high_resolution_clock::now();
float distance = calculate_distance_images(gpuImageNew, cpuImageNew);
std::cout << "cpu time: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(t2-t1).count()
<< "ms" << std::endl;
std::cout << "Distance gpu - cpu: " << distance << std::endl;
cv::imshow("gpu", gpuImageNew);
cv::imshow("cpu", cpuImageNew);
cv::waitKey(0);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment