Skip to content

E
eos
Commit 470615a6 authored by Patrik Huber's avatar Patrik Huber
Browse files
Options

Changed PcaModel to use Eigen::Matrix instead of cv::Mat 

This is a big step towards being not requiring OpenCV anymore as a dependency.
All models will have to be converted and re-downloaded.

* PcaModel: Removed the RNG from the class, and added a RNG parameter to the draw_sample method
* PcaModel: Added a get_eigenvalues() accessor that returns all eigenvalues
* MorphableModel: Changed CEREAL_CLASS_VERSION from 0 to 1 (and added a version check to the serialize method)
* MorphableModel: Changed texture_coordinates from cv::Vec2f to array<double, 2>
* Added binary Eigen::Matrix serialisation for cereal
* Changed Blendshape to Eigen::Vector as well
* Adjusted blendshape_fitting.hpp accordingly - and it now takes most parameters by const&
* Adjusted fitting.hpp accordingly - now also takes most parameters by const&
* Adjusted linear_shape_fitting.hpp: It's not totally converted, still using cv::Mat for the core algorithm
parent 29817961
Hide whitespace changes
Inline Side-by-side
Showing with 247 additions and 141 deletions
include/eos/fitting/blendshape_fitting.hpp
View file @ 470615a6
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
#include "Eigen/Core" // for nnls.h #include "Eigen/Core" // for nnls.h
#include "nnls.h" #include "nnls.h"
#include "Eigen/Core"
#include "opencv2/core/core.hpp" #include "opencv2/core/core.hpp"
#include <vector> #include <vector>
...@@ -55,30 +56,27 @@ namespace eos { ...@@ -55,30 +56,27 @@ namespace eos {
* @param[in] lambda A regularisation parameter, constraining the L2-norm of the coefficients. * @param[in] lambda A regularisation parameter, constraining the L2-norm of the coefficients.
* @return The estimated blendshape-coefficients. * @return The estimated blendshape-coefficients.
*/ */
inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, float lambda=500.0f) inline std::vector<float> fit_blendshapes_to_landmarks_linear(const std::vector<eos::morphablemodel::Blendshape>& blendshapes, const Eigen::VectorXf& face_instance, cv::Mat affine_camera_matrix, const std::vector<cv::Vec2f>& landmarks, const std::vector<int>& vertex_ids, float lambda=500.0f)
{ {
using cv::Mat;
assert(landmarks.size() == vertex_ids.size()); assert(landmarks.size() == vertex_ids.size());
int num_coeffs_to_fit = blendshapes.size(); using cv::Mat;
int num_landmarks = static_cast<int>(landmarks.size()); using Eigen::MatrixXf;
const int num_blendshapes = blendshapes.size();
const int num_landmarks = static_cast<int>(landmarks.size());
// Copy all blendshapes into a "basis" matrix with each blendshape being a column: // Copy all blendshapes into a "basis" matrix with each blendshape being a column:
cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points // $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$: // And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1); Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_blendshapes, CV_32FC1);
int row_index = 0; int row_index = 0;
for (int i = 0; i < num_landmarks; ++i) { for (int i = 0; i < num_landmarks; ++i) {
//Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better. Mat blendshapes_basis_as_mat = Mat(blendshapes_as_basis.rows(), blendshapes_as_basis.cols(), CV_32FC1, blendshapes_as_basis.data());
Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3); Mat basis_rows = blendshapes_basis_as_mat.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
//basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3)); basis_rows.copyTo(V_hat_h.rowRange(row_index, row_index + 3));
basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
} }
// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal: // Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
...@@ -99,7 +97,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m ...@@ -99,7 +97,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m
Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1); Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
for (int i = 0; i < num_landmarks; ++i) { for (int i = 0; i < num_landmarks; ++i) {
//cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]); //cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f); cv::Vec4f model_mean(face_instance(vertex_ids[i]*3), face_instance(vertex_ids[i]*3 + 1), face_instance(vertex_ids[i]*3 + 2), 1.0f);
v_bar.at<float>(4 * i, 0) = model_mean[0]; v_bar.at<float>(4 * i, 0) = model_mean[0];
v_bar.at<float>((4 * i) + 1, 0) = model_mean[1]; v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
v_bar.at<float>((4 * i) + 2, 0) = model_mean[2]; v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
...@@ -111,7 +109,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m ...@@ -111,7 +109,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m
Mat A = P * V_hat_h; // camera matrix times the basis Mat A = P * V_hat_h; // camera matrix times the basis
Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks. Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks.
Mat AtAReg = A.t() * A + lambda * Mat::eye(num_coeffs_to_fit, num_coeffs_to_fit, CV_32FC1); Mat AtAReg = A.t() * A + lambda * Mat::eye(num_blendshapes, num_blendshapes, CV_32FC1);
// Solve using OpenCV: // Solve using OpenCV:
Mat c_s; Mat c_s;
bool non_singular = cv::solve(AtAReg, -A.t() * b, c_s, cv::DECOMP_SVD); // DECOMP_SVD calculates the pseudo-inverse if the matrix is not invertible. bool non_singular = cv::solve(AtAReg, -A.t() * b, c_s, cv::DECOMP_SVD); // DECOMP_SVD calculates the pseudo-inverse if the matrix is not invertible.
...@@ -136,28 +134,27 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m ...@@ -136,28 +134,27 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m
* @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points. * @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
* @return The estimated blendshape-coefficients. * @return The estimated blendshape-coefficients.
*/ */
inline std::vector<float> fit_blendshapes_to_landmarks_nnls(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids) inline std::vector<float> fit_blendshapes_to_landmarks_nnls(const std::vector<eos::morphablemodel::Blendshape>& blendshapes, const Eigen::VectorXf& face_instance, cv::Mat affine_camera_matrix, const std::vector<cv::Vec2f>& landmarks, const std::vector<int>& vertex_ids)
{ {
using cv::Mat;
assert(landmarks.size() == vertex_ids.size()); assert(landmarks.size() == vertex_ids.size());
int num_coeffs_to_fit = blendshapes.size(); using cv::Mat;
int num_landmarks = static_cast<int>(landmarks.size()); using Eigen::MatrixXf;
const int num_blendshapes = blendshapes.size();
const int num_landmarks = static_cast<int>(landmarks.size());
// Copy all blendshapes into a "basis" matrix with each blendshape being a column: // Copy all blendshapes into a "basis" matrix with each blendshape being a column:
cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points // $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$: // And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1); Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_blendshapes, CV_32FC1);
int row_index = 0; int row_index = 0;
for (int i = 0; i < num_landmarks; ++i) { for (int i = 0; i < num_landmarks; ++i) {
Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3); Mat blendshapes_basis_as_mat = Mat(blendshapes_as_basis.rows(), blendshapes_as_basis.cols(), CV_32FC1, blendshapes_as_basis.data());
basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3)); // Todo: I think we can remove colRange here, as we always want to use all given blendshapes Mat basis_rows = blendshapes_basis_as_mat.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
basis_rows.copyTo(V_hat_h.rowRange(row_index, row_index + 3));
row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
} }
// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal: // Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
...@@ -177,7 +174,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_nnls(std::vector<eos::mor ...@@ -177,7 +174,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_nnls(std::vector<eos::mor
// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t // The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1); Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
for (int i = 0; i < num_landmarks; ++i) { for (int i = 0; i < num_landmarks; ++i) {
cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f); cv::Vec4f model_mean(face_instance(vertex_ids[i]*3), face_instance(vertex_ids[i]*3 + 1), face_instance(vertex_ids[i]*3 + 2), 1.0f);
v_bar.at<float>(4 * i, 0) = model_mean[0]; v_bar.at<float>(4 * i, 0) = model_mean[0];
v_bar.at<float>((4 * i) + 1, 0) = model_mean[1]; v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
v_bar.at<float>((4 * i) + 2, 0) = model_mean[2]; v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
......
include/eos/fitting/fitting.hpp
View file @ 470615a6
...@@ -65,39 +65,37 @@ namespace eos { ...@@ -65,39 +65,37 @@ namespace eos {
* @param[out] blendshape_coefficients Output parameter that will contain the resulting blendshape coefficients. * @param[out] blendshape_coefficients Output parameter that will contain the resulting blendshape coefficients.
* @return The fitted model shape instance. * @return The fitted model shape instance.
*/ */
inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::MorphableModel morphable_model, std::vector<morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda, boost::optional<int> num_coefficients_to_fit, std::vector<float>& pca_shape_coefficients, std::vector<float>& blendshape_coefficients) inline Eigen::VectorXf fit_shape(cv::Mat affine_camera_matrix, const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<cv::Vec2f>& image_points, const std::vector<int>& vertex_indices, float lambda, boost::optional<int> num_coefficients_to_fit, std::vector<float>& pca_shape_coefficients, std::vector<float>& blendshape_coefficients)
{ {
using cv::Mat; using Eigen::VectorXf;
using Eigen::MatrixXf;
Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
std::vector<float> last_blendshape_coeffs, current_blendshape_coeffs; std::vector<float> last_blendshape_coeffs, current_blendshape_coeffs;
std::vector<float> last_pca_coeffs, current_pca_coeffs; std::vector<float> last_pca_coeffs, current_pca_coeffs;
current_blendshape_coeffs.resize(blendshapes.size()); // starting values t_0, all zeros current_blendshape_coeffs.resize(blendshapes.size()); // starting values t_0, all zeros
// no starting values for current_pca_coeffs required, since we start with the shape fitting, and cv::norm of an empty vector is 0. // no starting values for current_pca_coeffs required, since we start with the shape fitting, and cv::norm of an empty vector is 0.
Mat combined_shape;
VectorXf combined_shape;
do // run at least once: do // run at least once:
{ {
last_blendshape_coeffs = current_blendshape_coeffs; last_blendshape_coeffs = current_blendshape_coeffs;
last_pca_coeffs = current_pca_coeffs; last_pca_coeffs = current_pca_coeffs;
// Estimate the PCA shape coefficients with the current blendshape coefficients (0 in the first iteration): // Estimate the PCA shape coefficients with the current blendshape coefficients (0 in the first iteration):
Mat mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Mat(last_blendshape_coeffs); VectorXf mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(last_blendshape_coeffs.data(), last_blendshape_coeffs.size());
current_pca_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_camera_matrix, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_coefficients_to_fit); current_pca_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_camera_matrix, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_coefficients_to_fit);
// Estimate the blendshape coefficients with the current PCA model estimate: // Estimate the blendshape coefficients with the current PCA model estimate:
Mat pca_model_shape = morphable_model.get_shape_model().draw_sample(current_pca_coeffs); VectorXf pca_model_shape = morphable_model.get_shape_model().draw_sample(current_pca_coeffs);
current_blendshape_coeffs = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, pca_model_shape, affine_camera_matrix, image_points, vertex_indices); current_blendshape_coeffs = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, pca_model_shape, affine_camera_matrix, image_points, vertex_indices);
combined_shape = pca_model_shape + blendshapes_as_basis * Mat(current_blendshape_coeffs); combined_shape = pca_model_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(current_blendshape_coeffs.data(), current_blendshape_coeffs.size()); // Todo/Note: Could move outside the loop, not needed in here actually
} while (std::abs(cv::norm(current_pca_coeffs) - cv::norm(last_pca_coeffs)) >= 0.01 || std::abs(cv::norm(current_blendshape_coeffs) - cv::norm(last_blendshape_coeffs)) >= 0.01); } while (std::abs(cv::norm(current_pca_coeffs) - cv::norm(last_pca_coeffs)) >= 0.01 || std::abs(cv::norm(current_blendshape_coeffs) - cv::norm(last_blendshape_coeffs)) >= 0.01);
pca_shape_coefficients = current_pca_coeffs; pca_shape_coefficients = current_pca_coeffs;
blendshape_coefficients = current_blendshape_coeffs; blendshape_coefficients = current_blendshape_coeffs;
return combined_shape; return combined_shape;
}; };
...@@ -115,7 +113,7 @@ inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::Morphable ...@@ -115,7 +113,7 @@ inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::Morphable
* @param[in] num_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients. * @param[in] num_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients.
* @return The fitted model shape instance. * @return The fitted model shape instance.
*/ */
inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::MorphableModel morphable_model, std::vector<morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda = 3.0f, boost::optional<int> num_coefficients_to_fit = boost::optional<int>()) inline Eigen::VectorXf fit_shape(cv::Mat affine_camera_matrix, const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<cv::Vec2f>& image_points, const std::vector<int>& vertex_indices, float lambda = 3.0f, boost::optional<int> num_coefficients_to_fit = boost::optional<int>())
{ {
std::vector<float> unused; std::vector<float> unused;
return fit_shape(affine_camera_matrix, morphable_model, blendshapes, image_points, vertex_indices, lambda, num_coefficients_to_fit, unused, unused); return fit_shape(affine_camera_matrix, morphable_model, blendshapes, image_points, vertex_indices, lambda, num_coefficients_to_fit, unused, unused);
...@@ -243,7 +241,8 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co ...@@ -243,7 +241,8 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
using std::vector; using std::vector;
using cv::Vec2f; using cv::Vec2f;
using cv::Vec4f; using cv::Vec4f;
using cv::Mat; using Eigen::VectorXf;
using Eigen::MatrixXf;
if (!num_shape_coefficients_to_fit) if (!num_shape_coefficients_to_fit)
{ {
...@@ -262,11 +261,11 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co ...@@ -262,11 +261,11 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
blendshape_coefficients.resize(blendshapes.size()); blendshape_coefficients.resize(blendshapes.size());
} }
Mat blendshapes_as_basis = morphablemodel::to_matrix(blendshapes); MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
// Current mesh - either from the given coefficients, or the mean: // Current mesh - either from the given coefficients, or the mean:
Mat current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients); VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
Mat current_combined_shape = current_pca_shape + blendshapes_as_basis * Mat(blendshape_coefficients); VectorXf current_combined_shape = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
auto current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); auto current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
// The 2D and 3D point correspondences used for the fitting: // The 2D and 3D point correspondences used for the fitting:
...@@ -294,11 +293,11 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co ...@@ -294,11 +293,11 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
current_pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image_height); current_pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image_height);
fitting::RenderingParameters rendering_params(current_pose, image_width, image_height); fitting::RenderingParameters rendering_params(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height); cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height);
blendshape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points, vertex_indices); blendshape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points, vertex_indices);
// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given): // Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
current_combined_shape = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Mat(blendshape_coefficients); current_combined_shape = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
// The static (fixed) landmark correspondences which will stay the same throughout // The static (fixed) landmark correspondences which will stay the same throughout
...@@ -346,17 +345,17 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co ...@@ -346,17 +345,17 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
current_pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image_height); current_pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image_height);
rendering_params = fitting::RenderingParameters(current_pose, image_width, image_height); rendering_params = fitting::RenderingParameters(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height); cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height);
// Estimate the PCA shape coefficients with the current blendshape coefficients: // Estimate the PCA shape coefficients with the current blendshape coefficients:
Mat mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Mat(blendshape_coefficients); VectorXf mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_shape_coefficients_to_fit); pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_shape_coefficients_to_fit);
// Estimate the blendshape coefficients with the current PCA model estimate: // Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients); current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
blendshape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points, vertex_indices); blendshape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points, vertex_indices);
current_combined_shape = current_pca_shape + blendshapes_as_basis * Mat(blendshape_coefficients); current_combined_shape = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates()); current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
} }
......
include/eos/fitting/linear_shape_fitting.hpp
View file @ 470615a6
...@@ -57,7 +57,7 @@ namespace eos { ...@@ -57,7 +57,7 @@ namespace eos {
* @param[in] model_standard_deviation The standard deviation of the 3D vertex points in the 3D model, projected to 2D (so the value is in pixels). * @param[in] model_standard_deviation The standard deviation of the 3D vertex points in the 3D model, projected to 2D (so the value is in pixels).
* @return The estimated shape-coefficients (alphas). * @return The estimated shape-coefficients (alphas).
*/ */
inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::MorphableModel& morphable_model, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, cv::Mat base_face=cv::Mat(), float lambda=3.0f, boost::optional<int> num_coefficients_to_fit=boost::optional<int>(), boost::optional<float> detector_standard_deviation=boost::optional<float>(), boost::optional<float> model_standard_deviation=boost::optional<float>()) inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::MorphableModel& morphable_model, cv::Mat affine_camera_matrix, const std::vector<cv::Vec2f>& landmarks, const std::vector<int>& vertex_ids, Eigen::VectorXf base_face=Eigen::VectorXf(), float lambda=3.0f, boost::optional<int> num_coefficients_to_fit=boost::optional<int>(), boost::optional<float> detector_standard_deviation=boost::optional<float>(), boost::optional<float> model_standard_deviation=boost::optional<float>())
{ {
using cv::Mat; using cv::Mat;
assert(landmarks.size() == vertex_ids.size()); assert(landmarks.size() == vertex_ids.size());
...@@ -65,7 +65,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo ...@@ -65,7 +65,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo
int num_coeffs_to_fit = num_coefficients_to_fit.get_value_or(morphable_model.get_shape_model().get_num_principal_components()); int num_coeffs_to_fit = num_coefficients_to_fit.get_value_or(morphable_model.get_shape_model().get_num_principal_components());
int num_landmarks = static_cast<int>(landmarks.size()); int num_landmarks = static_cast<int>(landmarks.size());
if (base_face.empty()) if (base_face.size() == 0)
{ {
base_face = morphable_model.get_shape_model().get_mean(); base_face = morphable_model.get_shape_model().get_mean();
} }
...@@ -75,7 +75,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo ...@@ -75,7 +75,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1); Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
int row_index = 0; int row_index = 0;
for (int i = 0; i < num_landmarks; ++i) { for (int i = 0; i < num_landmarks; ++i) {
Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better. auto basis_rows_ = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
Mat basis_rows = Mat(basis_rows_.rows(), basis_rows_.cols(), CV_32FC1, basis_rows_.data());
//basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3)); //basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3));
basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3)); basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
...@@ -108,7 +109,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo ...@@ -108,7 +109,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo
Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1); Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
for (int i = 0; i < num_landmarks; ++i) { for (int i = 0; i < num_landmarks; ++i) {
//cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]); //cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
cv::Vec4f model_mean(base_face.at<float>(vertex_ids[i] * 3), base_face.at<float>(vertex_ids[i] * 3 + 1), base_face.at<float>(vertex_ids[i] * 3 + 2), 1.0f); cv::Vec4f model_mean(base_face(vertex_ids[i] * 3), base_face(vertex_ids[i] * 3 + 1), base_face(vertex_ids[i] * 3 + 2), 1.0f);
v_bar.at<float>(4 * i, 0) = model_mean[0]; v_bar.at<float>(4 * i, 0) = model_mean[0];
v_bar.at<float>((4 * i) + 1, 0) = model_mean[1]; v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
v_bar.at<float>((4 * i) + 2, 0) = model_mean[2]; v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
......
include/eos/morphablemodel/Blendshape.hpp
View file @ 470615a6
...@@ -22,11 +22,11 @@ ...@@ -22,11 +22,11 @@
#ifndef BLENDSHAPE_HPP_ #ifndef BLENDSHAPE_HPP_
#define BLENDSHAPE_HPP_ #define BLENDSHAPE_HPP_
#include "eos/morphablemodel/io/mat_cerealisation.hpp" #include "eos/morphablemodel/io/eigen_cerealisation.hpp"
#include "cereal/types/string.hpp" #include "cereal/types/string.hpp"
#include "cereal/archives/binary.hpp" #include "cereal/archives/binary.hpp"
#include "opencv2/core/core.hpp" #include "Eigen/Core"
#include <string> #include <string>
#include <vector> #include <vector>
...@@ -46,7 +46,7 @@ namespace eos { ...@@ -46,7 +46,7 @@ namespace eos {
struct Blendshape struct Blendshape
{ {
std::string name; ///< Name of the blendshape. std::string name; ///< Name of the blendshape.
cv::Mat deformation; ///< A 3m x 1 col-vector (xyzxyz...)', where m is the number of model-vertices. Has the same format as PcaModel::mean. Eigen::VectorXf deformation; ///< A 3m x 1 col-vector (xyzxyz...)', where m is the number of model-vertices. Has the same format as PcaModel::mean.
friend class cereal::access; friend class cereal::access;
/** /**
...@@ -89,14 +89,15 @@ inline std::vector<Blendshape> load_blendshapes(std::string filename) ...@@ -89,14 +89,15 @@ inline std::vector<Blendshape> load_blendshapes(std::string filename)
* @param[in] blendshapes Vector of blendshapes. * @param[in] blendshapes Vector of blendshapes.
* @return The resulting matrix. * @return The resulting matrix.
*/ */
inline cv::Mat to_matrix(const std::vector<Blendshape>& blendshapes) inline Eigen::VectorXf to_matrix(const std::vector<Blendshape>& blendshapes)
{ {
// assert: all blendshapes have to have the same number of rows, and one col
assert(blendshapes.size() > 0); assert(blendshapes.size() > 0);
cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // Todo: Assert all blendshapes have to have the same number of rows, and one col
Eigen::MatrixXf blendshapes_as_basis(blendshapes[0].deformation.rows(), blendshapes.size());
for (int i = 0; i < blendshapes.size(); ++i) for (int i = 0; i < blendshapes.size(); ++i)
{ {
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i)); blendshapes_as_basis.col(i) = blendshapes[i].deformation;
} }
return blendshapes_as_basis; return blendshapes_as_basis;
}; };
......
include/eos/morphablemodel/MorphableModel.hpp
View file @ 470615a6
...@@ -36,18 +36,18 @@ ...@@ -36,18 +36,18 @@
#include "glm/vec3.hpp" #include "glm/vec3.hpp"
#include "glm/vec4.hpp" #include "glm/vec4.hpp"
#include "Eigen/Core"
#include <vector> #include <vector>
#include <array> #include <array>
#include <cstdint> #include <cstdint>
// Forward declaration:
namespace eos { namespace morphablemodel {
eos::core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance, std::vector<std::array<int, 3>> tvi, std::vector<std::array<int, 3>> tci, std::vector<cv::Vec2f> texture_coordinates = std::vector<cv::Vec2f>());
} }
namespace eos { namespace eos {
namespace morphablemodel { namespace morphablemodel {
// Forward declaration:
core::Mesh sample_to_mesh(const Eigen::VectorXf& shape_instance, const Eigen::VectorXf& color_instance, const std::vector<std::array<int, 3>>& tvi, const std::vector<std::array<int, 3>>& tci, const std::vector<std::array<double, 2>>& texture_coordinates = std::vector<std::array<double, 2>>());
/** /**
* @brief A class representing a 3D Morphable Model, consisting * @brief A class representing a 3D Morphable Model, consisting
* of a shape- and colour (albedo) PCA model. * of a shape- and colour (albedo) PCA model.
...@@ -61,14 +61,14 @@ public: ...@@ -61,14 +61,14 @@ public:
MorphableModel() = default; MorphableModel() = default;
/** /**
* Create a Morphable Model from a shape and a color PCA model, and optional * Create a Morphable Model from a shape and a colour PCA model, and optional
* texture coordinates. * texture coordinates.
* *
* @param[in] shape_model A PCA model over the shape. * @param[in] shape_model A PCA model over the shape.
* @param[in] color_model A PCA model over the colour (albedo). * @param[in] color_model A PCA model over the colour (albedo).
* @param[in] texture_coordinates Optional texture coordinates for every vertex. * @param[in] texture_coordinates Optional texture coordinates for every vertex.
*/ */
MorphableModel(PcaModel shape_model, PcaModel color_model, std::vector<cv::Vec2f> texture_coordinates = std::vector<cv::Vec2f>()) : shape_model(shape_model), color_model(color_model), texture_coordinates(texture_coordinates) MorphableModel(PcaModel shape_model, PcaModel color_model, std::vector<std::array<double, 2>> texture_coordinates = std::vector<std::array<double, 2>>()) : shape_model(shape_model), color_model(color_model), texture_coordinates(texture_coordinates)
{ {
}; };
...@@ -129,8 +129,8 @@ public: ...@@ -129,8 +129,8 @@ public:
{ {
assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model. assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
cv::Mat shape = shape_model.get_mean(); Eigen::VectorXf shape = shape_model.get_mean();
cv::Mat color = color_model.get_mean(); Eigen::VectorXf color = color_model.get_mean();
core::Mesh mesh; core::Mesh mesh;
if (has_texture_coordinates()) { if (has_texture_coordinates()) {
...@@ -147,16 +147,21 @@ public: ...@@ -147,16 +147,21 @@ public:
* for the shape- and colour models are both drawn from a standard * for the shape- and colour models are both drawn from a standard
* normal (or with the given standard deviation). * normal (or with the given standard deviation).
* *
* If the Morphable Model is a shape-only model, the returned mesh will
* not contain any colour data.
*
* @param[in] engine Random number engine used to draw random coefficients.
* @param[in] shape_sigma The shape model standard deviation. * @param[in] shape_sigma The shape model standard deviation.
* @param[in] color_sigma The colour model standard deviation. * @param[in] color_sigma The colour model standard deviation.
* @return A random sample from the model. * @return A random sample from the model.
*/ */
core::Mesh draw_sample(float shape_sigma = 1.0f, float color_sigma = 1.0f) template <class RNG>
core::Mesh draw_sample(RNG& engine, float shape_sigma = 1.0f, float color_sigma = 1.0f) const
{ {
assert(shape_model.get_data_dimension() == color_model.get_data_dimension()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models. assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
cv::Mat shape_sample = shape_model.draw_sample(shape_sigma); Eigen::VectorXf shape_sample = shape_model.draw_sample(engine, shape_sigma);
cv::Mat color_sample = color_model.draw_sample(color_sigma); Eigen::VectorXf color_sample = color_model.draw_sample(engine, color_sigma);
core::Mesh mesh; core::Mesh mesh;
if (has_texture_coordinates()) { if (has_texture_coordinates()) {
...@@ -186,8 +191,8 @@ public: ...@@ -186,8 +191,8 @@ public:
{ {
assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model. assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
cv::Mat shape_sample; Eigen::VectorXf shape_sample;
cv::Mat color_sample; Eigen::VectorXf color_sample;
if (shape_coefficients.empty()) { if (shape_coefficients.empty()) {
shape_sample = shape_model.get_mean(); shape_sample = shape_model.get_mean();
...@@ -220,7 +225,7 @@ public: ...@@ -220,7 +225,7 @@ public:
*/ */
bool has_color_model() const bool has_color_model() const
{ {
return !color_model.get_mean().empty(); return (color_model.get_mean().size() > 0);
}; };
/** /**
...@@ -228,7 +233,7 @@ public: ...@@ -228,7 +233,7 @@ public:
* *
* @return The texture coordinates for the model vertices. * @return The texture coordinates for the model vertices.
*/ */
std::vector<cv::Vec2f> get_texture_coordinates() const std::vector<std::array<double, 2>> get_texture_coordinates() const
{ {
return texture_coordinates; return texture_coordinates;
}; };
...@@ -236,7 +241,7 @@ public: ...@@ -236,7 +241,7 @@ public:
private: private:
PcaModel shape_model; ///< A PCA model of the shape PcaModel shape_model; ///< A PCA model of the shape
PcaModel color_model; ///< A PCA model of vertex colour information PcaModel color_model; ///< A PCA model of vertex colour information
std::vector<cv::Vec2f> texture_coordinates; ///< uv-coordinates for every vertex std::vector<std::array<double, 2>> texture_coordinates; ///< uv-coordinates for every vertex
/** /**
* Returns whether the model has texture mapping coordinates, i.e. * Returns whether the model has texture mapping coordinates, i.e.
...@@ -253,10 +258,15 @@ private: ...@@ -253,10 +258,15 @@ private:
* Serialises this class using cereal. * Serialises this class using cereal.
* *
* @param[in] ar The archive to serialise to (or to serialise from). * @param[in] ar The archive to serialise to (or to serialise from).
* @throw std::runtime_error When the model file doesn't have the most recent version (=1).
*/ */
template<class Archive> template<class Archive>
void serialize(Archive& archive, const std::uint32_t version) void serialize(Archive& archive, const std::uint32_t version)
{ {
if (version != 1)
{
throw std::runtime_error("The model file you are trying to load is in an old format. Please download the most recent model files.");
}
archive(CEREAL_NVP(shape_model), CEREAL_NVP(color_model), CEREAL_NVP(texture_coordinates)); archive(CEREAL_NVP(shape_model), CEREAL_NVP(color_model), CEREAL_NVP(texture_coordinates));
}; };
}; };
...@@ -311,25 +321,25 @@ inline void save_model(MorphableModel model, std::string filename) ...@@ -311,25 +321,25 @@ inline void save_model(MorphableModel model, std::string filename)
* @param[in] texture_coordinates Optional texture coordinates for each vertex. * @param[in] texture_coordinates Optional texture coordinates for each vertex.
* @return A mesh created from given parameters. * @return A mesh created from given parameters.
*/ */
inline core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance, std::vector<std::array<int, 3>> tvi, std::vector<std::array<int, 3>> tci, std::vector<cv::Vec2f> texture_coordinates /* = std::vector<cv::Vec2f>() */) inline core::Mesh sample_to_mesh(const Eigen::VectorXf& shape_instance, const Eigen::VectorXf& color_instance, const std::vector<std::array<int, 3>>& tvi, const std::vector<std::array<int, 3>>& tci, const std::vector<std::array<double, 2>>& texture_coordinates /* = std::vector<std::array<double, 2>>() */)
{ {
assert(shape_instance.rows == color_instance.rows || color_instance.empty()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model. assert(shape_instance.rows() == color_instance.rows() || color_instance.size() == 0); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
auto num_vertices = shape_instance.rows / 3; auto num_vertices = shape_instance.rows() / 3;
core::Mesh mesh; core::Mesh mesh;
// Construct the mesh vertices: // Construct the mesh vertices:
mesh.vertices.resize(num_vertices); mesh.vertices.resize(num_vertices);
for (auto i = 0; i < num_vertices; ++i) { for (auto i = 0; i < num_vertices; ++i) {
mesh.vertices[i] = glm::tvec4<float>(shape_instance.at<float>(i * 3 + 0), shape_instance.at<float>(i * 3 + 1), shape_instance.at<float>(i * 3 + 2), 1.0f); mesh.vertices[i] = glm::tvec4<float>(shape_instance(i * 3 + 0), shape_instance(i * 3 + 1), shape_instance(i * 3 + 2), 1.0f);
} }
// Assign the vertex colour information if it's not a shape-only model: // Assign the vertex colour information if it's not a shape-only model:
if (!color_instance.empty()) { if (color_instance.size() > 0) {
mesh.colors.resize(num_vertices); mesh.colors.resize(num_vertices);
for (auto i = 0; i < num_vertices; ++i) { for (auto i = 0; i < num_vertices; ++i) {
mesh.colors[i] = glm::tvec3<float>(color_instance.at<float>(i * 3 + 0), color_instance.at<float>(i * 3 + 1), color_instance.at<float>(i * 3 + 2)); // order in hdf5: RGB. Order in OCV: BGR. But order in vertex.color: RGB mesh.colors[i] = glm::tvec3<float>(color_instance(i * 3 + 0), color_instance(i * 3 + 1), color_instance(i * 3 + 2)); // We use RGB order everywhere
} }
} }
...@@ -351,6 +361,6 @@ inline core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance, ...@@ -351,6 +361,6 @@ inline core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance,
} /* namespace morphablemodel */ } /* namespace morphablemodel */
} /* namespace eos */ } /* namespace eos */
CEREAL_CLASS_VERSION(eos::morphablemodel::MorphableModel, 0); CEREAL_CLASS_VERSION(eos::morphablemodel::MorphableModel, 1);
#endif /* MORPHABLEMODEL_HPP_ */ #endif /* MORPHABLEMODEL_HPP_ */
include/eos/morphablemodel/PcaModel.hpp
View file @ 470615a6
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
* *
* File: include/eos/morphablemodel/PcaModel.hpp * File: include/eos/morphablemodel/PcaModel.hpp
* *
* Copyright 2014, 2015 Patrik Huber * Copyright 2014-2017 Patrik Huber
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
...@@ -22,11 +22,13 @@ ...@@ -22,11 +22,13 @@
#ifndef PCAMODEL_HPP_ #ifndef PCAMODEL_HPP_
#define PCAMODEL_HPP_ #define PCAMODEL_HPP_
#include "eos/morphablemodel/io/mat_cerealisation.hpp" #include "eos/morphablemodel/io/eigen_cerealisation.hpp"
#include "cereal/access.hpp" #include "cereal/access.hpp"
#include "cereal/types/array.hpp" #include "cereal/types/array.hpp"
#include "cereal/types/vector.hpp" #include "cereal/types/vector.hpp"
#include "Eigen/Core"
#include "opencv2/core/core.hpp" #include "opencv2/core/core.hpp"
#include <string> #include <string>
...@@ -39,8 +41,8 @@ namespace eos { ...@@ -39,8 +41,8 @@ namespace eos {
namespace morphablemodel { namespace morphablemodel {
// Forward declarations of free functions: // Forward declarations of free functions:
cv::Mat normalise_pca_basis(cv::Mat unnormalised_basis, cv::Mat eigenvalues); Eigen::MatrixXf normalise_pca_basis(const Eigen::MatrixXf& unnormalised_basis, const Eigen::VectorXf& eigenvalues);
cv::Mat unnormalise_pca_basis(cv::Mat normalised_basis, cv::Mat eigenvalues); Eigen::MatrixXf unnormalise_pca_basis(const Eigen::MatrixXf& normalised_basis, const Eigen::VectorXf& eigenvalues);
/** /**
* @brief This class represents a PCA-model that consists of: * @brief This class represents a PCA-model that consists of:
...@@ -68,10 +70,8 @@ public: ...@@ -68,10 +70,8 @@ public:
* @param[in] eigenvalues The eigenvalues used to build the PCA model. * @param[in] eigenvalues The eigenvalues used to build the PCA model.
* @param[in] triangle_list An index list of how to assemble the mesh. * @param[in] triangle_list An index list of how to assemble the mesh.
*/ */
PcaModel(cv::Mat mean, cv::Mat pca_basis, cv::Mat eigenvalues, std::vector<std::array<int, 3>> triangle_list) : mean(mean), normalised_pca_basis(pca_basis), eigenvalues(eigenvalues), triangle_list(triangle_list) PcaModel(Eigen::VectorXf mean, Eigen::MatrixXf pca_basis, Eigen::VectorXf eigenvalues, std::vector<std::array<int, 3>> triangle_list) : mean(mean), normalised_pca_basis(pca_basis), eigenvalues(eigenvalues), triangle_list(triangle_list)
{ {
const auto seed = std::random_device()();
engine.seed(seed);
unnormalised_pca_basis = unnormalise_pca_basis(normalised_pca_basis, eigenvalues); unnormalised_pca_basis = unnormalise_pca_basis(normalised_pca_basis, eigenvalues);
}; };
...@@ -83,7 +83,7 @@ public: ...@@ -83,7 +83,7 @@ public:
int get_num_principal_components() const int get_num_principal_components() const
{ {
// Note: we could assert(normalised_pca_basis.cols==unnormalised_pca_basis.cols) // Note: we could assert(normalised_pca_basis.cols==unnormalised_pca_basis.cols)
return normalised_pca_basis.cols; return normalised_pca_basis.cols();
}; };
/** /**
...@@ -97,7 +97,7 @@ public: ...@@ -97,7 +97,7 @@ public:
int get_data_dimension() const int get_data_dimension() const
{ {
// Note: we could assert(normalised_pca_basis.rows==unnormalised_pca_basis.rows) // Note: we could assert(normalised_pca_basis.rows==unnormalised_pca_basis.rows)
return normalised_pca_basis.rows; return normalised_pca_basis.rows();
}; };
/** /**
...@@ -113,9 +113,11 @@ public: ...@@ -113,9 +113,11 @@ public:
/** /**
* Returns the mean of the model. * Returns the mean of the model.
* *
* Todo: Return a const-ref to avoid copying of the vector?
*
* @return The mean of the model. * @return The mean of the model.
*/ */
cv::Mat get_mean() const Eigen::VectorXf get_mean() const
{ {
return mean; return mean;
}; };
...@@ -123,23 +125,28 @@ public: ...@@ -123,23 +125,28 @@ public:
/** /**
* Return the value of the mean at a given vertex index. * Return the value of the mean at a given vertex index.
* *
* Todo: Rename to get_mean? The other getters are overloaded on the vertex index too.
* I also think we should just return an Eigen::Vector3f - homogenous coords have no place here?
*
* @param[in] vertex_index A vertex index. * @param[in] vertex_index A vertex index.
* @return A homogeneous vector containing the values at the given vertex index. * @return A homogeneous vector containing the values at the given vertex index.
*/ */
cv::Vec4f get_mean_at_point(int vertex_index) const cv::Vec4f get_mean_at_point(int vertex_index) const
{ {
vertex_index *= 3; vertex_index *= 3;
return cv::Vec4f(mean.at<float>(vertex_index), mean.at<float>(vertex_index + 1), mean.at<float>(vertex_index + 2), 1.0f); return cv::Vec4f(mean(vertex_index), mean(vertex_index + 1), mean(vertex_index + 2), 1.0f);
}; };
/** /**
* Draws a random sample from the model, where the coefficients are drawn * Draws a random sample from the model, where the coefficients are drawn
* from a standard normal (or with the given standard deviation). * from a standard normal (or with the given standard deviation).
* *
* @param[in] engine Random number engine used to draw random coefficients.
* @param[in] sigma The standard deviation. * @param[in] sigma The standard deviation.
* @return A random sample from the model. * @return A random sample from the model.
*/ */
cv::Mat draw_sample(float sigma = 1.0f) template <class RNG>
Eigen::VectorXf draw_sample(RNG& engine, float sigma = 1.0f) const
{ {
std::normal_distribution<float> distribution(0.0f, sigma); // this constructor takes the stddev std::normal_distribution<float> distribution(0.0f, sigma); // this constructor takes the stddev
...@@ -160,15 +167,15 @@ public: ...@@ -160,15 +167,15 @@ public:
* @param[in] coefficients The PCA coefficients used to generate the sample. * @param[in] coefficients The PCA coefficients used to generate the sample.
* @return A model instance with given coefficients. * @return A model instance with given coefficients.
*/ */
cv::Mat draw_sample(std::vector<float> coefficients) const Eigen::VectorXf draw_sample(std::vector<float> coefficients) const
{ {
// Fill the rest with zeros if not all coefficients are given: // Fill the rest with zeros if not all coefficients are given:
if (coefficients.size() < get_num_principal_components()) { if (coefficients.size() < get_num_principal_components()) {
coefficients.resize(get_num_principal_components()); coefficients.resize(get_num_principal_components());
} }
cv::Mat alphas(coefficients); Eigen::Map<Eigen::VectorXf> alphas(coefficients.data(), coefficients.size());
cv::Mat model_sample = mean + normalised_pca_basis * alphas; Eigen::VectorXf model_sample = mean + normalised_pca_basis * alphas;
return model_sample; return model_sample;
}; };
...@@ -176,7 +183,7 @@ public: ...@@ -176,7 +183,7 @@ public:
/** /**
* @copydoc PcaModel::draw_sample(std::vector<float>) const * @copydoc PcaModel::draw_sample(std::vector<float>) const
*/ */
cv::Mat draw_sample(std::vector<double> coefficients) const Eigen::VectorXf draw_sample(std::vector<double> coefficients) const
{ {
// We have to convert the vector of doubles to float: // We have to convert the vector of doubles to float:
std::vector<float> coeffs_float(std::begin(coefficients), std::end(coefficients)); std::vector<float> coeffs_float(std::begin(coefficients), std::end(coefficients));
...@@ -189,14 +196,14 @@ public: ...@@ -189,14 +196,14 @@ public:
* The returned basis is normalised, i.e. every eigenvector * The returned basis is normalised, i.e. every eigenvector
* is normalised by multiplying it with the square root of its eigenvalue. * is normalised by multiplying it with the square root of its eigenvalue.
* *
* Returns a clone of the matrix so that the original cannot * Returns a copy of the matrix so that the original cannot
* be modified. TODO: No, don't return a clone. * be modified. TODO: No, don't return a clone.
* *
* @return Returns the normalised PCA basis matrix. * @return Returns the normalised PCA basis matrix.
*/ */
cv::Mat get_normalised_pca_basis() const Eigen::MatrixXf get_normalised_pca_basis() const
{ {
return normalised_pca_basis.clone(); return normalised_pca_basis;
}; };
/** /**
...@@ -204,14 +211,16 @@ public: ...@@ -204,14 +211,16 @@ public:
* The returned basis is normalised, i.e. every eigenvector * The returned basis is normalised, i.e. every eigenvector
* is normalised by multiplying it with the square root of its eigenvalue. * is normalised by multiplying it with the square root of its eigenvalue.
* *
* Todo: Can we return a const & view that points into the original data?
*
* @param[in] vertex_id A vertex index. Make sure it is valid. * @param[in] vertex_id A vertex index. Make sure it is valid.
* @return A Mat that points to the rows in the original basis. * @return A 1x3? 3x1? matrix that points to the rows in the original basis.
*/ */
cv::Mat get_normalised_pca_basis(int vertex_id) const Eigen::MatrixXf get_normalised_pca_basis(int vertex_id) const
{ {
vertex_id *= 3; // the basis is stored in the format [x y z x y z ...] vertex_id *= 3; // the basis is stored in the format [x y z x y z ...]
assert(vertex_id < get_data_dimension()); // Make sure the given vertex index isn't larger than the number of model vertices. assert(vertex_id < get_data_dimension()); // Make sure the given vertex index isn't larger than the number of model vertices.
return normalised_pca_basis.rowRange(vertex_id, vertex_id + 3); return normalised_pca_basis.block(vertex_id, 0, 3, normalised_pca_basis.cols());
}; };
/** /**
...@@ -224,9 +233,9 @@ public: ...@@ -224,9 +233,9 @@ public:
* *
* @return Returns the unnormalised PCA basis matrix. * @return Returns the unnormalised PCA basis matrix.
*/ */
cv::Mat get_unnormalised_pca_basis() const Eigen::MatrixXf get_unnormalised_pca_basis() const
{ {
return unnormalised_pca_basis.clone(); return unnormalised_pca_basis;
}; };
/** /**
...@@ -236,32 +245,40 @@ public: ...@@ -236,32 +245,40 @@ public:
* @param[in] vertex_id A vertex index. Make sure it is valid. * @param[in] vertex_id A vertex index. Make sure it is valid.
* @return A Mat that points to the rows in the original basis. * @return A Mat that points to the rows in the original basis.
*/ */
cv::Mat get_unnormalised_pca_basis(int vertex_id) const Eigen::MatrixXf get_unnormalised_pca_basis(int vertex_id) const
{ {
vertex_id *= 3; // the basis is stored in the format [x y z x y z ...] vertex_id *= 3; // the basis is stored in the format [x y z x y z ...]
assert(vertex_id < get_data_dimension()); // Make sure the given vertex index isn't larger than the number of model vertices. assert(vertex_id < get_data_dimension()); // Make sure the given vertex index isn't larger than the number of model vertices.
return unnormalised_pca_basis.rowRange(vertex_id, vertex_id + 3); return unnormalised_pca_basis.block(vertex_id, 0, 3, unnormalised_pca_basis.cols());
}; };
/** /**
* Returns an eigenvalue. * Returns the models eigenvalues.
*
* @return The eigenvalues.
*/
Eigen::VectorXf get_eigenvalues() const
{
return eigenvalues;
};
/**
* Returns a specific eigenvalue.
* *
* @param[in] index The index of the eigenvalue to return. * @param[in] index The index of the eigenvalue to return.
* @return The eigenvalue. * @return The eigenvalue.
*/ */
float get_eigenvalue(int index) const float get_eigenvalue(int index) const
{ {
// no assert - OpenCV checks ::at in debug builds // no assert - Eigen checks access with an assert in debug builds
return eigenvalues.at<float>(index); return eigenvalues(index);
}; };
private: private:
std::mt19937 engine; ///< Random number engine used to draw random coefficients. Eigen::VectorXf mean; ///< A 3m x 1 col-vector (xyzxyz...)', where m is the number of model-vertices.
Eigen::MatrixXf normalised_pca_basis; ///< The normalised PCA basis matrix. m x n (rows x cols) = numShapeDims x numShapePcaCoeffs, (=eigenvector matrix V). Each column is an eigenvector.
cv::Mat mean; ///< A 3m x 1 col-vector (xyzxyz...)', where m is the number of model-vertices. Eigen::MatrixXf unnormalised_pca_basis; ///< The unnormalised PCA basis matrix. m x n (rows x cols) = numShapeDims x numShapePcaCoeffs, (=eigenvector matrix V). Each column is an eigenvector.
cv::Mat normalised_pca_basis; ///< The normalised PCA basis matrix. m x n (rows x cols) = numShapeDims x numShapePcaCoeffs, (=eigenvector matrix V). Each column is an eigenvector. Eigen::VectorXf eigenvalues; ///< A col-vector of the eigenvalues (variances in the PCA space).
cv::Mat unnormalised_pca_basis; ///< The unnormalised PCA basis matrix. m x n (rows x cols) = numShapeDims x numShapePcaCoeffs, (=eigenvector matrix V). Each column is an eigenvector.
cv::Mat eigenvalues; ///< A col-vector of the eigenvalues (variances in the PCA space).
std::vector<std::array<int, 3>> triangle_list; ///< List of triangles that make up the mesh of the model. std::vector<std::array<int, 3>> triangle_list; ///< List of triangles that make up the mesh of the model.
...@@ -291,18 +308,14 @@ private: ...@@ -291,18 +308,14 @@ private:
* @param[in] eigenvalues A row or column vector of eigenvalues. * @param[in] eigenvalues A row or column vector of eigenvalues.
* @return The normalised PCA basis matrix. * @return The normalised PCA basis matrix.
*/ */
inline cv::Mat normalise_pca_basis(cv::Mat unnormalised_basis, cv::Mat eigenvalues) inline Eigen::MatrixXf normalise_pca_basis(const Eigen::MatrixXf& unnormalised_basis, const Eigen::VectorXf& eigenvalues)
{ {
using cv::Mat; using Eigen::MatrixXf;
Mat normalised_basis(unnormalised_basis.size(), unnormalised_basis.type()); // empty matrix with the same dimensions MatrixXf normalised_basis(unnormalised_basis.rows(), unnormalised_basis.cols()); // empty matrix with the same dimensions
Mat sqrt_of_eigenvalues = eigenvalues.clone(); MatrixXf sqrt_of_eigenvalues = eigenvalues.array().sqrt(); // using eigenvalues.sqrt() directly gives a compile error. These are all Eigen::Array functions? I don't think we copy here, do we?
for (int i = 0; i < eigenvalues.rows; ++i) {
sqrt_of_eigenvalues.at<float>(i) = std::sqrt(eigenvalues.at<float>(i));
}
// Normalise the basis: We multiply each eigenvector (i.e. each column) with the square root of its corresponding eigenvalue // Normalise the basis: We multiply each eigenvector (i.e. each column) with the square root of its corresponding eigenvalue
for (int basis = 0; basis < unnormalised_basis.cols; ++basis) { for (int basis = 0; basis < unnormalised_basis.cols(); ++basis) {
Mat normalised_eigenvector = unnormalised_basis.col(basis).mul(sqrt_of_eigenvalues.at<float>(basis)); normalised_basis.col(basis) = unnormalised_basis.col(basis) * sqrt_of_eigenvalues(basis);
normalised_eigenvector.copyTo(normalised_basis.col(basis));
} }
return normalised_basis; return normalised_basis;
...@@ -318,18 +331,14 @@ inline cv::Mat normalise_pca_basis(cv::Mat unnormalised_basis, cv::Mat eigenvalu ...@@ -318,18 +331,14 @@ inline cv::Mat normalise_pca_basis(cv::Mat unnormalised_basis, cv::Mat eigenvalu
* @param[in] eigenvalues A row or column vector of eigenvalues. * @param[in] eigenvalues A row or column vector of eigenvalues.
* @return The unnormalised PCA basis matrix. * @return The unnormalised PCA basis matrix.
*/ */
inline cv::Mat unnormalise_pca_basis(cv::Mat normalised_basis, cv::Mat eigenvalues) inline Eigen::MatrixXf unnormalise_pca_basis(const Eigen::MatrixXf& normalised_basis, const Eigen::VectorXf& eigenvalues)
{ {
using cv::Mat; using Eigen::MatrixXf;
Mat unnormalised_basis(normalised_basis.size(), normalised_basis.type()); // empty matrix with the same dimensions MatrixXf unnormalised_basis(normalised_basis.rows(), normalised_basis.cols()); // empty matrix with the same dimensions
Mat one_over_sqrt_of_eigenvalues = eigenvalues.clone(); Eigen::VectorXf one_over_sqrt_of_eigenvalues = eigenvalues.array().rsqrt();
for (int i = 0; i < eigenvalues.rows; ++i) {
one_over_sqrt_of_eigenvalues.at<float>(i) = 1.0f / std::sqrt(eigenvalues.at<float>(i));
}
// De-normalise the basis: We multiply each eigenvector (i.e. each column) with 1 over the square root of its corresponding eigenvalue // De-normalise the basis: We multiply each eigenvector (i.e. each column) with 1 over the square root of its corresponding eigenvalue
for (int basis = 0; basis < normalised_basis.cols; ++basis) { for (int basis = 0; basis < normalised_basis.cols(); ++basis) {
Mat unnormalised_eigenvector = normalised_basis.col(basis).mul(one_over_sqrt_of_eigenvalues.at<float>(basis)); unnormalised_basis.col(basis) = normalised_basis.col(basis) * one_over_sqrt_of_eigenvalues(basis);
unnormalised_eigenvector.copyTo(unnormalised_basis.col(basis));
} }
return unnormalised_basis; return unnormalised_basis;
......
include/eos/morphablemodel/io/eigen_cerealisation.hpp 0 → 100644
View file @ 470615a6
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: include/eos/morphablemodel/io/eigen_cerealisation.hpp
*
* Copyright 2017 Patrik Huber
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifndef EIGENCEREALISATION_HPP_
#define EIGENCEREALISATION_HPP_
#include "cereal/cereal.hpp"
#include "Eigen/Core"
#include <cstdint>
/**
* @brief Serialisation of Eigen matrices for the serialisation
* library cereal (http://uscilab.github.io/cereal/index.html).
*
* Contains serialisation for Eigen matrices to binary archives, i.e. matrices like
* \c Eigen::MatrixXf, \c Eigen::Matrix4d, or \c Eigen::Vector3f.
*
* Todo: Add serialisation to and from JSON. Need to find out how to combine the two
* variants of SFINAE that are used.
*/
namespace cereal {
/**
* @brief Serialise an Eigen::Matrix using cereal.
*
* Note: Writes the binary data from Matrix::data(), so not sure what happens if a matrix ever has
* non-contiguous data (if that can ever happen with Eigen).
*
* @param[in] ar The archive to serialise to.
* @param[in] matrix The matrix to serialise.
*/
template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
inline
typename std::enable_if<traits::is_output_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
save(Archive& ar, const Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& matrix)
{
std::int32_t rows = matrix.rows();
std::int32_t cols = matrix.cols();
ar(rows);
ar(cols);
ar(binary_data(matrix.data(), rows * cols * sizeof(_Scalar)));
};
/**
* @brief De-serialise an Eigen::Matrix using cereal.
*
* Reads the block of binary data back from a cereal archive into the Eigen::Matrix.
*
* @param[in] ar The archive to deserialise from.
* @param[in] matrix The matrix to deserialise into.
*/
template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
inline
typename std::enable_if<traits::is_input_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
load(Archive& ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& matrix)
{
std::int32_t rows;
std::int32_t cols;
ar(rows);
ar(cols);
matrix.resize(rows, cols);
ar(binary_data(matrix.data(), static_cast<std::size_t>(rows * cols * sizeof(_Scalar))));
};
} /* namespace cereal */
#endif /* EIGENCEREALISATION_HPP_ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment