Skip to content

E
eos
Commit 470615a6 authored by Patrik Huber's avatar Patrik Huber
Browse files
Options

Changed PcaModel to use Eigen::Matrix instead of cv::Mat 

This is a big step towards being not requiring OpenCV anymore as a dependency.
All models will have to be converted and re-downloaded.

* PcaModel: Removed the RNG from the class, and added a RNG parameter to the draw_sample method
* PcaModel: Added a get_eigenvalues() accessor that returns all eigenvalues
* MorphableModel: Changed CEREAL_CLASS_VERSION from 0 to 1 (and added a version check to the serialize method)
* MorphableModel: Changed texture_coordinates from cv::Vec2f to array<double, 2>
* Added binary Eigen::Matrix serialisation for cereal
* Changed Blendshape to Eigen::Vector as well
* Adjusted blendshape_fitting.hpp accordingly - and it now takes most parameters by const&
* Adjusted fitting.hpp accordingly - now also takes most parameters by const&
* Adjusted linear_shape_fitting.hpp: It's not totally converted, still using cv::Mat for the core algorithm
parent 29817961
Expand all Show whitespace changes
Inline Side-by-side
Showing with 247 additions and 141 deletions
include/eos/fitting/blendshape_fitting.hpp
View file @ 470615a6
......@@ -27,6 +27,7 @@
#include "Eigen/Core" // for nnls.h
#include "nnls.h"
#include "Eigen/Core"
#include "opencv2/core/core.hpp"
#include <vector>
......@@ -55,30 +56,27 @@ namespace eos {
* @param[in] lambda A regularisation parameter, constraining the L2-norm of the coefficients.
* @return The estimated blendshape-coefficients.
*/
inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, float lambda=500.0f)
inline std::vector<float> fit_blendshapes_to_landmarks_linear(const std::vector<eos::morphablemodel::Blendshape>& blendshapes, const Eigen::VectorXf& face_instance, cv::Mat affine_camera_matrix, const std::vector<cv::Vec2f>& landmarks, const std::vector<int>& vertex_ids, float lambda=500.0f)
{
using cv::Mat;
assert(landmarks.size() == vertex_ids.size());
int num_coeffs_to_fit = blendshapes.size();
int num_landmarks = static_cast<int>(landmarks.size());
using cv::Mat;
using Eigen::MatrixXf;
const int num_blendshapes = blendshapes.size();
const int num_landmarks = static_cast<int>(landmarks.size());
// Copy all blendshapes into a "basis" matrix with each blendshape being a column:
cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_blendshapes, CV_32FC1);
int row_index = 0;
for (int i = 0; i < num_landmarks; ++i) {
//Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
//basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3));
basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
Mat blendshapes_basis_as_mat = Mat(blendshapes_as_basis.rows(), blendshapes_as_basis.cols(), CV_32FC1, blendshapes_as_basis.data());
Mat basis_rows = blendshapes_basis_as_mat.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
basis_rows.copyTo(V_hat_h.rowRange(row_index, row_index + 3));
row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
}
// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
......@@ -99,7 +97,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m
Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
for (int i = 0; i < num_landmarks; ++i) {
//cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f);
cv::Vec4f model_mean(face_instance(vertex_ids[i]*3), face_instance(vertex_ids[i]*3 + 1), face_instance(vertex_ids[i]*3 + 2), 1.0f);
v_bar.at<float>(4 * i, 0) = model_mean[0];
v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
......@@ -111,7 +109,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m
Mat A = P * V_hat_h; // camera matrix times the basis
Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks.
Mat AtAReg = A.t() * A + lambda * Mat::eye(num_coeffs_to_fit, num_coeffs_to_fit, CV_32FC1);
Mat AtAReg = A.t() * A + lambda * Mat::eye(num_blendshapes, num_blendshapes, CV_32FC1);
// Solve using OpenCV:
Mat c_s;
bool non_singular = cv::solve(AtAReg, -A.t() * b, c_s, cv::DECOMP_SVD); // DECOMP_SVD calculates the pseudo-inverse if the matrix is not invertible.
......@@ -136,28 +134,27 @@ inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::m
* @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
* @return The estimated blendshape-coefficients.
*/
inline std::vector<float> fit_blendshapes_to_landmarks_nnls(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids)
inline std::vector<float> fit_blendshapes_to_landmarks_nnls(const std::vector<eos::morphablemodel::Blendshape>& blendshapes, const Eigen::VectorXf& face_instance, cv::Mat affine_camera_matrix, const std::vector<cv::Vec2f>& landmarks, const std::vector<int>& vertex_ids)
{
using cv::Mat;
assert(landmarks.size() == vertex_ids.size());
int num_coeffs_to_fit = blendshapes.size();
int num_landmarks = static_cast<int>(landmarks.size());
using cv::Mat;
using Eigen::MatrixXf;
const int num_blendshapes = blendshapes.size();
const int num_landmarks = static_cast<int>(landmarks.size());
// Copy all blendshapes into a "basis" matrix with each blendshape being a column:
cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_blendshapes, CV_32FC1);
int row_index = 0;
for (int i = 0; i < num_landmarks; ++i) {
Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3)); // Todo: I think we can remove colRange here, as we always want to use all given blendshapes
Mat blendshapes_basis_as_mat = Mat(blendshapes_as_basis.rows(), blendshapes_as_basis.cols(), CV_32FC1, blendshapes_as_basis.data());
Mat basis_rows = blendshapes_basis_as_mat.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
basis_rows.copyTo(V_hat_h.rowRange(row_index, row_index + 3));
row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
}
// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
......@@ -177,7 +174,7 @@ inline std::vector<float> fit_blendshapes_to_landmarks_nnls(std::vector<eos::mor
// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
for (int i = 0; i < num_landmarks; ++i) {
cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f);
cv::Vec4f model_mean(face_instance(vertex_ids[i]*3), face_instance(vertex_ids[i]*3 + 1), face_instance(vertex_ids[i]*3 + 2), 1.0f);
v_bar.at<float>(4 * i, 0) = model_mean[0];
v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
......
include/eos/fitting/fitting.hpp
View file @ 470615a6
......@@ -65,39 +65,37 @@ namespace eos {
* @param[out] blendshape_coefficients Output parameter that will contain the resulting blendshape coefficients.
* @return The fitted model shape instance.
*/
inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::MorphableModel morphable_model, std::vector<morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda, boost::optional<int> num_coefficients_to_fit, std::vector<float>& pca_shape_coefficients, std::vector<float>& blendshape_coefficients)
inline Eigen::VectorXf fit_shape(cv::Mat affine_camera_matrix, const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<cv::Vec2f>& image_points, const std::vector<int>& vertex_indices, float lambda, boost::optional<int> num_coefficients_to_fit, std::vector<float>& pca_shape_coefficients, std::vector<float>& blendshape_coefficients)
{
using cv::Mat;
using Eigen::VectorXf;
using Eigen::MatrixXf;
Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
}
MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
std::vector<float> last_blendshape_coeffs, current_blendshape_coeffs;
std::vector<float> last_pca_coeffs, current_pca_coeffs;
current_blendshape_coeffs.resize(blendshapes.size()); // starting values t_0, all zeros
// no starting values for current_pca_coeffs required, since we start with the shape fitting, and cv::norm of an empty vector is 0.
Mat combined_shape;
VectorXf combined_shape;
do // run at least once:
{
last_blendshape_coeffs = current_blendshape_coeffs;
last_pca_coeffs = current_pca_coeffs;
// Estimate the PCA shape coefficients with the current blendshape coefficients (0 in the first iteration):
Mat mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Mat(last_blendshape_coeffs);
VectorXf mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(last_blendshape_coeffs.data(), last_blendshape_coeffs.size());
current_pca_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_camera_matrix, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_coefficients_to_fit);
// Estimate the blendshape coefficients with the current PCA model estimate:
Mat pca_model_shape = morphable_model.get_shape_model().draw_sample(current_pca_coeffs);
VectorXf pca_model_shape = morphable_model.get_shape_model().draw_sample(current_pca_coeffs);
current_blendshape_coeffs = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, pca_model_shape, affine_camera_matrix, image_points, vertex_indices);
combined_shape = pca_model_shape + blendshapes_as_basis * Mat(current_blendshape_coeffs);
combined_shape = pca_model_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(current_blendshape_coeffs.data(), current_blendshape_coeffs.size()); // Todo/Note: Could move outside the loop, not needed in here actually
} while (std::abs(cv::norm(current_pca_coeffs) - cv::norm(last_pca_coeffs)) >= 0.01 || std::abs(cv::norm(current_blendshape_coeffs) - cv::norm(last_blendshape_coeffs)) >= 0.01);
pca_shape_coefficients = current_pca_coeffs;
blendshape_coefficients = current_blendshape_coeffs;
return combined_shape;
};
......@@ -115,7 +113,7 @@ inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::Morphable
* @param[in] num_coefficients_to_fit How many shape-coefficients to fit (all others will stay 0). Should be bigger than zero, or boost::none to fit all coefficients.
* @return The fitted model shape instance.
*/
inline cv::Mat fit_shape(cv::Mat affine_camera_matrix, morphablemodel::MorphableModel morphable_model, std::vector<morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda = 3.0f, boost::optional<int> num_coefficients_to_fit = boost::optional<int>())
inline Eigen::VectorXf fit_shape(cv::Mat affine_camera_matrix, const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<cv::Vec2f>& image_points, const std::vector<int>& vertex_indices, float lambda = 3.0f, boost::optional<int> num_coefficients_to_fit = boost::optional<int>())
{
std::vector<float> unused;
return fit_shape(affine_camera_matrix, morphable_model, blendshapes, image_points, vertex_indices, lambda, num_coefficients_to_fit, unused, unused);
......@@ -243,7 +241,8 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
using std::vector;
using cv::Vec2f;
using cv::Vec4f;
using cv::Mat;
using Eigen::VectorXf;
using Eigen::MatrixXf;
if (!num_shape_coefficients_to_fit)
{
......@@ -262,11 +261,11 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
blendshape_coefficients.resize(blendshapes.size());
}
Mat blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
MatrixXf blendshapes_as_basis = morphablemodel::to_matrix(blendshapes);
// Current mesh - either from the given coefficients, or the mean:
Mat current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
Mat current_combined_shape = current_pca_shape + blendshapes_as_basis * Mat(blendshape_coefficients);
VectorXf current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
VectorXf current_combined_shape = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
auto current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
// The 2D and 3D point correspondences used for the fitting:
......@@ -294,11 +293,11 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
current_pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image_height);
fitting::RenderingParameters rendering_params(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height);
cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height);
blendshape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points, vertex_indices);
// Mesh with same PCA coeffs as before, but new expression fit (this is relevant if no initial blendshape coeffs have been given):
current_combined_shape = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Mat(blendshape_coefficients);
current_combined_shape = current_pca_shape + morphablemodel::to_matrix(blendshapes) * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
// The static (fixed) landmark correspondences which will stay the same throughout
......@@ -346,17 +345,17 @@ inline std::pair<core::Mesh, fitting::RenderingParameters> fit_shape_and_pose(co
current_pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image_height);
rendering_params = fitting::RenderingParameters(current_pose, image_width, image_height);
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height);
cv::Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image_width, image_height);
// Estimate the PCA shape coefficients with the current blendshape coefficients:
Mat mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Mat(blendshape_coefficients);
VectorXf mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
pca_shape_coefficients = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_shape_coefficients_to_fit);
// Estimate the blendshape coefficients with the current PCA model estimate:
current_pca_shape = morphable_model.get_shape_model().draw_sample(pca_shape_coefficients);
blendshape_coefficients = fitting::fit_blendshapes_to_landmarks_nnls(blendshapes, current_pca_shape, affine_from_ortho, image_points, vertex_indices);
current_combined_shape = current_pca_shape + blendshapes_as_basis * Mat(blendshape_coefficients);
current_combined_shape = current_pca_shape + blendshapes_as_basis * Eigen::Map<const Eigen::VectorXf>(blendshape_coefficients.data(), blendshape_coefficients.size());
current_mesh = morphablemodel::sample_to_mesh(current_combined_shape, morphable_model.get_color_model().get_mean(), morphable_model.get_shape_model().get_triangle_list(), morphable_model.get_color_model().get_triangle_list(), morphable_model.get_texture_coordinates());
}
......
include/eos/fitting/linear_shape_fitting.hpp
View file @ 470615a6
......@@ -57,7 +57,7 @@ namespace eos {
* @param[in] model_standard_deviation The standard deviation of the 3D vertex points in the 3D model, projected to 2D (so the value is in pixels).
* @return The estimated shape-coefficients (alphas).
*/
inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::MorphableModel& morphable_model, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, cv::Mat base_face=cv::Mat(), float lambda=3.0f, boost::optional<int> num_coefficients_to_fit=boost::optional<int>(), boost::optional<float> detector_standard_deviation=boost::optional<float>(), boost::optional<float> model_standard_deviation=boost::optional<float>())
inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::MorphableModel& morphable_model, cv::Mat affine_camera_matrix, const std::vector<cv::Vec2f>& landmarks, const std::vector<int>& vertex_ids, Eigen::VectorXf base_face=Eigen::VectorXf(), float lambda=3.0f, boost::optional<int> num_coefficients_to_fit=boost::optional<int>(), boost::optional<float> detector_standard_deviation=boost::optional<float>(), boost::optional<float> model_standard_deviation=boost::optional<float>())
{
using cv::Mat;
assert(landmarks.size() == vertex_ids.size());
......@@ -65,7 +65,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo
int num_coeffs_to_fit = num_coefficients_to_fit.get_value_or(morphable_model.get_shape_model().get_num_principal_components());
int num_landmarks = static_cast<int>(landmarks.size());
if (base_face.empty())
if (base_face.size() == 0)
{
base_face = morphable_model.get_shape_model().get_mean();
}
......@@ -75,7 +75,8 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo
Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
int row_index = 0;
for (int i = 0; i < num_landmarks; ++i) {
Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
auto basis_rows_ = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
Mat basis_rows = Mat(basis_rows_.rows(), basis_rows_.cols(), CV_32FC1, basis_rows_.data());
//basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3));
basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
......@@ -108,7 +109,7 @@ inline std::vector<float> fit_shape_to_landmarks_linear(const morphablemodel::Mo
Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
for (int i = 0; i < num_landmarks; ++i) {
//cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
cv::Vec4f model_mean(base_face.at<float>(vertex_ids[i] * 3), base_face.at<float>(vertex_ids[i] * 3 + 1), base_face.at<float>(vertex_ids[i] * 3 + 2), 1.0f);
cv::Vec4f model_mean(base_face(vertex_ids[i] * 3), base_face(vertex_ids[i] * 3 + 1), base_face(vertex_ids[i] * 3 + 2), 1.0f);
v_bar.at<float>(4 * i, 0) = model_mean[0];
v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
......
include/eos/morphablemodel/Blendshape.hpp
View file @ 470615a6
......@@ -22,11 +22,11 @@
#ifndef BLENDSHAPE_HPP_
#define BLENDSHAPE_HPP_
#include "eos/morphablemodel/io/mat_cerealisation.hpp"
#include "eos/morphablemodel/io/eigen_cerealisation.hpp"
#include "cereal/types/string.hpp"
#include "cereal/archives/binary.hpp"
#include "opencv2/core/core.hpp"
#include "Eigen/Core"
#include <string>
#include <vector>
......@@ -46,7 +46,7 @@ namespace eos {
struct Blendshape
{
std::string name; ///< Name of the blendshape.
cv::Mat deformation; ///< A 3m x 1 col-vector (xyzxyz...)', where m is the number of model-vertices. Has the same format as PcaModel::mean.
Eigen::VectorXf deformation; ///< A 3m x 1 col-vector (xyzxyz...)', where m is the number of model-vertices. Has the same format as PcaModel::mean.
friend class cereal::access;
/**
......@@ -89,14 +89,15 @@ inline std::vector<Blendshape> load_blendshapes(std::string filename)
* @param[in] blendshapes Vector of blendshapes.
* @return The resulting matrix.
*/
inline cv::Mat to_matrix(const std::vector<Blendshape>& blendshapes)
inline Eigen::VectorXf to_matrix(const std::vector<Blendshape>& blendshapes)
{
// assert: all blendshapes have to have the same number of rows, and one col
assert(blendshapes.size() > 0);
cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1);
// Todo: Assert all blendshapes have to have the same number of rows, and one col
Eigen::MatrixXf blendshapes_as_basis(blendshapes[0].deformation.rows(), blendshapes.size());
for (int i = 0; i < blendshapes.size(); ++i)
{
blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
blendshapes_as_basis.col(i) = blendshapes[i].deformation;
}
return blendshapes_as_basis;
};
......
include/eos/morphablemodel/MorphableModel.hpp
View file @ 470615a6
......@@ -36,18 +36,18 @@
#include "glm/vec3.hpp"
#include "glm/vec4.hpp"
#include "Eigen/Core"
#include <vector>
#include <array>
#include <cstdint>
// Forward declaration:
namespace eos { namespace morphablemodel {
eos::core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance, std::vector<std::array<int, 3>> tvi, std::vector<std::array<int, 3>> tci, std::vector<cv::Vec2f> texture_coordinates = std::vector<cv::Vec2f>());
} }
namespace eos {
namespace morphablemodel {
// Forward declaration:
core::Mesh sample_to_mesh(const Eigen::VectorXf& shape_instance, const Eigen::VectorXf& color_instance, const std::vector<std::array<int, 3>>& tvi, const std::vector<std::array<int, 3>>& tci, const std::vector<std::array<double, 2>>& texture_coordinates = std::vector<std::array<double, 2>>());
/**
* @brief A class representing a 3D Morphable Model, consisting
* of a shape- and colour (albedo) PCA model.
......@@ -61,14 +61,14 @@ public:
MorphableModel() = default;
/**
* Create a Morphable Model from a shape and a color PCA model, and optional
* Create a Morphable Model from a shape and a colour PCA model, and optional
* texture coordinates.
*
* @param[in] shape_model A PCA model over the shape.
* @param[in] color_model A PCA model over the colour (albedo).
* @param[in] texture_coordinates Optional texture coordinates for every vertex.
*/
MorphableModel(PcaModel shape_model, PcaModel color_model, std::vector<cv::Vec2f> texture_coordinates = std::vector<cv::Vec2f>()) : shape_model(shape_model), color_model(color_model), texture_coordinates(texture_coordinates)
MorphableModel(PcaModel shape_model, PcaModel color_model, std::vector<std::array<double, 2>> texture_coordinates = std::vector<std::array<double, 2>>()) : shape_model(shape_model), color_model(color_model), texture_coordinates(texture_coordinates)
{
};
......@@ -129,8 +129,8 @@ public:
{
assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
cv::Mat shape = shape_model.get_mean();
cv::Mat color = color_model.get_mean();
Eigen::VectorXf shape = shape_model.get_mean();
Eigen::VectorXf color = color_model.get_mean();
core::Mesh mesh;
if (has_texture_coordinates()) {
......@@ -147,16 +147,21 @@ public:
* for the shape- and colour models are both drawn from a standard
* normal (or with the given standard deviation).
*
* If the Morphable Model is a shape-only model, the returned mesh will
* not contain any colour data.
*
* @param[in] engine Random number engine used to draw random coefficients.
* @param[in] shape_sigma The shape model standard deviation.
* @param[in] color_sigma The colour model standard deviation.
* @return A random sample from the model.
*/
core::Mesh draw_sample(float shape_sigma = 1.0f, float color_sigma = 1.0f)
template <class RNG>
core::Mesh draw_sample(RNG& engine, float shape_sigma = 1.0f, float color_sigma = 1.0f) const
{
assert(shape_model.get_data_dimension() == color_model.get_data_dimension()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models.
assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
cv::Mat shape_sample = shape_model.draw_sample(shape_sigma);
cv::Mat color_sample = color_model.draw_sample(color_sigma);
Eigen::VectorXf shape_sample = shape_model.draw_sample(engine, shape_sigma);
Eigen::VectorXf color_sample = color_model.draw_sample(engine, color_sigma);
core::Mesh mesh;
if (has_texture_coordinates()) {
......@@ -186,8 +191,8 @@ public:
{
assert(shape_model.get_data_dimension() == color_model.get_data_dimension() || !has_color_model()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
cv::Mat shape_sample;
cv::Mat color_sample;
Eigen::VectorXf shape_sample;
Eigen::VectorXf color_sample;
if (shape_coefficients.empty()) {
shape_sample = shape_model.get_mean();
......@@ -220,7 +225,7 @@ public:
*/
bool has_color_model() const
{
return !color_model.get_mean().empty();
return (color_model.get_mean().size() > 0);
};
/**
......@@ -228,7 +233,7 @@ public:
*
* @return The texture coordinates for the model vertices.
*/
std::vector<cv::Vec2f> get_texture_coordinates() const
std::vector<std::array<double, 2>> get_texture_coordinates() const
{
return texture_coordinates;
};
......@@ -236,7 +241,7 @@ public:
private:
PcaModel shape_model; ///< A PCA model of the shape
PcaModel color_model; ///< A PCA model of vertex colour information
std::vector<cv::Vec2f> texture_coordinates; ///< uv-coordinates for every vertex
std::vector<std::array<double, 2>> texture_coordinates; ///< uv-coordinates for every vertex
/**
* Returns whether the model has texture mapping coordinates, i.e.
......@@ -253,10 +258,15 @@ private:
* Serialises this class using cereal.
*
* @param[in] ar The archive to serialise to (or to serialise from).
* @throw std::runtime_error When the model file doesn't have the most recent version (=1).
*/
template<class Archive>
void serialize(Archive& archive, const std::uint32_t version)
{
if (version != 1)
{
throw std::runtime_error("The model file you are trying to load is in an old format. Please download the most recent model files.");
}
archive(CEREAL_NVP(shape_model), CEREAL_NVP(color_model), CEREAL_NVP(texture_coordinates));
};
};
......@@ -311,25 +321,25 @@ inline void save_model(MorphableModel model, std::string filename)
* @param[in] texture_coordinates Optional texture coordinates for each vertex.
* @return A mesh created from given parameters.
*/
inline core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance, std::vector<std::array<int, 3>> tvi, std::vector<std::array<int, 3>> tci, std::vector<cv::Vec2f> texture_coordinates /* = std::vector<cv::Vec2f>() */)
inline core::Mesh sample_to_mesh(const Eigen::VectorXf& shape_instance, const Eigen::VectorXf& color_instance, const std::vector<std::array<int, 3>>& tvi, const std::vector<std::array<int, 3>>& tci, const std::vector<std::array<double, 2>>& texture_coordinates /* = std::vector<std::array<double, 2>>() */)
{
assert(shape_instance.rows == color_instance.rows || color_instance.empty()); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
assert(shape_instance.rows() == color_instance.rows() || color_instance.size() == 0); // The number of vertices (= model.getDataDimension() / 3) has to be equal for both models, or, alternatively, it has to be a shape-only model.
auto num_vertices = shape_instance.rows / 3;
auto num_vertices = shape_instance.rows() / 3;
core::Mesh mesh;
// Construct the mesh vertices:
mesh.vertices.resize(num_vertices);
for (auto i = 0; i < num_vertices; ++i) {
mesh.vertices[i] = glm::tvec4<float>(shape_instance.at<float>(i * 3 + 0), shape_instance.at<float>(i * 3 + 1), shape_instance.at<float>(i * 3 + 2), 1.0f);
mesh.vertices[i] = glm::tvec4<float>(shape_instance(i * 3 + 0), shape_instance(i * 3 + 1), shape_instance(i * 3 + 2), 1.0f);
}
// Assign the vertex colour information if it's not a shape-only model:
if (!color_instance.empty()) {
if (color_instance.size() > 0) {
mesh.colors.resize(num_vertices);
for (auto i = 0; i < num_vertices; ++i) {
mesh.colors[i] = glm::tvec3<float>(color_instance.at<float>(i * 3 + 0), color_instance.at<float>(i * 3 + 1), color_instance.at<float>(i * 3 + 2)); // order in hdf5: RGB. Order in OCV: BGR. But order in vertex.color: RGB
mesh.colors[i] = glm::tvec3<float>(color_instance(i * 3 + 0), color_instance(i * 3 + 1), color_instance(i * 3 + 2)); // We use RGB order everywhere
}
}
......@@ -351,6 +361,6 @@ inline core::Mesh sample_to_mesh(cv::Mat shape_instance, cv::Mat color_instance,
} /* namespace morphablemodel */
} /* namespace eos */
CEREAL_CLASS_VERSION(eos::morphablemodel::MorphableModel, 0);
CEREAL_CLASS_VERSION(eos::morphablemodel::MorphableModel, 1);
#endif /* MORPHABLEMODEL_HPP_ */
include/eos/morphablemodel/PcaModel.hpp
View file @ 470615a6
This diff is collapsed.
include/eos/morphablemodel/io/eigen_cerealisation.hpp 0 → 100644
View file @ 470615a6
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: include/eos/morphablemodel/io/eigen_cerealisation.hpp
*
* Copyright 2017 Patrik Huber
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifndef EIGENCEREALISATION_HPP_
#define EIGENCEREALISATION_HPP_
#include "cereal/cereal.hpp"
#include "Eigen/Core"
#include <cstdint>
/**
* @brief Serialisation of Eigen matrices for the serialisation
* library cereal (http://uscilab.github.io/cereal/index.html).
*
* Contains serialisation for Eigen matrices to binary archives, i.e. matrices like
* \c Eigen::MatrixXf, \c Eigen::Matrix4d, or \c Eigen::Vector3f.
*
* Todo: Add serialisation to and from JSON. Need to find out how to combine the two
* variants of SFINAE that are used.
*/
namespace cereal {
/**
* @brief Serialise an Eigen::Matrix using cereal.
*
* Note: Writes the binary data from Matrix::data(), so not sure what happens if a matrix ever has
* non-contiguous data (if that can ever happen with Eigen).
*
* @param[in] ar The archive to serialise to.
* @param[in] matrix The matrix to serialise.
*/
template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
inline
typename std::enable_if<traits::is_output_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
save(Archive& ar, const Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& matrix)
{
std::int32_t rows = matrix.rows();
std::int32_t cols = matrix.cols();
ar(rows);
ar(cols);
ar(binary_data(matrix.data(), rows * cols * sizeof(_Scalar)));
};
/**
* @brief De-serialise an Eigen::Matrix using cereal.
*
* Reads the block of binary data back from a cereal archive into the Eigen::Matrix.
*
* @param[in] ar The archive to deserialise from.
* @param[in] matrix The matrix to deserialise into.
*/
template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
inline
typename std::enable_if<traits::is_input_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
load(Archive& ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>& matrix)
{
std::int32_t rows;
std::int32_t cols;
ar(rows);
ar(cols);
matrix.resize(rows, cols);
ar(binary_data(matrix.data(), static_cast<std::size_t>(rows * cols * sizeof(_Scalar))));
};
} /* namespace cereal */
#endif /* EIGENCEREALISATION_HPP_ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment