eos  0.7.1
blendshape_fitting.hpp
1 /*
2  * Eos - A 3D Morphable Model fitting library written in modern C++11/14.
3  *
4  * File: include/eos/fitting/blendshape_fitting.hpp
5  *
6  * Copyright 2015 Patrik Huber
7  *
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  */
20 #pragma once
21 
22 #ifndef BLENDSHAPEFITTING_HPP_
23 #define BLENDSHAPEFITTING_HPP_
24 
25 #include "eos/morphablemodel/Blendshape.hpp"
26 
27 #include "opencv2/core/core.hpp"
28 
29 #include <vector>
30 #include <cassert>
31 
32 namespace eos {
33  namespace fitting {
34 
50 inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, float lambda=500.0f)
51 {
52  using cv::Mat;
53  assert(landmarks.size() == vertex_ids.size());
54 
55  int num_coeffs_to_fit = blendshapes.size();
56  int num_landmarks = static_cast<int>(landmarks.size());
57 
58  // Copy all blendshapes into a "basis" matrix with each blendshape being a column:
59  cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
60  for (int i = 0; i < blendshapes.size(); ++i)
61  {
62  blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
63  }
64 
65  // $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
66  // And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
67  Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
68  int row_index = 0;
69  for (int i = 0; i < num_landmarks; ++i) {
70  //Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
71  Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
72  //basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3));
73  basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
74  row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
75  }
76  // Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
77  Mat P = Mat::zeros(3 * num_landmarks, 4 * num_landmarks, CV_32FC1);
78  for (int i = 0; i < num_landmarks; ++i) {
79  Mat submatrix_to_replace = P.colRange(4 * i, (4 * i) + 4).rowRange(3 * i, (3 * i) + 3);
80  affine_camera_matrix.copyTo(submatrix_to_replace);
81  }
82 
83  // The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
84  Mat y = Mat::ones(3 * num_landmarks, 1, CV_32FC1);
85  for (int i = 0; i < num_landmarks; ++i) {
86  y.at<float>(3 * i, 0) = landmarks[i][0];
87  y.at<float>((3 * i) + 1, 0) = landmarks[i][1];
88  //y.at<float>((3 * i) + 2, 0) = 1; // already 1, stays (homogeneous coordinate)
89  }
90  // The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
91  Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
92  for (int i = 0; i < num_landmarks; ++i) {
93  //cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
94  cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f);
95  v_bar.at<float>(4 * i, 0) = model_mean[0];
96  v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
97  v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
98  //v_bar.at<float>((4 * i) + 3, 0) = 1; // already 1, stays (homogeneous coordinate)
99  // note: now that a Vec4f is returned, we could use copyTo?
100  }
101 
102  // Bring into standard regularised quadratic form:
103  Mat A = P * V_hat_h; // camera matrix times the basis
104  Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks.
105 
106  Mat AtAReg = A.t() * A + lambda * Mat::eye(num_coeffs_to_fit, num_coeffs_to_fit, CV_32FC1);
107  // Solve using OpenCV:
108  Mat c_s;
109  bool non_singular = cv::solve(AtAReg, -A.t() * b, c_s, cv::DECOMP_SVD); // DECOMP_SVD calculates the pseudo-inverse if the matrix is not invertible.
110  // Because we're using SVD, non_singular will always be true. If we were to use e.g. Cholesky, we could return an expected<T>.
111 
112  return std::vector<float>(c_s);
113 };
114 
115  } /* namespace fitting */
116 } /* namespace eos */
117 
118 #endif /* BLENDSHAPEFITTING_HPP_ */
Namespace containing all of eos&#39;s 3D model fitting functionality.
std::vector< float > fit_blendshapes_to_landmarks_linear(std::vector< eos::morphablemodel::Blendshape > blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector< cv::Vec2f > landmarks, std::vector< int > vertex_ids, float lambda=500.0f)
Definition: blendshape_fitting.hpp:50