eos  0.7.1
fitting.hpp
1 /*
2  * Eos - A 3D Morphable Model fitting library written in modern C++11/14.
3  *
4  * File: include/eos/fitting/fitting.hpp
5  *
6  * Copyright 2015 Patrik Huber
7  *
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  */
20 #pragma once
21 
22 #ifndef FITTING_HPP_
23 #define FITTING_HPP_
24 
25 #include "eos/morphablemodel/MorphableModel.hpp"
26 #include "eos/morphablemodel/Blendshape.hpp"
27 #include "eos/fitting/linear_shape_fitting.hpp"
28 #include "eos/fitting/blendshape_fitting.hpp"
29 
30 #include "opencv2/core/core.hpp"
31 
32 #include <vector>
33 
34 namespace eos {
35  namespace fitting {
36 
58 cv::Mat fit_shape(cv::Mat affine_camera_matrix, eos::morphablemodel::MorphableModel morphable_model, std::vector<eos::morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda, boost::optional<int> num_coefficients_to_fit, std::vector<float>& pca_shape_coefficients, std::vector<float>& blendshape_coefficients)
59 {
60  using cv::Mat;
61 
62  Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
63  for (int i = 0; i < blendshapes.size(); ++i)
64  {
65  blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
66  }
67 
68  std::vector<float> last_blendshape_coeffs, current_blendshape_coeffs;
69  std::vector<float> last_pca_coeffs, current_pca_coeffs;
70  current_blendshape_coeffs.resize(blendshapes.size()); // starting values t_0, all zeros
71  // no starting values for current_pca_coeffs required, since we start with the shape fitting, and cv::norm of an empty vector is 0.
72  Mat combined_shape;
73 
74  do // run at least once:
75  {
76  last_blendshape_coeffs = current_blendshape_coeffs;
77  last_pca_coeffs = current_pca_coeffs;
78  // Estimate the PCA shape coefficients with the current blendshape coefficients (0 in the first iteration):
79  Mat mean_plus_blendshapes = morphable_model.get_shape_model().get_mean() + blendshapes_as_basis * Mat(last_blendshape_coeffs);
80  current_pca_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_camera_matrix, image_points, vertex_indices, mean_plus_blendshapes, lambda, num_coefficients_to_fit);
81 
82  // Estimate the blendshape coefficients with the current PCA model estimate:
83  Mat pca_model_shape = morphable_model.get_shape_model().draw_sample(current_pca_coeffs);
84  current_blendshape_coeffs = eos::fitting::fit_blendshapes_to_landmarks_linear(blendshapes, pca_model_shape, affine_camera_matrix, image_points, vertex_indices, 0.0f);
85 
86  combined_shape = pca_model_shape + blendshapes_as_basis * Mat(current_blendshape_coeffs);
87  } while (std::abs(cv::norm(current_pca_coeffs) - cv::norm(last_pca_coeffs)) >= 0.01 || std::abs(cv::norm(current_blendshape_coeffs) - cv::norm(last_blendshape_coeffs)) >= 0.01);
88 
89  pca_shape_coefficients = current_pca_coeffs;
90  blendshape_coefficients = current_blendshape_coeffs;
91  return combined_shape;
92 };
93 
108 cv::Mat fit_shape(cv::Mat affine_camera_matrix, eos::morphablemodel::MorphableModel morphable_model, std::vector<eos::morphablemodel::Blendshape> blendshapes, std::vector<cv::Vec2f> image_points, std::vector<int> vertex_indices, float lambda = 3.0f, boost::optional<int> num_coefficients_to_fit = boost::optional<int>())
109 {
110  std::vector<float> unused;
111  return fit_shape(affine_camera_matrix, morphable_model, blendshapes, image_points, vertex_indices, lambda, num_coefficients_to_fit, unused, unused);
112 };
113 
114  } /* namespace fitting */
115 } /* namespace eos */
116 
117 #endif /* FITTING_HPP_ */
cv::Mat draw_sample(float sigma=1.0f)
Definition: PcaModel.hpp:143
Namespace containing all of eos&#39;s 3D model fitting functionality.
std::vector< float > fit_shape_to_landmarks_linear(morphablemodel::MorphableModel morphable_model, cv::Mat affine_camera_matrix, std::vector< cv::Vec2f > landmarks, std::vector< int > vertex_ids, cv::Mat base_face=cv::Mat(), float lambda=3.0f, boost::optional< int > num_coefficients_to_fit=boost::optional< int >(), boost::optional< float > detector_standard_deviation=boost::optional< float >(), boost::optional< float > model_standard_deviation=boost::optional< float >())
Definition: linear_shape_fitting.hpp:60
cv::Mat get_mean() const
Definition: PcaModel.hpp:119
PcaModel get_shape_model() const
Definition: MorphableModel.hpp:77
std::vector< float > fit_blendshapes_to_landmarks_linear(std::vector< eos::morphablemodel::Blendshape > blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector< cv::Vec2f > landmarks, std::vector< int > vertex_ids, float lambda=500.0f)
Definition: blendshape_fitting.hpp:50
cv::Mat fit_shape(cv::Mat affine_camera_matrix, eos::morphablemodel::MorphableModel morphable_model, std::vector< eos::morphablemodel::Blendshape > blendshapes, std::vector< cv::Vec2f > image_points, std::vector< int > vertex_indices, float lambda, boost::optional< int > num_coefficients_to_fit, std::vector< float > &pca_shape_coefficients, std::vector< float > &blendshape_coefficients)
Definition: fitting.hpp:58
A class representing a 3D Morphable Model, consisting of a shape- and colour (albedo) PCA model...
Definition: MorphableModel.hpp:54