eos  0.7.1
contour_correspondence.hpp
1 /*
2  * Eos - A 3D Morphable Model fitting library written in modern C++11/14.
3  *
4  * File: include/eos/fitting/contour_correspondence.hpp
5  *
6  * Copyright 2015, 2016 Patrik Huber
7  *
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  *
12  * http://www.apache.org/licenses/LICENSE-2.0
13  *
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  */
20 #pragma once
21 
22 #ifndef CONTOURCORRESPONDENCE_HPP_
23 #define CONTOURCORRESPONDENCE_HPP_
24 
25 #include "eos/core/Landmark.hpp"
26 #include "eos/morphablemodel/MorphableModel.hpp"
27 
28 #include "cereal/archives/json.hpp"
29 
30 #include "glm/gtc/matrix_transform.hpp"
31 
32 #include "opencv2/core/core.hpp"
33 
34 #include "boost/property_tree/ptree.hpp"
35 #include "boost/property_tree/info_parser.hpp"
36 
37 #include <vector>
38 #include <string>
39 #include <algorithm>
40 #include <fstream>
41 
42 namespace eos {
43  namespace fitting {
44 
45 // Forward declarations of later used functions and types:
46 struct ModelContour;
47 struct ContourLandmarks;
48 std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_angle, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour);
49 std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_nearest_contour_correspondences(const eos::core::LandmarkCollection<cv::Vec2f>& landmarks, const std::vector<std::string>& landmark_contour_identifiers, const std::vector<int>& model_contour_indices, const morphablemodel::MorphableModel& morphable_model, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport);
50 
51 
65 {
66  // starting from right side, eyebrow-height: (I think the order matters here)
67  std::vector<int> right_contour;
68  /* 23 = middle, below chin - not included in the contour here */
69  // starting from left side, eyebrow-height:
70  std::vector<int> left_contour;
71 
72  // We store r/l separately because we currently only fit to the contour facing the camera.
73  // Also if we were to fit to the whole contour: Be careful not to just fit to the closest. The
74  // "invisible" ones behind might be closer on an e.g 90° angle. Store CNT for left/right side separately?
75 
86  static ModelContour load(std::string filename)
87  {
88  ModelContour contour;
89 
90  std::ifstream file(filename);
91  if (file.fail()) {
92  throw std::runtime_error("Error opening given file: " + filename);
93  }
94  cereal::JSONInputArchive input_archive(file);
95  input_archive(contour);
96 
97  return contour;
98  };
99 
100  friend class cereal::access;
106  template<class Archive>
107  void serialize(Archive& archive)
108  {
109  archive(cereal::make_nvp("right_contour", right_contour), cereal::make_nvp("left_contour", left_contour));
110  };
111 };
112 
125 {
126  // starting from right side, eyebrow-height.
127  std::vector<std::string> right_contour;
128  // Chin point is not included in the contour here.
129  // starting from left side, eyebrow-height. Order doesn't matter here.
130  std::vector<std::string> left_contour;
131 
132  // Note: We store r/l separately because we currently only fit to the contour facing the camera.
133 
142  static ContourLandmarks load(std::string filename)
143  {
144  ContourLandmarks contour;
145 
146  using std::string;
147  using boost::property_tree::ptree;
148  ptree configtree;
149  try {
150  boost::property_tree::info_parser::read_info(filename, configtree);
151  }
152  catch (const boost::property_tree::ptree_error& error) {
153  throw std::runtime_error(string("ContourLandmarks: Error reading landmark-mappings file: ") + error.what());
154  }
155 
156  try {
157  // Todo: I think we should improve error handling here. When there's no contour_landmarks in the file, it
158  // should still work, the returned vectors should just be empty.
159  ptree right_contour = configtree.get_child("contour_landmarks.right");
160  for (auto&& landmark : right_contour) {
161  contour.right_contour.emplace_back(landmark.first);
162  }
163  ptree left_contour = configtree.get_child("contour_landmarks.left");
164  for (auto&& landmark : left_contour) {
165  contour.left_contour.emplace_back(landmark.first);
166  }
167  }
168  catch (const boost::property_tree::ptree_error& error) {
169  throw std::runtime_error(string("ContourLandmarks: Error while parsing the mappings file: ") + error.what());
170  }
171  catch (const std::runtime_error& error) {
172  throw std::runtime_error(string("ContourLandmarks: Error while parsing the mappings file: ") + error.what());
173  }
174 
175  return contour;
176  };
177 };
178 
202 std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_contour_correspondences(const eos::core::LandmarkCollection<cv::Vec2f>& landmarks, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour, float yaw_angle, const morphablemodel::MorphableModel& morphable_model, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport)
203 {
204  // Select which side of the contour we'll use:
205  std::vector<int> model_contour_indices;
206  std::vector<std::string> landmark_contour_identifiers;
207  std::tie(landmark_contour_identifiers, model_contour_indices) = select_contour(glm::degrees(yaw_angle), contour_landmarks, model_contour);
208 
209  // These are the additional contour-correspondences we're going to find and then use:
210  std::vector<cv::Vec4f> model_points_contour; // the points in the 3D shape model
211  std::vector<int> vertex_indices_contour; // their vertex indices
212  std::vector<cv::Vec2f> image_points_contour; // the corresponding 2D landmark points
213 
214  // For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
215  // Note/Todo: Loop here instead of calling this function where we have no idea what it's doing? What does its documentation say?
216  return get_nearest_contour_correspondences(landmarks, landmark_contour_identifiers, model_contour_indices, morphable_model, view_model, ortho_projection, viewport);
217 };
218 
234 std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_angle, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour)
235 {
236  std::vector<int> model_contour_indices;
237  std::vector<std::string> contour_landmark_identifiers;
238  if (yaw_angle >= 0.0f) { // positive yaw = subject looking to the left
239  model_contour_indices = model_contour.right_contour; // ==> we use the right cnt-lms
240  contour_landmark_identifiers = contour_landmarks.right_contour;
241  }
242  else {
243  model_contour_indices = model_contour.left_contour;
244  contour_landmark_identifiers = contour_landmarks.left_contour;
245  }
246  return std::make_pair(contour_landmark_identifiers, model_contour_indices);
247 };
248 
273 std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_nearest_contour_correspondences(const eos::core::LandmarkCollection<cv::Vec2f>& landmarks, const std::vector<std::string>& landmark_contour_identifiers, const std::vector<int>& model_contour_indices, const morphablemodel::MorphableModel& morphable_model, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport)
274 {
275  // These are the additional contour-correspondences we're going to find and then use!
276  std::vector<cv::Vec4f> model_points_cnt; // the points in the 3D shape model
277  std::vector<int> vertex_indices_cnt; // their vertex indices
278  std::vector<cv::Vec2f> image_points_cnt; // the corresponding 2D landmark points
279 
280  // For each 2D-CNT-LM, find the closest 3DMM-CNT-LM and add to correspondences:
281  // Note: If we were to do this for all 3DMM vertices, then ray-casting (i.e. glm::unproject) would be quicker to find the closest vertex)
282  for (auto&& ibug_idx : landmark_contour_identifiers)
283  {
284  // Check if the contour landmark is amongst the landmarks given to us (from detector or ground truth):
285  // (Note: Alternatively, we could filter landmarks beforehand and then just loop over landmarks => means one less function param here. Separate filtering from actual algorithm.)
286  auto result = std::find_if(begin(landmarks), end(landmarks), [&ibug_idx](auto&& e) { return e.name == ibug_idx; }); // => this can go outside the loop
287  // TODO Check for ::end!!! if it's not found!
288  cv::Vec2f screen_point_2d_contour_landmark = result->coordinates;
289 
290  std::vector<float> distances_2d;
291  for (auto&& model_contour_vertex_idx : model_contour_indices) // we could actually pre-project them, i.e. only project them once, not for each landmark newly...
292  {
293  auto vertex = morphable_model.get_shape_model().get_mean_at_point(model_contour_vertex_idx);
294  glm::vec3 proj = glm::project(glm::vec3{ vertex[0], vertex[1], vertex[2] }, view_model, ortho_projection, viewport);
295  cv::Vec2f screen_point_model_contour(proj.x, proj.y);
296 
297  double dist = cv::norm(screen_point_model_contour, screen_point_2d_contour_landmark, cv::NORM_L2);
298  distances_2d.emplace_back(dist);
299  }
300  auto min_ele = std::min_element(begin(distances_2d), end(distances_2d));
301  // Todo: Cover the case when cnt_indices_to_use.size() is 0.
302  auto min_ele_idx = std::distance(begin(distances_2d), min_ele);
303  auto the_3dmm_vertex_id_that_is_closest = model_contour_indices[min_ele_idx];
304 
305  cv::Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(the_3dmm_vertex_id_that_is_closest);
306  model_points_cnt.emplace_back(vertex);
307  vertex_indices_cnt.emplace_back(the_3dmm_vertex_id_that_is_closest);
308  image_points_cnt.emplace_back(screen_point_2d_contour_landmark);
309  }
310 
311  return std::make_tuple(image_points_cnt, model_points_cnt, vertex_indices_cnt);
312 };
313 
314  } /* namespace fitting */
315 } /* namespace eos */
316 
317 #endif /* CONTOURCORRESPONDENCE_HPP_ */
static ContourLandmarks load(std::string filename)
Definition: contour_correspondence.hpp:142
std::tuple< std::vector< cv::Vec2f >, std::vector< cv::Vec4f >, std::vector< int > > get_contour_correspondences(const eos::core::LandmarkCollection< cv::Vec2f > &landmarks, const ContourLandmarks &contour_landmarks, const ModelContour &model_contour, float yaw_angle, const morphablemodel::MorphableModel &morphable_model, const glm::mat4x4 &view_model, const glm::mat4x4 &ortho_projection, const glm::vec4 &viewport)
Definition: contour_correspondence.hpp:202
std::pair< std::vector< std::string >, std::vector< int > > select_contour(float yaw_angle, const ContourLandmarks &contour_landmarks, const ModelContour &model_contour)
Definition: contour_correspondence.hpp:234
std::vector< Landmark< LandmarkType >> LandmarkCollection
A trivial collection of landmarks that belong together.
Definition: Landmark.hpp:46
Definition of the vertex indices that define the right and left model contour.
Definition: contour_correspondence.hpp:64
Namespace containing all of eos&#39;s 3D model fitting functionality.
Defines which 2D landmarks comprise the right and left face contour.
Definition: contour_correspondence.hpp:124
void serialize(Archive &archive)
Definition: contour_correspondence.hpp:107
static ModelContour load(std::string filename)
Definition: contour_correspondence.hpp:86
std::tuple< std::vector< cv::Vec2f >, std::vector< cv::Vec4f >, std::vector< int > > get_nearest_contour_correspondences(const eos::core::LandmarkCollection< cv::Vec2f > &landmarks, const std::vector< std::string > &landmark_contour_identifiers, const std::vector< int > &model_contour_indices, const morphablemodel::MorphableModel &morphable_model, const glm::mat4x4 &view_model, const glm::mat4x4 &ortho_projection, const glm::vec4 &viewport)
Definition: contour_correspondence.hpp:273
PcaModel get_shape_model() const
Definition: MorphableModel.hpp:77
cv::Vec4f get_mean_at_point(int vertex_index) const
Definition: PcaModel.hpp:130
A class representing a 3D Morphable Model, consisting of a shape- and colour (albedo) PCA model...
Definition: MorphableModel.hpp:54