Improved documentation of contour landmarks fitting and classes

include/eos/fitting/contour_correspondence.hpp

@@ -3,7 +3,7 @@
  *
  * File: include/eos/fitting/contour_correspondence.hpp
  *
- * Copyright 2015 Patrik Huber
+ * Copyright 2015, 2016 Patrik Huber
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -48,9 +48,17 @@ struct ContourLandmarks;
 std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_angle, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour);
 std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_nearest_contour_correspondences(const eos::core::LandmarkCollection<cv::Vec2f>& landmarks, const std::vector<std::string>& landmark_contour_identifiers, const std::vector<int>& model_contour_indices, const morphablemodel::MorphableModel& morphable_model, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport);
 
-// The contour (outline) on the right and left side of the reference face model.
-// We should extend that to the 1724 model to get a few more points, it should
-// improve the contour fitting.
+
+/**
+ * This class holds definitions for the contour (outline) on the right and left
+ * side of the reference 3D face model. These can be found in the file
+ * share/model_contours.json. The Surrey model's boundaries are conveniently
+ * approximately located near the actual 2D image contour, for the front-facing
+ * contour.
+ *
+ * Note: We should extend that to the 1724 model to get a few more points, this
+ * should improve the contour fitting.
+ */
 struct ModelContour
 {
 	// starting from right side, eyebrow-height: (I think the order matters here)
@@ -65,7 +73,7 @@ struct ModelContour
 	
 	/**
 	 * Helper method to load a ModelContour from
-	 * a json file from the harddisk.
+	 * a json file from the hard drive.
 	 *
 	 * Eventually, it might be included in the MorphableModel class.
 	 *
@@ -100,6 +108,15 @@ struct ModelContour
 	};
 };
 
+/**
+ * This class holds 2D image contour landmark information. More specifically,
+ * it defines which 2D landmark IDs correspond to the right contour and which
+ * to the left. These definitions are loaded from a file, for example from
+ * the "contour_landmarks" part of share/ibug2did.txt.
+ *
+ * Note: Better names could be ContourDefinition or ImageContourLandmarks, to
+ * disambiguate 3D and 2D landmarks?
+ */
 struct ContourLandmarks
 {
 	// starting from right side, eyebrow-height.
@@ -108,7 +125,7 @@ struct ContourLandmarks
 	// starting from left side, eyebrow-height. Order doesn't matter here.
 	std::vector<std::string> left_contour;
 
-	// We store r/l separately because we currently only fit to the contour facing the camera.
+	// Note: We store r/l separately because we currently only fit to the contour facing the camera.
 	
 	/**
 	 * Helper method to load contour landmarks from a text file with landmark
@@ -156,20 +173,26 @@ struct ContourLandmarks
 };
 
 /**
- * Given a set of 2D image landmarks, finds the closest (in a L2 sense) 3D vertex from a list of vertices.... and... todo
+ * Given a set of 2D image landmarks, finds the closest (in a L2 sense) 3D vertex
+ * from a list of vertices pre-defined in \p model_contour. \p landmarks can contain
+ * all landmarks, and the function will sub-select the relevant contour landmarks with
+ * the help of the given \p contour_landmarks. This function choses the front-facing
+ * contour and only fits this contour to the 3D model, since these correspondences
+ * are approximately static and do not move with changing pose-angle.
  *
  * It's the main contour fitting function that calls all other functions.
  *
  * Note: Maybe rename to find_contour_correspondences, to highlight that there is (potentially a lot) computational cost involved?
+ * Note: Does ortho_projection have to be specifically orthographic? Otherwise, if it works with perspective too, rename to just "projection".
  *
  * @param[in] landmarks All image landmarks.
- * @param[in] contour_landmarks ibug contour ids of left or right side.
- * @param[in] model_contour The model contour indices that should be used/considered to find the closest corresponding 3D vertex.
- * @param[in] yaw_angle X.
+ * @param[in] contour_landmarks 2D image contour ids of left or right side (for example for ibug landmarks).
+ * @param[in] model_contour The model contour indices that should be considered to find the closest corresponding 3D vertex.
+ * @param[in] yaw_angle Yaw angle of the current fitting. The front-facing contour will be chosen depending on this yaw angle.
  * @param[in] morphable_model A Morphable Model whose mean is used.
- * @param[in] view_model x.
- * @param[in] ortho_projection Note: Does this depend on ortho? Maybe not? If it works with persp too, then rename param & doc.
- * @param[in] viewport X.
+ * @param[in] view_model Model-view matrix of the current fitting to project the 3D model vertices to 2D.
+ * @param[in] ortho_projection Projection matrix to project the 3D model vertices to 2D.
+ * @param[in] viewport Current viewport to use.
  * @return A tuple with the 2D contour landmark points, the corresponding points in the 3D shape model and their vertex indices.
  */
 std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_contour_correspondences(const eos::core::LandmarkCollection<cv::Vec2f>& landmarks, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour, float yaw_angle, const morphablemodel::MorphableModel& morphable_model, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport)
@@ -185,18 +208,24 @@ std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get
 	std::vector<cv::Vec2f> image_points_contour; // the corresponding 2D landmark points
 	
 	// For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
+	// Note/Todo: Loop here instead of calling this function where we have no idea what it's doing? What does its documentation say?
 	return get_nearest_contour_correspondences(landmarks, landmark_contour_identifiers, model_contour_indices, morphable_model, view_model, ortho_projection, viewport);
 };
 
 /**
- * Takes a ... returns two vectors... can have different size. Does not establish correspondence. Use get_nearest_contour_points() for that.
+ * Takes a set of 2D and 3D contour landmarks and a yaw angle and returns two
+ * vectors with either the right or the left 2D and 3D contour indices. This
+ * function does not establish correspondence between the 2D and 3D landmarks,
+ * it just selects the front-facing contour. The two returned vectors can thus
+ * have different size. 
+ * Correspondence can be established using get_nearest_contour_correspondences().
  *
  * Note: Maybe rename to find_nearest_contour_points, to highlight that there is (potentially a lot) computational cost involved?
  *
  * @param[in] yaw_angle yaw angle in degrees.
- * @param[in] contour_landmarks E.g. ibug contours.
- * @param[in] model_contour X.
- * @return A tuple/two vectors... with X. returns ... model_cnt_idx different size than 2d_cnt. Not in correspondence.
+ * @param[in] contour_landmarks 2D image contour ids of left or right side (for example for ibug landmarks).
+ * @param[in] model_contour The model contour indices that should be used/considered to find the closest corresponding 3D vertex.
+ * @return A pair with two vectors containing the selected 2D image contour landmark ids and the 3D model contour indices.
  */
 std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_angle, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour)
 {
@@ -214,25 +243,29 @@ std::pair<std::vector<std::string>, std::vector<int>> select_contour(float yaw_a
 };
 
 /**
- * Given a set of 2D image landmarks, finds the closest (in a L2 sense) 3D vertex from a list of vertices.
+ * Given a set of 2D image landmarks, finds the closest (in a L2 sense) 3D vertex
+ * from a list of vertices pre-defined in \p model_contour. Assumes to be given
+ * contour correspondences of the front-facing contour.
  *
  * Note: Maybe rename to find_nearest_contour_points, to highlight that there is (potentially a lot) computational cost involved?
+ * Note: Does ortho_projection have to be specifically orthographic? Otherwise, if it works with perspective too, rename to just "projection".
+ * More notes:
+ *   Actually, only return the vertex id, not the point? Same with get_corresponding_pointset? Because
+ *   then it's much easier to use the current shape estimate instead of the mean! But this function needs to project.
+ *   So... it should take a Mesh actually? But creating a Mesh is a lot of computation?
+ *   When we want to use the non-mean, then we need to use draw_sample() anyway? So overhead of Mesh is only if we use the mean?
+ *   Maybe two overloads?
+ *   Note: Uses the mean to calculate.
  *
  * @param[in] landmarks All image landmarks.
- * @param[in] landmark_contour_identifiers ibug contour ids of left or right side.
- * @param[in] model_contour_indices The model contour indices that should be used/considered to find the closest corresponding 3D vertex.
+ * @param[in] landmark_contour_identifiers 2D image contour ids of left or right side (for example for ibug landmarks).
+ * @param[in] model_contour_indices The model contour indices that should be considered to find the closest corresponding 3D vertex.
  * @param[in] morphable_model The Morphable Model whose shape (coefficients) are estimated.
- * @param[in] view_model x.
- * @param[in] ortho_projection Note: Does this depend on ortho? Maybe not? If it works with persp too, then rename param & doc.
- * @param[in] viewport X.
+ * @param[in] view_model Model-view matrix of the current fitting to project the 3D model vertices to 2D.
+ * @param[in] ortho_projection Projection matrix to project the 3D model vertices to 2D.
+ * @param[in] viewport Current viewport to use.
  * @return A tuple with the 2D contour landmark points, the corresponding points in the 3D shape model and their vertex indices.
  */
-// actually, only return the vertex id, not the point? Same with get_corresponding_pointset? Because
-// then it's much easier to use the current shape estimate instead of the mean! But this function needs to project.
-// So... it should take a Mesh actually? But creating a Mesh is a lot of computation?
-// When we want to use the non-mean, then we need to use draw_sample() anyway? So overhead of Mesh is only if we use the mean?
-// Maybe two overloads?
-// Note: Uses the mean to calculate.
 std::tuple<std::vector<cv::Vec2f>, std::vector<cv::Vec4f>, std::vector<int>> get_nearest_contour_correspondences(const eos::core::LandmarkCollection<cv::Vec2f>& landmarks, const std::vector<std::string>& landmark_contour_identifiers, const std::vector<int>& model_contour_indices, const morphablemodel::MorphableModel& morphable_model, const glm::mat4x4& view_model, const glm::mat4x4& ortho_projection, const glm::vec4& viewport)
 {
 	// These are the additional contour-correspondences we're going to find and then use!