Merge branch 'python-bindings' into devel

pybind11 @ 6ae68fe3

-Subproject commit 425b4970b28cedfb8c4567dded6378c0e33c117b
+Subproject commit 6ae68fe3013c85cb53986d65ac0dd885af724559

python/CMakeLists.txt

@@ -37,4 +37,6 @@ endif()
 pybind11_add_module(python-bindings generate-python-bindings.cpp pybind11_glm.hpp)
 target_link_libraries(python-bindings PRIVATE ${OpenCV_LIBS} ${Boost_LIBRARIES})
 set_target_properties(python-bindings PROPERTIES OUTPUT_NAME eos)
+
 install(TARGETS python-bindings DESTINATION python)
+install(FILES ${CMAKE_SOURCE_DIR}/python/demo.py DESTINATION python)

python/demo.py

+import eos
+import numpy as np
+
+def main():
+    """Demo for running the eos fitting from Python."""
+    landmarks = read_pts('../bin/data/image_0010.pts')
+    landmark_ids = list(map(str, range(1, 69))) # generates the numbers 1 to 68, as strings
+    image_width = 1280 # Make sure to adjust these when using your own images!
+    image_height = 1024
+
+    model = eos.morphablemodel.load_model("../share/sfm_shape_3448.bin")
+    blendshapes = eos.morphablemodel.load_blendshapes("../share/expression_blendshapes_3448.bin")
+    landmark_mapper = eos.core.LandmarkMapper('../share/ibug2did.txt')
+    edge_topology = eos.morphablemodel.load_edge_topology('../share/sfm_3448_edge_topology.json')
+    contour_landmarks = eos.fitting.ContourLandmarks.load('../share/ibug2did.txt')
+    model_contour = eos.fitting.ModelContour.load('../share/model_contours.json')
+
+    (mesh, pose, shape_coeffs, blendshape_coeffs) = eos.fitting.fit_shape_and_pose(model, blendshapes,
+        landmarks, landmark_ids, landmark_mapper,
+        image_width, image_height, edge_topology, contour_landmarks, model_contour)
+
+    # Now you can use your favourite plotting/rendering library to display the fitted mesh, using the rendering
+    # parameters in the 'pose' variable.
+
+
+def read_pts(filename):
+    """A helper function to read ibug .pts landmarks from a file."""
+    lines = open(filename).read().splitlines()
+    lines = lines[3:71]
+
+    landmarks = []
+    for l in lines:
+        coords = l.split()
+        landmarks.append([float(coords[0]), float(coords[1])])
+
+    return landmarks
+
+if __name__ == "__main__":
+    main()

python/generate-python-bindings.cpp

@@ -17,11 +17,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include "eos/core/LandmarkMapper.hpp"
 #include "eos/morphablemodel/PcaModel.hpp"
 #include "eos/morphablemodel/MorphableModel.hpp"
 #include "eos/morphablemodel/Blendshape.hpp"
+#include "eos/morphablemodel/EdgeTopology.hpp"
+#include "eos/fitting/contour_correspondence.hpp"
+#include "eos/fitting/fitting.hpp"
 #include "eos/fitting/orthographic_camera_estimation_linear.hpp"
 #include "eos/fitting/RenderingParameters.hpp"
+#include "eos/render/Mesh.hpp"
 
 #include "opencv2/core/core.hpp"
 
@@ -33,6 +38,7 @@
 #include <stdexcept>
 #include <string>
 #include <algorithm>
+#include <cassert>
 
 namespace py = pybind11;
 using namespace eos;
@@ -193,6 +199,19 @@ PYBIND11_PLUGIN(eos) {
 		})
 	;
 
+	/**
+	 * Bindings for the eos::core namespace:
+	 *  - LandmarkMapper
+	 */
+	py::module core_module = eos_module.def_submodule("core", "Essential functions and classes to work with 3D face models and landmarks.");
+	py::class_<core::LandmarkMapper>(core_module, "LandmarkMapper", "Represents a mapping from one kind of landmarks to a different format(e.g.model vertices).")
+		.def(py::init<>(), "Constructs a new landmark mapper that performs an identity mapping, that is, its output is the same as the input.")
+		.def("__init__", [](core::LandmarkMapper& instance, std::string filename) { // wrap the fs::path c'tor with std::string
+				new (&instance) core::LandmarkMapper(filename);
+			}, "Constructs a new landmark mapper from a file containing mappings from one set of landmark identifiers to another.")
+		// We can't expose the convert member function yet - need std::optional (or some trick with self/this and a lambda)
+		;
+
 	/**
 	 * Bindings for the eos::morphablemodel namespace:
 	 *  - PcaModel
@@ -206,8 +225,8 @@ PYBIND11_PLUGIN(eos) {
 		.def("get_data_dimension", &morphablemodel::PcaModel::get_data_dimension, "Returns the dimension of the data, i.e. the number of shape dimensions.")
 		.def("get_triangle_list", &morphablemodel::PcaModel::get_triangle_list, "Returns a list of triangles on how to assemble the vertices into a mesh.")
 		.def("get_mean", &morphablemodel::PcaModel::get_mean, "Returns the mean of the model.")
-		.def("get_mean_at_point", &morphablemodel::PcaModel::get_mean_at_point, "Return the value of the mean at a given vertex index.")
-		.def("draw_sample", (cv::Mat (morphablemodel::PcaModel::*)(std::vector<float>) const)&morphablemodel::PcaModel::draw_sample, "Returns a sample from the model with the given PCA coefficients. The given coefficients should follow a standard normal distribution, i.e. not be \"normalised\" with their eigenvalues/variances.")
+		.def("get_mean_at_point", &morphablemodel::PcaModel::get_mean_at_point, "Return the value of the mean at a given vertex index.", py::arg("vertex_index"))
+		.def("draw_sample", (cv::Mat (morphablemodel::PcaModel::*)(std::vector<float>) const)&morphablemodel::PcaModel::draw_sample, "Returns a sample from the model with the given PCA coefficients. The given coefficients should follow a standard normal distribution, i.e. not be scaled with their eigenvalues/variances.", py::arg("coefficients"))
 		;
 
 	py::class_<morphablemodel::MorphableModel>(morphablemodel_module, "MorphableModel", "A class representing a 3D Morphable Model, consisting of a shape- and colour (albedo) PCA model, as well as texture (uv) coordinates.")
@@ -228,11 +247,37 @@ PYBIND11_PLUGIN(eos) {
 
 	morphablemodel_module.def("load_blendshapes", &morphablemodel::load_blendshapes, "Load a file with blendshapes from a cereal::BinaryInputArchive (.bin) from the harddisk.");
 
+	/**
+	 *  - EdgeTopology
+	 *  - load_edge_topology()
+	 */
+	py::class_<morphablemodel::EdgeTopology>(morphablemodel_module, "EdgeTopology", "A struct containing a 3D shape model's edge topology.");
+
+	morphablemodel_module.def("load_edge_topology", &morphablemodel::load_edge_topology, "Load a 3DMM edge topology file from a json file.");
+
+	/**
+	 * Bindings for the eos::render namespace:
+	 * (Note: Defining Mesh before using it below in fitting::fit_shape_and_pose)
+	 *  - Mesh
+	 */
+	py::module render_module = eos_module.def_submodule("render", "3D mesh and texture extraction functionality.");
+
+	py::class_<render::Mesh>(render_module, "Mesh", "This class represents a 3D mesh consisting of vertices, vertex colour information and texture coordinates.")
+		.def_readwrite("vertices", &render::Mesh::vertices, "Vertices")
+		.def_readwrite("tvi", &render::Mesh::tvi, "Triangle vertex indices")
+		.def_readwrite("colors", &render::Mesh::colors, "Colour data")
+		.def_readwrite("tci", &render::Mesh::tci, "Triangle colour indices (usually the same as tvi)")
+		.def_readwrite("texcoords", &render::Mesh::texcoords, "Texture coordinates")
+		;
+
 	/**
 	 * Bindings for the eos::fitting namespace:
 	 *  - ScaledOrthoProjectionParameters
 	 *  - RenderingParameters
 	 *  - estimate_orthographic_projection_linear()
+	 *  - ContourLandmarks
+	 *  - ModelContour
+	 *  - fit_shape_and_pose()
 	 */
 	py::module fitting_module = eos_module.def_submodule("fitting", "Pose and shape fitting of a 3D Morphable Model.");
 
@@ -245,7 +290,7 @@ PYBIND11_PLUGIN(eos) {
 
 	py::class_<fitting::RenderingParameters>(fitting_module, "RenderingParameters", "Represents a set of estimated model parameters (rotation, translation) and camera parameters (viewing frustum).")
 		.def(py::init<fitting::ScaledOrthoProjectionParameters, int, int>(), "Create a RenderingParameters object from an instance of estimated ScaledOrthoProjectionParameters.")
-		.def("get_rotation", [](const fitting::RenderingParameters& p) { return glm::vec4(p.get_rotation().w, p.get_rotation().x, p.get_rotation().y, p.get_rotation().z); }, "Returns the rotation quaternion [w x y z].")
+		.def("get_rotation", [](const fitting::RenderingParameters& p) { return glm::vec4(p.get_rotation().x, p.get_rotation().y, p.get_rotation().z, p.get_rotation().w); }, "Returns the rotation quaternion [x y z w].")
 		.def("get_rotation_euler_angles", [](const fitting::RenderingParameters& p) { return glm::eulerAngles(p.get_rotation()); }, "Returns the rotation's Euler angles (in radians) as [pitch, yaw, roll].")
 		.def("get_modelview", &fitting::RenderingParameters::get_modelview, "Returns the 4x4 model-view matrix.")
 		.def("get_projection", &fitting::RenderingParameters::get_projection, "Returns the 4x4 projection matrix.")
@@ -262,5 +307,30 @@ PYBIND11_PLUGIN(eos) {
 		}, "This algorithm estimates the parameters of a scaled orthographic projection, given a set of corresponding 2D-3D points.", py::arg("image_points"), py::arg("model_points"), py::arg("is_viewport_upsidedown"), py::arg("viewport_height") = 0)
 		;
 
+	py::class_<fitting::ContourLandmarks>(fitting_module, "ContourLandmarks", "Defines which 2D landmarks comprise the right and left face contour.")
+		.def_static("load", &fitting::ContourLandmarks::load, "Helper method to load contour landmarks from a text file with landmark mappings, like ibug2did.txt.")
+		;
+
+	py::class_<fitting::ModelContour>(fitting_module, "ModelContour", "Definition of the vertex indices that define the right and left model contour.")
+		.def_static("load", &fitting::ModelContour::load, "Helper method to load a ModelContour from a json file from the hard drive.")
+		;
+	
+	fitting_module.def("fit_shape_and_pose", [](const morphablemodel::MorphableModel& morphable_model, const std::vector<morphablemodel::Blendshape>& blendshapes, const std::vector<glm::vec2>& landmarks, const std::vector<std::string>& landmark_ids, const core::LandmarkMapper& landmark_mapper, int image_width, int image_height, const morphablemodel::EdgeTopology& edge_topology, const fitting::ContourLandmarks& contour_landmarks, const fitting::ModelContour& model_contour, int num_iterations, int num_shape_coefficients_to_fit, float lambda) {
+			assert(landmarks.size() == landmark_ids.size());
+			std::vector<float> pca_coeffs;
+			std::vector<float> blendshape_coeffs;
+			std::vector<cv::Vec2f> fitted_image_points;
+			// We can change this to std::optional as soon as we switch to VS2017 and pybind supports std::optional
+			const boost::optional<int> num_shape_coefficients_opt = num_shape_coefficients_to_fit == -1 ? boost::none : boost::optional<int>(num_shape_coefficients_to_fit);
+			core::LandmarkCollection<cv::Vec2f> landmark_collection;
+			for (int i = 0; i < landmarks.size(); ++i)
+			{
+				landmark_collection.push_back(core::Landmark<cv::Vec2f>{ landmark_ids[i], cv::Vec2f(landmarks[i].x, landmarks[i].y) });
+			}
+			auto result = fitting::fit_shape_and_pose(morphable_model, blendshapes, landmark_collection, landmark_mapper, image_width, image_height, edge_topology, contour_landmarks, model_contour, num_iterations, num_shape_coefficients_opt, lambda, boost::none, pca_coeffs, blendshape_coeffs, fitted_image_points);
+			return std::make_tuple(result.first, result.second, pca_coeffs, blendshape_coeffs);
+		}, "Fit the pose (camera), shape model, and expression blendshapes to landmarks, in an iterative way. Returns a tuple (mesh, rendering_parameters, shape_coefficients, blendshape_coefficients).", py::arg("morphable_model"), py::arg("blendshapes"), py::arg("landmarks"), py::arg("landmark_ids"), py::arg("landmark_mapper"), py::arg("image_width"), py::arg("image_height"), py::arg("edge_topology"), py::arg("contour_landmarks"), py::arg("model_contour"), py::arg("num_iterations") = 5, py::arg("num_shape_coefficients_to_fit") = -1, py::arg("lambda") = 30.0f)
+		;
+
     return eos_module.ptr();
 };

python/pybind11_glm.hpp

@@ -44,7 +44,7 @@ struct type_caster<glm::tvec2<T, P>>
 	bool load(handle src, bool)
 	{
 		array_t<Scalar> buf(src, true);
-		if (!buf.check())
+		if (!buf)
 			return false;
 
 		if (buf.ndim() == 1) // a 1-dimensional vector
@@ -88,7 +88,7 @@ struct type_caster<glm::tvec3<T, P>>
 	bool load(handle src, bool)
 	{
 		array_t<Scalar> buf(src, true);
-		if (!buf.check())
+		if (!buf)
 			return false;
 
 		if (buf.ndim() == 1) // a 1-dimensional vector
@@ -132,7 +132,7 @@ struct type_caster<glm::tvec4<T, P>>
 	bool load(handle src, bool)
 	{
 		array_t<Scalar> buf(src, true);
-		if (!buf.check())
+		if (!buf)
 			return false;
 
 		if (buf.ndim() == 1) // a 1-dimensional vector
@@ -177,7 +177,7 @@ struct type_caster<glm::tmat3x3<T, P>>
 	bool load(handle src, bool)
 	{
 		array_t<Scalar> buf(src, true);
-		if (!buf.check())
+		if (!buf)
 			return false;
 
 		if (buf.ndim() == 2) // a 2-dimensional matrix
@@ -225,7 +225,7 @@ struct type_caster<glm::tmat4x3<T, P>>
 	bool load(handle src, bool)
 	{
 		array_t<Scalar> buf(src, true);
-		if (!buf.check())
+		if (!buf)
 			return false;
 
 		if (buf.ndim() == 2) // a 2-dimensional matrix
@@ -273,7 +273,7 @@ struct type_caster<glm::tmat4x4<T, P>>
 	bool load(handle src, bool)
 	{
 		array_t<Scalar> buf(src, true);
-		if (!buf.check())
+		if (!buf)
 			return false;
 
 		if (buf.ndim() == 2) // a 2-dimensional matrix