Testing with eos reconstruction

.gitignore

@@ -2,6 +2,7 @@ data/*
 src/.cache/*
 src/*/.cache
 src/reconstruction/*.c
+scripts/PublicMM1/
 *.o
 *.so
 src/reconstruction/build/

Dockerfile

@@ -4,32 +4,51 @@ FROM smvanveen/computer-vision:20161109143812
 COPY requirements.txt /tmp
 RUN pip install -r /tmp/requirements.txt
 
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:bzindovic/suitesparse-bugfix-1319687 -y
+RUN apt-get update -y
+RUN apt-get install libsuitesparse-dev -y
+
 # extra packages:
 # graphviz: for cProfiling using pycallgraph.
 # libeigen3-dev: for eos: 3D morphable face model fitting library.
 RUN apt-get install -y \
     graphviz \
+    libeigen3-dev \
+    libgoogle-glog-dev \
+    libatlas-base-dev \
     libeigen3-dev
 
+
 WORKDIR /libs
 
 # install dlib
 RUN git clone https://github.com/davisking/dlib.git
 RUN (cd dlib; python setup.py install --yes USE_AVX_INSTRUCTIONS)
 
+RUN git clone https://ceres-solver.googlesource.com/ceres-solver
+RUN (cd ceres-solver; make -j3)
+RUN (cd ceres-solver; make install)
+
 # install eos (face-recosntruction, (3D Morphable Face Model fitting library)
-RUN git clone --recursive https://github.com/patrikhuber/eos.git
+RUN git clone --recursive \
+    https://github.com/patrikhuber/eos.git
 
 # remove dependency on opencv 2.4.3, opencv 3.0 works fine
 WORKDIR /libs/eos
-RUN sed -i 's/2.4.3//g' CMakeLists.txt
+RUN git checkout devel
+
+# needed for master branch
+#RUN sed -i 's/2.4.3//g' CMakeLists.txt
+
 RUN mkdir build
 WORKDIR /libs/eos/build
 RUN cmake ../ \
     -DCMAKE_INSTALL_PREFIX=/usr/local/eos \
-    -DGENERATE_PYTHON_BINDINGS=on \
-    -DBUILD_UTILS=on \
-    -DPYTHON_EXECUTABLE=/usr/bin/python
+    -DEOS_GENERATE_PYTHON_BINDINGS=on \
+    -DEOS_BUILD_CERES_EXAMPLE=on \
+    -DEOS_BUILD_UTILS=on \
+    -DEOS_BUILD_EXAMPLES=on
 
 RUN make && make install
 
@@ -40,4 +59,6 @@ RUN git clone https://github.com/pybind/pybind11.git
 RUN (cd pybind11; mkdir build; cd build; cmake -DPYBIND11_PYTHON_VERSION=2.7 ..);
 RUN (cd pybind11/build; make -j4 && make install);
 
+#TODO, remove the tmp libs folder in production?
+
 WORKDIR /src

actions.mk

 .PHONY := train_model show_pca test_model show_reconstruction
 DEBUG_LEVEL=*
 
-data/imm_face_db: data/imm_face_db.tar.gz
+data/imm_face_db: # data/imm_face_db.tar.gz
 	(cd data; mkdir -p imm_face_db; \
 		tar -xvzf imm_face_db.tar.gz -C imm_face_db \
 	)

makefile

@@ -13,7 +13,7 @@ BASE_DOCKER_CMD:= docker run $(DOCKER_RUN_FLAGS) $(IMAGE_TAG)
 $(info $(TARGETS))
 
 DEPENDENCIES:= data/imm_face_db
-TARGETS:= data/shape_predictor_68_face_landmarks.dat\
+TARGETS:= data/shape_predictor_68_face_landmarks.dat \
 	src/reconstruction/texture.so \
 	src/reconstruction/fit.so \
 	data/pca_ibug_shape_model.npy \

scripts/PublicMM1/python/bfm_to_obj.py

@@ -44,7 +44,7 @@ def output_shape_and_texture(shape, texture, f):
     for i in range(0, len(shape), 3):
         f.write('v {} {} {} {} {} {}\n'.format(
             shape[i], shape[i + 1], shape[i + 2],
-            int(texture[i]), int(texture[i + 1]), int(texture[i + 2])
+            texture[i], texture[i + 1], texture[i + 2]
             )
         )
 

scripts/PublicMM1/python/random_face.py

@@ -131,8 +131,8 @@ def weight_face_shape(attr, alpha, n=30):
 
 def gen_random_shape_coef(attr, dim):
     alpha = np.random.randn(dim, 1)
-    alpha = age_face_shape(attr, alpha, 1)
-    alpha = gender_face_shape(attr, alpha, -5)
+    alpha = age_face_shape(attr, alpha, 10)
+    alpha = gender_face_shape(attr, alpha, 5)
     alpha = weight_face_shape(attr, alpha, -10)
 
     return alpha
@@ -140,8 +140,8 @@ def gen_random_shape_coef(attr, dim):
 
 def gen_random_tex_coef(attr, dim):
     betha = np.random.randn(dim, 1)
-    betha = age_face_tex(attr, betha, 1)
-    betha = gender_face_tex(attr, betha, -5)
+    betha = age_face_tex(attr, betha, 10)
+    betha = gender_face_tex(attr, betha, 5)
     betha = weight_face_tex(attr, betha, -10)
 
     return betha

src/datasets/ibug.py

@@ -12,6 +12,7 @@ import numpy as np
 import aam
 import landmarks
 from settings import logger
+from .ibug_to_bfm import ibug_mapping
 
 # load detector (this loads the datafile from disk, so needs to be done once).
 detector = landmarks.Detector()

src/datasets/ibug_to_bfm.py

+# Filename: MPEG4_FDP_face05.fp
+# Format: (vertex_nr) (x y z) (x y) (name)
+
+# mapping from ibug idx to bfm vertex
+ibug_mapping = {
+    # nose
+    30: 8309,  # -72.8548 18986.1 121850 0 0 9.12
+    31: 8319,  # 13.2323 4430.29 131560 0 0 9.3
+    # 6611 # -12555.7 29870.6 96949.7 0 0 9.6 --
+    # 6244 # -15010.6 13787.1 106131 0 0 9.14 --
+    33: 5879,  # -15364.7 -7915.79 105577 0 0 9.4
+    32: 7032,  # -13710 -543.09 119384 0 0 9.2
+    34: 8334,  # -174.404 -12553.8 115099 0 0 9.15
+    # eye left
+    # 4410 # -31025.3 33492.2 94793.8 0 0 3.6 -- center eye
+    39: 4404,  # -31993.5 37990.5 94498 0 0 3.2 -- eye, not perfect
+    41: 4416,  # -30242.7 29444 93887.9 0 0 3.4
+    # 4402 -32257.8 38510.2 95919.1 0 0 3.14 --
+    # 4418 -30262.3 28855.5 94215.7 0 0 3.10 --
+    37: 2088,  # -43304.9 33261.1 85744.4 0 0 3.12
+    40: 5959,  # -18690.6 32105.7 91502.5 0 0 3.8
+    52: 8344,  # -192.025 -24960.5 116208 0 0 8.1
+    51: 7570,  # -5897.69 -23762.2 115642 0 0 8.9
+    50: 5768,  # -18543.4 -28311.2 107964 0 0 8.6
+    49: 5006,  # -29800 -34598.4 97775.3 0 0 8.4
+    58: 8374,  # -154.684 -40559.2 112944 0 0 8.2
+    60: 6697,  # -17838.4 -37488.3 106002 0 0 8.8
+    1: 21628,  # -77447 30661.5 12096.9 0 0 10.10
+    # 19963 -88262.2 36394.8 -4947.64 0 0 10.2 -- ears
+    # 19330 -85528.2 2072.17 -10931.6 0 0 10.4 -- ears
+    # 20203 -73194.6 -20281.3 11874.6 0 0 10.6 -- ears
+    # 20547 -67842.2 -17904 16423.6 0 0 10.8 -- ears
+    22: 40514,  # -14438.3 49609.7 107232 0 0 4.2 -- eyebrow
+    20: 40087,  # -37475.1 56446.7 100681 0 0 4.4 -- eyebrow
+    37: 38792,  # -57986.8 45943.2 78728.1 0 0 4.6
+    # 9965# 12130.9 29795.3 96702.2 0 0 9.7 -- nose
+    # 10372# 14681.1 13742.5 105748 0 0 9.13 -- nose
+    35: 10781,  # 14916.3 -7662.06 105539 0 0 9.5 -- nose
+    36: 9612,  # 13568 -523.124 119279 0 0 9.1 -- nose
+    # 12150# 30157.7 33516.8 94540 0 0 3.5 -- center eye
+    44: 12144,  # 31037.6 38025.8 94139.1 0 0 3.1 -- eye, not perfect
+    48: 12156,  # 29458.6 29444.7 93770.8 0 0 3.3 -- eye, not perfect
+    # 12142 31349.6 38584.3 95585.3 0 0 3.13 -- eye
+    # 12158 29537.5 28834.2 94046.5 0 0 3.9 -- eye
+    43: 10603,  # 17667.8 32012.7 91252.6 0 0 3.11 -- eye
+    55: 11714,  # 29517.6 -34275.5 97471.5 0 0 8.3
+    17: 33496,  # 76722.4 30669.8 12318.6 0 0 10.9
+    # 35151 87810.5 36294.1 -4628.25 0 0 10.1 -- ear
+    # 35807 85139.7 2150.69 -10665 0 0 10.3
+    # 34981 72909.8 -20461.1 12009.1 0 0 10.5
+    # 34656 67686.9 -18200 16564.5 0 0 10.7
+    63: 8354,  # -275.894 -31384.5 110683 0 0 2.2 -- mouth
+    67: 8366,  # -257.674 -31426.9 110736 0 0 2.3 -- mouth
+    # 40777 -802.905 96304.2 99265.5 0 0 11.1 -- upside
+    46: 14472,  # 42745.8 33135.4 85724.1 0 0 3.7 -- eye left
+    53: 9118,   # 5483.92 -23769.2 115655 0 0 8.10 -- mouth
+    54: 10928,  # 18155.1 -28245 107864 0 0 8.5 -- mouth
+    56: 10051,  # 17625.3 -37478 105917 0 0 8.7 -- not perfect
+    23: 41091,  # 13583.4 49767.2 107218 0 0 4.1
+    25: 41511,  # 36713.9 56793.8 100463 0 0 4.3
+    46: 42825,  # 57424.9 46059.1 78590.1 0 0 4.5
+    9: 48187  # -48.2388 -76505.2 99442.1 0 0 gn
+    # 19963 -88262.2 36394.8 -4947.64 0 0 sa --
+    # 20205 -71257.4 -20598.4 13258.3 0 0 sba --
+    # 21629 -77516 30127.9 12058.9 0 0 pra --
+    # 19325 -88087.9 14538.9 -14157.6 0 0 pa --
+    # 20983 -79905 34425.4 8525.19 0 0 obs --
+    # 20786 -66866.5 -16819 17784.3 0 0 obi
+    # 21776 -72643.3 11413.3 11140.1 0 0 t
+    # 26140 -65298.1 23275.2 64256.6 0 0 zy
+    # 43070 -59848.7 -42669.3 41668.6 0 0 go
+    # 8374 -154.684 -40559.2 112944 0 0 li
+    # 8381 -140.066 -49939.9 106747 0 0 sl
+    # 48187 -48.2388 -76505.2 99442.1 0 0 gn
+    # 23188 -71385.1 15971.4 28332 0 0 cdl
+    # 8261 -369.986 47259.7 108662 0 0 g
+    # 26236 -60480.3 43221.1 72106.9 0 0 ft
+    # 7441 -6628.37 -23768.2 115463 0 0 cph
+    # 8344 -192.025 -24960.5 116208 0 0 ls
+    # 8366 -257.674 -31426.9 110736 0 0 sto
+    # 5392 -26244.2 -33187.9 98582 0 0 ch
+    # 8275 -190.815 39984 108192 0 0 n
+    # 8286 -144.161 35988.3 109453 0 0 se
+    # 8311 -47.9918 16757.4 123572 0 0 se
+    # 6389 -16981.4 -3046.09 111901 0 0 al
+    # 8320 20.1202 2232.26 131944 0 0 prn
+    # 8332 -175.672 -10527.6 116009 0 0 sn
+    # 6912 -9848.33 -8505.34 111144 0 0 sbal
+    # 5488 -17814.8 -3696.18 103442 0 0 ac
+    # 7809 -6093.17 -3557.61 124863 0 0 c'
+    # 7032 -13710 -543.09 119384 0 0 al'
+    # 7165 -10444.4 -2520.99 117527 0 0 al'
+    # 7814 -4855.38 -5635.67 114857 0 0 sn'
+    # 5959 -18690.6 32105.7 91502.5 0 0 en
+    # 2088 -43304.9 33261.1 85744.4 0 0 ex
+    # 4280 -32240.9 34083.1 94614 0 0 p
+    # 3393 -38037.4 23596.3 91323.3 0 0 or
+    # 4158 -32467.7 29315.4 93418 0 0 pi
+    # 40087 -37475.1 56446.7 100681 0 0 sci
+    # 4646 -37373.6 48977.9 99911.2 0 0 os
+    # 6796 -10468.9 -24402.7 113952 0 0 ls'
+    # 4404 -31993.5 37990.5 94498 0 0 ps
+    # 48180 -41.2938 -65127.6 108493 0 0 pg
+    # 35151 87810.5 36294.1 -4628.25 0 0 sa*
+    # 34983 71040 -20787.8 13418.5 0 0 sba*
+    # 33497 76793.9 30128.9 12280.4 0 0 pra*
+    # 35802 87874.7 14663.6 -13676.7 0 0 pa*
+    # 34132 79275.1 34409.8 8855 0 0 obs*
+    # 34425 66654.4 -17219.2 17867.2 0 0 obi*
+    # 33386 71894.4 11245.4 11118.2 0 0 t*
+    # 28978 64756.8 23136.4 64176 0 0 zy*
+    # 53307 59692.2 -42997 41678.6 0 0 go*
+    # 31960 70614.6 15771.9 28283.9 0 0 cdl*
+    # 28816 59843 43216.9 71989.8 0 0 ft*
+    # 9247 6180.74 -23794.4 115481 0 0 cph*
+    # 11326 25887.4 -32810.2 98262 0 0 ch*
+    # 10259 16732.3 -2973.56 111813 0 0 al*
+    # 9750 9386.51 -8462.76 111119 0 0 sbal*
+    # 11164 17421.1 -3511.99 103224 0 0 ac*
+    # 8841 5890.83 -3559.51 124776 0 0 c'*
+    # 9612 13568 -523.124 119279 0 0 al'*
+    # 9487 10105.6 -2556.09 117492 0 0 al'*
+    # 8846 4183.15 -5680.2 114730 0 0 sn'*
+    # 10603 17667.8 32012.7 91252.6 0 0 en*
+    # 14472 42745.8 33135.4 85724.1 0 0 ex*
+    # 12278 31373.5 34100.5 94353.7 0 0 p*
+    # 13197 37566.8 23529.1 91177.5 0 0 or*
+    # 12414 31723.8 29338.3 93324.8 0 0 pi*
+    # 41511 36713.9 56793.8 100463 0 0 sci*
+    # 11870 36697.5 49438 99579.8 0 0 os*
+    # 9892 9980 -24508.4 113969 0 0 ls'*
+    # 12144 31037.6 38025.8 94139.1 0 0 ps*
+}

src/fitting/camera.py

+import cv2
+
+def calc_affine_projection_matrix(landmarks, vertices, image):
+    """
+    returns the 3x4 camera matrix based on the given locations of the landmarks
+    and matching 3d vertices.
+    """
+    cv2.initCameraMatrix2D(vertices, landmarks, image.shape)

src/reconstruction/fit-model.cpp

@@ -5,6 +5,7 @@
 #include "eos/fitting/linear_shape_fitting.hpp"
 #include "eos/render/utils.hpp"
 #include "eos/render/texture_extraction.hpp"
+//#include "eos/pybind11_glm.hpp"
 
 #include "opencv2/core/core.hpp"
 #include "opencv2/highgui/highgui.hpp"
@@ -19,8 +20,7 @@
 #include <pybind11/numpy.h>
 
 namespace py = pybind11;
-using namespace eos;
-
+using namespace eos; 
 namespace po = boost::program_options;
 namespace fs = boost::filesystem;
 
@@ -79,175 +79,122 @@ float add(py::array_t<uint32_t> image_input,
 
 
 void fit(Mat image, Vec2f landmarks) {
-    std::cout << image.at<uint32_t>(0,0) << std::endl;
+    // std::cout << image.at<uint32_t>(0,0) << std::endl;
+    std::cout << landmarks << std::endl;
+    return;
+    LandmarkCollection<cv::Vec2f> landmarkCollection;
+    morphablemodel::MorphableModel morphable_model;
+
+    try {
+        morphable_model = morphablemodel::load_model("/usr/local/eos/share/sfm_shape_3448.bin");
+    }
+    catch (const std::runtime_error& e) {
+        cout << "Error loading the Morphable Model: " << e.what() << endl;
+        return;
+    }
 
-    //LandmarkCollection<cv::Vec2f> landmarks;
-}
+    core::LandmarkMapper landmark_mapper = core::LandmarkMapper("/usr/local/eos/share/bug2did.txt");
+
+    // Draw the loaded landmarks:
+    Mat outimg = image.clone();
+    for (auto&& lm : landmarkCollection) {
+        cv::rectangle(
+            outimg,
+            cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
+            cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f),
+            { 255, 0, 0 }
+        );
+    }
 
-//    Mat image = cv::imread(imagefile.string());
-//    LandmarkCollection<cv::Vec2f> landmarks;
-//    try {
-//        landmarks = read_pts_landmarks(landmarksfile.string());
-//    }
-//    catch (const std::runtime_error& e) {
-//        cout << "Error reading the landmarks: " << e.what() << endl;
-//        return EXIT_FAILURE;
-//    }
-//    morphablemodel::MorphableModel morphable_model;
-//    try {
-//        morphable_model = morphablemodel::load_model(modelfile.string());
-//    }
-//    catch (const std::runtime_error& e) {
-//        cout << "Error loading the Morphable Model: " << e.what() << endl;
-//        return EXIT_FAILURE;
-//    }
-//    core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
-//
-//    // Draw the loaded landmarks:
-//    Mat outimg = image.clone();
-//    for (auto&& lm : landmarks) {
-//        cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 });
-//    }
-//
-//    // These will be the final 2D and 3D points used for the fitting:
-//    vector<Vec4f> model_points; // the points in the 3D shape model
-//    vector<int> vertex_indices; // their vertex indices
-//    vector<Vec2f> image_points; // the corresponding 2D landmark points
-//
-//    // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
-//    for (int i = 0; i < landmarks.size(); ++i) {
-//        auto converted_name = landmark_mapper.convert(landmarks[i].name);
-//        if (!converted_name) { // no mapping defined for the current landmark
-//            continue;
-//        }
-//        int vertex_idx = std::stoi(converted_name.get());
-//        Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
-//        model_points.emplace_back(vertex);
-//        vertex_indices.emplace_back(vertex_idx);
-//        image_points.emplace_back(landmarks[i].coordinates);
-//    }
-//
-//    // Estimate the camera (pose) from the 2D - 3D point correspondences
-//    fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
-//    fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
-//
-//    // The 3D head pose can be recovered as follows:
-//    float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
-//    // and similarly for pitch and roll.
-//
-//    // Estimate the shape coefficients by fitting the shape to the landmarks:
-//    Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
-//    vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
-//
-//    // Obtain the full mesh with the estimated coefficients:
-//    render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
-//
-//    // Extract the texture from the image using given mesh and camera parameters:
-//    Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
-//
-//    // Save the mesh as textured obj:
-//    outputfile += fs::path(".obj");
-//    render::write_textured_obj(mesh, outputfile.string());
-//
-//    // And save the isomap:
-//    outputfile.replace_extension(".isomap.png");
-//    cv::imwrite(outputfile.string(), isomap);
-//
-//    cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
-//
-//    return EXIT_SUCCESS;
-//}
+    // These will be the final 2D and 3D points used for the fitting:
+    vector<Vec4f> model_points; // the points in the 3D shape model
+    vector<int> vertex_indices; // their vertex indices
+    vector<Vec2f> image_points; // the corresponding 2D landmark points
+
+    // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
+    for (int i = 0; i < landmarkCollection.size(); ++i) {
+        auto converted_name = landmark_mapper.convert(landmarkCollection[i].name);
+        if (!converted_name) { // no mapping defined for the current landmark
+            continue;
+        }
+        int vertex_idx = std::stoi(converted_name.get());
+        Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
+        model_points.emplace_back(vertex);
+        vertex_indices.emplace_back(vertex_idx);
+        image_points.emplace_back(landmarkCollection[i].coordinates);
+    }
+
+    // Estimate the camera (pose) from the 2D - 3D point correspondences
+    fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
+    fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
+
+    // The 3D head pose can be recovered as follows:
+    float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+    // and similarly for pitch and roll.
+
+    // Estimate the shape coefficients by fitting the shape to the landmarks:
+    Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
+    vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
+
+    // Obtain the full mesh with the estimated coefficients:
+    render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
+
+    // Extract the texture from the image using given mesh and camera parameters:
+    Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
+
+    // Save the mesh as textured obj:
+    render::write_textured_obj(mesh, "/data/fit-model-test.obj");
+
+    // And save the isomap:
+    cv::imwrite("/data/fit-model-test.isomap.png", isomap);
+}
 
-/**
- * This app demonstrates estimation of the camera and fitting of the shape
- * model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
- *
- * First, the 68 ibug landmarks are loaded from the .pts file and converted
- * to vertex indices using the LandmarkMapper. Then, an affine camera matrix
- * is estimated, and then, using this camera matrix, the shape is fitted
- * to the landmarks.
- */
-//int fit(int argc, char *argv[])
-//{
-//    // Load the image, landmarks, LandmarkMapper and the Morphable Model:
-//    Mat image = cv::imread(imagefile.string());
-//    LandmarkCollection<cv::Vec2f> landmarks;
-//    try {
-//        landmarks = read_pts_landmarks(landmarksfile.string());
-//    }
-//    catch (const std::runtime_error& e) {
-//        cout << "Error reading the landmarks: " << e.what() << endl;
-//        return EXIT_FAILURE;
-//    }
-//    morphablemodel::MorphableModel morphable_model;
-//    try {
-//        morphable_model = morphablemodel::load_model(modelfile.string());
-//    }
-//    catch (const std::runtime_error& e) {
-//        cout << "Error loading the Morphable Model: " << e.what() << endl;
-//        return EXIT_FAILURE;
-//    }
-//    core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
-//
-//    // Draw the loaded landmarks:
-//    Mat outimg = image.clone();
-//    for (auto&& lm : landmarks) {
-//        cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 });
-//    }
-//
-//    // These will be the final 2D and 3D points used for the fitting:
-//    vector<Vec4f> model_points; // the points in the 3D shape model
-//    vector<int> vertex_indices; // their vertex indices
-//    vector<Vec2f> image_points; // the corresponding 2D landmark points
-//
-//    // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
-//    for (int i = 0; i < landmarks.size(); ++i) {
-//        auto converted_name = landmark_mapper.convert(landmarks[i].name);
-//        if (!converted_name) { // no mapping defined for the current landmark
-//            continue;
-//        }
-//        int vertex_idx = std::stoi(converted_name.get());
-//        Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
-//        model_points.emplace_back(vertex);
-//        vertex_indices.emplace_back(vertex_idx);
-//        image_points.emplace_back(landmarks[i].coordinates);
-//    }
-//
-//    // Estimate the camera (pose) from the 2D - 3D point correspondences
-//    fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
-//    fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
-//
-//    // The 3D head pose can be recovered as follows:
-//    float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
-//    // and similarly for pitch and roll.
-//
-//    // Estimate the shape coefficients by fitting the shape to the landmarks:
-//    Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
-//    vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
-//
-//    // Obtain the full mesh with the estimated coefficients:
-//    render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
-//
-//    // Extract the texture from the image using given mesh and camera parameters:
-//    Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
-//
-//    // Save the mesh as textured obj:
-//    outputfile += fs::path(".obj");
-//    render::write_textured_obj(mesh, outputfile.string());
-//
-//    // And save the isomap:
-//    outputfile.replace_extension(".isomap.png");
-//    cv::imwrite(outputfile.string(), isomap);
-//
-//    cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
-//
-//    return EXIT_SUCCESS;
-//}
 
 PYBIND11_PLUGIN(fit) {
     py::module m("fit", "fit example");
 
+    /**
+    * General bindings, for OpenCV vector types and cv::Mat:
+    *  - cv::Vec2f
+    *  - cv::Vec4f
+    *  - cv::Mat (only 1-channel matrices and only conversion of CV_32F C++ matrices to Python, and conversion of CV_32FC1 and CV_64FC1 matrices from Python to C++)
+    */
+    //py::class_<cv::Vec2f>(fit, "Vec2f", "Wrapper for OpenCV's cv::Vec2f type.")
+    //.def("__init__", [](cv::Vec2f& vec, py::buffer b) {
+    //    py::buffer_info info = b.request();
+
+    //    if (info.ndim != 1)
+    //        throw std::runtime_error("Buffer ndim is " + std::to_string(info.ndim) + ", please hand a buffer with dimension == 1 to create a Vec2f.");
+    //    if (info.strides.size() != 1)
+    //        throw std::runtime_error("strides.size() is " + std::to_string(info.strides.size()) + ", please hand a buffer with strides.size() == 1 to create a Vec2f.");
+    //    // Todo: Should add a check that checks for default stride sizes, everything else would not work yet I think.
+    //    if (info.shape.size() != 1)
+    //        throw std::runtime_error("shape.size() is " + std::to_string(info.shape.size()) + ", please hand a buffer with shape dimension == 1 to create a Vec2f.");
+    //    if (info.shape[0] != 2)
+    //        throw std::runtime_error("shape[0] is " + std::to_string(info.shape[0]) + ", please hand a buffer with 2 entries to create a Vec2f.");
+
+    //    if (info.format == py::format_descriptor<float>::format()) {
+    //        cv::Mat temp(1, 2, CV_32FC1, info.ptr);
+    //        // std::cout << temp << std::endl;
+    //        new (&vec) cv::Vec2f(temp);
+    //    } else {
+    //        throw std::runtime_error("Not given a buffer of type float - please hand a buffer of type float to create a Vec2f.");
+    //    }
+    //})
+    //.def_buffer([](cv::Vec2f& vec) -> py::buffer_info {
+    //return py::buffer_info(
+    //    &vec.val,                               /* Pointer to buffer */
+    //    sizeof(float),                          /* Size of one scalar */
+    //    py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
+    //    2,                                      /* Number of dimensions */
+    //    { vec.rows, vec.cols },                 /* Buffer dimensions */
+    //    { sizeof(float),             /* Strides (in bytes) for each index */
+    //    sizeof(float) }             //=> both sizeof(float), since the data is hold in an array, i.e. contiguous memory */
+    //);
+    //});
+
     m.def("add", &add, "A function which adds two numbers");
+    m.def("fit", &fit, "Fit");
 
     return m.ptr();
 }

src/test/reconstruction_test.py

@@ -50,13 +50,45 @@ def fit_model():
 
     input_points = dataset_module.factory(filename=image_filename)
     input_image = input_points.get_image()
+    input_points.get_points()
+
+    print dir(eos)
 
     # scale points to output image shape. We MUST do this.
     points = input_points.get_scaled_points(input_image.shape)
-    print(input_image.shape)
-    print dir(eos)
+    fit.fit(input_image, points)
+
+    # fit.add(input_image, points)
 
-    #fit.add(input_image, points)
 
 if __name__ == '__main__':
     fit_model()
+
+
+## Try seo python bindings
+#    model = eos.morphablemodel.load_model(
+#        '/usr/local/eos/share/sfm_shape_3448.bin')
+#    blend_shapes = eos.morphablemodel.load_blendshapes(
+#        '/usr/local/eos/share/expression_blendshapes_3448.bin'
+#    )
+#
+#    s = model.get_shape_model().draw_sample([1.0, -0.5, 0.7, 0.1])
+#
+#    sample = np.array(s)
+#    tri = model.get_shape_model().get_triangle_list()
+#    mean = model.get_shape_model().get_mean()
+#    dims = model.get_shape_model().get_data_dimension()
+#
+#    mean = np.array(mean)
+
+#    with open('/data/test.obj', 'w') as f:
+#        for i in range(0, len(sample), 3):
+#            f.write('v {} {} {}\n'.format(sample[i], sample[i + 1], sample[i + 2])
+#        )
+#
+#        for i in range(0, len(tri)):
+#            f.write('f {} {} {}\n'.format(
+#                tri[i][0], tri[i][1], tri[i][2],
+#            )
+#        )
+