Add some experiments for using the BFM

scripts/PublicMM1/python/bfm_to_obj.py

+# todo-list:
+# 1. Create obj. file.
+# 2. Create isomap of 3D object.
+#   https://nl.mathworks.com/matlabcentral/fileexchange/5355-toolbox-graph/content/toolbox_graph/toolbox/isomap.m
+
+# keys BFM model:
+#   'segbin',
+#   '__version__',
+#   'texEV',
+#   'shapeMU',
+#   'texMU',
+#   '__header__',
+#   '__globals__',
+#   'shapeEV',
+#   'tl',
+#   'segMM',
+#   'shapePC',
+#   'texPC',
+#   'segMB'
+import scipy.io as io
+
+
+def load_bfm_model():
+    print('BFM: Loading model file...')
+    return io.loadmat('../01_MorphableModel.mat')
+
+
+def load_bfm_attributes():
+    print('BFM: Loading attributes file...')
+    return io.loadmat('../04_attributes.mat')
+
+
+def output_shape_and_texture(shape, texture, f):
+    """
+    Write shape in .obj format
+
+    Args:
+        shape(list): [[x_0, y_0, z_0], ..., [x_n, y_n, z_n]]
+        f(filepointer): pointer to .obj file
+    """
+    shape = shape.T[0]
+    texture = texture.T[0]
+
+    for i in range(0, len(shape), 3):
+        f.write('v {} {} {} {} {} {}\n'.format(
+            shape[i], shape[i + 1], shape[i + 2],
+            int(texture[i]), int(texture[i + 1]), int(texture[i + 2])
+            )
+        )
+
+
+def output_triangles(triangles, f):
+    """
+    Write shape in .obj format
+
+    Args:
+        shape(list): [[x_0, y_0, z_0], ..., [x_n, y_n, z_n]]
+        f(filepointer): pointer to .obj file
+    """
+
+    for i in range(0, len(triangles)):
+        f.write('f {} {} {}\n'.format(
+            triangles[i][2],
+            triangles[i][1],
+            triangles[i][0]
+        ))
+
+
+def save_to_obj(shape, texture, triangles, filename):
+    with open(filename, 'w') as f:
+        output_shape_and_texture(shape, texture, f)
+        output_triangles(triangles, f)
+
+
+def main():
+    dmm = load_bfm_model()
+
+    with open('output_model.obj', 'w') as f:
+        output_shape_and_texture(dmm['shapeMU'], dmm['texMU'], f)
+        output_triangles(dmm['tl'], f)
+
+
+if __name__ == '__main__':
+    main()

scripts/PublicMM1/python/random_face.py

+import numpy as np
+from bfm_to_obj import load_bfm_model, save_to_obj, load_bfm_attributes
+
+
+def gen_random_shape(mean, Vt, eigenvalues, alpha):
+    # generate influences eigenvalues, with a random sample
+    # of a normal distribution.
+    infl_eigenvalues = alpha * eigenvalues
+    random_face = mean + np.dot(Vt, infl_eigenvalues)
+
+    return random_face
+
+
+def gen_random_texture(mean, Vt, eigenvalues, betha):
+    # generate influences eigenvalues, with a random sample
+    # of a normal distribution.
+    infl_eigenvalues = betha * eigenvalues
+    random_face = mean + np.dot(Vt, infl_eigenvalues)
+
+    return random_face
+
+
+def gender_face_tex(attr, betha, n=5):
+    """
+    Influence gender.
+
+    Args:
+        attr(attributes model) - see BFM documentation
+        alpha(ndarray) - random range of floats generates with np.randn for
+            example with dim of the eigenvalues of the shape
+        n(integer) - amount of influence
+
+    Notes:
+        Higher n is more male.
+        Lower n is more female.
+        Minus **n** is allowed.
+    """
+    print 'tex', len(betha)
+    return betha + n * attr['gender_tex'][:len(betha)]
+
+
+def age_face_tex(attr, betha, n=50):
+    """
+    Influence age.
+
+    Args:
+        attr(attributes model) - see BFM documentation.
+        alpha(ndarray) - random range of floats generates with np.randn for
+            example with dim of the eigenvalues of the shape.
+        n(integer) - amount of influence.
+
+    Notes:
+        Higher n is older.
+        Lower n is younger.
+        Minus **n** is allowed.
+    """
+    return betha + n * attr['age_tex'][:199]
+
+
+def weight_face_tex(attr, betha, n=30):
+    """
+    Influence weight.
+
+    Args:
+        attr(attributes model) - see BFM documentation.
+        alpha(ndarray) - random range of floats generates with np.randn for
+            example with dim of the eigenvalues of the shape.
+        n(integer) - amount of influence
+
+    Notes:
+        Higher n is fatter.
+        Lower n is thinner.
+        Minus **n** is allowed.
+    """
+    return betha + n * attr['weight_tex'][:199]
+
+
+def gender_face_shape(attr, alpha, n=5):
+    """
+    Influence gender.
+
+    Args:
+        attr(attributes model) - see BFM documentation
+        alpha(ndarray) - random range of floats generates with np.randn for
+            example with dim of the eigenvalues of the shape
+        n(integer) - amount of influence
+
+    Notes:
+        Higher n is more male.
+        Lower n is more female.
+        Minus **n** is allowed.
+    """
+    return alpha + n * attr['gender_shape'][:199]
+
+
+def age_face_shape(attr, alpha, n=50):
+    """
+    Influence age.
+
+    Args:
+        attr(attributes model) - see BFM documentation.
+        alpha(ndarray) - random range of floats generates with np.randn for
+            example with dim of the eigenvalues of the shape.
+        n(integer) - amount of influence.
+
+    Notes:
+        Higher n is older.
+        Lower n is younger.
+        Minus **n** is allowed.
+    """
+    return alpha + n * attr['age_shape'][:199]
+
+
+def weight_face_shape(attr, alpha, n=30):
+    """
+    Influence weight.
+
+    Args:
+        attr(attributes model) - see BFM documentation.
+        alpha(ndarray) - random range of floats generates with np.randn for
+            example with dim of the eigenvalues of the shape.
+        n(integer) - amount of influence
+
+    Notes:
+        Higher n is fatter.
+        Lower n is thinner.
+        Minus **n** is allowed.
+    """
+    return alpha + n * attr['weight_shape'][:199]
+
+
+def gen_random_shape_coef(attr, dim):
+    alpha = np.random.randn(dim, 1)
+    alpha = age_face_shape(attr, alpha, 1)
+    alpha = gender_face_shape(attr, alpha, -5)
+    alpha = weight_face_shape(attr, alpha, -10)
+
+    return alpha
+
+
+def gen_random_tex_coef(attr, dim):
+    betha = np.random.randn(dim, 1)
+    betha = age_face_tex(attr, betha, 1)
+    betha = gender_face_tex(attr, betha, -5)
+    betha = weight_face_tex(attr, betha, -10)
+
+    return betha
+
+
+def main():
+    dmm = load_bfm_model()
+    attr = load_bfm_attributes()
+
+    Vt_shape = dmm['shapePC']
+    mean_shape = dmm['shapeMU']
+    eigenv_shape = dmm['shapeEV']
+
+    alpha = gen_random_shape_coef(attr, eigenv_shape.shape[0])
+
+    random_face_shape = gen_random_shape(
+        mean_shape, Vt_shape, eigenv_shape, alpha
+    )
+
+    Vt_tex = dmm['texPC']
+    mean_tex = dmm['texMU']
+    eigenv_tex = dmm['texEV']
+
+    betha = gen_random_tex_coef(attr, eigenv_tex.shape[0])
+
+    random_face_texture = gen_random_texture(
+        mean_tex, Vt_tex, eigenv_tex, betha
+    )
+
+    save_to_obj(
+        random_face_shape,
+        random_face_texture,
+        dmm['tl'], 'random_out_1.obj'
+    )
+
+
+if __name__ == '__main__':
+    main()

src/reconstruction/fit-model.cpp

-#include <pybind11/pybind11.h>
-
-namespace py = pybind11;
-
-int add(int i, int j) {
-    return i + j;
-
-}
-
-/*
- * eos - A 3D Morphable Model fitting library written in modern C++11/14.
- *
- * File: examples/fit-model.cpp
- *
- * Copyright 2015 Patrik Huber
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 #include "eos/core/Landmark.hpp"
 #include "eos/core/LandmarkMapper.hpp"
 #include "eos/fitting/orthographic_camera_estimation_linear.hpp"
@@ -43,7 +15,10 @@ int add(int i, int j) {
 #include <vector>
 #include <iostream>
 #include <fstream>
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
 
+namespace py = pybind11;
 using namespace eos;
 
 namespace po = boost::program_options;
@@ -60,6 +35,128 @@ using std::endl;
 using std::vector;
 using std::string;
 
+/**
+ * Print colored image in grayscale
+ */
+void print_image(uint32_t *image, py::buffer_info *image_info) {
+    size_t width = image_info->shape[1];
+    size_t height = image_info->shape[0];
+    size_t color = image_info->shape[2];
+
+    for (size_t y = 0; y < height; y++) {
+        for (size_t x = 0; x < width; x++) {
+            auto B = image[y * width + x + 0];
+            auto G = image[y * width + x + 1];
+            auto R = image[y * width + x + 2];
+
+            std::cout << (int)((B + G + R) / 3.0) << " ";
+        }
+
+        std::cout << std::endl;
+    }
+}
+
+float add(py::array_t<uint32_t> image_input,
+          py::array_t<float> input_points) {
+    auto image_info = image_input.request();
+    auto points_info = input_points.request();
+
+    uint32_t *image = reinterpret_cast<uint32_t *>(image_info.ptr);
+    float *points = reinterpret_cast<float *>(points_info.ptr);
+
+    //std::cout << image_info.shape << std::endl;
+    std::cout << image_info.itemsize << std::endl;
+    std::cout << image_info.ndim << std::endl;
+    std::cout << image_info.format << std::endl;
+
+    std::cout << image_info.shape[0] << std::endl;
+    std::cout << image_info.shape[1] << std::endl;
+
+    print_image(image, &image_info);
+
+    return 1.0;
+}
+
+
+void fit(Mat image, Vec2f landmarks) {
+    std::cout << image.at<uint32_t>(0,0) << std::endl;
+
+    //LandmarkCollection<cv::Vec2f> landmarks;
+}
+
+//    Mat image = cv::imread(imagefile.string());
+//    LandmarkCollection<cv::Vec2f> landmarks;
+//    try {
+//        landmarks = read_pts_landmarks(landmarksfile.string());
+//    }
+//    catch (const std::runtime_error& e) {
+//        cout << "Error reading the landmarks: " << e.what() << endl;
+//        return EXIT_FAILURE;
+//    }
+//    morphablemodel::MorphableModel morphable_model;
+//    try {
+//        morphable_model = morphablemodel::load_model(modelfile.string());
+//    }
+//    catch (const std::runtime_error& e) {
+//        cout << "Error loading the Morphable Model: " << e.what() << endl;
+//        return EXIT_FAILURE;
+//    }
+//    core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
+//
+//    // Draw the loaded landmarks:
+//    Mat outimg = image.clone();
+//    for (auto&& lm : landmarks) {
+//        cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 });
+//    }
+//
+//    // These will be the final 2D and 3D points used for the fitting:
+//    vector<Vec4f> model_points; // the points in the 3D shape model
+//    vector<int> vertex_indices; // their vertex indices
+//    vector<Vec2f> image_points; // the corresponding 2D landmark points
+//
+//    // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
+//    for (int i = 0; i < landmarks.size(); ++i) {
+//        auto converted_name = landmark_mapper.convert(landmarks[i].name);
+//        if (!converted_name) { // no mapping defined for the current landmark
+//            continue;
+//        }
+//        int vertex_idx = std::stoi(converted_name.get());
+//        Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
+//        model_points.emplace_back(vertex);
+//        vertex_indices.emplace_back(vertex_idx);
+//        image_points.emplace_back(landmarks[i].coordinates);
+//    }
+//
+//    // Estimate the camera (pose) from the 2D - 3D point correspondences
+//    fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
+//    fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
+//
+//    // The 3D head pose can be recovered as follows:
+//    float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
+//    // and similarly for pitch and roll.
+//
+//    // Estimate the shape coefficients by fitting the shape to the landmarks:
+//    Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
+//    vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
+//
+//    // Obtain the full mesh with the estimated coefficients:
+//    render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
+//
+//    // Extract the texture from the image using given mesh and camera parameters:
+//    Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
+//
+//    // Save the mesh as textured obj:
+//    outputfile += fs::path(".obj");
+//    render::write_textured_obj(mesh, outputfile.string());
+//
+//    // And save the isomap:
+//    outputfile.replace_extension(".isomap.png");
+//    cv::imwrite(outputfile.string(), isomap);
+//
+//    cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
+//
+//    return EXIT_SUCCESS;
+//}
 
 /**
  * This app demonstrates estimation of the camera and fitting of the shape

src/test/reconstruction_test.py

 import cv2
+import numpy as np
+import eos
 
 import pca as pca
 from settings import logger
@@ -49,7 +51,12 @@ def fit_model():
     input_points = dataset_module.factory(filename=image_filename)
     input_image = input_points.get_image()
 
-    print(fit.add(1, 3))
+    # scale points to output image shape. We MUST do this.
+    points = input_points.get_scaled_points(input_image.shape)
+    print(input_image.shape)
+    print dir(eos)
+
+    #fit.add(input_image, points)
 
 if __name__ == '__main__':
     fit_model()