Commit 426b9854 authored by Richard Torenvliet's avatar Richard Torenvliet

Testing with eos reconstruction

parent 482b13b3
...@@ -2,6 +2,7 @@ data/* ...@@ -2,6 +2,7 @@ data/*
src/.cache/* src/.cache/*
src/*/.cache src/*/.cache
src/reconstruction/*.c src/reconstruction/*.c
scripts/PublicMM1/
*.o *.o
*.so *.so
src/reconstruction/build/ src/reconstruction/build/
......
...@@ -4,32 +4,51 @@ FROM smvanveen/computer-vision:20161109143812 ...@@ -4,32 +4,51 @@ FROM smvanveen/computer-vision:20161109143812
COPY requirements.txt /tmp COPY requirements.txt /tmp
RUN pip install -r /tmp/requirements.txt RUN pip install -r /tmp/requirements.txt
RUN apt-get install software-properties-common -y
RUN add-apt-repository ppa:bzindovic/suitesparse-bugfix-1319687 -y
RUN apt-get update -y
RUN apt-get install libsuitesparse-dev -y
# extra packages: # extra packages:
# graphviz: for cProfiling using pycallgraph. # graphviz: for cProfiling using pycallgraph.
# libeigen3-dev: for eos: 3D morphable face model fitting library. # libeigen3-dev: for eos: 3D morphable face model fitting library.
RUN apt-get install -y \ RUN apt-get install -y \
graphviz \ graphviz \
libeigen3-dev \
libgoogle-glog-dev \
libatlas-base-dev \
libeigen3-dev libeigen3-dev
WORKDIR /libs WORKDIR /libs
# install dlib # install dlib
RUN git clone https://github.com/davisking/dlib.git RUN git clone https://github.com/davisking/dlib.git
RUN (cd dlib; python setup.py install --yes USE_AVX_INSTRUCTIONS) RUN (cd dlib; python setup.py install --yes USE_AVX_INSTRUCTIONS)
RUN git clone https://ceres-solver.googlesource.com/ceres-solver
RUN (cd ceres-solver; make -j3)
RUN (cd ceres-solver; make install)
# install eos (face-recosntruction, (3D Morphable Face Model fitting library) # install eos (face-recosntruction, (3D Morphable Face Model fitting library)
RUN git clone --recursive https://github.com/patrikhuber/eos.git RUN git clone --recursive \
https://github.com/patrikhuber/eos.git
# remove dependency on opencv 2.4.3, opencv 3.0 works fine # remove dependency on opencv 2.4.3, opencv 3.0 works fine
WORKDIR /libs/eos WORKDIR /libs/eos
RUN sed -i 's/2.4.3//g' CMakeLists.txt RUN git checkout devel
# needed for master branch
#RUN sed -i 's/2.4.3//g' CMakeLists.txt
RUN mkdir build RUN mkdir build
WORKDIR /libs/eos/build WORKDIR /libs/eos/build
RUN cmake ../ \ RUN cmake ../ \
-DCMAKE_INSTALL_PREFIX=/usr/local/eos \ -DCMAKE_INSTALL_PREFIX=/usr/local/eos \
-DGENERATE_PYTHON_BINDINGS=on \ -DEOS_GENERATE_PYTHON_BINDINGS=on \
-DBUILD_UTILS=on \ -DEOS_BUILD_CERES_EXAMPLE=on \
-DPYTHON_EXECUTABLE=/usr/bin/python -DEOS_BUILD_UTILS=on \
-DEOS_BUILD_EXAMPLES=on
RUN make && make install RUN make && make install
...@@ -40,4 +59,6 @@ RUN git clone https://github.com/pybind/pybind11.git ...@@ -40,4 +59,6 @@ RUN git clone https://github.com/pybind/pybind11.git
RUN (cd pybind11; mkdir build; cd build; cmake -DPYBIND11_PYTHON_VERSION=2.7 ..); RUN (cd pybind11; mkdir build; cd build; cmake -DPYBIND11_PYTHON_VERSION=2.7 ..);
RUN (cd pybind11/build; make -j4 && make install); RUN (cd pybind11/build; make -j4 && make install);
#TODO, remove the tmp libs folder in production?
WORKDIR /src WORKDIR /src
.PHONY := train_model show_pca test_model show_reconstruction .PHONY := train_model show_pca test_model show_reconstruction
DEBUG_LEVEL=* DEBUG_LEVEL=*
data/imm_face_db: data/imm_face_db.tar.gz data/imm_face_db: # data/imm_face_db.tar.gz
(cd data; mkdir -p imm_face_db; \ (cd data; mkdir -p imm_face_db; \
tar -xvzf imm_face_db.tar.gz -C imm_face_db \ tar -xvzf imm_face_db.tar.gz -C imm_face_db \
) )
......
...@@ -13,7 +13,7 @@ BASE_DOCKER_CMD:= docker run $(DOCKER_RUN_FLAGS) $(IMAGE_TAG) ...@@ -13,7 +13,7 @@ BASE_DOCKER_CMD:= docker run $(DOCKER_RUN_FLAGS) $(IMAGE_TAG)
$(info $(TARGETS)) $(info $(TARGETS))
DEPENDENCIES:= data/imm_face_db DEPENDENCIES:= data/imm_face_db
TARGETS:= data/shape_predictor_68_face_landmarks.dat\ TARGETS:= data/shape_predictor_68_face_landmarks.dat \
src/reconstruction/texture.so \ src/reconstruction/texture.so \
src/reconstruction/fit.so \ src/reconstruction/fit.so \
data/pca_ibug_shape_model.npy \ data/pca_ibug_shape_model.npy \
......
...@@ -44,7 +44,7 @@ def output_shape_and_texture(shape, texture, f): ...@@ -44,7 +44,7 @@ def output_shape_and_texture(shape, texture, f):
for i in range(0, len(shape), 3): for i in range(0, len(shape), 3):
f.write('v {} {} {} {} {} {}\n'.format( f.write('v {} {} {} {} {} {}\n'.format(
shape[i], shape[i + 1], shape[i + 2], shape[i], shape[i + 1], shape[i + 2],
int(texture[i]), int(texture[i + 1]), int(texture[i + 2]) texture[i], texture[i + 1], texture[i + 2]
) )
) )
......
...@@ -131,8 +131,8 @@ def weight_face_shape(attr, alpha, n=30): ...@@ -131,8 +131,8 @@ def weight_face_shape(attr, alpha, n=30):
def gen_random_shape_coef(attr, dim): def gen_random_shape_coef(attr, dim):
alpha = np.random.randn(dim, 1) alpha = np.random.randn(dim, 1)
alpha = age_face_shape(attr, alpha, 1) alpha = age_face_shape(attr, alpha, 10)
alpha = gender_face_shape(attr, alpha, -5) alpha = gender_face_shape(attr, alpha, 5)
alpha = weight_face_shape(attr, alpha, -10) alpha = weight_face_shape(attr, alpha, -10)
return alpha return alpha
...@@ -140,8 +140,8 @@ def gen_random_shape_coef(attr, dim): ...@@ -140,8 +140,8 @@ def gen_random_shape_coef(attr, dim):
def gen_random_tex_coef(attr, dim): def gen_random_tex_coef(attr, dim):
betha = np.random.randn(dim, 1) betha = np.random.randn(dim, 1)
betha = age_face_tex(attr, betha, 1) betha = age_face_tex(attr, betha, 10)
betha = gender_face_tex(attr, betha, -5) betha = gender_face_tex(attr, betha, 5)
betha = weight_face_tex(attr, betha, -10) betha = weight_face_tex(attr, betha, -10)
return betha return betha
......
...@@ -12,6 +12,7 @@ import numpy as np ...@@ -12,6 +12,7 @@ import numpy as np
import aam import aam
import landmarks import landmarks
from settings import logger from settings import logger
from .ibug_to_bfm import ibug_mapping
# load detector (this loads the datafile from disk, so needs to be done once). # load detector (this loads the datafile from disk, so needs to be done once).
detector = landmarks.Detector() detector = landmarks.Detector()
......
# Filename: MPEG4_FDP_face05.fp
# Format: (vertex_nr) (x y z) (x y) (name)
# mapping from ibug idx to bfm vertex
ibug_mapping = {
# nose
30: 8309, # -72.8548 18986.1 121850 0 0 9.12
31: 8319, # 13.2323 4430.29 131560 0 0 9.3
# 6611 # -12555.7 29870.6 96949.7 0 0 9.6 --
# 6244 # -15010.6 13787.1 106131 0 0 9.14 --
33: 5879, # -15364.7 -7915.79 105577 0 0 9.4
32: 7032, # -13710 -543.09 119384 0 0 9.2
34: 8334, # -174.404 -12553.8 115099 0 0 9.15
# eye left
# 4410 # -31025.3 33492.2 94793.8 0 0 3.6 -- center eye
39: 4404, # -31993.5 37990.5 94498 0 0 3.2 -- eye, not perfect
41: 4416, # -30242.7 29444 93887.9 0 0 3.4
# 4402 -32257.8 38510.2 95919.1 0 0 3.14 --
# 4418 -30262.3 28855.5 94215.7 0 0 3.10 --
37: 2088, # -43304.9 33261.1 85744.4 0 0 3.12
40: 5959, # -18690.6 32105.7 91502.5 0 0 3.8
52: 8344, # -192.025 -24960.5 116208 0 0 8.1
51: 7570, # -5897.69 -23762.2 115642 0 0 8.9
50: 5768, # -18543.4 -28311.2 107964 0 0 8.6
49: 5006, # -29800 -34598.4 97775.3 0 0 8.4
58: 8374, # -154.684 -40559.2 112944 0 0 8.2
60: 6697, # -17838.4 -37488.3 106002 0 0 8.8
1: 21628, # -77447 30661.5 12096.9 0 0 10.10
# 19963 -88262.2 36394.8 -4947.64 0 0 10.2 -- ears
# 19330 -85528.2 2072.17 -10931.6 0 0 10.4 -- ears
# 20203 -73194.6 -20281.3 11874.6 0 0 10.6 -- ears
# 20547 -67842.2 -17904 16423.6 0 0 10.8 -- ears
22: 40514, # -14438.3 49609.7 107232 0 0 4.2 -- eyebrow
20: 40087, # -37475.1 56446.7 100681 0 0 4.4 -- eyebrow
37: 38792, # -57986.8 45943.2 78728.1 0 0 4.6
# 9965# 12130.9 29795.3 96702.2 0 0 9.7 -- nose
# 10372# 14681.1 13742.5 105748 0 0 9.13 -- nose
35: 10781, # 14916.3 -7662.06 105539 0 0 9.5 -- nose
36: 9612, # 13568 -523.124 119279 0 0 9.1 -- nose
# 12150# 30157.7 33516.8 94540 0 0 3.5 -- center eye
44: 12144, # 31037.6 38025.8 94139.1 0 0 3.1 -- eye, not perfect
48: 12156, # 29458.6 29444.7 93770.8 0 0 3.3 -- eye, not perfect
# 12142 31349.6 38584.3 95585.3 0 0 3.13 -- eye
# 12158 29537.5 28834.2 94046.5 0 0 3.9 -- eye
43: 10603, # 17667.8 32012.7 91252.6 0 0 3.11 -- eye
55: 11714, # 29517.6 -34275.5 97471.5 0 0 8.3
17: 33496, # 76722.4 30669.8 12318.6 0 0 10.9
# 35151 87810.5 36294.1 -4628.25 0 0 10.1 -- ear
# 35807 85139.7 2150.69 -10665 0 0 10.3
# 34981 72909.8 -20461.1 12009.1 0 0 10.5
# 34656 67686.9 -18200 16564.5 0 0 10.7
63: 8354, # -275.894 -31384.5 110683 0 0 2.2 -- mouth
67: 8366, # -257.674 -31426.9 110736 0 0 2.3 -- mouth
# 40777 -802.905 96304.2 99265.5 0 0 11.1 -- upside
46: 14472, # 42745.8 33135.4 85724.1 0 0 3.7 -- eye left
53: 9118, # 5483.92 -23769.2 115655 0 0 8.10 -- mouth
54: 10928, # 18155.1 -28245 107864 0 0 8.5 -- mouth
56: 10051, # 17625.3 -37478 105917 0 0 8.7 -- not perfect
23: 41091, # 13583.4 49767.2 107218 0 0 4.1
25: 41511, # 36713.9 56793.8 100463 0 0 4.3
46: 42825, # 57424.9 46059.1 78590.1 0 0 4.5
9: 48187 # -48.2388 -76505.2 99442.1 0 0 gn
# 19963 -88262.2 36394.8 -4947.64 0 0 sa --
# 20205 -71257.4 -20598.4 13258.3 0 0 sba --
# 21629 -77516 30127.9 12058.9 0 0 pra --
# 19325 -88087.9 14538.9 -14157.6 0 0 pa --
# 20983 -79905 34425.4 8525.19 0 0 obs --
# 20786 -66866.5 -16819 17784.3 0 0 obi
# 21776 -72643.3 11413.3 11140.1 0 0 t
# 26140 -65298.1 23275.2 64256.6 0 0 zy
# 43070 -59848.7 -42669.3 41668.6 0 0 go
# 8374 -154.684 -40559.2 112944 0 0 li
# 8381 -140.066 -49939.9 106747 0 0 sl
# 48187 -48.2388 -76505.2 99442.1 0 0 gn
# 23188 -71385.1 15971.4 28332 0 0 cdl
# 8261 -369.986 47259.7 108662 0 0 g
# 26236 -60480.3 43221.1 72106.9 0 0 ft
# 7441 -6628.37 -23768.2 115463 0 0 cph
# 8344 -192.025 -24960.5 116208 0 0 ls
# 8366 -257.674 -31426.9 110736 0 0 sto
# 5392 -26244.2 -33187.9 98582 0 0 ch
# 8275 -190.815 39984 108192 0 0 n
# 8286 -144.161 35988.3 109453 0 0 se
# 8311 -47.9918 16757.4 123572 0 0 se
# 6389 -16981.4 -3046.09 111901 0 0 al
# 8320 20.1202 2232.26 131944 0 0 prn
# 8332 -175.672 -10527.6 116009 0 0 sn
# 6912 -9848.33 -8505.34 111144 0 0 sbal
# 5488 -17814.8 -3696.18 103442 0 0 ac
# 7809 -6093.17 -3557.61 124863 0 0 c'
# 7032 -13710 -543.09 119384 0 0 al'
# 7165 -10444.4 -2520.99 117527 0 0 al'
# 7814 -4855.38 -5635.67 114857 0 0 sn'
# 5959 -18690.6 32105.7 91502.5 0 0 en
# 2088 -43304.9 33261.1 85744.4 0 0 ex
# 4280 -32240.9 34083.1 94614 0 0 p
# 3393 -38037.4 23596.3 91323.3 0 0 or
# 4158 -32467.7 29315.4 93418 0 0 pi
# 40087 -37475.1 56446.7 100681 0 0 sci
# 4646 -37373.6 48977.9 99911.2 0 0 os
# 6796 -10468.9 -24402.7 113952 0 0 ls'
# 4404 -31993.5 37990.5 94498 0 0 ps
# 48180 -41.2938 -65127.6 108493 0 0 pg
# 35151 87810.5 36294.1 -4628.25 0 0 sa*
# 34983 71040 -20787.8 13418.5 0 0 sba*
# 33497 76793.9 30128.9 12280.4 0 0 pra*
# 35802 87874.7 14663.6 -13676.7 0 0 pa*
# 34132 79275.1 34409.8 8855 0 0 obs*
# 34425 66654.4 -17219.2 17867.2 0 0 obi*
# 33386 71894.4 11245.4 11118.2 0 0 t*
# 28978 64756.8 23136.4 64176 0 0 zy*
# 53307 59692.2 -42997 41678.6 0 0 go*
# 31960 70614.6 15771.9 28283.9 0 0 cdl*
# 28816 59843 43216.9 71989.8 0 0 ft*
# 9247 6180.74 -23794.4 115481 0 0 cph*
# 11326 25887.4 -32810.2 98262 0 0 ch*
# 10259 16732.3 -2973.56 111813 0 0 al*
# 9750 9386.51 -8462.76 111119 0 0 sbal*
# 11164 17421.1 -3511.99 103224 0 0 ac*
# 8841 5890.83 -3559.51 124776 0 0 c'*
# 9612 13568 -523.124 119279 0 0 al'*
# 9487 10105.6 -2556.09 117492 0 0 al'*
# 8846 4183.15 -5680.2 114730 0 0 sn'*
# 10603 17667.8 32012.7 91252.6 0 0 en*
# 14472 42745.8 33135.4 85724.1 0 0 ex*
# 12278 31373.5 34100.5 94353.7 0 0 p*
# 13197 37566.8 23529.1 91177.5 0 0 or*
# 12414 31723.8 29338.3 93324.8 0 0 pi*
# 41511 36713.9 56793.8 100463 0 0 sci*
# 11870 36697.5 49438 99579.8 0 0 os*
# 9892 9980 -24508.4 113969 0 0 ls'*
# 12144 31037.6 38025.8 94139.1 0 0 ps*
}
import cv2
def calc_affine_projection_matrix(landmarks, vertices, image):
"""
returns the 3x4 camera matrix based on the given locations of the landmarks
and matching 3d vertices.
"""
cv2.initCameraMatrix2D(vertices, landmarks, image.shape)
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#include "eos/fitting/linear_shape_fitting.hpp" #include "eos/fitting/linear_shape_fitting.hpp"
#include "eos/render/utils.hpp" #include "eos/render/utils.hpp"
#include "eos/render/texture_extraction.hpp" #include "eos/render/texture_extraction.hpp"
//#include "eos/pybind11_glm.hpp"
#include "opencv2/core/core.hpp" #include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp" #include "opencv2/highgui/highgui.hpp"
...@@ -19,8 +20,7 @@ ...@@ -19,8 +20,7 @@
#include <pybind11/numpy.h> #include <pybind11/numpy.h>
namespace py = pybind11; namespace py = pybind11;
using namespace eos; using namespace eos;
namespace po = boost::program_options; namespace po = boost::program_options;
namespace fs = boost::filesystem; namespace fs = boost::filesystem;
...@@ -79,175 +79,122 @@ float add(py::array_t<uint32_t> image_input, ...@@ -79,175 +79,122 @@ float add(py::array_t<uint32_t> image_input,
void fit(Mat image, Vec2f landmarks) { void fit(Mat image, Vec2f landmarks) {
std::cout << image.at<uint32_t>(0,0) << std::endl; // std::cout << image.at<uint32_t>(0,0) << std::endl;
std::cout << landmarks << std::endl;
return;
LandmarkCollection<cv::Vec2f> landmarkCollection;
morphablemodel::MorphableModel morphable_model;
try {
morphable_model = morphablemodel::load_model("/usr/local/eos/share/sfm_shape_3448.bin");
}
catch (const std::runtime_error& e) {
cout << "Error loading the Morphable Model: " << e.what() << endl;
return;
}
//LandmarkCollection<cv::Vec2f> landmarks; core::LandmarkMapper landmark_mapper = core::LandmarkMapper("/usr/local/eos/share/bug2did.txt");
}
// Draw the loaded landmarks:
Mat outimg = image.clone();
for (auto&& lm : landmarkCollection) {
cv::rectangle(
outimg,
cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f),
{ 255, 0, 0 }
);
}
// Mat image = cv::imread(imagefile.string()); // These will be the final 2D and 3D points used for the fitting:
// LandmarkCollection<cv::Vec2f> landmarks; vector<Vec4f> model_points; // the points in the 3D shape model
// try { vector<int> vertex_indices; // their vertex indices
// landmarks = read_pts_landmarks(landmarksfile.string()); vector<Vec2f> image_points; // the corresponding 2D landmark points
// }
// catch (const std::runtime_error& e) { // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
// cout << "Error reading the landmarks: " << e.what() << endl; for (int i = 0; i < landmarkCollection.size(); ++i) {
// return EXIT_FAILURE; auto converted_name = landmark_mapper.convert(landmarkCollection[i].name);
// } if (!converted_name) { // no mapping defined for the current landmark
// morphablemodel::MorphableModel morphable_model; continue;
// try { }
// morphable_model = morphablemodel::load_model(modelfile.string()); int vertex_idx = std::stoi(converted_name.get());
// } Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
// catch (const std::runtime_error& e) { model_points.emplace_back(vertex);
// cout << "Error loading the Morphable Model: " << e.what() << endl; vertex_indices.emplace_back(vertex_idx);
// return EXIT_FAILURE; image_points.emplace_back(landmarkCollection[i].coordinates);
// } }
// core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
// // Estimate the camera (pose) from the 2D - 3D point correspondences
// // Draw the loaded landmarks: fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
// Mat outimg = image.clone(); fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
// for (auto&& lm : landmarks) {
// cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 }); // The 3D head pose can be recovered as follows:
// } float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// // and similarly for pitch and roll.
// // These will be the final 2D and 3D points used for the fitting:
// vector<Vec4f> model_points; // the points in the 3D shape model // Estimate the shape coefficients by fitting the shape to the landmarks:
// vector<int> vertex_indices; // their vertex indices Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
// vector<Vec2f> image_points; // the corresponding 2D landmark points vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
//
// // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM): // Obtain the full mesh with the estimated coefficients:
// for (int i = 0; i < landmarks.size(); ++i) { render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
// auto converted_name = landmark_mapper.convert(landmarks[i].name);
// if (!converted_name) { // no mapping defined for the current landmark // Extract the texture from the image using given mesh and camera parameters:
// continue; Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
// }
// int vertex_idx = std::stoi(converted_name.get()); // Save the mesh as textured obj:
// Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx); render::write_textured_obj(mesh, "/data/fit-model-test.obj");
// model_points.emplace_back(vertex);
// vertex_indices.emplace_back(vertex_idx); // And save the isomap:
// image_points.emplace_back(landmarks[i].coordinates); cv::imwrite("/data/fit-model-test.isomap.png", isomap);
// } }
//
// // Estimate the camera (pose) from the 2D - 3D point correspondences
// fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
// fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
//
// // The 3D head pose can be recovered as follows:
// float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// // and similarly for pitch and roll.
//
// // Estimate the shape coefficients by fitting the shape to the landmarks:
// Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
// vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
//
// // Obtain the full mesh with the estimated coefficients:
// render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
//
// // Extract the texture from the image using given mesh and camera parameters:
// Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
//
// // Save the mesh as textured obj:
// outputfile += fs::path(".obj");
// render::write_textured_obj(mesh, outputfile.string());
//
// // And save the isomap:
// outputfile.replace_extension(".isomap.png");
// cv::imwrite(outputfile.string(), isomap);
//
// cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
//
// return EXIT_SUCCESS;
//}
/**
* This app demonstrates estimation of the camera and fitting of the shape
* model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
*
* First, the 68 ibug landmarks are loaded from the .pts file and converted
* to vertex indices using the LandmarkMapper. Then, an affine camera matrix
* is estimated, and then, using this camera matrix, the shape is fitted
* to the landmarks.
*/
//int fit(int argc, char *argv[])
//{
// // Load the image, landmarks, LandmarkMapper and the Morphable Model:
// Mat image = cv::imread(imagefile.string());
// LandmarkCollection<cv::Vec2f> landmarks;
// try {
// landmarks = read_pts_landmarks(landmarksfile.string());
// }
// catch (const std::runtime_error& e) {
// cout << "Error reading the landmarks: " << e.what() << endl;
// return EXIT_FAILURE;
// }
// morphablemodel::MorphableModel morphable_model;
// try {
// morphable_model = morphablemodel::load_model(modelfile.string());
// }
// catch (const std::runtime_error& e) {
// cout << "Error loading the Morphable Model: " << e.what() << endl;
// return EXIT_FAILURE;
// }
// core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile);
//
// // Draw the loaded landmarks:
// Mat outimg = image.clone();
// for (auto&& lm : landmarks) {
// cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 });
// }
//
// // These will be the final 2D and 3D points used for the fitting:
// vector<Vec4f> model_points; // the points in the 3D shape model
// vector<int> vertex_indices; // their vertex indices
// vector<Vec2f> image_points; // the corresponding 2D landmark points
//
// // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM):
// for (int i = 0; i < landmarks.size(); ++i) {
// auto converted_name = landmark_mapper.convert(landmarks[i].name);
// if (!converted_name) { // no mapping defined for the current landmark
// continue;
// }
// int vertex_idx = std::stoi(converted_name.get());
// Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx);
// model_points.emplace_back(vertex);
// vertex_indices.emplace_back(vertex_idx);
// image_points.emplace_back(landmarks[i].coordinates);
// }
//
// // Estimate the camera (pose) from the 2D - 3D point correspondences
// fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
// fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
//
// // The 3D head pose can be recovered as follows:
// float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// // and similarly for pitch and roll.
//
// // Estimate the shape coefficients by fitting the shape to the landmarks:
// Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
// vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
//
// // Obtain the full mesh with the estimated coefficients:
// render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>());
//
// // Extract the texture from the image using given mesh and camera parameters:
// Mat isomap = render::extract_texture(mesh, affine_from_ortho, image);
//
// // Save the mesh as textured obj:
// outputfile += fs::path(".obj");
// render::write_textured_obj(mesh, outputfile.string());
//
// // And save the isomap:
// outputfile.replace_extension(".isomap.png");
// cv::imwrite(outputfile.string(), isomap);
//
// cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl;
//
// return EXIT_SUCCESS;
//}
PYBIND11_PLUGIN(fit) { PYBIND11_PLUGIN(fit) {
py::module m("fit", "fit example"); py::module m("fit", "fit example");
/**
* General bindings, for OpenCV vector types and cv::Mat:
* - cv::Vec2f
* - cv::Vec4f
* - cv::Mat (only 1-channel matrices and only conversion of CV_32F C++ matrices to Python, and conversion of CV_32FC1 and CV_64FC1 matrices from Python to C++)
*/
//py::class_<cv::Vec2f>(fit, "Vec2f", "Wrapper for OpenCV's cv::Vec2f type.")
//.def("__init__", [](cv::Vec2f& vec, py::buffer b) {
// py::buffer_info info = b.request();
// if (info.ndim != 1)
// throw std::runtime_error("Buffer ndim is " + std::to_string(info.ndim) + ", please hand a buffer with dimension == 1 to create a Vec2f.");
// if (info.strides.size() != 1)
// throw std::runtime_error("strides.size() is " + std::to_string(info.strides.size()) + ", please hand a buffer with strides.size() == 1 to create a Vec2f.");
// // Todo: Should add a check that checks for default stride sizes, everything else would not work yet I think.
// if (info.shape.size() != 1)
// throw std::runtime_error("shape.size() is " + std::to_string(info.shape.size()) + ", please hand a buffer with shape dimension == 1 to create a Vec2f.");
// if (info.shape[0] != 2)
// throw std::runtime_error("shape[0] is " + std::to_string(info.shape[0]) + ", please hand a buffer with 2 entries to create a Vec2f.");
// if (info.format == py::format_descriptor<float>::format()) {
// cv::Mat temp(1, 2, CV_32FC1, info.ptr);
// // std::cout << temp << std::endl;
// new (&vec) cv::Vec2f(temp);
// } else {
// throw std::runtime_error("Not given a buffer of type float - please hand a buffer of type float to create a Vec2f.");
// }
//})
//.def_buffer([](cv::Vec2f& vec) -> py::buffer_info {
//return py::buffer_info(
// &vec.val, /* Pointer to buffer */
// sizeof(float), /* Size of one scalar */
// py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
// 2, /* Number of dimensions */
// { vec.rows, vec.cols }, /* Buffer dimensions */
// { sizeof(float), /* Strides (in bytes) for each index */
// sizeof(float) } //=> both sizeof(float), since the data is hold in an array, i.e. contiguous memory */
//);
//});
m.def("add", &add, "A function which adds two numbers"); m.def("add", &add, "A function which adds two numbers");
m.def("fit", &fit, "Fit");
return m.ptr(); return m.ptr();
} }
...@@ -50,13 +50,45 @@ def fit_model(): ...@@ -50,13 +50,45 @@ def fit_model():
input_points = dataset_module.factory(filename=image_filename) input_points = dataset_module.factory(filename=image_filename)
input_image = input_points.get_image() input_image = input_points.get_image()
input_points.get_points()
print dir(eos)
# scale points to output image shape. We MUST do this. # scale points to output image shape. We MUST do this.
points = input_points.get_scaled_points(input_image.shape) points = input_points.get_scaled_points(input_image.shape)
print(input_image.shape) fit.fit(input_image, points)
print dir(eos)
# fit.add(input_image, points)
#fit.add(input_image, points)
if __name__ == '__main__': if __name__ == '__main__':
fit_model() fit_model()
## Try seo python bindings
# model = eos.morphablemodel.load_model(
# '/usr/local/eos/share/sfm_shape_3448.bin')
# blend_shapes = eos.morphablemodel.load_blendshapes(
# '/usr/local/eos/share/expression_blendshapes_3448.bin'
# )
#
# s = model.get_shape_model().draw_sample([1.0, -0.5, 0.7, 0.1])
#
# sample = np.array(s)
# tri = model.get_shape_model().get_triangle_list()
# mean = model.get_shape_model().get_mean()
# dims = model.get_shape_model().get_data_dimension()
#
# mean = np.array(mean)
# with open('/data/test.obj', 'w') as f:
# for i in range(0, len(sample), 3):
# f.write('v {} {} {}\n'.format(sample[i], sample[i + 1], sample[i + 2])
# )
#
# for i in range(0, len(tri)):
# f.write('f {} {} {}\n'.format(
# tri[i][0], tri[i][1], tri[i][2],
# )
# )
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment