Testing with eos reconstruction

.gitignore

@@ -2,6 +2,7 @@ data/*
 src/.cache/*
 src/*/.cache
 src/reconstruction/*.c
+scripts/PublicMM1/
 *.o
 *.so
 src/reconstruction/build/

Dockerfile

@@ -4,32 +4,51 @@ FROM smvanveen/computer-vision:20161109143812
 COPY requirements.txt /tmp
 RUN pip install -r /tmp/requirements.txt
 
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:bzindovic/suitesparse-bugfix-1319687 -y
+RUN apt-get update -y
+RUN apt-get install libsuitesparse-dev -y
+
 # extra packages:
 # graphviz: for cProfiling using pycallgraph.
 # libeigen3-dev: for eos: 3D morphable face model fitting library.
 RUN apt-get install -y \
     graphviz \
+    libeigen3-dev \
+    libgoogle-glog-dev \
+    libatlas-base-dev \
     libeigen3-dev
 
+
 WORKDIR /libs
 
 # install dlib
 RUN git clone https://github.com/davisking/dlib.git
 RUN (cd dlib; python setup.py install --yes USE_AVX_INSTRUCTIONS)
 
+RUN git clone https://ceres-solver.googlesource.com/ceres-solver
+RUN (cd ceres-solver; make -j3)
+RUN (cd ceres-solver; make install)
+
 # install eos (face-recosntruction, (3D Morphable Face Model fitting library)
-RUN git clone --recursive https://github.com/patrikhuber/eos.git
+RUN git clone --recursive \
+    https://github.com/patrikhuber/eos.git
 
 # remove dependency on opencv 2.4.3, opencv 3.0 works fine
 WORKDIR /libs/eos
-RUN sed -i 's/2.4.3//g' CMakeLists.txt
+RUN git checkout devel
+
+# needed for master branch
+#RUN sed -i 's/2.4.3//g' CMakeLists.txt
+
 RUN mkdir build
 WORKDIR /libs/eos/build
 RUN cmake ../ \
     -DCMAKE_INSTALL_PREFIX=/usr/local/eos \
-    -DGENERATE_PYTHON_BINDINGS=on \
-    -DBUILD_UTILS=on \
-    -DPYTHON_EXECUTABLE=/usr/bin/python
+    -DEOS_GENERATE_PYTHON_BINDINGS=on \
+    -DEOS_BUILD_CERES_EXAMPLE=on \
+    -DEOS_BUILD_UTILS=on \
+    -DEOS_BUILD_EXAMPLES=on
 
 RUN make && make install
 
@@ -40,4 +59,6 @@ RUN git clone https://github.com/pybind/pybind11.git
 RUN (cd pybind11; mkdir build; cd build; cmake -DPYBIND11_PYTHON_VERSION=2.7 ..);
 RUN (cd pybind11/build; make -j4 && make install);
 
+#TODO, remove the tmp libs folder in production?
+
 WORKDIR /src

actions.mk

 .PHONY := train_model show_pca test_model show_reconstruction
 DEBUG_LEVEL=*
 
-data/imm_face_db: data/imm_face_db.tar.gz
+data/imm_face_db: # data/imm_face_db.tar.gz
 	(cd data; mkdir -p imm_face_db; \
 		tar -xvzf imm_face_db.tar.gz -C imm_face_db \
 	)

makefile

@@ -13,7 +13,7 @@ BASE_DOCKER_CMD:= docker run $(DOCKER_RUN_FLAGS) $(IMAGE_TAG)
 $(info $(TARGETS))
 
 DEPENDENCIES:= data/imm_face_db
-TARGETS:= data/shape_predictor_68_face_landmarks.dat\
+TARGETS:= data/shape_predictor_68_face_landmarks.dat \
 	src/reconstruction/texture.so \
 	src/reconstruction/fit.so \
 	data/pca_ibug_shape_model.npy \

scripts/PublicMM1/python/bfm_to_obj.py

@@ -44,7 +44,7 @@ def output_shape_and_texture(shape, texture, f):
     for i in range(0, len(shape), 3):
         f.write('v {} {} {} {} {} {}\n'.format(
             shape[i], shape[i + 1], shape[i + 2],
-            int(texture[i]), int(texture[i + 1]), int(texture[i + 2])
+            texture[i], texture[i + 1], texture[i + 2]
             )
         )
 

scripts/PublicMM1/python/random_face.py

@@ -131,8 +131,8 @@ def weight_face_shape(attr, alpha, n=30):
 
 def gen_random_shape_coef(attr, dim):
     alpha = np.random.randn(dim, 1)
-    alpha = age_face_shape(attr, alpha, 1)
-    alpha = gender_face_shape(attr, alpha, -5)
+    alpha = age_face_shape(attr, alpha, 10)
+    alpha = gender_face_shape(attr, alpha, 5)
     alpha = weight_face_shape(attr, alpha, -10)
 
     return alpha
@@ -140,8 +140,8 @@ def gen_random_shape_coef(attr, dim):
 
 def gen_random_tex_coef(attr, dim):
     betha = np.random.randn(dim, 1)
-    betha = age_face_tex(attr, betha, 1)
-    betha = gender_face_tex(attr, betha, -5)
+    betha = age_face_tex(attr, betha, 10)
+    betha = gender_face_tex(attr, betha, 5)
     betha = weight_face_tex(attr, betha, -10)
 
     return betha

src/datasets/ibug.py

@@ -12,6 +12,7 @@ import numpy as np
 import aam
 import landmarks
 from settings import logger
+from .ibug_to_bfm import ibug_mapping
 
 # load detector (this loads the datafile from disk, so needs to be done once).
 detector = landmarks.Detector()

src/datasets/ibug_to_bfm.py

+# Filename: MPEG4_FDP_face05.fp
+# Format: (vertex_nr) (x y z) (x y) (name)
+
+# mapping from ibug idx to bfm vertex
+ibug_mapping = {
+    # nose
+    30: 8309,  # -72.8548 18986.1 121850 0 0 9.12
+    31: 8319,  # 13.2323 4430.29 131560 0 0 9.3
+    # 6611 # -12555.7 29870.6 96949.7 0 0 9.6 --
+    # 6244 # -15010.6 13787.1 106131 0 0 9.14 --
+    33: 5879,  # -15364.7 -7915.79 105577 0 0 9.4
+    32: 7032,  # -13710 -543.09 119384 0 0 9.2
+    34: 8334,  # -174.404 -12553.8 115099 0 0 9.15
+    # eye left
+    # 4410 # -31025.3 33492.2 94793.8 0 0 3.6 -- center eye
+    39: 4404,  # -31993.5 37990.5 94498 0 0 3.2 -- eye, not perfect
+    41: 4416,  # -30242.7 29444 93887.9 0 0 3.4
+    # 4402 -32257.8 38510.2 95919.1 0 0 3.14 --
+    # 4418 -30262.3 28855.5 94215.7 0 0 3.10 --
+    37: 2088,  # -43304.9 33261.1 85744.4 0 0 3.12
+    40: 5959,  # -18690.6 32105.7 91502.5 0 0 3.8
+    52: 8344,  # -192.025 -24960.5 116208 0 0 8.1
+    51: 7570,  # -5897.69 -23762.2 115642 0 0 8.9
+    50: 5768,  # -18543.4 -28311.2 107964 0 0 8.6
+    49: 5006,  # -29800 -34598.4 97775.3 0 0 8.4
+    58: 8374,  # -154.684 -40559.2 112944 0 0 8.2
+    60: 6697,  # -17838.4 -37488.3 106002 0 0 8.8
+    1: 21628,  # -77447 30661.5 12096.9 0 0 10.10
+    # 19963 -88262.2 36394.8 -4947.64 0 0 10.2 -- ears
+    # 19330 -85528.2 2072.17 -10931.6 0 0 10.4 -- ears
+    # 20203 -73194.6 -20281.3 11874.6 0 0 10.6 -- ears
+    # 20547 -67842.2 -17904 16423.6 0 0 10.8 -- ears
+    22: 40514,  # -14438.3 49609.7 107232 0 0 4.2 -- eyebrow
+    20: 40087,  # -37475.1 56446.7 100681 0 0 4.4 -- eyebrow
+    37: 38792,  # -57986.8 45943.2 78728.1 0 0 4.6
+    # 9965# 12130.9 29795.3 96702.2 0 0 9.7 -- nose
+    # 10372# 14681.1 13742.5 105748 0 0 9.13 -- nose
+    35: 10781,  # 14916.3 -7662.06 105539 0 0 9.5 -- nose
+    36: 9612,  # 13568 -523.124 119279 0 0 9.1 -- nose
+    # 12150# 30157.7 33516.8 94540 0 0 3.5 -- center eye
+    44: 12144,  # 31037.6 38025.8 94139.1 0 0 3.1 -- eye, not perfect
+    48: 12156,  # 29458.6 29444.7 93770.8 0 0 3.3 -- eye, not perfect
+    # 12142 31349.6 38584.3 95585.3 0 0 3.13 -- eye
+    # 12158 29537.5 28834.2 94046.5 0 0 3.9 -- eye
+    43: 10603,  # 17667.8 32012.7 91252.6 0 0 3.11 -- eye
+    55: 11714,  # 29517.6 -34275.5 97471.5 0 0 8.3
+    17: 33496,  # 76722.4 30669.8 12318.6 0 0 10.9
+    # 35151 87810.5 36294.1 -4628.25 0 0 10.1 -- ear
+    # 35807 85139.7 2150.69 -10665 0 0 10.3
+    # 34981 72909.8 -20461.1 12009.1 0 0 10.5
+    # 34656 67686.9 -18200 16564.5 0 0 10.7
+    63: 8354,  # -275.894 -31384.5 110683 0 0 2.2 -- mouth
+    67: 8366,  # -257.674 -31426.9 110736 0 0 2.3 -- mouth
+    # 40777 -802.905 96304.2 99265.5 0 0 11.1 -- upside
+    46: 14472,  # 42745.8 33135.4 85724.1 0 0 3.7 -- eye left
+    53: 9118,   # 5483.92 -23769.2 115655 0 0 8.10 -- mouth
+    54: 10928,  # 18155.1 -28245 107864 0 0 8.5 -- mouth
+    56: 10051,  # 17625.3 -37478 105917 0 0 8.7 -- not perfect
+    23: 41091,  # 13583.4 49767.2 107218 0 0 4.1
+    25: 41511,  # 36713.9 56793.8 100463 0 0 4.3
+    46: 42825,  # 57424.9 46059.1 78590.1 0 0 4.5
+    9: 48187  # -48.2388 -76505.2 99442.1 0 0 gn
+    # 19963 -88262.2 36394.8 -4947.64 0 0 sa --
+    # 20205 -71257.4 -20598.4 13258.3 0 0 sba --
+    # 21629 -77516 30127.9 12058.9 0 0 pra --
+    # 19325 -88087.9 14538.9 -14157.6 0 0 pa --
+    # 20983 -79905 34425.4 8525.19 0 0 obs --
+    # 20786 -66866.5 -16819 17784.3 0 0 obi
+    # 21776 -72643.3 11413.3 11140.1 0 0 t
+    # 26140 -65298.1 23275.2 64256.6 0 0 zy
+    # 43070 -59848.7 -42669.3 41668.6 0 0 go
+    # 8374 -154.684 -40559.2 112944 0 0 li
+    # 8381 -140.066 -49939.9 106747 0 0 sl
+    # 48187 -48.2388 -76505.2 99442.1 0 0 gn
+    # 23188 -71385.1 15971.4 28332 0 0 cdl
+    # 8261 -369.986 47259.7 108662 0 0 g
+    # 26236 -60480.3 43221.1 72106.9 0 0 ft
+    # 7441 -6628.37 -23768.2 115463 0 0 cph
+    # 8344 -192.025 -24960.5 116208 0 0 ls
+    # 8366 -257.674 -31426.9 110736 0 0 sto
+    # 5392 -26244.2 -33187.9 98582 0 0 ch
+    # 8275 -190.815 39984 108192 0 0 n
+    # 8286 -144.161 35988.3 109453 0 0 se
+    # 8311 -47.9918 16757.4 123572 0 0 se
+    # 6389 -16981.4 -3046.09 111901 0 0 al
+    # 8320 20.1202 2232.26 131944 0 0 prn
+    # 8332 -175.672 -10527.6 116009 0 0 sn
+    # 6912 -9848.33 -8505.34 111144 0 0 sbal
+    # 5488 -17814.8 -3696.18 103442 0 0 ac
+    # 7809 -6093.17 -3557.61 124863 0 0 c'
+    # 7032 -13710 -543.09 119384 0 0 al'
+    # 7165 -10444.4 -2520.99 117527 0 0 al'
+    # 7814 -4855.38 -5635.67 114857 0 0 sn'
+    # 5959 -18690.6 32105.7 91502.5 0 0 en
+    # 2088 -43304.9 33261.1 85744.4 0 0 ex
+    # 4280 -32240.9 34083.1 94614 0 0 p
+    # 3393 -38037.4 23596.3 91323.3 0 0 or
+    # 4158 -32467.7 29315.4 93418 0 0 pi
+    # 40087 -37475.1 56446.7 100681 0 0 sci
+    # 4646 -37373.6 48977.9 99911.2 0 0 os
+    # 6796 -10468.9 -24402.7 113952 0 0 ls'
+    # 4404 -31993.5 37990.5 94498 0 0 ps
+    # 48180 -41.2938 -65127.6 108493 0 0 pg
+    # 35151 87810.5 36294.1 -4628.25 0 0 sa*
+    # 34983 71040 -20787.8 13418.5 0 0 sba*
+    # 33497 76793.9 30128.9 12280.4 0 0 pra*
+    # 35802 87874.7 14663.6 -13676.7 0 0 pa*
+    # 34132 79275.1 34409.8 8855 0 0 obs*
+    # 34425 66654.4 -17219.2 17867.2 0 0 obi*
+    # 33386 71894.4 11245.4 11118.2 0 0 t*
+    # 28978 64756.8 23136.4 64176 0 0 zy*
+    # 53307 59692.2 -42997 41678.6 0 0 go*
+    # 31960 70614.6 15771.9 28283.9 0 0 cdl*
+    # 28816 59843 43216.9 71989.8 0 0 ft*
+    # 9247 6180.74 -23794.4 115481 0 0 cph*
+    # 11326 25887.4 -32810.2 98262 0 0 ch*
+    # 10259 16732.3 -2973.56 111813 0 0 al*
+    # 9750 9386.51 -8462.76 111119 0 0 sbal*
+    # 11164 17421.1 -3511.99 103224 0 0 ac*
+    # 8841 5890.83 -3559.51 124776 0 0 c'*
+    # 9612 13568 -523.124 119279 0 0 al'*
+    # 9487 10105.6 -2556.09 117492 0 0 al'*
+    # 8846 4183.15 -5680.2 114730 0 0 sn'*
+    # 10603 17667.8 32012.7 91252.6 0 0 en*
+    # 14472 42745.8 33135.4 85724.1 0 0 ex*
+    # 12278 31373.5 34100.5 94353.7 0 0 p*
+    # 13197 37566.8 23529.1 91177.5 0 0 or*
+    # 12414 31723.8 29338.3 93324.8 0 0 pi*
+    # 41511 36713.9 56793.8 100463 0 0 sci*
+    # 11870 36697.5 49438 99579.8 0 0 os*
+    # 9892 9980 -24508.4 113969 0 0 ls'*
+    # 12144 31037.6 38025.8 94139.1 0 0 ps*
+}

src/fitting/camera.py

+import cv2
+
+def calc_affine_projection_matrix(landmarks, vertices, image):
+    """
+    returns the 3x4 camera matrix based on the given locations of the landmarks
+    and matching 3d vertices.
+    """
+    cv2.initCameraMatrix2D(vertices, landmarks, image.shape)

src/test/reconstruction_test.py

@@ -50,13 +50,45 @@ def fit_model():
 
     input_points = dataset_module.factory(filename=image_filename)
     input_image = input_points.get_image()
+    input_points.get_points()
+
+    print dir(eos)
 
     # scale points to output image shape. We MUST do this.
     points = input_points.get_scaled_points(input_image.shape)
-    print(input_image.shape)
-    print dir(eos)
+    fit.fit(input_image, points)
+
+    # fit.add(input_image, points)
 
-    #fit.add(input_image, points)
 
 if __name__ == '__main__':
     fit_model()
+
+
+## Try seo python bindings
+#    model = eos.morphablemodel.load_model(
+#        '/usr/local/eos/share/sfm_shape_3448.bin')
+#    blend_shapes = eos.morphablemodel.load_blendshapes(
+#        '/usr/local/eos/share/expression_blendshapes_3448.bin'
+#    )
+#
+#    s = model.get_shape_model().draw_sample([1.0, -0.5, 0.7, 0.1])
+#
+#    sample = np.array(s)
+#    tri = model.get_shape_model().get_triangle_list()
+#    mean = model.get_shape_model().get_mean()
+#    dims = model.get_shape_model().get_data_dimension()
+#
+#    mean = np.array(mean)
+
+#    with open('/data/test.obj', 'w') as f:
+#        for i in range(0, len(sample), 3):
+#            f.write('v {} {} {}\n'.format(sample[i], sample[i + 1], sample[i + 2])
+#        )
+#
+#        for i in range(0, len(tri)):
+#            f.write('f {} {} {}\n'.format(
+#                tri[i][0], tri[i][1], tri[i][2],
+#            )
+#        )
+