Fix the errors of roundings issues for now

actions.mk

@@ -20,6 +20,7 @@ runnit:
 src/reconstruction/texture.so: src/reconstruction/texture.pyx
 	$(BASE_DOCKER_CMD) /bin/bash -c '(cd reconstruction; python setup.py build_ext --inplace)'
 
+## IMM Dataset
 data/pca_imm_shape_model.npy:
 	$(BASE_DOCKER_CMD) python main.py \
 		--save_pca_shape \
@@ -27,6 +28,16 @@ data/pca_imm_shape_model.npy:
 		--model_shape_file /data/pca_imm_shape_model \
 		--shape_type imm
 
+data/pca_imm_texture_model.npy:
+	$(BASE_DOCKER_CMD) python main.py \
+		--save_pca_texture \
+		--files `./scripts/imm_train_set.sh` \
+		--model_texture_file /data/pca_imm_texture_model \
+		--model_shape_file /data/pca_imm_shape_model.npy \
+		--shape_type imm
+## END OF IMM
+
+## IBUG using dlib landmark detector
 data/pca_ibug_shape_model.npy:
 	$(BASE_DOCKER_CMD) python main.py \
 		--save_pca_shape \
@@ -41,14 +52,8 @@ data/pca_ibug_texture_model.npy:
 		--model_texture_file /data/pca_ibug_texture_model \
 		--model_shape_file /data/pca_ibug_shape_model.npy \
 		--shape_type ibug
+## END OF IBUG
 
-data/pca_imm_texture_model.npy:
-	$(BASE_DOCKER_CMD) python main.py \
-		--save_pca_texture \
-		--files `./scripts/imm_train_set.sh` \
-		--model_texture_file /data/pca_imm_texture_model \
-		--model_shape_file /data/pca_imm_shape_model.npy \
-		--shape_type imm
 
 test_model:
 	$(BASE_DOCKER_CMD) python main.py \
@@ -105,7 +110,9 @@ test:
 
 .PHONY:= server
 server:
-	$(BASE_DOCKER_CMD) python -m tornado.autoreload server.py
+	docker run $(DOCKER_RUN_FLAGS) $(EXTRA_FLAGS) \
+		-p 6930:8888 $(IMAGE_TAG) \
+		python -m tornado.autoreload server.py
 
 .PHONY:= ember
 ember:
@@ -113,4 +120,4 @@ ember:
 
 .PHONY:= ctags
 ctags:
-	ctags -R --python-kinds=-i src
+	ctags --python-kinds=-i src

makefile

@@ -5,15 +5,15 @@ HERE:=$(shell pwd)
 DOCKER_RUN_FLAGS:= --rm \
 	--volume $(HERE)/data:/data \
 	--volume $(HERE)/src:/src \
-	-e "DEBUG=$(DEBUG)" \
-	-p 6930:8888
+	--volume ~/.bash_history:/root/.bash_history \
+	-e "DEBUG=$(DEBUG)"
 
 BASE_DOCKER_CMD:= docker run $(DOCKER_RUN_FLAGS) $(IMAGE_TAG)
 
 $(info $(TARGETS))
 
 DEPENDENCIES:= data/imm_face_db
-TARGETS:= shape_predictor_68_face_landmarks.dat\
+TARGETS:= data/shape_predictor_68_face_landmarks.dat\
 	src/reconstruction/texture.so \
 	data/pca_ibug_shape_model.npy \
 	data/pca_ibug_texture_model.npy
@@ -31,6 +31,10 @@ build: requirements.txt
 run-bash:
 	docker run --interactive --tty $(DOCKER_RUN_FLAGS) $(IMAGE_TAG) /bin/bash
 
+run-bash-cmd:
+	docker run --interactive --tty $(DOCKER_RUN_FLAGS) $(IMAGE_TAG) \
+		/bin/bash -c "$(CMD)"
+
 $(VIRTUALENV):
 	virtualenv -p $(PYTHON_BIN_PATH) venv
 

src/reconstruction/reconstruction.py

@@ -3,45 +3,7 @@ import numpy as np
 
 import pca
 import aam
-
-
-def cartesian2barycentric(r1, r2, r3, r):
-    """
-    Given a triangle spanned by three cartesion points
-    r1, r2, r2, and point r, return the barycentric weights l1, l2, l3.
-
-    Returns:
-        ndarray (of dim 3) weights of the barycentric coordinates
-
-    """
-    x, y = r
-    x1, y1 = r1
-    x2, y2 = r2
-    x3, y3 = r3
-
-    a = np.array([[x1, x2, x3], [y1, y2, y3], [1, 1, 1]])
-    b = np.array([x, y, 1])
-
-    return np.linalg.solve(a, b)
-
-
-def barycentric2cartesian(r1, r2, r3, L):
-    """
-    Given the barycentric weights in L, and cartesian r1, r2, r3 coordinates of
-    points that span the triangle, return the cartesian coordinate of the
-    points that is located at the weights of L.
-
-    Returns:
-        ndarray [x,y] cartesian points.
-    """
-    x1, y1 = r1
-    x2, y2 = r2
-    x3, y3 = r3
-
-    a = np.array([[x1, x2, x3], [y1, y2, y3], [1, 1, 1]])
-    b = np.array(L)
-
-    return np.asarray(np.dot(a, b), dtype=np.uint32)
+from utility import import_dataset_module
 
 
 def draw_shape(image, points, triangles, multiply=True):
@@ -173,3 +135,60 @@ def reconstruct_texture(src_image, dst_image, texture_model,
         triangles,
         dst_image
     )
+
+
+def reconstruct_shape_texture(dataset_name, shape_model, texture_model,
+                              image_filename, shape_components,
+                              shape_eigenvalues_multiplier=[],
+                              image_as_background=False):
+    """Reconstructs shape and texture"""
+    dataset_module = import_dataset_module(dataset_name)
+    input_points = dataset_module.factory(filename=image_filename)
+    input_image = input_points.get_image()
+
+    mean_points = dataset_module.factory(points_list=shape_model.mean_values)
+    mean_points.get_scaled_points(input_image.shape)
+
+    shape_eigenvalues_multiplier = np.asarray(
+        shape_eigenvalues_multiplier, dtype=np.float32
+    )
+
+    # set dst image to an empty image if value is None
+    if image_as_background is False:
+        h, w, _ = input_image.shape
+        dst_image = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
+    else:
+        dst_image = input_image
+
+    # get the location of the landmarks in a list of [x,y, ... x_n, y_n]
+    output_points = dataset_module.factory(
+        points_list=input_points.get_points()
+    )
+
+    # get the pca components (ie., V^T)
+    shape_Vt = shape_model.Vt
+
+    # if a eigen value multiplier array is given, scale the Vt with this.
+    # the chosen PCA components will have more impact then others.
+    if len(shape_eigenvalues_multiplier):
+        shape_Vt = scale_eigenvalues(shape_Vt, shape_eigenvalues_multiplier)
+
+    # recontruct the shape
+    reconstruct_shape(
+        output_points,
+        shape_model,
+        shape_Vt=shape_Vt  # overwrite by scaled Vt
+    )
+
+    reconstruct_texture(
+        input_image,  # src image
+        dst_image,  # dst image
+        texture_model,
+        input_points,  # shape points input
+        mean_points,   # shape points mean
+        output_points
+    )
+
+    # output_points.draw_triangles(image=dst_image, show_points=False)
+
+    return dst_image

src/reconstruction/texture.pyx

@@ -116,42 +116,6 @@ cdef inline np.ndarray[double, ndim=2] barycentric2cartesian(
     output[1] = y1 * lambdas[0] + y2 * lambdas[1] + y3 * lambdas[2]
 
 
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def fill_triangle(np.ndarray[unsigned char, ndim=3] src,
-                  np.ndarray[unsigned char, ndim=3] dst,
-                  int x1, int y1, int x2, int y2, int x3, int y3):
-    """
-    Fill a triangle by applying the Barycentric Algorithm for deciding if a
-    point lies inside or outside a triangle.
-    """
-
-    # Get the bounding box of the triangle
-    cdef int x_min = min(x1, min(x2, x3))
-    cdef int x_max = max(x1, max(x2, x3))
-    cdef int y_min = min(y1, min(y2, y3))
-    cdef int y_max = max(y1, max(y2, y3))
-
-    cdef int w = x_max - x_min
-    cdef int h = y_max - y_min
-    cdef int new_offset
-
-    cdef c_array.array dst_loc = c_array.array('f', [0., 0., 0.])
-
-    for j, y in enumerate(xrange(y_min, y_max)):
-        for i, x in enumerate(xrange(x_min, x_max)):
-            cartesian2barycentric(
-                x1, y1, x2, y2, x3, y3, x, y, dst_loc
-            )
-
-            s = dst_loc[0]
-            t = dst_loc[1]
-
-            # In or out the triangle (with a soft margin)
-            if s >= -0.01 and t >= -0.01 and s + t <= 1.001:
-                dst[y, x, :] = src[y, x, :]
-
-
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
@@ -177,10 +141,16 @@ def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
     cdef int y_min = np.argmin(triangle_y)
     cdef int y_max = np.argmax(triangle_y)
 
+    cdef int src_max_dim_x = src.shape[1]
+    cdef int src_max_dim_y = src.shape[0]
+
+    cdef int dst_max_dim_x = dst.shape[1]
+    cdef int dst_max_dim_y = dst.shape[0]
+
     # walk over x and y values of this bounding box see if the
     # pixel is in or out the boudning box
-    for y in xrange(triangle_y[y_min] + 1, triangle_y[y_max] - 1):
-        for x in xrange(triangle_x[x_min] + 1, triangle_x[x_max] - 1):
+    for y in xrange(triangle_y[y_min], triangle_y[y_max]):
+        for x in xrange(triangle_x[x_min], triangle_x[x_max]):
             cartesian2barycentric(
                 triangle_x[0], triangle_y[0],
                 triangle_x[1], triangle_y[1],
@@ -192,11 +162,66 @@ def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
             t = dst_loc[1]
 
             # In or out the triangle (with a soft margin)
-            if s >= -0.01 and t >= -0.01 and s + t <= 1.001:
+            if s >= -0.01 and t >= -0.01 and s + t <= 1.01:
                 barycentric2cartesian(
                     src_x1, src_x2, src_x3,
                     src_y1, src_y2, src_y3,
                     dst_loc, src_loc
                 )
 
-                dst[y, x, :] = src[src_loc[1], src_loc[0], :]
+                #max_dim_y:  188 104.592903137 316 True
+                #max_dim_x:  201 188.147247314 357 True
+                #IndexError: index 188 is out of bounds for axis 1 with size 188
+                #print 'max_dim_y: ', max_dim_y, src_loc[1], y, src_loc[1] < max_dim_y
+                #print 'max_dim_x: ', max_dim_x, src_loc[0], x, src_loc[0] < max_dim_x
+                #print 'together:', src_loc[1] < max_dim_y and src_loc[0] < max_dim_x
+
+                #if src_loc[1] < src_max_dim_y and src_loc[0] < src_max_dim_x:
+                #    print 'yo'
+                #    print src_loc[1], src_loc[0]
+                #    print y, x
+                #    print dst_max_dim_y
+                #    print dst_max_dim_x
+                #    print src_max_dim_y
+                #    print src_max_dim_x
+
+                if src_loc[1] < src_max_dim_y and src_loc[0] < src_max_dim_x \
+                        and y < dst_max_dim_y and x < dst_max_dim_x:
+                    dst[y, x, :] = src[src_loc[1], src_loc[0], :]
+
+
+#@cython.boundscheck(False)
+#@cython.wraparound(False)
+#def fill_triangle(np.ndarray[unsigned char, ndim=3] src,
+#                  np.ndarray[unsigned char, ndim=3] dst,
+#                  int x1, int y1, int x2, int y2, int x3, int y3):
+#    """
+#    Fill a triangle by applying the Barycentric Algorithm for deciding if a
+#    point lies inside or outside a triangle.
+#    """
+#
+#    # Get the bounding box of the triangle
+#    cdef int x_min = min(x1, min(x2, x3))
+#    cdef int x_max = max(x1, max(x2, x3))
+#    cdef int y_min = min(y1, min(y2, y3))
+#    cdef int y_max = max(y1, max(y2, y3))
+#
+#    cdef int w = x_max - x_min
+#    cdef int h = y_max - y_min
+#    cdef int new_offset
+#
+#    cdef c_array.array dst_loc = c_array.array('f', [0., 0., 0.])
+#
+#    for j, y in enumerate(xrange(y_min, y_max)):
+#        for i, x in enumerate(xrange(x_min, x_max)):
+#            cartesian2barycentric(
+#                x1, y1, x2, y2, x3, y3, x, y, dst_loc
+#            )
+#
+#            s = dst_loc[0]
+#            t = dst_loc[1]
+#
+#            # In or out the triangle (with a soft margin)
+#            if s >= -0.01 and t >= -0.01 and s + t <= 1.01:
+#                dst[y, x, :] = src[y, x, :]
+#

src/server.py

+import os
 import json
 import traceback
 import os.path
@@ -8,16 +9,17 @@ import cv2
 import numpy as np
 from tornado import websocket, web, ioloop, autoreload
 
-
 import pca
 from datasets import imm
 from reconstruction import reconstruction
 from settings import logger
 from utility import import_dataset_module
 
+
 FILES_DIR = '/data/'
 FACE_DB_NAME = 'imm_face_db'
 FACE_DB = '{}{}'.format(FILES_DIR, FACE_DB_NAME)
+DATASET = os.environ.get('DATASET', 'ibug')  # see src/datasets for options
 
 
 class ImageWebSocketHandler(websocket.WebSocketHandler):
@@ -31,8 +33,10 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
         self.asf = glob('{}/*.asf'.format(FACE_DB))
 
         # todo get from settings
-        model_texture_file = '{}/pca_ibug_texture_model.npy'.format(FILES_DIR)
-        model_shape_file = '{}/pca_ibug_shape_model.npy'.format(FILES_DIR)
+        model_texture_file = '{}/pca_{}_texture_model.npy'.format(
+            FILES_DIR, DATASET)
+        model_shape_file = '{}/pca_{}_shape_model.npy'.format(
+            FILES_DIR, DATASET)
 
         self.shape_model = pca.PCAModel(model_shape_file)
         self.texture_model = pca.PCAModel(model_texture_file)
@@ -67,55 +71,23 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
         #image = message.get('image')
         #input_image = base64.b64decode(image)
 
-        shape_eigenvalues_multiplier = np.asarray(
-            shape_eigenvalues_multiplier, dtype=np.float32
-        )
-
         logger.info('using %s shape_components', shape_components)
 
-        image_filename = self.images[image_index]
-
-        dataset_module = import_dataset_module('ibug')
-        input_points = dataset_module.factory(filename=image_filename)
-        input_image = input_points.get_image()
-
-        mean_points = dataset_module.factory(points_list=self.shape_model.mean_values)
-        mean_points.get_scaled_points(input_image.shape)
-
-        # set dst image to an empty image if value is None
-        if image_as_background is False:
-            h, w, _ = input_image.shape
-            dst_image = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
+        if DATASET == 'imm':
+            image_filename = self.asf[image_index]
         else:
-            dst_image = input_image
+            image_filename = self.images[image_index]
 
-        output_points = dataset_module.factory(
-            points_list=input_points.get_points()
-        )
-
-        shape_Vt = self.shape_model.Vt
-        shape_Vt = reconstruction.scale_eigenvalues(
-            shape_Vt, shape_eigenvalues_multiplier
-        )
-
-        # recontruct the shape
-        reconstruction.reconstruct_shape(
-            output_points,
+        dst_image = reconstruction.reconstruct_shape_texture(
+            DATASET,
             self.shape_model,
-            shape_Vt=shape_Vt  # overwrite by scaled Vt
-        )
-
-        reconstruction.reconstruct_texture(
-            input_image,  # src image
-            dst_image,  # dst image
             self.texture_model,
-            input_points,  # shape points input
-            mean_points,   # shape points mean
-            output_points
+            image_filename,
+            shape_components,
+            shape_eigenvalues_multiplier,
+            image_as_background=image_as_background
         )
 
-        output_points.draw_triangles(image=dst_image, show_points=False)
-
         _, reconstructed = cv2.imencode('.jpg', dst_image)
         reconstructed = base64.b64encode(reconstructed)
 
@@ -165,16 +137,20 @@ class ApiHandler(web.RequestHandler):
 class FaceHandler(ApiHandler):
     @web.asynchronous
     def get(self, *args):
+        """
+        Get's all faces in the given FACE_DB and returns the url to be able
+        to do requests with it in the webapplication.
+        """
         data = []
 
         for id, filename in enumerate(self.asf_files):
-            Points = imm.IMMPoints(filename)
             data.append({
                 'type': 'faces',
                 'id': id,
                 'attributes': {
-                    'filename': '{}/{}'.format(FACE_DB_NAME, os.path.basename(self.images[id])),
-                    'shape': Points.get_scaled_points((480, 640)).tolist()
+                    'filename': '{}/{}'.format(
+                        FACE_DB_NAME, os.path.basename(self.images[id])
+                    ),
                 }
             })
 

src/test/reconstruction_test.py

+import cv2
+
+import pca as pca
+from settings import logger
+from reconstruction import reconstruction
+
+#model_texture_file = '/data/pca_ibug_texture_model.npy'
+#model_shape_file = '/data/pca_ibug_shape_model.npy'
+
+model_texture_file = '/data/pca_imm_texture_model.npy'
+model_shape_file = '/data/pca_imm_shape_model.npy'
+
+def main():
+    shape_components = 58
+
+    shape_model = pca.PCAModel(model_shape_file)
+    texture_model = pca.PCAModel(model_texture_file)
+
+    logger.info('using %s shape_components', shape_components)
+    image_filename = '/data/imm_face_db/01-1m.asf'
+
+    dst_image = reconstruction.reconstruct_shape_texture(
+        'imm',
+        shape_model,
+        texture_model,
+        image_filename,
+        shape_components
+    )
+
+    cv2.imwrite('/data/reconstructed.png', dst_image)
+
+
+if __name__ == '__main__':
+    main()