Do the reconstruction in a test environment

actions.mk

@@ -6,7 +6,7 @@ data/imm_face_db: data/imm_face_db.tar.gz
 		tar -xvzf imm_face_db.tar.gz -C imm_face_db
 	)
 
-train_model:train_shape
+train_model:train_shape train_texture
 train_texture: data/pca_texture_model.npy
 train_shape: data/pca_shape_model.npy
 

src/aam.py

@@ -5,8 +5,7 @@ import cv2
 
 # local imports
 import pca
-from utils.generate_head_texture import fill_triangle, get_colors_triangle, \
-    get_row_colors_triangle
+from utils.generate_head_texture import fill_triangle, get_row_colors_triangle
 import utils.triangles as tu
 
 logging.basicConfig(level=logging.INFO,
@@ -72,17 +71,29 @@ def build_shape_feature_vectors(files, get_points, flattened=False):
     return points
 
 
-def sample_from_triangles(src, points2d_src, points2d_dst, triangles):
-    # texture = np.asarray(texture, dtype=np.uint8).reshape((-1, 3))
+def sample_from_triangles(src, points2d_src, points2d_dst, triangles, dst):
+    """
+    Get pixels from within the  triangles [[p1, p2, p3]_0, .. [p1, p2, p3]_n].
+    Args:
+        src(ndarray, dtype=uint8): input image
+        points2d_src(ndarray, dtype=np.int32): shape array [[x, y], ... [x, y]]
+        points2d_dst(ndarray, dtype=np.int32): shape array [[x, y], ... [x, y]]
+        triangles(ndarray, ndim=3, dtype=np.int32): shape array [[p1, p2, p3]_0, .. [p1, p2, p3]_n].
+
+    returns:
+        ndarray(dtype=uint8): flattened array of bounding boxes around the
+        given triangles(in order).
+
+    """
+
     triangles_pixels = []
-    pixels = 0
 
     for tri in triangles:
         src_p1, src_p2, src_p3 = points2d_src[tri]
         dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
 
-        dst = get_row_colors_triangle(
-            src,
+        get_row_colors_triangle(
+            src, dst,
             src_p1[0], src_p1[1],
             src_p2[0], src_p2[1],
             src_p3[0], src_p3[1],
@@ -91,13 +102,9 @@ def sample_from_triangles(src, points2d_src, points2d_dst, triangles):
             dst_p3[0], dst_p3[1]
         )
 
-        pixels += dst.flatten().shape[0]
-
-        triangles_pixels.extend(dst.flatten())
+        #triangles_pixels.extend(dst.flatten())
 
-    result = np.asarray(triangles_pixels, dtype=np.uint8)
-
-    return result
+    #return np.asarray(triangles_pixels, dtype=np.uint8)
 
 
 def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, triangles):
@@ -116,6 +123,9 @@ def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, trian
     mean_shape_scaled[:, 0] = mean_shape_scaled[:, 0] * 640
     mean_shape_scaled[:, 1] = mean_shape_scaled[:, 1] * 480
 
+    hull = cv2.convexHull(mean_shape_scaled, returnPoints=True)
+    x, y, w_slice, h_slice = cv2.boundingRect(hull)
+
     for i, f in enumerate(files):
         image, shape = get_image_with_shape(f)
         h, w, c = image.shape
@@ -123,25 +133,22 @@ def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, trian
         shape[:, 0] = shape[:, 0] * w
         shape[:, 1] = shape[:, 1] * h
 
+        dst = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
+
         triangles_colors = sample_from_triangles(
-            image, shape, mean_shape_scaled, triangles
+            image, shape, mean_shape_scaled, triangles, dst
         )
 
-        mean_texture.append(triangles_colors)
-        logger.info('processed file: {} {}/{}'.format(f, i, len(files)))
-
-        # cv2.imshow('image', image)
-        # k = cv2.waitKey(0) & 0xFF
+        dst_flattened = dst[y: y + h_slice, x: x + w_slice].flatten()
+        mean_texture.append(dst_flattened)
 
-        # if k == 27:
-        #     break
-    mean_texture = np.asarray(mean_texture)
-    #mean_texture = pca.flatten_feature_vectors(mean_texture)
+        logger.info('processed file: {} {}/{}'.format(f, i, len(files)))
 
-    return mean_texture
+    return np.asarray(mean_texture)
 
 
 def get_pixel_values(image, points):
+    """ docstring """
     h, w, c = image.shape
 
     points[:, 0] = points[:, 0] * w

src/main.py

@@ -186,9 +186,11 @@ def show_pca_model(args):
     mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
 
     while True:
+        #draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
+        #             mean_values_texture, triangles, n_texture_components)
+        #draw_shape(image, mean_values_shape, triangles, multiply=False)
         draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
                      mean_values_texture, triangles, n_texture_components)
-        #draw_shape(image, mean_values_shape, triangles, multiply=False)
 
         cv2.imshow('input_image', input_image)
         cv2.imshow('image', image)
@@ -200,6 +202,53 @@ def show_pca_model(args):
     cv2.destroyAllWindows()
 
 
+def show_reconstruction(args):
+    assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
+    assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
+
+    from utils.triangles import draw_shape, draw_texture, reconstruct_texture
+
+    Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
+    Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
+
+    image = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
+    image_2 = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
+
+    imm_points = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
+    input_image = imm_points.get_image()
+    input_points = imm_points.get_points()
+    h, w, c = input_image.shape
+
+    input_points[:, 0] = input_points[:, 0] * w
+    input_points[:, 1] = input_points[:, 1] * h
+
+    mean_values_shape = mean_values_shape.reshape((58, 2))
+    mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
+    mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
+
+    while True:
+        #reconstruct_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
+        #                    mean_values_texture, triangles, n_texture_components)
+        reconstruct_texture(input_image, input_image, Vt_texture, input_points, mean_values_shape,
+                            mean_values_texture, triangles, n_texture_components)
+        draw_texture(input_image, image_2, Vt_texture, input_points, mean_values_shape,
+                     mean_values_texture, triangles, n_texture_components)
+
+        #draw_shape(image_2, mean_values_shape, triangles, multiply=False)
+        #draw_shape(input_image, input_points, triangles, multiply=False)
+
+        cv2.imshow('original', imm_points.get_image())
+        cv2.imshow('reconstructed', input_image)
+        cv2.imshow('image_2', image_2)
+
+        k = cv2.waitKey(0) & 0xFF
+
+        if k == 27:
+            break
+
+    cv2.destroyAllWindows()
+
+
 def main():
     """main"""
     parser = add_parser_options()
@@ -212,7 +261,8 @@ def main():
     elif args.save_pca_texture:
         save_pca_model_texture(args)
     elif args.reconstruct:
-        reconstruct_with_model(args)
+        #reconstruct_with_model(args)
+        show_reconstruction(args)
 
 if __name__ == '__main__':
     main()

src/utils/generate_head_texture.pyx

@@ -28,7 +28,12 @@ cdef inline np.ndarray[double, ndim=1] cartesian2barycentric(
         ndarray (of dim 3) weights of the barycentric coordinates
 
     """
-    a = np.array([[r1_x, r2_x, r3_x], [r1_y, r2_y, r3_y], [1, 1, 1]])
+    a = np.array([
+        [r1_x, r2_x, r3_x],
+        [r1_y, r2_y, r3_y],
+        [1, 1, 1]
+    ])
+
     b = np.array([r_x, r_y, 1])
 
     return np.linalg.solve(a, b)
@@ -77,10 +82,7 @@ def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
     cdef int h = y_max - y_min
     cdef int new_offset
 
-    cdef np.ndarray src_reshaped = src[offset:offset + (w * h * 3)].reshape((h, w, 3))
-
-    #print src_reshaped
-    #print '(', w, '*', h, '*', 3, ') * ', ' = ', offset
+    #cdef np.ndarray src_reshaped = src[offset:offset + (w * h * 3)].reshape((h, w, 3))
 
     for j, y in enumerate(xrange(y_min, y_max)):
         for i, x in enumerate(xrange(x_min, x_max)):
@@ -93,40 +95,38 @@ def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
 
             # notice we have a soft margin of -0.00001, which makes sure there are no
             # gaps due to rounding issues
-            if s >= -0.000000001 and t >= -0.000000001 and s + t <= 1.0:
+            if s >= -0.01 and t >= -0.01 and s + t <= 1.0:
                 dst[y, x, :] = src_reshaped[j, i, :]
 
-    new_offset = (w * h * 3)
-    return new_offset
+    return (w * h * 3)
 
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
+def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
+                       np.ndarray[unsigned char, ndim=3] dst,
                        int src_x1, int src_y1, int src_x2, int src_y2,
                        int src_x3, int src_y3,
                        int dst_x1, int dst_y1, int dst_x2, int dst_y2,
-                            int dst_x3, int dst_y3):
-    """
-    Fill a triangle by applying the Barycentric Algorithm for deciding if a
-    point lies inside or outside a triangle.
-    """
-    cdef int x_min = min(dst_x1, min(dst_x2, dst_x3))
-    cdef int x_max = max(dst_x1, max(dst_x2, dst_x3))
-    cdef int y_min = min(dst_y1, min(dst_y2, dst_y3))
-    cdef int y_max = max(dst_y1, max(dst_y2, dst_y3))
+                       int dst_x3, int dst_y3,
+                       int offset_x, int offset_y):
+
+    cdef np.ndarray triangle_x = np.array([dst_x1, dst_x2, dst_x3])
+    cdef np.ndarray triangle_y = np.array([dst_y1, dst_y2, dst_y3])
+
+    cdef int x_min = np.argmin(triangle_x)
+    cdef int x_max = np.argmax(triangle_x)
+    cdef int y_min = np.argmin(triangle_y)
+    cdef int y_max = np.argmax(triangle_y)
 
     cdef np.ndarray L = np.zeros([3, 1], dtype=DTYPE_float32)
     cdef np.ndarray matrix = np.full([3, 3], fill_value=1, dtype=DTYPE_int)
 
     cdef np.ndarray src_loc = np.zeros([3, 1], dtype=DTYPE_float64)
     cdef np.ndarray dst_loc = np.zeros([3, 1], dtype=DTYPE_float64)
-    cdef np.ndarray dst = np.full(
-        [y_max - y_min, x_max - x_min, 3], fill_value=255, dtype=DTYPE_float64
-    )
 
-    for j, y in enumerate(xrange(y_min, y_max)):
-        for i, x in enumerate(xrange(x_min, x_max)):
+    for j, y in enumerate(xrange(triangle_y[y_min], triangle_y[y_max])):
+        for i, x in enumerate(xrange(triangle_x[x_min], triangle_x[x_max])):
             dst_loc = cartesian2barycentric(
                 dst_x1, dst_y1, dst_x2, dst_y2, dst_x3, dst_y3, x, y
             )
@@ -134,9 +134,7 @@ def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
             s = dst_loc[0]
             t = dst_loc[1]
 
-            # notice we have a soft margin of -0.00001, which makes sure there are no
-            # gaps due to rounding issues
-            if s >= -0.000001 and t >= -0.000001 and s + t <= 1.0:
+            if s >= -0.001 and t >= -0.001 and s + t <= 1.001:
                 L[0] = s
                 L[1] = t
                 L[2] = 1 - s - t
@@ -148,14 +146,12 @@ def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
                     L
                 )
 
-                dst[j, i, :] = src[src_loc[1][0], src_loc[0][0], :]
-
-    return dst
+                dst[y, x, :] = src[src_loc[1][0], src_loc[0][0], :]
 
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def get_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
+def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
                             np.ndarray[unsigned char, ndim=3] dst,
                             int src_x1, int src_y1, int src_x2, int src_y2,
                             int src_x3, int src_y3,
@@ -165,13 +161,13 @@ def get_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
     Fill a triangle by applying the Barycentric Algorithm for deciding if a
     point lies inside or outside a triangle.
     """
-    cdef int x_min = min(dst_x1, min(dst_x2, dst_x3))
-    cdef int x_max = max(dst_x1, max(dst_x2, dst_x3))
-    cdef int y_min = min(dst_y1, min(dst_y2, dst_y3))
-    cdef int y_max = max(dst_y1, max(dst_y2, dst_y3))
+    cdef np.ndarray triangle_x = np.array([dst_x1, dst_x2, dst_x3])
+    cdef np.ndarray triangle_y = np.array([dst_y1, dst_y2, dst_y3])
 
-    cdef float s
-    cdef float t
+    cdef int x_min = np.argmin(triangle_x)
+    cdef int x_max = np.argmax(triangle_x)
+    cdef int y_min = np.argmin(triangle_y)
+    cdef int y_max = np.argmax(triangle_y)
 
     cdef np.ndarray L = np.zeros([3, 1], dtype=DTYPE_float32)
     cdef np.ndarray matrix = np.full([3, 3], fill_value=1, dtype=DTYPE_int)
@@ -179,18 +175,19 @@ def get_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
     cdef np.ndarray src_loc = np.zeros([3, 1], dtype=DTYPE_float64)
     cdef np.ndarray dst_loc = np.zeros([3, 1], dtype=DTYPE_float64)
 
-    for y in xrange(y_min, y_max):
-        for x in xrange(x_min, x_max):
+    for j, y in enumerate(xrange(triangle_y[y_min], triangle_y[y_max])):
+        for i, x in enumerate(xrange(triangle_x[x_min], triangle_x[x_max])):
             dst_loc = cartesian2barycentric(
-                dst_x1, dst_y1, dst_x2, dst_y2, dst_x3, dst_y3, x, y
+                triangle_x[0], triangle_y[0],
+                triangle_x[1], triangle_y[1],
+                triangle_x[2], triangle_y[2],
+                x, y
             )
 
             s = dst_loc[0]
             t = dst_loc[1]
 
-            # notice we have a soft margin of -0.00001, which makes sure there are no
-            # gaps due to rounding issues
-            if s >= -0.000001 and t >= -0.000001 and s + t <= 1.0:
+            if s >= -0.01 and t >= -0.01 and s + t <= 1.001:
                 L[0] = s
                 L[1] = t
                 L[2] = 1 - s - t

src/utils/triangles.py

 import numpy as np
 import cv2
 
-from utils.generate_head_texture import fill_triangle, get_colors_triangle
+from utils.generate_head_texture import fill_triangle, fill_triangle_src_dst
 
+import pca
+import aam
 
 def cartesian2barycentric(r1, r2, r3, r):
     """
@@ -62,25 +64,66 @@ def draw_shape(image, points, triangles, multiply=True):
 
     for i, p in enumerate(points):
         point_index = int(point_indices[i])
-        cv2.putText(image, str(point_index), (p[0], p[1]),
+        #cv2.putText(image, str(point_index), (p[0], p[1]),
+        #            cv2.FONT_HERSHEY_SIMPLEX, .5, (100, 0, 255))
+        cv2.putText(image, str(i), (p[0], p[1]),
                     cv2.FONT_HERSHEY_SIMPLEX, .5, (100, 0, 255))
         cv2.circle(image, tuple(p), 3, color=(0, 255, 100))
 
 
-def draw_texture(src, dest, Vt, points2d_src, points2d_dst, texture, triangles, n_components):
-    # texture = np.asarray(texture, dtype=np.uint8).reshape((-1, 3))
+def draw_texture(src, dst, Vt, points2d_src, points2d_dst, texture, triangles, n_components):
     texture = np.asarray(texture, np.uint8)
     offset = 0
 
     for t, tri in enumerate(triangles):
-        src_p1, src_p2, src_p3 = points2d_src[tri]
         dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
 
         offset += fill_triangle(
-            texture, dest,
+            texture, dst,
             dst_p1[0], dst_p1[1],
             dst_p2[0], dst_p2[1],
             dst_p3[0], dst_p3[1],
             offset,
             t
         )
+
+
+def reconstruct_texture(src, dst, Vt, points2d_src, points2d_dst, mean_texture, triangles, n_components):
+    # S_mean format
+    h, w, c = src.shape
+    input_texture = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
+    aam.sample_from_triangles(src, points2d_src, points2d_dst, triangles, input_texture)
+
+    hull = cv2.convexHull(points2d_dst, returnPoints=True)
+    offset_x, offset_y, w_slice, h_slice = cv2.boundingRect(hull)
+
+    input_texture = input_texture[offset_y: offset_y + h_slice,
+                                  offset_x: offset_x + w_slice].flatten()
+
+    # Still in  S_mean format
+    r_texture = pca.reconstruct(input_texture, Vt, mean_texture)
+
+    # Make an image from the float data
+    r_texture = np.asarray(r_texture, np.uint8).reshape((h_slice, w_slice, 3))
+
+    offset = 0
+
+    # subtract the offset
+    points2d_dst[:, 0] -= offset_x
+    points2d_dst[:, 1] -= offset_y
+
+    for t, tri in enumerate(triangles):
+        src_p1, src_p2, src_p3 = points2d_src[tri]
+        dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
+
+        fill_triangle_src_dst(
+            r_texture, src,
+            dst_p1[0], dst_p1[1],
+            dst_p2[0], dst_p2[1],
+            dst_p3[0], dst_p3[1],
+            src_p1[0], src_p1[1],
+            src_p2[0], src_p2[1],
+            src_p3[0], src_p3[1],
+            offset_x,
+            offset_y
+        )

src/view/reconstruct.py

@@ -77,42 +77,43 @@ class ImageCanvas(Widget):
         self.image.source = self.filename_image
         self.canvas.ask_update()
 
-    def build_texture(self, r_shape, r_texture, triangles):
-        return
-        #self.texture.clear()
-
-        #image_width, image_height = self.get_rendered_size()
-
-        #bary_centric_range = np.linspace(0, 1, num=20)
-        #texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
-        #buf = np.zeros((image_width, image_height, 3), dtype=np.uint8)
-
-        #offset = 0
-
-        #for t, tri in enumerate(triangles):
-        #    src_p1, src_p2, src_p3 = r_shape[tri]
-
-        #    x = r_shape[:, 0] * image_width + self.get_image_left(image_width)
-        #    y = (1.0 - r_shape[:, 1]) * image_height + self.get_image_bottom(image_height)
-
-        #    for t, tri in enumerate(triangles):
-        #        offset += fill_triangle(
-        #            r_texture, buf,
-        #            src_p1[0], src_p1[1],
-        #            src_p2[0], src_p2[1],
-        #            src_p3[0], src_p3[1],
-        #            dst_p1[0], dst_p1[1],
-        #            dst_p2[0], dst_p2[1],
-        #            dst_p3[0], dst_p3[1],
-        #            offset,
-        #            t
-        #        )
-
-        #        print offset
-
-        #    #p1 = [x[0], y[0]]
-        #    #p2 = [x[1], y[1]]
-        #    #p3 = [x[2], y[2]]
+    def build_texture(self, mean_shape, r_shape, r_texture, triangles):
+        self.texture.clear()
+        image_width, image_height = self.get_rendered_size()
+        bary_centric_range = np.linspace(0, 1, num=20)
+
+        texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
+        buf = np.zeros((image_width, image_height, 3), dtype=np.uint8)
+
+        offset = 0
+
+        for t, tri in enumerate(triangles):
+            points = r_shape[tri]
+
+            x = points[:, 0] * image_width + self.get_image_left(image_width)
+            y = (1.0 - points[:, 1]) * image_height + self.get_image_bottom(image_height)
+
+            src_p1_x, src_p2_x, src_p3_x = x
+            src_p1_y, src_p2_y, src_p3_y = y
+
+            for t, tri in enumerate(triangles):
+                offset += fill_triangle(
+                    r_texture, buf,
+                    src_p1_x, src_p1_y,
+                    src_p2_x, src_p2_y,
+                    src_p3_x, src_p3_y,
+                    dst_p1_x, dst_p1_y,
+                    dst_p2_x, dst_p2_y,
+                    dst_p3_x, dst_p3_y,
+                    offset,
+                    t
+                )
+
+                print offset
+
+            #p1 = [x[0], y[0]]
+            #p2 = [x[1], y[1]]
+            #p3 = [x[2], y[2]]
 
 
         ##buf = b''.join(map(chr, buf))