Made the scaling operations transparent by means of abstraction

makefile

@@ -11,9 +11,9 @@ all: $(TARGETS)
 include actions.mk
 
 data: data/imm_face_db
-utils: generate_head_texture.so
+utils: texture.so
 
-generate_head_texture.so: src/utils/generate_head_texture.pyx
+texture.so: src/utils/texture.pyx
 	(cd src/utils; python setup.py build_ext --inplace)
 
 

src/aam.py

@@ -5,7 +5,7 @@ import cv2
 
 # local imports
 import pca
-from utils.generate_head_texture import fill_triangle, get_row_colors_triangle
+import utils.texture as tx
 import utils.triangles as tu
 
 logging.basicConfig(level=logging.INFO,
@@ -13,6 +13,56 @@ logging.basicConfig(level=logging.INFO,
 logger = logging.getLogger(__name__)
 
 
+class AAMPoints():
+    """
+    Object to store AAM points / landmarks. Tries to keep the scaling of
+    the shape parameters transparent.
+    """
+    def __init__(self, normalized_flattened_points_list=None, points_list=None, actual_shape=()):
+        self.normalized_flattened_points_list = normalized_flattened_points_list
+        self.points_list = points_list
+        self.actual_shape = actual_shape
+        self.bounding_box = None
+
+    def get_bounding_box(self):
+        if self.bounding_box is None:
+            return self.calculate_bounding_box()
+        return self.bounding_box
+
+    def get_scaled_points(self, shape, rescale=True):
+        if self.points_list is None:
+            self.points_list = self.normalized_flattened_points_list
+
+            if len(self.actual_shape):
+                self.points_list = self.points_list.reshape(self.actual_shape)
+
+            h = shape[0]
+            w = shape[1]
+
+            self.points_list[:, 0] = self.points_list[:, 0] * w
+            self.points_list[:, 1] = self.points_list[:, 1] * h
+
+        return self.points_list
+
+    def calculate_bounding_box(self):
+        """
+        Calculate bounding box in the **scaled** points list
+        The empasis on on scaled because the convexHull does not support
+        small values, the normalized_flattened_points_list does not work.
+        """
+        assert self.points_list is not None, \
+            'the list points already need to be scaled order to correctly work'
+
+        hull = cv2.convexHull(self.points_list, returnPoints=True)
+        return cv2.boundingRect(hull)
+
+    def get_bounding_box_2(self):
+        pass
+        #hull = cv2.convexHull(self.points_list, returnPoints=True)
+        #x, y, w_slice, h_slice = cv2.boundingRect(hull)
+
+        #return cv2.boundingRect()
+
 def get_mean(vector):
     """ construct a mean from a matrix of x,y values
     Args:
@@ -80,19 +130,12 @@ def sample_from_triangles(src, points2d_src, points2d_dst, triangles, dst):
         points2d_dst(ndarray, dtype=np.int32): shape array [[x, y], ... [x, y]]
         triangles(ndarray, ndim=3, dtype=np.int32): shape array [[p1, p2, p3]_0, .. [p1, p2, p3]_n].
 
-    returns:
-        ndarray(dtype=uint8): flattened array of bounding boxes around the
-        given triangles(in order).
-
     """
-
-    triangles_pixels = []
-
     for tri in triangles:
         src_p1, src_p2, src_p3 = points2d_src[tri]
         dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
 
-        get_row_colors_triangle(
+        tx.fill_triangle_src_dst(
             src, dst,
             src_p1[0], src_p1[1],
             src_p2[0], src_p2[1],
@@ -102,41 +145,38 @@ def sample_from_triangles(src, points2d_src, points2d_dst, triangles, dst):
             dst_p3[0], dst_p3[1]
         )
 
-        #triangles_pixels.extend(dst.flatten())
 
-    #return np.asarray(triangles_pixels, dtype=np.uint8)
-
-
-def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, triangles):
+def build_texture_feature_vectors(files, get_image_with_points, MeanPoints, triangles):
     """
     Args:
         files (list): list files
         flattened (bool): Flatten the inner feature vectors, see
             flatten_feature_vectors.
+        MeanPoints(AAMPoints): AAMPoints object
 
     Returns:
         list: list of feature vectors
     """
     mean_texture = []
-
-    mean_shape_scaled = mean_shape.reshape((58, 2))
-    mean_shape_scaled[:, 0] = mean_shape_scaled[:, 0] * 640
-    mean_shape_scaled[:, 1] = mean_shape_scaled[:, 1] * 480
-
-    hull = cv2.convexHull(mean_shape_scaled, returnPoints=True)
-    x, y, w_slice, h_slice = cv2.boundingRect(hull)
+    x, y, w_slice, h_slice = MeanPoints.get_bounding_box()
 
     for i, f in enumerate(files):
-        image, shape = get_image_with_shape(f)
-        h, w, c = image.shape
+        image, points = get_image_with_points(f)
 
-        shape[:, 0] = shape[:, 0] * w
-        shape[:, 1] = shape[:, 1] * h
+        Points = AAMPoints(
+            normalized_flattened_points_list=points,
+            actual_shape=(58, 2)
+        )
 
-        dst = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
+        # empty colored image
+        dst = np.full((image.shape[0], image.shape[1], 3), fill_value=0, dtype=np.uint8)
 
-        triangles_colors = sample_from_triangles(
-            image, shape, mean_shape_scaled, triangles, dst
+        sample_from_triangles(
+            image,
+            Points.get_scaled_points(image.shape),
+            MeanPoints(image.shape),
+            triangles,
+            dst
         )
 
         dst_flattened = dst[y: y + h_slice, x: x + w_slice].flatten()

src/aam_test.py

@@ -2,10 +2,11 @@ import numpy as np
 import cv2
 import pytest
 
-import imm_points
 import aam
 import pca
+import imm_points as imm
 
+from utils import triangles as tri
 
 def test_build_mean_aan():
     imm_points = np.array([
@@ -45,10 +46,39 @@ def test_zero_mean_aan():
     np.testing.assert_array_equal(zero_mean, expected)
 
 
+def test_build_texture_feature_vectors():
+    Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load('data/test_data/pca_shape_model.npy')
+    Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load('data/test_data/pca_texture_model.npy')
+
+    InputPoints = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
+    input_image = InputPoints.get_image()
+
+    MeanPoints = imm.IMMPoints(points_list=mean_value_points)
+    mean_points = MeanPoints.get_scaled_points(input_image.shape)
+    input_points = InputPoints.get_scaled_points(input_image.shape)
+
+    tri.reconstruct_texture(input_image, input_image, Vt_texture, input_points, mean_points,
+                            mean_values_texture, triangles, n_texture_components)
+    dst = tri.get_texture(mean_points, mean_values_texture)
+
+    assert np.mean(input_points) > 1.0, 'should be greater than 1.0, because \
+        it array should be scaled to the image width and height'
+    assert np.mean(mean_points) > 1.0, 'should be greater than 1.0, because \
+        it array should be scaled to the image width and height'
+
+    #cv2.imshow('original', imm_points.get_image())
+    #cv2.imshow('reconstructed', input_image)
+    #cv2.imshow('main face', dst)
+
+    #cv2.waitKey(0) & 0xFF
+
+    #cv2.destroyAllWindows()
+
+
 @pytest.mark.skipif(True, reason='not suitable for pytest')
 def test_get_pixel_values():
     asf_file = '../data/imm_face_db/40-2m.asf'
-    imm = imm_points.IMMPoints(filename=asf_file)
+    Vt, s, n_components, mean_shape, triangles = pca.load(args.model_shape_file)
 
     points = imm.get_points()
     image = imm.get_image()

src/imm_points.py

@@ -5,28 +5,39 @@ import numpy as np
 import argparse
 import os
 
+import aam
 
-class IMMPoints():
+
+class IMMPoints(aam.AAMPoints):
     """Accepts IMM datapoint file which can be shown or used"""
-    def __init__(self, filename=None, points=None):
+    def __init__(self, filename=None, points_list=None):
         """
         Args:
             filename: optional .asf file with the imm format
             points: optional list of x,y points
         """
-        self.points = points if points is not None else []
+        assert filename is not None or points_list is not None, 'filename or \
+         a ndarray of points list should be given'
+
         self.filename = filename
 
         if filename:
-            self.import_file(filename)
+            points_list = self.import_file(filename)
+
+        aam.AAMPoints.__init__(
+            self, normalized_flattened_points_list=points_list.flatten(),
+            actual_shape=(58, 2)
+        )
 
     def get_points(self):
-        return self.points
+        return self.points_list
 
     def get_image(self):
         return cv2.imread(self.image_file)
 
     def import_file(self, filename):
+        points_list = []
+
         with open(filename, 'r') as f:
             lines = f.readlines()
             data = lines[16:74]
@@ -34,9 +45,9 @@ class IMMPoints():
             self.image_file = "{}/{}".format(dir_name, lines[-1].strip())
 
             for d in data:
-                self.points.append(d.split()[2:4])
+                points_list.append(d.split()[2:4])
 
-        self.points = np.asarray(self.points, dtype='f')
+        return np.asarray(points_list, dtype='f')
 
     def draw_triangles(self, image, points, multiply=True):
         if multiply:
@@ -61,16 +72,16 @@ class IMMPoints():
             cv2.circle(image, tuple(p), 3, color=(0, 255, 100))
 
     def show_on_image(self, image, window_name='image', multiply=True):
-        self.draw_triangles(image, self.points, multiply=multiply)
+        self.draw_triangles(image, self.points_list, multiply=multiply)
 
     def show(self, window_name='image'):
         """show the image and datapoints on the image"""
-        assert(len(self.points) > 0)
+        assert(len(self.points_list) > 0)
         assert(len(self.filename) > 0)
 
         image = self.get_image()
 
-        self.draw_triangles(image, self.points)
+        self.draw_triangles(image, self.points_list)
 
 
 def get_imm_points(files):

src/main.py

@@ -86,8 +86,13 @@ def save_pca_model_texture(args):
 
     Vt, s, n_components, mean_shape, triangles = pca.load(args.model_shape_file)
 
+    MeanPoints = aam.AAMPoints(
+        normalized_flattened_points_list=mean_value_points,
+        actual_shape=(58, 2)
+    )
+
     textures = aam.build_texture_feature_vectors(
-        args.files, imm.get_imm_image_with_landmarks, mean_shape, triangles
+        args.files, imm.get_imm_image_with_landmarks, MeanPoints, triangles
     )
 
     mean_texture = aam.get_mean(textures)
@@ -166,13 +171,11 @@ def show_pca_model(args):
     assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
     assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
 
-    from utils.triangles import draw_shape, draw_texture
+    from utils.triangles import draw_shape, get_texture
 
     Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
     Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
 
-    image = np.full((480, 640, 3), fill_value=255, dtype=np.uint8)
-
     imm_points = imm.IMMPoints(filename='data/imm_face_db/40-1m.asf')
     input_image = imm_points.get_image()
     input_points = imm_points.get_points()
@@ -186,14 +189,10 @@ def show_pca_model(args):
     mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
 
     while True:
-        #draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
-        #             mean_values_texture, triangles, n_texture_components)
-        #draw_shape(image, mean_values_shape, triangles, multiply=False)
-        draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
-                     mean_values_texture, triangles, n_texture_components)
+        dst = get_texture(mean_values_shape, mean_values_texture)
 
         cv2.imshow('input_image', input_image)
-        cv2.imshow('image', image)
+        cv2.imshow('image', dst)
         k = cv2.waitKey(0) & 0xFF
 
         if k == 27:
@@ -206,40 +205,26 @@ def show_reconstruction(args):
     assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
     assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
 
-    from utils.triangles import draw_shape, draw_texture, reconstruct_texture
+    from utils import triangles as tri
 
-    Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
+    Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
     Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
 
-    image = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
-    image_2 = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
+    InputPoints = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
+    input_image = InputPoints.get_image()
 
-    imm_points = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
-    input_image = imm_points.get_image()
-    input_points = imm_points.get_points()
-    h, w, c = input_image.shape
-
-    input_points[:, 0] = input_points[:, 0] * w
-    input_points[:, 1] = input_points[:, 1] * h
-
-    mean_values_shape = mean_values_shape.reshape((58, 2))
-    mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
-    mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
+    MeanPoints = imm.IMMPoints(points_list=mean_value_points)
+    MeanPoints.get_scaled_points(input_image.shape)
 
     while True:
-        #reconstruct_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
-        #                    mean_values_texture, triangles, n_texture_components)
-        reconstruct_texture(input_image, input_image, Vt_texture, input_points, mean_values_shape,
-                            mean_values_texture, triangles, n_texture_components)
-        draw_texture(input_image, image_2, Vt_texture, input_points, mean_values_shape,
-                     mean_values_texture, triangles, n_texture_components)
-
-        #draw_shape(image_2, mean_values_shape, triangles, multiply=False)
-        #draw_shape(input_image, input_points, triangles, multiply=False)
+        tri.reconstruct_texture(input_image, input_image, Vt_texture,
+                                InputPoints, MeanPoints,
+                                mean_values_texture, triangles, n_texture_components)
+        dst = tri.get_texture(MeanPoints, mean_values_texture)
 
-        cv2.imshow('original', imm_points.get_image())
+        cv2.imshow('original', InputPoints.get_image())
         cv2.imshow('reconstructed', input_image)
-        cv2.imshow('image_2', image_2)
+        cv2.imshow('main face', dst)
 
         k = cv2.waitKey(0) & 0xFF
 

src/utils/setup.py

@@ -5,8 +5,8 @@ from distutils.extension import Extension
 from Cython.Build import cythonize
 extensions = [
     Extension(
-        'generate_head_texture',
-        ['generate_head_texture.pyx'],
+        'texture',
+        ['texture.pyx'],
         include_dirs=[np.get_include()], )
 ]
 

src/utils/generate_head_texture.pyx → src/utils/texture.pyx

@@ -57,10 +57,9 @@ cdef inline np.ndarray[double, ndim=2] barycentric2cartesian(
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
+def fill_triangle(np.ndarray[unsigned char, ndim=3] src,
                   np.ndarray[unsigned char, ndim=3] dst,
-                  int x1, int y1, int x2, int y2, int x3, int y3, int offset,
-                  int index):
+                  int x1, int y1, int x2, int y2, int x3, int y3):
     """
     Fill a triangle by applying the Barycentric Algorithm for deciding if a
     point lies inside or outside a triangle.
@@ -96,20 +95,18 @@ def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
             # notice we have a soft margin of -0.00001, which makes sure there are no
             # gaps due to rounding issues
             if s >= -0.01 and t >= -0.01 and s + t <= 1.0:
-                dst[y, x, :] = src_reshaped[j, i, :]
-
-    return (w * h * 3)
+                dst[y, x, :] = src[y, x, :]
 
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
-                       np.ndarray[unsigned char, ndim=3] dst,
-                       int src_x1, int src_y1, int src_x2, int src_y2,
-                       int src_x3, int src_y3,
-                       int dst_x1, int dst_y1, int dst_x2, int dst_y2,
-                       int dst_x3, int dst_y3,
-                       int offset_x, int offset_y):
+                          np.ndarray[unsigned char, ndim=3] dst,
+                          int src_x1, int src_y1, int src_x2, int src_y2,
+                          int src_x3, int src_y3,
+                          int dst_x1, int dst_y1, int dst_x2, int dst_y2,
+                          int dst_x3, int dst_y3,
+                          int offset_x, int offset_y):
 
     cdef np.ndarray triangle_x = np.array([dst_x1, dst_x2, dst_x3])
     cdef np.ndarray triangle_y = np.array([dst_y1, dst_y2, dst_y3])
@@ -151,12 +148,12 @@ def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
-                            np.ndarray[unsigned char, ndim=3] dst,
-                            int src_x1, int src_y1, int src_x2, int src_y2,
-                            int src_x3, int src_y3,
-                            int dst_x1, int dst_y1, int dst_x2, int dst_y2,
-                            int dst_x3, int dst_y3):
+def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
+                          np.ndarray[unsigned char, ndim=3] dst,
+                          int src_x1, int src_y1, int src_x2, int src_y2,
+                          int src_x3, int src_y3,
+                          int dst_x1, int dst_y1, int dst_x2, int dst_y2,
+                          int dst_x3, int dst_y3):
     """
     Fill a triangle by applying the Barycentric Algorithm for deciding if a
     point lies inside or outside a triangle.

src/utils/triangles.py

@@ -71,48 +71,45 @@ def draw_shape(image, points, triangles, multiply=True):
         cv2.circle(image, tuple(p), 3, color=(0, 255, 100))
 
 
-def draw_texture(src, dst, Vt, points2d_src, points2d_dst, texture, triangles, n_components):
-    texture = np.asarray(texture, np.uint8)
-    offset = 0
+def get_texture(Points, flattened_texture):
+    offset_x, offset_y, w_slice, h_slice = Points.get_bounding_box()
 
-    for t, tri in enumerate(triangles):
-        dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
-
-        offset += fill_triangle(
-            texture, dst,
-            dst_p1[0], dst_p1[1],
-            dst_p2[0], dst_p2[1],
-            dst_p3[0], dst_p3[1],
-            offset,
-            t
-        )
+    # Make a rectangle image from the flattened texture array
+    return np.asarray(flattened_texture, np.uint8).reshape((h_slice, w_slice, 3))
 
 
-def reconstruct_texture(src, dst, Vt, points2d_src, points2d_dst, mean_texture, triangles, n_components):
+def reconstruct_texture(src, dst, Vt, SrcPoints, DstPoints,
+                        mean_texture, triangles, n_components):
     # S_mean format
     h, w, c = src.shape
     input_texture = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
-    aam.sample_from_triangles(src, points2d_src, points2d_dst, triangles, input_texture)
 
-    hull = cv2.convexHull(points2d_dst, returnPoints=True)
-    offset_x, offset_y, w_slice, h_slice = cv2.boundingRect(hull)
+    points2d_src = SrcPoints.get_scaled_points(src.shape)
+    points2d_dst = DstPoints.get_scaled_points(dst.shape)
+
+    aam.sample_from_triangles(
+        src,
+        points2d_src,
+        points2d_dst,
+        triangles,
+        input_texture
+    )
 
+    offset_x, offset_y, w_slice, h_slice = DstPoints.get_bounding_box()
     input_texture = input_texture[offset_y: offset_y + h_slice,
                                   offset_x: offset_x + w_slice].flatten()
 
-    # Still in  S_mean format
+    ## Still in  S_mean format
     r_texture = pca.reconstruct(input_texture, Vt, mean_texture)
 
     # Make an image from the float data
     r_texture = np.asarray(r_texture, np.uint8).reshape((h_slice, w_slice, 3))
 
-    offset = 0
-
-    # subtract the offset
+    ## subtract the offset
     points2d_dst[:, 0] -= offset_x
     points2d_dst[:, 1] -= offset_y
 
-    for t, tri in enumerate(triangles):
+    for tri in triangles:
         src_p1, src_p2, src_p3 = points2d_src[tri]
         dst_p1, dst_p2, dst_p3 = points2d_dst[tri]