Renamed, refactored and attempt to make kivy work with texturing

actions.mk

@@ -49,6 +49,14 @@ show_reconstruction:
 		--model_shape_file data/pca_shape_model.npy \
 		--n_components 6
 
+show_kivy:
+	python src/main.py \
+		--show_kivy \
+		--files data/imm_face_db/*.asf \
+		--model_texture_file data/pca_texture_model.npy \
+		--model_shape_file data/pca_shape_model.npy \
+		--n_components 6
+
 test:
 	python -m py.test -f src/*_test.py
 

src/aam.py

@@ -6,17 +6,19 @@ import cv2
 # local imports
 import pca
 import utils.texture as tx
-import utils.triangles as tu
+from utils import utils
 
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s %(levelname)s %(name)s: %(message)s')
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s %(levelname)s %(name)s: %(message)s'
+)
 logger = logging.getLogger(__name__)
 
 
 class AAMPoints():
     """
     Object to store AAM points / landmarks. Tries to keep the scaling of
-    the shape parameters transparent.
+    these points transparent.
     """
     def __init__(self, normalized_flattened_points_list=None, points_list=None, actual_shape=()):
         self.normalized_flattened_points_list = normalized_flattened_points_list
@@ -25,12 +27,27 @@ class AAMPoints():
         self.bounding_box = None
 
     def get_bounding_box(self):
+        """
+        Get the bounding box around the points.
+        """
         if self.bounding_box is None:
             return self.calculate_bounding_box()
+
         return self.bounding_box
 
-    def get_scaled_points(self, shape, rescale=True):
-        if self.points_list is None:
+    def get_scaled_points(self, shape, rescale=False):
+        """
+        Scale the normalized flattened points list to a scale given by 'shape'.
+        The x and y values should be scaled to the width and height of the image.
+        Args:
+            shape(tuple): (height, width)
+            rescal(boolean): flag if we should rescale or not because if we
+                already scaled, we are not going to do it again by
+                default.
+        Returns:
+            ndarray scaled to 'shape' width and height.
+        """
+        if self.points_list is None or rescale:
             self.points_list = self.normalized_flattened_points_list
 
             if len(self.actual_shape):
@@ -49,6 +66,10 @@ class AAMPoints():
         Calculate bounding box in the **scaled** points list
         The empasis on on scaled because the convexHull does not support
         small values, the normalized_flattened_points_list does not work.
+
+        Use get_scaled_points first, with a shape that is needed. The shape
+        should be the dimensions of the out image, example (480, 640), ie., (height,
+        width)
         """
         assert self.points_list is not None, \
             'the list points already need to be scaled order to correctly work'
@@ -201,7 +222,6 @@ def get_pixel_values(image, points):
 
     x, y, w, h = rect
 
-    # pixels = np.zeros((h, w, c), dtype=np.uint8)
     for i in np.linspace(0, 1, num=150):
         for j in np.linspace(0, 1, num=150):
             y_loc_g = int(i * h + y)

src/imm_points.py

@@ -30,7 +30,7 @@ class IMMPoints(aam.AAMPoints):
         )
 
     def get_points(self):
-        return self.points_list
+        return self.normalized_flattened_points_list
 
     def get_image(self):
         return cv2.imread(self.image_file)

src/main.py

@@ -5,13 +5,14 @@ import sys
 
 # installed packages
 import cv2
-import numpy as np
 
 # local imports
 import pca
 import aam
 import imm_points as imm
 
+from utils import utils
+
 logging.basicConfig(level=logging.INFO,
                     format='%(asctime)s %(levelname)s %(name)s: %(message)s')
 logger = logging.getLogger(__name__)
@@ -26,6 +27,11 @@ def add_parser_options():
         help='Reconstruct one face with a given pca model'
     )
 
+    pca_group.add_argument(
+        '--show_kivy', action='store_true',
+        help='Reconstruct using kivy as a GUI'
+    )
+
     pca_group.add_argument(
         '--save_pca_shape', action='store_true',
         help='save the pca shape model'
@@ -148,7 +154,7 @@ def reconstruct_with_model(args):
 
     from view.reconstruct import ReconstructApp
 
-    Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
+    Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
     Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
 
     app = ReconstructApp()
@@ -157,7 +163,7 @@ def reconstruct_with_model(args):
         args=args,
         eigenv_shape=Vt_shape,
         eigenv_texture=Vt_texture,
-        mean_values_shape=mean_values_shape,
+        mean_value_points=mean_value_points,
         n_shape_components=n_shape_components,
         mean_values_texture=mean_values_texture,
         n_texture_components=n_texture_components,
@@ -173,7 +179,7 @@ def show_pca_model(args):
 
     from utils.triangles import draw_shape, get_texture
 
-    Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
+    Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
     Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
 
     imm_points = imm.IMMPoints(filename='data/imm_face_db/40-1m.asf')
@@ -184,12 +190,12 @@ def show_pca_model(args):
     input_points[:, 0] = input_points[:, 0] * w
     input_points[:, 1] = input_points[:, 1] * h
 
-    mean_values_shape = mean_values_shape.reshape((58, 2))
-    mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
-    mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
+    mean_value_points = mean_value_points.reshape((58, 2))
+    mean_value_points[:, 0] = mean_value_points[:, 0] * w
+    mean_value_points[:, 1] = mean_value_points[:, 1] * h
 
     while True:
-        dst = get_texture(mean_values_shape, mean_values_texture)
+        dst = get_texture(mean_value_points, mean_values_texture)
 
         cv2.imshow('input_image', input_image)
         cv2.imshow('image', dst)
@@ -205,8 +211,6 @@ def show_reconstruction(args):
     assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
     assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
 
-    from utils import triangles as tri
-
     Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
     Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
 
@@ -217,10 +221,18 @@ def show_reconstruction(args):
     MeanPoints.get_scaled_points(input_image.shape)
 
     while True:
-        tri.reconstruct_texture(input_image, input_image, Vt_texture,
-                                InputPoints, MeanPoints,
-                                mean_values_texture, triangles, n_texture_components)
-        dst = tri.get_texture(MeanPoints, mean_values_texture)
+        utils.reconstruct_texture(
+            input_image,  # src image
+            input_image,  # dst image
+            Vt_texture,   # Vt
+            InputPoints,  # shape points input
+            MeanPoints,   # shape points mean
+            mean_values_texture,  # mean texture
+            triangles,  # triangles
+            n_texture_components  # learned n_texture_components
+        )
+
+        dst = utils.get_texture(MeanPoints, mean_values_texture)
 
         cv2.imshow('original', InputPoints.get_image())
         cv2.imshow('reconstructed', input_image)
@@ -248,6 +260,8 @@ def main():
     elif args.reconstruct:
         #reconstruct_with_model(args)
         show_reconstruction(args)
+    elif args.show_kivy:
+        reconstruct_with_model(args)
 
 if __name__ == '__main__':
     main()

src/utils/generate_head_texture_test.py

-import numpy as np
-import cv2
-import utils.generate_head_texture as ht
-import time
-
-blue_image = cv2.imread('data/test_data/blue.png')
-red_image = cv2.imread('data/test_data/red.png')
-
-
-def test_get_colors_triangle():
-    src = blue_image
-    dst = red_image
-
-    points2d_src = np.array([
-        [20, 20], [50, 160], [160, 20],
-        [50, 20], [60, 200], [180, 20]
-    ])
-
-    points2d_dst = np.array([
-        [40, 80], [130, 150], [40, 150],
-        [40, 80], [60, 82], [60, 100]
-    ])
-
-    triangles = [[0, 1, 2]]
-
-    t1 = time.clock()
-
-    for i in range(10):
-        for t, tri in enumerate(triangles):
-            src_p1, src_p2, src_p3 = points2d_src[tri]
-            dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
-
-            ht.get_colors_triangle(
-                src, dst,
-                src_p1[0], src_p1[1],
-                src_p2[0], src_p2[1],
-                src_p3[0], src_p3[1],
-                dst_p1[0], dst_p1[1],
-                dst_p2[0], dst_p2[1],
-                dst_p3[0], dst_p3[1]
-            )
-
-    print (time.clock() - t1) / 10.
-
-    #cv2.imshow('blue_image', blue_image)
-    #cv2.imshow('red_image', red_image)
-    #cv2.waitKey(0)
-
-    assert False

src/utils/triangles.py → src/utils/utils.py

 import numpy as np
 import cv2
 
-from utils.generate_head_texture import fill_triangle, fill_triangle_src_dst
+from .texture import fill_triangle, fill_triangle_src_dst
 
 import pca
 import aam
@@ -102,10 +102,10 @@ def reconstruct_texture(src, dst, Vt, SrcPoints, DstPoints,
     ## Still in  S_mean format
     r_texture = pca.reconstruct(input_texture, Vt, mean_texture)
 
-    # Make an image from the float data
+    ## Make an image from the float data
     r_texture = np.asarray(r_texture, np.uint8).reshape((h_slice, w_slice, 3))
 
-    ## subtract the offset
+    ### subtract the offset
     points2d_dst[:, 0] -= offset_x
     points2d_dst[:, 1] -= offset_y
 
@@ -114,13 +114,11 @@ def reconstruct_texture(src, dst, Vt, SrcPoints, DstPoints,
         dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
 
         fill_triangle_src_dst(
-            r_texture, src,
+            r_texture, dst,
             dst_p1[0], dst_p1[1],
             dst_p2[0], dst_p2[1],
             dst_p3[0], dst_p3[1],
             src_p1[0], src_p1[1],
             src_p2[0], src_p2[1],
-            src_p3[0], src_p3[1],
-            offset_x,
-            offset_y
+            src_p3[0], src_p3[1]
         )

src/view/reconstruct.py

@@ -20,8 +20,8 @@ from functools import partial
 from math import cos, sin, pi
 
 import imm_points as imm
-from utils.triangles import draw_texture
-from utils.generate_head_texture import fill_triangle
+from utils import utils
+from utils.texture import fill_triangle
 #import IMMPoints, build_feature_vectors, \
 #    flatten_feature_vectors
 import pca
@@ -77,55 +77,47 @@ class ImageCanvas(Widget):
         self.image.source = self.filename_image
         self.canvas.ask_update()
 
-    def build_texture(self, mean_shape, r_shape, r_texture, triangles):
+    def build_texture(self, filename, Vt_texture, input_points, mean_value_points,
+                      mean_values_texture, triangles, n_texture_components):
         self.texture.clear()
-        image_width, image_height = self.get_rendered_size()
-        bary_centric_range = np.linspace(0, 1, num=20)
-
-        texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
-        buf = np.zeros((image_width, image_height, 3), dtype=np.uint8)
 
-        offset = 0
-
-        for t, tri in enumerate(triangles):
-            points = r_shape[tri]
+        image_width, image_height = self.get_rendered_size()
 
-            x = points[:, 0] * image_width + self.get_image_left(image_width)
-            y = (1.0 - points[:, 1]) * image_height + self.get_image_bottom(image_height)
+        InputPoints = imm.IMMPoints(filename=filename)
+        input_image = InputPoints.get_image()
+        buf = np.zeros(input_image.shape, dtype=np.uint8)
 
-            src_p1_x, src_p2_x, src_p3_x = x
-            src_p1_y, src_p2_y, src_p3_y = y
+        InputPoints.get_scaled_points((input_image.shape))
+        offset_x, offset_y, w, h = InputPoints.get_bounding_box()
 
-            for t, tri in enumerate(triangles):
-                offset += fill_triangle(
-                    r_texture, buf,
-                    src_p1_x, src_p1_y,
-                    src_p2_x, src_p2_y,
-                    src_p3_x, src_p3_y,
-                    dst_p1_x, dst_p1_y,
-                    dst_p2_x, dst_p2_y,
-                    dst_p3_x, dst_p3_y,
-                    offset,
-                    t
-                )
+        MeanPoints = imm.IMMPoints(points_list=mean_value_points)
 
-                print offset
+        utils.reconstruct_texture(
+            input_image,  # src image
+            buf,          # dst image
+            Vt_texture,   # Vt
+            InputPoints,  # shape points input
+            MeanPoints,   # shape points mean
+            mean_values_texture,  # mean texture
+            triangles,    # triangles
+            n_texture_components  # learned n_texture_components
+        )
 
-            #p1 = [x[0], y[0]]
-            #p2 = [x[1], y[1]]
-            #p3 = [x[2], y[2]]
+        ratio_x = image_width / w
+        ratio_y = image_height / h
 
+        buf = cv2.resize(np.asarray(buf, np.uint8), (int(image_width), int(image_height)))
+        #buf = buf[offset_y: offset_y + h, offset_x: offset_x + w]
 
-        ##buf = b''.join(map(chr, buf))
-        #cv2.imshow('image', buf)
-        #cv2.waitKey(0)
+        cv2.imshow('original', buf)
+        buf = b''.join(map(chr, buf.flatten()))
 
-        #texture.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
-        #self.texture.add(Rectangle(texture=texture, pos=(0, 100),
-        #                 size=(image_width, image_height)))
+        texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
+        texture.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
+        self.texture.add(Rectangle(texture=texture, pos=(offset_x, offset_y), size=(w, h)))
 
-        #self.canvas.add(self.texture)
-        #self.canvas.ask_update()
+        self.canvas.add(self.texture)
+        self.canvas.ask_update()
 
 
     def build_line_grid(self, r_shape, triangles):
@@ -178,24 +170,27 @@ class RootWidget(BoxLayout):
         super(RootWidget, self).__init__(**kwargs)
 
         self.files = kwargs['args'].files
-        self.mean_values_shape = kwargs['mean_values_shape']
+        self.mean_value_points = kwargs['mean_value_points']
         self.mean_values_texture = kwargs['mean_values_texture']
         self.eigenv_shape = kwargs['eigenv_shape']
         self.eigenv_texture = kwargs['eigenv_texture']
         self.triangles = kwargs['triangles']
         self.n_shape_components = kwargs['args'].n_components
         self.n_texture_components = kwargs['n_texture_components']
-        self.multipliers = np.ones(self.eigenv_shape.shape[1])
-        self.mean_values_shape_reshaped = np.copy(self.mean_values_shape).reshape((58, 2))
+        self.multipliers = np.ones(self.eigenv_shape.shape[0])
+        self.texture_multipliers = np.ones(self.eigenv_texture.shape[0])
 
         # slider index
         self.index = 0
         self.filename = ''
 
-        self.shape_list = aam.build_shape_feature_vectors(self.files, imm.get_imm_points, flattened=True)
+        self.shape_list = aam.build_shape_feature_vectors(
+            self.files, imm.get_imm_points, flattened=True
+        )
 
         # sliders
         self.add_shape_sliders()
+        self.add_texture_sliders()
         self.add_image_slider()
         self.add_components_slider()
         # ======= #
@@ -213,6 +208,15 @@ class RootWidget(BoxLayout):
         image_slider.max = len(self.files) - 1
         image_slider.bind(value=self.update_image)
 
+    def add_texture_sliders(self):
+        box_layout = self.ids['texture_eigenvalues']
+
+        for c in range(10):
+            slider = Slider(min=-10, max=10, value=0, id=str(c))
+            box_layout.add_widget(slider)
+            slider.bind(value=self.update_texture_eigenvalues)
+
+
     def add_shape_sliders(self):
         box_layout = self.ids['eigenvalues']
 
@@ -222,51 +226,43 @@ class RootWidget(BoxLayout):
             slider.bind(value=self.update_eigenvalues)
 
     def reset_sliders(self):
-        self.multipliers = np.ones(self.eigenv_shape.shape[1])
+        self.multipliers = np.ones(self.eigenv_shape.shape[0])
+        self.texture_multipliers = np.ones(self.texture_multipliers.shape[0])
+
         box_layout = self.ids['eigenvalues']
 
         for c in box_layout.children:
             c.value = 0
 
+        texture_eigenvalues = self.ids['texture_eigenvalues']
+
+        for c in texture_eigenvalues.children:
+            c.value = 0
+
     def update_image_viewer(self):
         self.filename = self.files[self.index].split('.')[0] + '.jpg'
         Vt_shape = np.dot(np.diag(self.multipliers), self.eigenv_shape)
-        # Vt_texture = np.dot(np.diag(self.multipliers), self.eigenv_texture)
+        Vt_texture = np.dot(np.diag(self.texture_multipliers), self.eigenv_texture)
+
+        input_points = self.shape_list[self.index]
 
         r_shape = pca.reconstruct(
-            self.shape_list[self.index], Vt_shape, self.mean_values_shape,
+            input_points, Vt_shape, self.mean_value_points,
             n_components=self.n_shape_components
         ).reshape((-1, 2))
 
-        #image = cv2.imread(self.filename)
-        #h, w = image.shape[0], image.shape[1]
-        #mean_values_shape = np.copy(self.mean_values_shape_reshaped)
-
-        #mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
-        #mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
-
-        #texture = aam.sample_from_triangles(
-        #    image, r_shape, mean_values_shape, self.triangles
-        #)
-
-        #r_texture = pca.reconstruct(
-        #    texture, self.eigenv_texture, self.mean_values_texture
-        #)
-
-        #r_texture = np.asarray(texture, np.uint8)
-
-        # pixels = aam.sample_from_triangles(image, r_shape, self.triangles)
-        # pixels = np.ndarray.flatten(pixels)
-
-        # r_texture = pca.reconstruct(
-        #     pixels, self.eigenv_texture, self.mean_values_texture,
-        #     n_components=50000).reshape((95, -1))
-        # self.ids['image_viewer'].build_texture(r_shape, r_texture, self.triangles)
-
         self.ids['image_viewer'].update_rect()
         self.ids['image_viewer'].update_image(self.filename)
         self.ids['image_viewer'].build_line_grid(r_shape, self.triangles)
-        #self.ids['image_viewer'].build_texture(r_shape, r_texture, self.triangles)
+        self.ids['image_viewer'].build_texture(
+                self.files[self.index],
+                self.eigenv_texture,
+                input_points,
+                self.mean_value_points,
+                self.mean_values_texture,
+                self.triangles,
+                self.n_texture_components
+        )
 
     def on_resize(self, *args):
         self.update_image_viewer()
@@ -280,6 +276,11 @@ class RootWidget(BoxLayout):
         self.reset_sliders()
         self.update_image_viewer()
 
+    def update_texture_eigenvalues(self, slider, value):
+        multiplier_index = int(slider.id)
+        self.texture_multipliers[multiplier_index] = value
+        self.update_image_viewer()
+
     def update_eigenvalues(self, slider, value):
         multiplier_index = int(slider.id)
         self.multipliers[multiplier_index] = value
@@ -301,7 +302,7 @@ class ReconstructApp(App):
             args=self.args,
             eigenv_shape=self.eigenv_shape,
             eigenv_texture=self.eigenv_texture,
-            mean_values_shape=self.mean_values_shape,
+            mean_value_points=self.mean_value_points,
             n_shape_components=self.n_shape_components,
             mean_values_texture=self.mean_values_texture,
             n_texture_components=self.n_texture_components,

src/view/templates/reconstruct.kv

@@ -34,8 +34,9 @@
                     max: 58
                     value: 0
             BoxLayout:
-                Label:
-                    text: "Eigenvalues"
+                BoxLayout:
+                    orientation: 'vertical'
+                    id: texture_eigenvalues
                 BoxLayout:
                     orientation: 'vertical'
                     id: eigenvalues