Made the shape_components sliders work with the multipliers in the backend,...

Made the shape_components sliders work with the multipliers in the backend, able to change the face now

src/main.py

@@ -10,6 +10,7 @@ import cv2
 # local imports
 import pca
 import aam
+import numpy as np
 # import imm
 
 from reconstruction import reconstruction
@@ -200,7 +201,7 @@ def show_reconstruction(args):
     texture_model = pca.PCAModel(args.model_texture_file)
 
     input_points = dataset_module.IMMPoints(
-        filename='data/imm_face_db/40-3m.asf'
+        filename='data/imm_face_db/01-1m.asf'
     )
 
     input_image = input_points.get_image()
@@ -210,41 +211,49 @@ def show_reconstruction(args):
 
     n_components = 58
     count = 0
+    shape_eigenvalues_multiplier = np.ones(15, dtype=np.float32)
 
     while True:
         input_image_copy = input_image.copy()
         input_points_copy = copy.deepcopy(input_points)
 
+        output_points = dataset_module.IMMPoints(
+            points_list=input_points.get_points()
+        )
+
+        # scale by scaling the Vt matrix
+        shape_Vt = shape_model.Vt
+        shape_Vt = reconstruction.scale_eigenvalues(
+            shape_Vt, shape_eigenvalues_multiplier
+        )
+
+        # recontruct the shape
         reconstruction.reconstruct_shape(
-            input_image_copy, input_points_copy, shape_model,
+            output_points,
+            shape_model,
+            shape_Vt=shape_Vt,  # overwrite by scaled Vt
             n_components=n_components - count
         )
 
+        # use the new shape ane mean points to reconstruct
         reconstruction.reconstruct_texture(
             input_image_copy,  # src image
             input_image_copy,  # dst image
             texture_model,
             input_points_copy,  # shape points input
-            mean_points    # shape points mean
+            mean_points,    # shape points mean
+            output_points
         )
 
-        input_points_copy.get_scaled_points(input_image.shape)
-        input_points_copy.draw_triangles(image=input_image_copy, show_points=False)
+        output_points.get_scaled_points(input_image.shape)
+        output_points.draw_triangles(image=input_image_copy, show_points=False)
 
         dst = reconstruction.get_texture(
             mean_points, texture_model.mean_values
         )
 
-        cv2.imshow('original', input_image)
-        cv2.imshow('reconstructed', input_image_copy)
-        cv2.imshow('main face', dst)
-
-        k = cv2.waitKey(0) & 0xFF
-
-        if k == 27:
-            break
-
-        count += 2
+        #count += 2
+        shape_eigenvalues_multiplier[0] += 0.1
 
     cv2.destroyAllWindows()
 

src/reconstruction/reconstruction.py

@@ -73,18 +73,26 @@ def draw_shape(image, points, triangles, multiply=True):
 
 def get_texture(Points, flattened_texture):
     offset_x, offset_y, w_slice, h_slice = Points.get_bounding_box()
-
     # Make a rectangle image from the flattened texture array
     return np.asarray(flattened_texture, np.uint8).reshape((h_slice, w_slice, 3))
 
 
-def reconstruct_shape(image, points, shape_model, n_components=None):
+def scale_eigenvalues(Vt, multiplier_array):
+    multipliers = np.ones(Vt.shape[1], dtype=np.float32)
+    multipliers[:len(multiplier_array)] = multiplier_array
+    Vt = np.dot(np.diag(multipliers), Vt)
+
+    return Vt
+
+
+def reconstruct_shape(points, shape_model, shape_Vt=None, n_components=None):
     input_points = points.get_points()
     mean_points = shape_model.mean_values
+    shape_Vt = shape_Vt if shape_Vt is not None else shape_model.Vt
 
     reconstructed = pca.reconstruct(
         input_points,
-            shape_model.Vt,
+        shape_Vt,
         mean_points,
         n_components=n_components
     )
@@ -92,7 +100,8 @@ def reconstruct_shape(image, points, shape_model, n_components=None):
     points.normalized_flattened_points_list = reconstructed
 
 
-def reconstruct_texture(src_image, dst_image, texture_model, src_points, dst_points):
+def reconstruct_texture(src_image, dst_image, texture_model,
+        src_points, dst_points, output_points):
     """
     Recontruct texture given the src and dst image
 
@@ -124,6 +133,7 @@ def reconstruct_texture(src_image, dst_image, texture_model, src_points, dst_poi
 
     points2d_src = src_points.get_scaled_points(src_image.shape)
     points2d_dst = dst_points.get_scaled_points(dst_image.shape)
+    points2d_output = output_points.get_scaled_points(src_image.shape)
 
     # get the texture from the rectangles.
     aam.sample_from_triangles(
@@ -154,16 +164,13 @@ def reconstruct_texture(src_image, dst_image, texture_model, src_points, dst_poi
     points2d_dst[:, 0] -= offset_x
     points2d_dst[:, 1] -= offset_y
 
-    for tri in triangles:
-        src_p1, src_p2, src_p3 = points2d_src[tri]
-        dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
-
-        fill_triangle_src_dst(
-            r_texture, dst_image,
-            dst_p1[0], dst_p1[1],
-            dst_p2[0], dst_p2[1],
-            dst_p3[0], dst_p3[1],
-            src_p1[0], src_p1[1],
-            src_p2[0], src_p2[1],
-            src_p3[0], src_p3[1]
+    points2d_src = points2d_src * 1.1
+
+    # get the texture from the rectangles.
+    aam.sample_from_triangles(
+        r_texture,
+        points2d_dst,  # turn src and dst around
+        points2d_output,  # turn src and dst around
+        triangles,
+        dst_image
     )

src/server.py

@@ -61,7 +61,11 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
         image_index = message['reconstruction_index']
         image_as_background = message.get('background_image', True)
         shape_components = message.get('shape_components', 58)
-        print message
+        shape_eigenvalues_multiplier = message.get('shape_eigenvalues')
+
+        shape_eigenvalues_multiplier = np.asarray(
+            shape_eigenvalues_multiplier, dtype=np.float32
+        )
 
         logger.info('using %s shape_components', shape_components)
 
@@ -80,8 +84,20 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
         else:
             dst_image = input_image
 
+        output_points = imm.IMMPoints(
+            points_list=input_points.get_points()
+        )
+
+        shape_Vt = self.shape_model.Vt
+        shape_Vt = reconstruction.scale_eigenvalues(
+            shape_Vt, shape_eigenvalues_multiplier
+        )
+
+        # recontruct the shape
         reconstruction.reconstruct_shape(
-            input_image, input_points, self.shape_model,
+            output_points,
+            self.shape_model,
+            shape_Vt=shape_Vt,  # overwrite by scaled Vt
             n_components=shape_components
         )
 
@@ -90,10 +106,11 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
             dst_image,  # dst image
             self.texture_model,
             input_points,  # shape points input
-            mean_points   # shape points mean
+            mean_points,   # shape points mean
+            output_points
         )
 
-        input_points.draw_triangles(show_points=False)
+        output_points.draw_triangles(image=dst_image, show_points=False)
 
         _, reconstructed = cv2.imencode('.jpg', dst_image)
         reconstructed = base64.b64encode(reconstructed)
@@ -112,12 +129,15 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
         for m in message.keys():
             try:
                 handler = getattr(self, self.handlers[m])
+                handler(message[m])
             except (AttributeError, KeyError) as e:
                 msg = 'no handler for {}'.format(m)
                 print(msg, e)
                 self.__return_error(msg)
-
-            handler(message[m])
+            except Exception as e:
+                msg = 'no handler for {}'.format(m)
+                print(msg, e)
+                self.__return_error(msg)
 
     def on_close(self):
         print("WebSocket closed")

viewer/app/components/three-js-reconstruction.js

@@ -20,25 +20,10 @@ const ThreeComponent = Ember.Component.extend({
 
         var renderer = new THREE.WebGLRenderer();
 
-        // the sidebar 'dat-gui' controls
-        var reconstructionControls = {
-            index: 0,
-            shape_components: 58,
-            background_image: true,
-        };
-
-        for(var i = 0; i < 15; i++) {
-            reconstructionControls['shape_eigen_value_' + i] = 0.0;
-        }
-
-        var shapeEigenValueSliders = {};
-
         this.set('scene', scene);
         this.set('camera', camera);
         this.set('renderer', renderer);
         this.set('gui', gui);
-        this.set('controls', reconstructionControls);
-        this.set('shapeEigenValueSliders', shapeEigenValueSliders);
 
         this.get('store').findAll('face').then((faces) => {
             this.set('faces', faces);
@@ -66,20 +51,19 @@ const ThreeComponent = Ember.Component.extend({
     addSliders() {
         var self = this;
         var gui = this.get('gui');
-        var reconstructionControls = this.get('controls');
-        var shapeEigenValueSliders = this.get('shapeEigenValueSliders');
+
+        // the sidebar 'dat-gui' controls
+        var reconstructionControls = {
+            index: 0,
+            shape_components: 58,
+            background_image: true
+        };
 
         var length = this.get('faces').get('length');
 
         var index = gui.add(reconstructionControls, 'index', 0, length - 1);
-        var shape_components = gui.add(reconstructionControls, 'shape_components', 0, 58);
-
+        var shapeComponents = gui.add(reconstructionControls, 'shape_components', 0, 58);
         var background = gui.add(reconstructionControls, 'background_image');
-        var shapeEigenValueControls = gui.addFolder('shape_eigen_values');
-
-        for(var i = 0; i < 15; i++) {
-            shapeEigenValueControls.add(reconstructionControls, 'shape_eigen_value_' + i, 0.0, 10.0);
-        }
 
         // on index change
         index.onChange(function(newValue) {
@@ -92,13 +76,35 @@ const ThreeComponent = Ember.Component.extend({
             self.sendAction('updateBackground', newValue);
         });
 
-        shape_components.onChange(function(newValue) {
+        shapeComponents.onChange(function(newValue) {
             self.sendAction('updateShapeComponents', newValue);
         });
 
-        reconstructionControls.onChange(function(newValue) {
-            console.log(newValue);
+        var shapeEigenValueControls = gui.addFolder('shape_eigen_values');
+
+        /**
+        * ShapeSlider callback function
+        */
+        var handleShapeSlidersCb = function(value) {
+            var sliderObject = this;
+
+            // slider index is the last character of the slider property string.
+            var sliderCharacterIndex = sliderObject.property.length - 1;
+            var sliderIndex = parseInt(sliderObject.property[sliderCharacterIndex]);
+
+            self.sendAction('updateShapeEigenValues', sliderIndex, value);
+        };
+
+        var shapeEigenValues = this.get('shape_eigenvalues');
+
+        shapeEigenValues.forEach(function(value, index) {
+            reconstructionControls['shape_eigen_value_' + index] = value;
+            var slider = shapeEigenValueControls.add(reconstructionControls, 'shape_eigen_value_' + index, 0.0, 10.0);
+
+            slider.onChange(handleShapeSlidersCb);
         });
+
+        console.log(gui.__controllers);
     }
 });
 

viewer/app/controllers/reconstruction.js

@@ -11,9 +11,10 @@ export default Ember.Controller.extend({
 
     image_index: 0,
     background_image: true,
-    shape_components: null,
+    shape_components: 58,
     n_images: null,
     reconstructed: null,
+    shape_eigenvalues: [],
 
     socketRef: null,
 
@@ -25,6 +26,16 @@ export default Ember.Controller.extend({
         socket.on('close', this.closeHandler, this);
 
         this.set('socketRef', socket);
+        this.initShapeEigenValues(15);
+    },
+
+    initShapeEigenValues(amountOfEigenValues) {
+        var shapeEigenValues = this.get('shape_eigenvalues');
+
+        shapeEigenValues.length = amountOfEigenValues;
+        shapeEigenValues.fill(1.0);
+
+        this.set('shape_eigenvalues', shapeEigenValues);
     },
 
     willDestroyElement() {
@@ -66,7 +77,9 @@ export default Ember.Controller.extend({
     },
 
     getReconstruction: Ember.observer(
-            'image_index', 'background_image', 'shape_components', function() {
+            'image_index', 'background_image',
+            'shape_components', 'shape_eigenvalues', function() {
+        console.log('updating');
         this.send('getReconstruction');
     }),
 
@@ -94,11 +107,11 @@ export default Ember.Controller.extend({
                     reconstruction: {
                         reconstruction_index: this.get('image_index'),
                         background_image: this.get('background_image'),
-                        shape_components: this.get('shape_components')
+                        shape_components: this.get('shape_components'),
+                        shape_eigenvalues: this.get('shape_eigenvalues')
                     }
-
-                }
-            ));
+                })
+            );
         },
 
         // connects components together
@@ -114,6 +127,19 @@ export default Ember.Controller.extend({
         updateShapeComponents(components) {
             console.log('shape_components', components);
             this.set('shape_components', components);
+        },
+
+        updateShapeEigenValues(eigenValueIndex, value) {
+            console.log('shape_eigenvalues', value);
+            var eigenValues = this.get('shape_eigenvalues');
+            eigenValues[eigenValueIndex] = value;
+
+            this.set('shape_eigenvalues', eigenValues);
+            this.send('getReconstruction');
+        },
+
+        resetShapeEigenValues(gui) {
+            this.initShapeEigenValues(15);
         }
     }
 });

viewer/app/templates/reconstruction.hbs

@@ -28,9 +28,12 @@
 
     <div class="col col-12 px2">
         {{three-js-reconstruction
+            shape_eigenvalues=shape_eigenvalues
             updateIndex=(action 'updateIndexComponentConnector')
             updateBackground=(action 'updateBackgroundComponentConnector')
             updateShapeComponents=(action 'updateShapeComponents')
+            updateShapeEigenValues=(action 'updateShapeEigenValues')
+            resetShapeEigenValues=(action 'resetShapeEigenValues')
         }}
     </div>
 </div>