Skip to content

P
py-3d-face-reconstruction
Commit 6ae2cffe authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Refactored code to keep things a bit more seperate

parent d63e9150
Hide whitespace changes
Inline Side-by-side
Showing with 199 additions and 236 deletions
makefile
View file @ 6ae2cffe
......@@ -37,5 +37,4 @@ show_reconstruction:
--n_components 6
test:
py.test -f src/*_test.py
python -m py.test -f src/*_test.py
src/aam.py
View file @ 6ae2cffe
from matplotlib.tri import Triangulation
import numpy as np
import cv2
import pca
def get_mean(imm_points):
......@@ -33,3 +37,57 @@ def get_mean(imm_points):
mean. [[x_mean_0, y_mean_0], ... [x_mean_n, y_mean_n]]
"""
return np.mean(imm_points, axis=0)
def get_triangles(x_vector, y_vector):
""" perform triangulation between two 2d vectors"""
return Triangulation(x_vector, y_vector).triangles
def build_feature_vectors(files, get_points, flattened=False):
"""
Gets the aam points from the files and appends them seperately to one
array.
Args:
files (list): list files
return:
list: list of feature vectors
"""
points = get_points(files)
if flattened:
points = pca.flatten_feature_vectors(points)
return points
def get_pixel_values(image, points):
h, w, c = image.shape
points[:, 0] = points[:, 0] * w
points[:, 1] = points[:, 1] * h
image = cv2.blur(image, (3, 3))
hull = cv2.convexHull(points, returnPoints=True)
rect = cv2.boundingRect(hull)
pixels = []
x, y, w, h = rect
# pixels = np.zeros((h, w, c), dtype=np.uint8)
for i in np.linspace(0, 1, num=100):
for j in np.linspace(0, 1, num=100):
y_loc_g = int(i * h + y)
x_loc_g = int(j * w + x)
y_loc = min(int(i * h), h - 1)
x_loc = min(int(j * w), w - 1)
if cv2.pointPolygonTest(hull, (x_loc_g, y_loc_g), measureDist=False) >= 0:
pixels.extend(image[y_loc_g][x_loc_g])
return np.asarray(pixels, dtype=np.uint8), hull
src/aam_test.py
View file @ 6ae2cffe
import numpy as np
from aam import get_mean
from aam import get_mean, get_pixel_values
import imm_points
import pca
def test_build_mean_aan():
......@@ -39,3 +41,15 @@ def test_zero_mean_aan():
)
np.testing.assert_array_equal(zero_mean, expected)
def test_get_pixel_values():
asf_file = '../data/imm_face_db/40-2m.asf'
imm = imm_points.IMMPoints(filename=asf_file)
points = imm.get_points()
image = imm.get_image()
pixels, hull = get_pixel_values(image, points)
assert False
src/imm_points.py
View file @ 6ae2cffe
from matplotlib.tri import Triangulation
import cv2
import numpy as np
import argparse
import os
class IMMPoints():
......@@ -21,12 +23,16 @@ class IMMPoints():
def get_points(self):
return self.points
def get_image(self):
cv2.imread(self.image_file)
return cv2.imread(self.image_file)
def import_file(self, filename):
with open(filename, 'r') as f:
lines = f.readlines()
# store the filename we've got
self.filename = lines[-1].strip()
data = lines[16:74]
dir_name = os.path.dirname(filename)
self.image_file = "{}/{}".format(dir_name, lines[-1].strip())
for d in data:
self.points.append(d.split()[2:4])
......@@ -34,7 +40,6 @@ class IMMPoints():
self.points = np.asarray(self.points, dtype='f')
def draw_triangles(self, img, points):
assert(len(self.points) > 0)
h, w, c = img.shape
points[:, 0] = points[:, 0] * w
......@@ -63,69 +68,19 @@ class IMMPoints():
assert(len(self.points) > 0)
assert(len(self.filename) > 0)
img = cv2.imread('data/imm_face_db/' + self.filename)
img = self.get_image()
self.draw_triangles(img, self.points)
def flatten_feature_vectors(data):
"""
Flattens the feature vectors inside a ndarray
Example:
input:
[
[[1, 2], [3, 4], [5, 6]],
...
[[1, 2], [3, 4], [5, 6]]
]
output:
[
[1, 2, 3, 4, 5, 6],
...
[1, 2, 3, 4, 5, 6]
]
Args:
data (numpy array): array of feature vectors
return:
array: (numpy array): array flattened feature vectors
"""
flattened = []
rows, _, _ = data.shape
for i in range(rows):
flattened.append(np.ndarray.flatten(data[i]))
return np.array(flattened)
def build_feature_vectors(files, flattened=False):
"""
Gets the aam points from the files and appends them seperately to one
array.
Args:
files (list): list files
return:
list: list of feature vectors
"""
imm_points = []
def get_imm_landmarks(files):
points = []
for f in files:
imm = IMMPoints(filename=f)
imm_points.append(imm.get_points())
imm_points = np.array(imm_points)
if flattened:
imm_points = flatten_feature_vectors(imm_points)
points.append(imm.get_points())
return imm_points
return np.asarray(points)
def add_parser_options():
......
src/imm_points_test.py
View file @ 6ae2cffe
import numpy as np
from imm_points import flatten_feature_vectors
def test_flatten_feature_vectors():
imm_points = np.array([
[[1, 2], [2, 4]],
[[2, 3], [3, 6]],
])
expected = np.array([
[1, 2, 2, 4],
[2, 3, 3, 6]
])
result = flatten_feature_vectors(imm_points)
np.testing.assert_array_equal(result, expected)
src/main.py
View file @ 6ae2cffe
import copy
import argparse
import logging
import sys
import cv2
import numpy as np
# local imports
import pca
from aam import get_mean
from imm_points import IMMPoints, build_feature_vectors, \
flatten_feature_vectors
import aam
import imm_points as imm
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
......@@ -20,7 +14,6 @@ logger = logging.getLogger(__name__)
def add_parser_options():
parser = argparse.ArgumentParser(description='IMMPoints tool')
pca_group = parser.add_argument_group('show_pca')
pca_group.add_argument(
......@@ -63,27 +56,26 @@ def save_pca_model(args):
It is saved in the following way:
np.load(filename, np.assary([U, s, Vt, mean_values])
np.load(filename, np.assary([Vt, mean_values])
And accessed by:
UsVtm = np.load(args.model_file)
Vtm = np.load(args.model_file)
U = UsVtm[0]
s = UsVtm[1]
Vt = UsVtm[2]
mean_values = UsVtm[3]
Vt = Vtm[0]
mean_values = Vtm[1][0]
"""
assert args.asf, '--asf files should be given'
assert args.model_file, '--model_file needs to be provided to save the pca model'
imm_points = build_feature_vectors(args.asf, flattened=True)
mean_values = get_mean(imm_points)
points = aam.build_feature_vectors(args.asf,
imm.get_imm_landmarks, flattened=True)
mean_values = aam.get_mean(points)
U, s, Vt = pca.pca(imm_points, mean_values)
_, _, Vt = pca.pca(points, mean_values)
pca.save(U, s, Vt, mean_values, args.model_file)
pca.save(Vt, mean_values, args.model_file)
logger.info('saved pca model in %s', args.model_file)
......@@ -92,59 +84,21 @@ def show_pca_model(args):
assert args.asf, '--asf files should be given to allow the image to be shown'
assert args.model_file, '--model_file needs to be provided to get the pca model'
U, s, Vt, mean_values = pca.load(args.model_file)
# init trackbars
# index = 0
# cv2.namedWindow('index')
# cv2.createTrackbar('index', 'index', index, len(args.asf) - 1, trackbarUpdate)
n_components = args.n_components
view.init_eigenvalue_trackbars(n_components, s, window='eigenvalues')
# use a copy of s to manipulate so we never touch the original
s_copy = copy.copy(s)
reconstruction = np.dot(Vt[:n_components], x - mean_values)
while True:
imm = IMMPoints(filename=args.asf[index])
reconstruction = np.dot(V[:n_components], x - mean_values)
# reconstruction = pca.reconstruct(U[index], s_copy, Vt, n_components, mean_values)
# reshape to x, y values
reconstructed = reconstruction.reshape((58, 2))
imm = IMMPoints(points=reconstructed)
img = np.full((480, 640, 3), 255, np.uint8)
imm.show_on_img(img)
for i in range(n_components):
s_copy[i] = s[i] + (
(cv2.getTrackbarPos(str(i), 'eigenvalues') - 50) / 10.0)
index = cv2.getTrackbarPos('index', 'index')
imm.show(window_name='original')
k = cv2.waitKey(1) & 0xFF
if k == 27:
break
cv2.destroyAllWindows()
Vt, mean_values = pca.load(args.model_file)
def reconstruct_with_model(args):
assert args.asf, '--asf files should be given to allow the image to be shown'
assert args.model_file, '--model_file needs to be provided to get the pca model'
# clear args. arguments are conflicting with parseargs
# clear sys args. arguments are conflicting with parseargs
# kivy will parse args upon import and will crash if it finds our
# 'unsuported by kivy' arguments.
# 'unsupported by kivy' arguments.
sys.argv[1:] = []
from view.reconstruct import ReconstructApp
U, s, Vt, mean_values = pca.load(args.model_file)
Vt, mean_values = pca.load(args.model_file)
ReconstructApp(
args=args, eigen_vectors=Vt, mean_values=mean_values,
).run()
......
src/pca.py
View file @ 6ae2cffe
......@@ -2,6 +2,14 @@ import numpy as np
def pca(data, mean_values):
"""
Perform Singlar Value Decomposition
Returns:
U (ndarray): U matrix
s (ndarray): 1d singular values (diagonal in array form)
Vt (ndarray): Vt matrix
"""
# subtract mean
zero_mean = data - mean_values
U, s, Vt = np.linalg.svd(zero_mean, full_matrices=False)
......@@ -28,24 +36,23 @@ def reconstruct(feature_vector, Vt, mean_values, n_components=10):
return np.dot(Vt[:n_components].T, yk) + mean_values
def save(U, s, Vt, mean_values, filename):
def save(Vt, mean_values, filename):
"""
Store the U, s, Vt and mean of all the asf datafiles given by the asf
files.
It is stored in the following way:
np.load(filename, np.assary([U, s, Vt, mean_values])
np.load(filename, np.assary([Vt, [mean_values]])
And accessed by:
UsVtm = np.load(args.model_file)
Vtm = np.load(args.model_file)
U = UsVtm[0]
s = UsVtm[1]
Vt = UsVtm[2]
mean_values = UsVtm[3]
Vt = Vtm[0]
mean_values = Vtm[1][0]
"""
np.save(filename, np.asarray([U, s, Vt, mean_values]))
saving = np.asarray([Vt, [mean_values]])
np.save(filename, saving)
def load(filename):
......@@ -53,28 +60,56 @@ def load(filename):
The model stored by pca.store (see ``pca.store`` method above) is loaded as:
UsVtm = np.load(args.model_file)
U = UsVtm[0]
s = UsVtm[1]
Vt = UsVtm[2]
mean_values = UsVtm[3]
Vt = Vtm[0]
mean_values = Vtm[1][0]
Returns:
(tuple): U, s, Vt, mean_values
(tuple): Vt, mean_values
U (numpy ndarray): One feature vector from the reduced SVD.
U should have shape (n_features,), (i.e., one dimensional)
s (numpy ndarray): The singular values as a one dimensional array
Vt (numpy ndarray): Two dimensional array with dimensions
(n_features, n_features)
mean_values (numpy ndarray): mean values of the features of the model,
this should have dimensions (n_featurs, )
"""
# load the stored model file
UsVtm = np.load(filename)
Vtm = np.load(filename)
Vt = Vtm[0]
mean_values = Vtm[1][0]
return Vt, mean_values
def flatten_feature_vectors(data):
"""
Flattens the feature vectors inside a ndarray
Example:
input:
[
[[1, 2], [3, 4], [5, 6]],
...
[[1, 2], [3, 4], [5, 6]]
]
output:
[
[1, 2, 3, 4, 5, 6],
...
[1, 2, 3, 4, 5, 6]
]
Args:
data (numpy array): array of feature vectors
return:
array: (numpy array): array flattened feature vectors
"""
flattened = []
rows, _, _ = data.shape
U = UsVtm[0]
s = UsVtm[1]
Vt = UsVtm[2]
mean_values = UsVtm[3]
for i in range(rows):
flattened.append(np.ndarray.flatten(data[i]))
return U, s, Vt, mean_values
return np.array(flattened)
src/pca_test.py
View file @ 6ae2cffe
import numpy as np
from pca import pca, reconstruct
from pca import flatten_feature_vectors
def test_flatten_feature_vectors():
imm_points = np.array([
[[1, 2], [2, 4]],
[[2, 3], [3, 6]],
])
expected = np.array([
[1, 2, 2, 4],
[2, 3, 3, 6]
])
result = flatten_feature_vectors(imm_points)
np.testing.assert_array_equal(result, expected)
src/view/reconstruct.py
View file @ 6ae2cffe
......@@ -2,7 +2,6 @@ import kivy
kivy.require('1.0.7')
import numpy as np
from matplotlib.tri import Triangulation
from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
......@@ -18,9 +17,11 @@ from kivy.graphics.context_instructions import Color
from functools import partial
from math import cos, sin, pi
from imm_points import IMMPoints, build_feature_vectors, \
flatten_feature_vectors
import imm_points as imm
#import IMMPoints, build_feature_vectors, \
# flatten_feature_vectors
import pca
import aam
class ImageCanvas(Widget):
......@@ -34,12 +35,13 @@ class ImageCanvas(Widget):
self.image = Image(pos=self.pos, size=self.size, source=self.filename_image)
self.mesh = Mesh(mode='triangle_fan')
self.triangles = InstructionGroup()
self.middle_group = InstructionGroup()
self.outline = InstructionGroup()
self.bind(pos=self.update_rect, size=self.update_rect)
def get_rendered_size(self):
"""get the rendered size of the image
"""
get the rendered size of the image
Returns:
(tuple) width, height in pixels
"""
......@@ -73,7 +75,7 @@ class ImageCanvas(Widget):
self.triangles.clear()
image_width, image_height = self.get_rendered_size()
triangles = Triangulation(reconstructed[:, 0], reconstructed[:, 1]).triangles
triangles = aam.get_triangles(reconstructed[:, 0], reconstructed[:, 1])
for tri in triangles:
self.triangles.add(Color(0, 0, 1, 1))
......@@ -92,7 +94,6 @@ class ImageCanvas(Widget):
self.triangles.add(Line(circle=(x[2], y[2], 3)))
self.canvas.add(self.triangles)
self.canvas.add(self.middle_group)
self.canvas.ask_update()
def build_mesh(self, reconstructed):
......@@ -109,8 +110,8 @@ class ImageCanvas(Widget):
xy_vertices = np.array(xy_vertices)
indices = []
indices = Triangulation(xy_vertices[:, 0], xy_vertices[:, 1]).triangles
indices = np.ndarray.flatten(indices)[:30]
indices = aam.get_triangles(xy_vertices[:, 0], xy_vertices[:, 1])
indices = np.ndarray.flatten(indices)
self.mesh.vertices = vertices
self.mesh.indices = indices
......@@ -126,6 +127,7 @@ class RootWidget(BoxLayout):
self.n_components = kwargs['args'].n_components
self.multipliers = np.ones(self.Vt.shape[1])
# slider index
self.index = 0
self.filename = ''
......@@ -137,11 +139,12 @@ class RootWidget(BoxLayout):
n_components_slider.value = self.n_components
n_components_slider.bind(value=self.update_n_components)
self.landmark_list = build_feature_vectors(self.images, flattened=True)
self.ids['image_viewer'].bind(size=self.on_resize)
box_layout = self.ids['eigenvalues']
self.landmark_list = aam.build_feature_vectors(self.images,
imm.get_imm_landmarks, flattened=True)
for c in range(self.n_components):
slider = Slider(min=-10, max=10, value=0, id=str(c))
box_layout.add_widget(slider)
......@@ -149,7 +152,6 @@ class RootWidget(BoxLayout):
def reset_sliders(self):
self.multipliers = np.ones(self.Vt.shape[1])
box_layout = self.ids['eigenvalues']
for c in box_layout.children:
......@@ -164,7 +166,7 @@ class RootWidget(BoxLayout):
n_components=self.n_components
)
reconstruction = reconstruction.reshape((58, 2))
reconstruction = reconstruction.reshape((-1, 2))
self.ids['image_viewer'].update_rect()
self.ids['image_viewer'].update_image(self.filename)
......@@ -199,7 +201,8 @@ class ReconstructApp(App):
super(ReconstructApp, self).__init__(**kwargs)
def build(self):
return RootWidget(args=self.args, eigen_vectors=self.eigen_vectors,
return RootWidget(
args=self.args, eigen_vectors=self.eigen_vectors,
mean_values=self.mean_values
)
......
src/view/templates/reconstruct.kv
View file @ 6ae2cffe
<Widget>
canvas.after:
Line:
rectangle: self.x+1,self.y+1,self.width-1,self.height-1
dash_offset: 5
dash_length: 3
# Draw lines between widgets for easy debugging
# <Widget>
# canvas.after:
# Line:
# rectangle: self.x+1,self.y+1,self.width-1,self.height-1
# dash_offset: 5
# dash_length: 3
<ImageCanvas>
Widget
<RootWidget>
BoxLayout:
......@@ -39,55 +39,3 @@
BoxLayout:
orientation: 'vertical'
id: eigenvalues
# size_hint_y: None
# height: sp(100)
# BoxLayout:
# orientation: 'vertical'
# Slider:
# id: e1
# min: -360.
# max: 360.
# Label:
# text: 'angle_start = {}'.format(e1.value)
# BoxLayout:
# orientation: 'vertical'
# Slider:
# id: e2
# min: -360.
# max: 360.
# value: 360
# Label:
# text: 'angle_end = {}'.format(e2.value)
# BoxLayout:
# size_hint_y: None
# height: sp(100)
# BoxLayout:
# orientation: 'vertical'
# Slider:
# id: wm
# min: 0
# max: 2
# value: 1
# Label:
# text: 'Width mult. = {}'.format(wm.value)
# BoxLayout:
# orientation: 'vertical'
# Slider:
# id: hm
# min: 0
# max: 2
# value: 1
# Label:
# text: 'Height mult. = {}'.format(hm.value)
# Button:
# text: 'Reset ratios'
# on_press: wm.value = 1; hm.value = 1
# FloatLayout:
# canvas:
# Color:
# rgb: 1, 1, 1
# Rectangle:
# pos: 100, 100
# size: 200 * wm.value, 201 * hm.value
# source: 'data/imm_face_db/16-2m.jpg'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment