Skip to content

P
py-3d-face-reconstruction
Commit 9c87511e authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Renamed, refactored and attempt to make kivy work with texturing

parent ba216925
Show whitespace changes
Inline Side-by-side
Showing with 147 additions and 154 deletions
actions.mk
View file @ 9c87511e
......@@ -49,6 +49,14 @@ show_reconstruction:
--model_shape_file data/pca_shape_model.npy \
--n_components 6
show_kivy:
python src/main.py \
--show_kivy \
--files data/imm_face_db/*.asf \
--model_texture_file data/pca_texture_model.npy \
--model_shape_file data/pca_shape_model.npy \
--n_components 6
test:
python -m py.test -f src/*_test.py
......
src/aam.py
View file @ 9c87511e
......@@ -6,17 +6,19 @@ import cv2
# local imports
import pca
import utils.texture as tx
import utils.triangles as tu
from utils import utils
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(levelname)s %(name)s: %(message)s'
)
logger = logging.getLogger(__name__)
class AAMPoints():
"""
Object to store AAM points / landmarks. Tries to keep the scaling of
the shape parameters transparent.
these points transparent.
"""
def __init__(self, normalized_flattened_points_list=None, points_list=None, actual_shape=()):
self.normalized_flattened_points_list = normalized_flattened_points_list
......@@ -25,12 +27,27 @@ class AAMPoints():
self.bounding_box = None
def get_bounding_box(self):
"""
Get the bounding box around the points.
"""
if self.bounding_box is None:
return self.calculate_bounding_box()
return self.bounding_box
def get_scaled_points(self, shape, rescale=True):
if self.points_list is None:
def get_scaled_points(self, shape, rescale=False):
"""
Scale the normalized flattened points list to a scale given by 'shape'.
The x and y values should be scaled to the width and height of the image.
Args:
shape(tuple): (height, width)
rescal(boolean): flag if we should rescale or not because if we
already scaled, we are not going to do it again by
default.
Returns:
ndarray scaled to 'shape' width and height.
"""
if self.points_list is None or rescale:
self.points_list = self.normalized_flattened_points_list
if len(self.actual_shape):
......@@ -49,6 +66,10 @@ class AAMPoints():
Calculate bounding box in the **scaled** points list
The empasis on on scaled because the convexHull does not support
small values, the normalized_flattened_points_list does not work.
Use get_scaled_points first, with a shape that is needed. The shape
should be the dimensions of the out image, example (480, 640), ie., (height,
width)
"""
assert self.points_list is not None, \
'the list points already need to be scaled order to correctly work'
......@@ -201,7 +222,6 @@ def get_pixel_values(image, points):
x, y, w, h = rect
# pixels = np.zeros((h, w, c), dtype=np.uint8)
for i in np.linspace(0, 1, num=150):
for j in np.linspace(0, 1, num=150):
y_loc_g = int(i * h + y)
......
src/imm_points.py
View file @ 9c87511e
......@@ -30,7 +30,7 @@ class IMMPoints(aam.AAMPoints):
)
def get_points(self):
return self.points_list
return self.normalized_flattened_points_list
def get_image(self):
return cv2.imread(self.image_file)
......
src/main.py
View file @ 9c87511e
......@@ -5,13 +5,14 @@ import sys
# installed packages
import cv2
import numpy as np
# local imports
import pca
import aam
import imm_points as imm
from utils import utils
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
logger = logging.getLogger(__name__)
......@@ -26,6 +27,11 @@ def add_parser_options():
help='Reconstruct one face with a given pca model'
)
pca_group.add_argument(
'--show_kivy', action='store_true',
help='Reconstruct using kivy as a GUI'
)
pca_group.add_argument(
'--save_pca_shape', action='store_true',
help='save the pca shape model'
......@@ -148,7 +154,7 @@ def reconstruct_with_model(args):
from view.reconstruct import ReconstructApp
Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
app = ReconstructApp()
......@@ -157,7 +163,7 @@ def reconstruct_with_model(args):
args=args,
eigenv_shape=Vt_shape,
eigenv_texture=Vt_texture,
mean_values_shape=mean_values_shape,
mean_value_points=mean_value_points,
n_shape_components=n_shape_components,
mean_values_texture=mean_values_texture,
n_texture_components=n_texture_components,
......@@ -173,7 +179,7 @@ def show_pca_model(args):
from utils.triangles import draw_shape, get_texture
Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
imm_points = imm.IMMPoints(filename='data/imm_face_db/40-1m.asf')
......@@ -184,12 +190,12 @@ def show_pca_model(args):
input_points[:, 0] = input_points[:, 0] * w
input_points[:, 1] = input_points[:, 1] * h
mean_values_shape = mean_values_shape.reshape((58, 2))
mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
mean_value_points = mean_value_points.reshape((58, 2))
mean_value_points[:, 0] = mean_value_points[:, 0] * w
mean_value_points[:, 1] = mean_value_points[:, 1] * h
while True:
dst = get_texture(mean_values_shape, mean_values_texture)
dst = get_texture(mean_value_points, mean_values_texture)
cv2.imshow('input_image', input_image)
cv2.imshow('image', dst)
......@@ -205,8 +211,6 @@ def show_reconstruction(args):
assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
from utils import triangles as tri
Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
......@@ -217,10 +221,18 @@ def show_reconstruction(args):
MeanPoints.get_scaled_points(input_image.shape)
while True:
tri.reconstruct_texture(input_image, input_image, Vt_texture,
InputPoints, MeanPoints,
mean_values_texture, triangles, n_texture_components)
dst = tri.get_texture(MeanPoints, mean_values_texture)
utils.reconstruct_texture(
input_image, # src image
input_image, # dst image
Vt_texture, # Vt
InputPoints, # shape points input
MeanPoints, # shape points mean
mean_values_texture, # mean texture
triangles, # triangles
n_texture_components # learned n_texture_components
)
dst = utils.get_texture(MeanPoints, mean_values_texture)
cv2.imshow('original', InputPoints.get_image())
cv2.imshow('reconstructed', input_image)
......@@ -248,6 +260,8 @@ def main():
elif args.reconstruct:
#reconstruct_with_model(args)
show_reconstruction(args)
elif args.show_kivy:
reconstruct_with_model(args)
if __name__ == '__main__':
main()
src/utils/generate_head_texture_test.py deleted 100644 → 0
View file @ ba216925
import numpy as np
import cv2
import utils.generate_head_texture as ht
import time
blue_image = cv2.imread('data/test_data/blue.png')
red_image = cv2.imread('data/test_data/red.png')
def test_get_colors_triangle():
src = blue_image
dst = red_image
points2d_src = np.array([
[20, 20], [50, 160], [160, 20],
[50, 20], [60, 200], [180, 20]
])
points2d_dst = np.array([
[40, 80], [130, 150], [40, 150],
[40, 80], [60, 82], [60, 100]
])
triangles = [[0, 1, 2]]
t1 = time.clock()
for i in range(10):
for t, tri in enumerate(triangles):
src_p1, src_p2, src_p3 = points2d_src[tri]
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
ht.get_colors_triangle(
src, dst,
src_p1[0], src_p1[1],
src_p2[0], src_p2[1],
src_p3[0], src_p3[1],
dst_p1[0], dst_p1[1],
dst_p2[0], dst_p2[1],
dst_p3[0], dst_p3[1]
)
print (time.clock() - t1) / 10.
#cv2.imshow('blue_image', blue_image)
#cv2.imshow('red_image', red_image)
#cv2.waitKey(0)
assert False
src/utils/triangles.py → src/utils/utils.py
View file @ 9c87511e
import numpy as np
import cv2
from utils.generate_head_texture import fill_triangle, fill_triangle_src_dst
from .texture import fill_triangle, fill_triangle_src_dst
import pca
import aam
......@@ -102,10 +102,10 @@ def reconstruct_texture(src, dst, Vt, SrcPoints, DstPoints,
## Still in S_mean format
r_texture = pca.reconstruct(input_texture, Vt, mean_texture)
# Make an image from the float data
## Make an image from the float data
r_texture = np.asarray(r_texture, np.uint8).reshape((h_slice, w_slice, 3))
## subtract the offset
### subtract the offset
points2d_dst[:, 0] -= offset_x
points2d_dst[:, 1] -= offset_y
......@@ -114,13 +114,11 @@ def reconstruct_texture(src, dst, Vt, SrcPoints, DstPoints,
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
fill_triangle_src_dst(
r_texture, src,
r_texture, dst,
dst_p1[0], dst_p1[1],
dst_p2[0], dst_p2[1],
dst_p3[0], dst_p3[1],
src_p1[0], src_p1[1],
src_p2[0], src_p2[1],
src_p3[0], src_p3[1],
offset_x,
offset_y
src_p3[0], src_p3[1]
)
src/view/reconstruct.py
View file @ 9c87511e
......@@ -20,8 +20,8 @@ from functools import partial
from math import cos, sin, pi
import imm_points as imm
from utils.triangles import draw_texture
from utils.generate_head_texture import fill_triangle
from utils import utils
from utils.texture import fill_triangle
#import IMMPoints, build_feature_vectors, \
# flatten_feature_vectors
import pca
......@@ -77,55 +77,47 @@ class ImageCanvas(Widget):
self.image.source = self.filename_image
self.canvas.ask_update()
def build_texture(self, mean_shape, r_shape, r_texture, triangles):
def build_texture(self, filename, Vt_texture, input_points, mean_value_points,
mean_values_texture, triangles, n_texture_components):
self.texture.clear()
image_width, image_height = self.get_rendered_size()
bary_centric_range = np.linspace(0, 1, num=20)
texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
buf = np.zeros((image_width, image_height, 3), dtype=np.uint8)
image_width, image_height = self.get_rendered_size()
offset = 0
InputPoints = imm.IMMPoints(filename=filename)
input_image = InputPoints.get_image()
buf = np.zeros(input_image.shape, dtype=np.uint8)
for t, tri in enumerate(triangles):
points = r_shape[tri]
InputPoints.get_scaled_points((input_image.shape))
offset_x, offset_y, w, h = InputPoints.get_bounding_box()
x = points[:, 0] * image_width + self.get_image_left(image_width)
y = (1.0 - points[:, 1]) * image_height + self.get_image_bottom(image_height)
MeanPoints = imm.IMMPoints(points_list=mean_value_points)
src_p1_x, src_p2_x, src_p3_x = x
src_p1_y, src_p2_y, src_p3_y = y
for t, tri in enumerate(triangles):
offset += fill_triangle(
r_texture, buf,
src_p1_x, src_p1_y,
src_p2_x, src_p2_y,
src_p3_x, src_p3_y,
dst_p1_x, dst_p1_y,
dst_p2_x, dst_p2_y,
dst_p3_x, dst_p3_y,
offset,
t
utils.reconstruct_texture(
input_image, # src image
buf, # dst image
Vt_texture, # Vt
InputPoints, # shape points input
MeanPoints, # shape points mean
mean_values_texture, # mean texture
triangles, # triangles
n_texture_components # learned n_texture_components
)
print offset
ratio_x = image_width / w
ratio_y = image_height / h
#p1 = [x[0], y[0]]
#p2 = [x[1], y[1]]
#p3 = [x[2], y[2]]
buf = cv2.resize(np.asarray(buf, np.uint8), (int(image_width), int(image_height)))
#buf = buf[offset_y: offset_y + h, offset_x: offset_x + w]
cv2.imshow('original', buf)
buf = b''.join(map(chr, buf.flatten()))
##buf = b''.join(map(chr, buf))
#cv2.imshow('image', buf)
#cv2.waitKey(0)
#texture.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
#self.texture.add(Rectangle(texture=texture, pos=(0, 100),
# size=(image_width, image_height)))
texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
texture.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
self.texture.add(Rectangle(texture=texture, pos=(offset_x, offset_y), size=(w, h)))
#self.canvas.add(self.texture)
#self.canvas.ask_update()
self.canvas.add(self.texture)
self.canvas.ask_update()
def build_line_grid(self, r_shape, triangles):
......@@ -178,24 +170,27 @@ class RootWidget(BoxLayout):
super(RootWidget, self).__init__(**kwargs)
self.files = kwargs['args'].files
self.mean_values_shape = kwargs['mean_values_shape']
self.mean_value_points = kwargs['mean_value_points']
self.mean_values_texture = kwargs['mean_values_texture']
self.eigenv_shape = kwargs['eigenv_shape']
self.eigenv_texture = kwargs['eigenv_texture']
self.triangles = kwargs['triangles']
self.n_shape_components = kwargs['args'].n_components
self.n_texture_components = kwargs['n_texture_components']
self.multipliers = np.ones(self.eigenv_shape.shape[1])
self.mean_values_shape_reshaped = np.copy(self.mean_values_shape).reshape((58, 2))
self.multipliers = np.ones(self.eigenv_shape.shape[0])
self.texture_multipliers = np.ones(self.eigenv_texture.shape[0])
# slider index
self.index = 0
self.filename = ''
self.shape_list = aam.build_shape_feature_vectors(self.files, imm.get_imm_points, flattened=True)
self.shape_list = aam.build_shape_feature_vectors(
self.files, imm.get_imm_points, flattened=True
)
# sliders
self.add_shape_sliders()
self.add_texture_sliders()
self.add_image_slider()
self.add_components_slider()
# ======= #
......@@ -213,6 +208,15 @@ class RootWidget(BoxLayout):
image_slider.max = len(self.files) - 1
image_slider.bind(value=self.update_image)
def add_texture_sliders(self):
box_layout = self.ids['texture_eigenvalues']
for c in range(10):
slider = Slider(min=-10, max=10, value=0, id=str(c))
box_layout.add_widget(slider)
slider.bind(value=self.update_texture_eigenvalues)
def add_shape_sliders(self):
box_layout = self.ids['eigenvalues']
......@@ -222,51 +226,43 @@ class RootWidget(BoxLayout):
slider.bind(value=self.update_eigenvalues)
def reset_sliders(self):
self.multipliers = np.ones(self.eigenv_shape.shape[1])
self.multipliers = np.ones(self.eigenv_shape.shape[0])
self.texture_multipliers = np.ones(self.texture_multipliers.shape[0])
box_layout = self.ids['eigenvalues']
for c in box_layout.children:
c.value = 0
texture_eigenvalues = self.ids['texture_eigenvalues']
for c in texture_eigenvalues.children:
c.value = 0
def update_image_viewer(self):
self.filename = self.files[self.index].split('.')[0] + '.jpg'
Vt_shape = np.dot(np.diag(self.multipliers), self.eigenv_shape)
# Vt_texture = np.dot(np.diag(self.multipliers), self.eigenv_texture)
Vt_texture = np.dot(np.diag(self.texture_multipliers), self.eigenv_texture)
input_points = self.shape_list[self.index]
r_shape = pca.reconstruct(
self.shape_list[self.index], Vt_shape, self.mean_values_shape,
input_points, Vt_shape, self.mean_value_points,
n_components=self.n_shape_components
).reshape((-1, 2))
#image = cv2.imread(self.filename)
#h, w = image.shape[0], image.shape[1]
#mean_values_shape = np.copy(self.mean_values_shape_reshaped)
#mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
#mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
#texture = aam.sample_from_triangles(
# image, r_shape, mean_values_shape, self.triangles
#)
#r_texture = pca.reconstruct(
# texture, self.eigenv_texture, self.mean_values_texture
#)
#r_texture = np.asarray(texture, np.uint8)
# pixels = aam.sample_from_triangles(image, r_shape, self.triangles)
# pixels = np.ndarray.flatten(pixels)
# r_texture = pca.reconstruct(
# pixels, self.eigenv_texture, self.mean_values_texture,
# n_components=50000).reshape((95, -1))
# self.ids['image_viewer'].build_texture(r_shape, r_texture, self.triangles)
self.ids['image_viewer'].update_rect()
self.ids['image_viewer'].update_image(self.filename)
self.ids['image_viewer'].build_line_grid(r_shape, self.triangles)
#self.ids['image_viewer'].build_texture(r_shape, r_texture, self.triangles)
self.ids['image_viewer'].build_texture(
self.files[self.index],
self.eigenv_texture,
input_points,
self.mean_value_points,
self.mean_values_texture,
self.triangles,
self.n_texture_components
)
def on_resize(self, *args):
self.update_image_viewer()
......@@ -280,6 +276,11 @@ class RootWidget(BoxLayout):
self.reset_sliders()
self.update_image_viewer()
def update_texture_eigenvalues(self, slider, value):
multiplier_index = int(slider.id)
self.texture_multipliers[multiplier_index] = value
self.update_image_viewer()
def update_eigenvalues(self, slider, value):
multiplier_index = int(slider.id)
self.multipliers[multiplier_index] = value
......@@ -301,7 +302,7 @@ class ReconstructApp(App):
args=self.args,
eigenv_shape=self.eigenv_shape,
eigenv_texture=self.eigenv_texture,
mean_values_shape=self.mean_values_shape,
mean_value_points=self.mean_value_points,
n_shape_components=self.n_shape_components,
mean_values_texture=self.mean_values_texture,
n_texture_components=self.n_texture_components,
......
src/view/templates/reconstruct.kv
View file @ 9c87511e
......@@ -34,8 +34,9 @@
max: 58
value: 0
BoxLayout:
Label:
text: "Eigenvalues"
BoxLayout:
orientation: 'vertical'
id: texture_eigenvalues
BoxLayout:
orientation: 'vertical'
id: eigenvalues
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment