Skip to content

P
py-3d-face-reconstruction
Commit c4418b4f authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Do the reconstruction in a test environment

parent f7da1d93
Hide whitespace changes
Inline Side-by-side
Showing with 218 additions and 120 deletions
actions.mk
View file @ c4418b4f
......@@ -6,7 +6,7 @@ data/imm_face_db: data/imm_face_db.tar.gz
tar -xvzf imm_face_db.tar.gz -C imm_face_db
)
train_model:train_shape
train_model:train_shape train_texture
train_texture: data/pca_texture_model.npy
train_shape: data/pca_shape_model.npy
......
src/aam.py
View file @ c4418b4f
......@@ -5,8 +5,7 @@ import cv2
# local imports
import pca
from utils.generate_head_texture import fill_triangle, get_colors_triangle, \
get_row_colors_triangle
from utils.generate_head_texture import fill_triangle, get_row_colors_triangle
import utils.triangles as tu
logging.basicConfig(level=logging.INFO,
......@@ -72,17 +71,29 @@ def build_shape_feature_vectors(files, get_points, flattened=False):
return points
def sample_from_triangles(src, points2d_src, points2d_dst, triangles):
# texture = np.asarray(texture, dtype=np.uint8).reshape((-1, 3))
def sample_from_triangles(src, points2d_src, points2d_dst, triangles, dst):
"""
Get pixels from within the triangles [[p1, p2, p3]_0, .. [p1, p2, p3]_n].
Args:
src(ndarray, dtype=uint8): input image
points2d_src(ndarray, dtype=np.int32): shape array [[x, y], ... [x, y]]
points2d_dst(ndarray, dtype=np.int32): shape array [[x, y], ... [x, y]]
triangles(ndarray, ndim=3, dtype=np.int32): shape array [[p1, p2, p3]_0, .. [p1, p2, p3]_n].
returns:
ndarray(dtype=uint8): flattened array of bounding boxes around the
given triangles(in order).
"""
triangles_pixels = []
pixels = 0
for tri in triangles:
src_p1, src_p2, src_p3 = points2d_src[tri]
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
dst = get_row_colors_triangle(
src,
get_row_colors_triangle(
src, dst,
src_p1[0], src_p1[1],
src_p2[0], src_p2[1],
src_p3[0], src_p3[1],
......@@ -91,13 +102,9 @@ def sample_from_triangles(src, points2d_src, points2d_dst, triangles):
dst_p3[0], dst_p3[1]
)
pixels += dst.flatten().shape[0]
triangles_pixels.extend(dst.flatten())
#triangles_pixels.extend(dst.flatten())
result = np.asarray(triangles_pixels, dtype=np.uint8)
return result
#return np.asarray(triangles_pixels, dtype=np.uint8)
def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, triangles):
......@@ -116,6 +123,9 @@ def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, trian
mean_shape_scaled[:, 0] = mean_shape_scaled[:, 0] * 640
mean_shape_scaled[:, 1] = mean_shape_scaled[:, 1] * 480
hull = cv2.convexHull(mean_shape_scaled, returnPoints=True)
x, y, w_slice, h_slice = cv2.boundingRect(hull)
for i, f in enumerate(files):
image, shape = get_image_with_shape(f)
h, w, c = image.shape
......@@ -123,25 +133,22 @@ def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, trian
shape[:, 0] = shape[:, 0] * w
shape[:, 1] = shape[:, 1] * h
triangles_colors = sample_from_triangles(
image, shape, mean_shape_scaled, triangles
)
dst = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
mean_texture.append(triangles_colors)
logger.info('processed file: {} {}/{}'.format(f, i, len(files)))
triangles_colors = sample_from_triangles(
image, shape, mean_shape_scaled, triangles, dst
)
# cv2.imshow('image', image)
# k = cv2.waitKey(0) & 0xFF
dst_flattened = dst[y: y + h_slice, x: x + w_slice].flatten()
mean_texture.append(dst_flattened)
# if k == 27:
# break
mean_texture = np.asarray(mean_texture)
#mean_texture = pca.flatten_feature_vectors(mean_texture)
logger.info('processed file: {} {}/{}'.format(f, i, len(files)))
return mean_texture
return np.asarray(mean_texture)
def get_pixel_values(image, points):
""" docstring """
h, w, c = image.shape
points[:, 0] = points[:, 0] * w
......
src/main.py
View file @ c4418b4f
......@@ -186,9 +186,11 @@ def show_pca_model(args):
mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
while True:
#draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
# mean_values_texture, triangles, n_texture_components)
#draw_shape(image, mean_values_shape, triangles, multiply=False)
draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
mean_values_texture, triangles, n_texture_components)
#draw_shape(image, mean_values_shape, triangles, multiply=False)
cv2.imshow('input_image', input_image)
cv2.imshow('image', image)
......@@ -200,6 +202,53 @@ def show_pca_model(args):
cv2.destroyAllWindows()
def show_reconstruction(args):
assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
from utils.triangles import draw_shape, draw_texture, reconstruct_texture
Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
image = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
image_2 = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
imm_points = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
input_image = imm_points.get_image()
input_points = imm_points.get_points()
h, w, c = input_image.shape
input_points[:, 0] = input_points[:, 0] * w
input_points[:, 1] = input_points[:, 1] * h
mean_values_shape = mean_values_shape.reshape((58, 2))
mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
while True:
#reconstruct_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
# mean_values_texture, triangles, n_texture_components)
reconstruct_texture(input_image, input_image, Vt_texture, input_points, mean_values_shape,
mean_values_texture, triangles, n_texture_components)
draw_texture(input_image, image_2, Vt_texture, input_points, mean_values_shape,
mean_values_texture, triangles, n_texture_components)
#draw_shape(image_2, mean_values_shape, triangles, multiply=False)
#draw_shape(input_image, input_points, triangles, multiply=False)
cv2.imshow('original', imm_points.get_image())
cv2.imshow('reconstructed', input_image)
cv2.imshow('image_2', image_2)
k = cv2.waitKey(0) & 0xFF
if k == 27:
break
cv2.destroyAllWindows()
def main():
"""main"""
parser = add_parser_options()
......@@ -212,7 +261,8 @@ def main():
elif args.save_pca_texture:
save_pca_model_texture(args)
elif args.reconstruct:
reconstruct_with_model(args)
#reconstruct_with_model(args)
show_reconstruction(args)
if __name__ == '__main__':
main()
src/utils/generate_head_texture.pyx
View file @ c4418b4f
......@@ -28,7 +28,12 @@ cdef inline np.ndarray[double, ndim=1] cartesian2barycentric(
ndarray (of dim 3) weights of the barycentric coordinates
"""
a = np.array([[r1_x, r2_x, r3_x], [r1_y, r2_y, r3_y], [1, 1, 1]])
a = np.array([
[r1_x, r2_x, r3_x],
[r1_y, r2_y, r3_y],
[1, 1, 1]
])
b = np.array([r_x, r_y, 1])
return np.linalg.solve(a, b)
......@@ -77,10 +82,7 @@ def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
cdef int h = y_max - y_min
cdef int new_offset
cdef np.ndarray src_reshaped = src[offset:offset + (w * h * 3)].reshape((h, w, 3))
#print src_reshaped
#print '(', w, '*', h, '*', 3, ') * ', ' = ', offset
#cdef np.ndarray src_reshaped = src[offset:offset + (w * h * 3)].reshape((h, w, 3))
for j, y in enumerate(xrange(y_min, y_max)):
for i, x in enumerate(xrange(x_min, x_max)):
......@@ -93,40 +95,38 @@ def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
# notice we have a soft margin of -0.00001, which makes sure there are no
# gaps due to rounding issues
if s >= -0.000000001 and t >= -0.000000001 and s + t <= 1.0:
if s >= -0.01 and t >= -0.01 and s + t <= 1.0:
dst[y, x, :] = src_reshaped[j, i, :]
new_offset = (w * h * 3)
return new_offset
return (w * h * 3)
@cython.boundscheck(False)
@cython.wraparound(False)
def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3):
"""
Fill a triangle by applying the Barycentric Algorithm for deciding if a
point lies inside or outside a triangle.
"""
cdef int x_min = min(dst_x1, min(dst_x2, dst_x3))
cdef int x_max = max(dst_x1, max(dst_x2, dst_x3))
cdef int y_min = min(dst_y1, min(dst_y2, dst_y3))
cdef int y_max = max(dst_y1, max(dst_y2, dst_y3))
def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3,
int offset_x, int offset_y):
cdef np.ndarray triangle_x = np.array([dst_x1, dst_x2, dst_x3])
cdef np.ndarray triangle_y = np.array([dst_y1, dst_y2, dst_y3])
cdef int x_min = np.argmin(triangle_x)
cdef int x_max = np.argmax(triangle_x)
cdef int y_min = np.argmin(triangle_y)
cdef int y_max = np.argmax(triangle_y)
cdef np.ndarray L = np.zeros([3, 1], dtype=DTYPE_float32)
cdef np.ndarray matrix = np.full([3, 3], fill_value=1, dtype=DTYPE_int)
cdef np.ndarray src_loc = np.zeros([3, 1], dtype=DTYPE_float64)
cdef np.ndarray dst_loc = np.zeros([3, 1], dtype=DTYPE_float64)
cdef np.ndarray dst = np.full(
[y_max - y_min, x_max - x_min, 3], fill_value=255, dtype=DTYPE_float64
)
for j, y in enumerate(xrange(y_min, y_max)):
for i, x in enumerate(xrange(x_min, x_max)):
for j, y in enumerate(xrange(triangle_y[y_min], triangle_y[y_max])):
for i, x in enumerate(xrange(triangle_x[x_min], triangle_x[x_max])):
dst_loc = cartesian2barycentric(
dst_x1, dst_y1, dst_x2, dst_y2, dst_x3, dst_y3, x, y
)
......@@ -134,9 +134,7 @@ def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
s = dst_loc[0]
t = dst_loc[1]
# notice we have a soft margin of -0.00001, which makes sure there are no
# gaps due to rounding issues
if s >= -0.000001 and t >= -0.000001 and s + t <= 1.0:
if s >= -0.001 and t >= -0.001 and s + t <= 1.001:
L[0] = s
L[1] = t
L[2] = 1 - s - t
......@@ -148,30 +146,28 @@ def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
L
)
dst[j, i, :] = src[src_loc[1][0], src_loc[0][0], :]
return dst
dst[y, x, :] = src[src_loc[1][0], src_loc[0][0], :]
@cython.boundscheck(False)
@cython.wraparound(False)
def get_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3):
def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3):
"""
Fill a triangle by applying the Barycentric Algorithm for deciding if a
point lies inside or outside a triangle.
"""
cdef int x_min = min(dst_x1, min(dst_x2, dst_x3))
cdef int x_max = max(dst_x1, max(dst_x2, dst_x3))
cdef int y_min = min(dst_y1, min(dst_y2, dst_y3))
cdef int y_max = max(dst_y1, max(dst_y2, dst_y3))
cdef np.ndarray triangle_x = np.array([dst_x1, dst_x2, dst_x3])
cdef np.ndarray triangle_y = np.array([dst_y1, dst_y2, dst_y3])
cdef float s
cdef float t
cdef int x_min = np.argmin(triangle_x)
cdef int x_max = np.argmax(triangle_x)
cdef int y_min = np.argmin(triangle_y)
cdef int y_max = np.argmax(triangle_y)
cdef np.ndarray L = np.zeros([3, 1], dtype=DTYPE_float32)
cdef np.ndarray matrix = np.full([3, 3], fill_value=1, dtype=DTYPE_int)
......@@ -179,18 +175,19 @@ def get_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
cdef np.ndarray src_loc = np.zeros([3, 1], dtype=DTYPE_float64)
cdef np.ndarray dst_loc = np.zeros([3, 1], dtype=DTYPE_float64)
for y in xrange(y_min, y_max):
for x in xrange(x_min, x_max):
for j, y in enumerate(xrange(triangle_y[y_min], triangle_y[y_max])):
for i, x in enumerate(xrange(triangle_x[x_min], triangle_x[x_max])):
dst_loc = cartesian2barycentric(
dst_x1, dst_y1, dst_x2, dst_y2, dst_x3, dst_y3, x, y
triangle_x[0], triangle_y[0],
triangle_x[1], triangle_y[1],
triangle_x[2], triangle_y[2],
x, y
)
s = dst_loc[0]
t = dst_loc[1]
# notice we have a soft margin of -0.00001, which makes sure there are no
# gaps due to rounding issues
if s >= -0.000001 and t >= -0.000001 and s + t <= 1.0:
if s >= -0.01 and t >= -0.01 and s + t <= 1.001:
L[0] = s
L[1] = t
L[2] = 1 - s - t
......
src/utils/triangles.py
View file @ c4418b4f
import numpy as np
import cv2
from utils.generate_head_texture import fill_triangle, get_colors_triangle
from utils.generate_head_texture import fill_triangle, fill_triangle_src_dst
import pca
import aam
def cartesian2barycentric(r1, r2, r3, r):
"""
......@@ -62,25 +64,66 @@ def draw_shape(image, points, triangles, multiply=True):
for i, p in enumerate(points):
point_index = int(point_indices[i])
cv2.putText(image, str(point_index), (p[0], p[1]),
#cv2.putText(image, str(point_index), (p[0], p[1]),
# cv2.FONT_HERSHEY_SIMPLEX, .5, (100, 0, 255))
cv2.putText(image, str(i), (p[0], p[1]),
cv2.FONT_HERSHEY_SIMPLEX, .5, (100, 0, 255))
cv2.circle(image, tuple(p), 3, color=(0, 255, 100))
def draw_texture(src, dest, Vt, points2d_src, points2d_dst, texture, triangles, n_components):
# texture = np.asarray(texture, dtype=np.uint8).reshape((-1, 3))
def draw_texture(src, dst, Vt, points2d_src, points2d_dst, texture, triangles, n_components):
texture = np.asarray(texture, np.uint8)
offset = 0
for t, tri in enumerate(triangles):
src_p1, src_p2, src_p3 = points2d_src[tri]
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
offset += fill_triangle(
texture, dest,
texture, dst,
dst_p1[0], dst_p1[1],
dst_p2[0], dst_p2[1],
dst_p3[0], dst_p3[1],
offset,
t
)
def reconstruct_texture(src, dst, Vt, points2d_src, points2d_dst, mean_texture, triangles, n_components):
# S_mean format
h, w, c = src.shape
input_texture = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
aam.sample_from_triangles(src, points2d_src, points2d_dst, triangles, input_texture)
hull = cv2.convexHull(points2d_dst, returnPoints=True)
offset_x, offset_y, w_slice, h_slice = cv2.boundingRect(hull)
input_texture = input_texture[offset_y: offset_y + h_slice,
offset_x: offset_x + w_slice].flatten()
# Still in S_mean format
r_texture = pca.reconstruct(input_texture, Vt, mean_texture)
# Make an image from the float data
r_texture = np.asarray(r_texture, np.uint8).reshape((h_slice, w_slice, 3))
offset = 0
# subtract the offset
points2d_dst[:, 0] -= offset_x
points2d_dst[:, 1] -= offset_y
for t, tri in enumerate(triangles):
src_p1, src_p2, src_p3 = points2d_src[tri]
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
fill_triangle_src_dst(
r_texture, src,
dst_p1[0], dst_p1[1],
dst_p2[0], dst_p2[1],
dst_p3[0], dst_p3[1],
src_p1[0], src_p1[1],
src_p2[0], src_p2[1],
src_p3[0], src_p3[1],
offset_x,
offset_y
)
src/view/reconstruct.py
View file @ c4418b4f
......@@ -77,42 +77,43 @@ class ImageCanvas(Widget):
self.image.source = self.filename_image
self.canvas.ask_update()
def build_texture(self, r_shape, r_texture, triangles):
return
#self.texture.clear()
#image_width, image_height = self.get_rendered_size()
#bary_centric_range = np.linspace(0, 1, num=20)
#texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
#buf = np.zeros((image_width, image_height, 3), dtype=np.uint8)
#offset = 0
#for t, tri in enumerate(triangles):
# src_p1, src_p2, src_p3 = r_shape[tri]
# x = r_shape[:, 0] * image_width + self.get_image_left(image_width)
# y = (1.0 - r_shape[:, 1]) * image_height + self.get_image_bottom(image_height)
# for t, tri in enumerate(triangles):
# offset += fill_triangle(
# r_texture, buf,
# src_p1[0], src_p1[1],
# src_p2[0], src_p2[1],
# src_p3[0], src_p3[1],
# dst_p1[0], dst_p1[1],
# dst_p2[0], dst_p2[1],
# dst_p3[0], dst_p3[1],
# offset,
# t
# )
# print offset
# #p1 = [x[0], y[0]]
# #p2 = [x[1], y[1]]
# #p3 = [x[2], y[2]]
def build_texture(self, mean_shape, r_shape, r_texture, triangles):
self.texture.clear()
image_width, image_height = self.get_rendered_size()
bary_centric_range = np.linspace(0, 1, num=20)
texture = Texture.create(size=(image_width, image_height), colorfmt='bgr')
buf = np.zeros((image_width, image_height, 3), dtype=np.uint8)
offset = 0
for t, tri in enumerate(triangles):
points = r_shape[tri]
x = points[:, 0] * image_width + self.get_image_left(image_width)
y = (1.0 - points[:, 1]) * image_height + self.get_image_bottom(image_height)
src_p1_x, src_p2_x, src_p3_x = x
src_p1_y, src_p2_y, src_p3_y = y
for t, tri in enumerate(triangles):
offset += fill_triangle(
r_texture, buf,
src_p1_x, src_p1_y,
src_p2_x, src_p2_y,
src_p3_x, src_p3_y,
dst_p1_x, dst_p1_y,
dst_p2_x, dst_p2_y,
dst_p3_x, dst_p3_y,
offset,
t
)
print offset
#p1 = [x[0], y[0]]
#p2 = [x[1], y[1]]
#p3 = [x[2], y[2]]
##buf = b''.join(map(chr, buf))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment