Skip to content

P
py-3d-face-reconstruction
Commit ba216925 authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Made the scaling operations transparent by means of abstraction

parent c4418b4f
Hide whitespace changes
Inline Side-by-side
Showing with 181 additions and 121 deletions
makefile
View file @ ba216925
......@@ -11,9 +11,9 @@ all: $(TARGETS)
include actions.mk
data: data/imm_face_db
utils: generate_head_texture.so
utils: texture.so
generate_head_texture.so: src/utils/generate_head_texture.pyx
texture.so: src/utils/texture.pyx
(cd src/utils; python setup.py build_ext --inplace)
......
src/aam.py
View file @ ba216925
......@@ -5,7 +5,7 @@ import cv2
# local imports
import pca
from utils.generate_head_texture import fill_triangle, get_row_colors_triangle
import utils.texture as tx
import utils.triangles as tu
logging.basicConfig(level=logging.INFO,
......@@ -13,6 +13,56 @@ logging.basicConfig(level=logging.INFO,
logger = logging.getLogger(__name__)
class AAMPoints():
"""
Object to store AAM points / landmarks. Tries to keep the scaling of
the shape parameters transparent.
"""
def __init__(self, normalized_flattened_points_list=None, points_list=None, actual_shape=()):
self.normalized_flattened_points_list = normalized_flattened_points_list
self.points_list = points_list
self.actual_shape = actual_shape
self.bounding_box = None
def get_bounding_box(self):
if self.bounding_box is None:
return self.calculate_bounding_box()
return self.bounding_box
def get_scaled_points(self, shape, rescale=True):
if self.points_list is None:
self.points_list = self.normalized_flattened_points_list
if len(self.actual_shape):
self.points_list = self.points_list.reshape(self.actual_shape)
h = shape[0]
w = shape[1]
self.points_list[:, 0] = self.points_list[:, 0] * w
self.points_list[:, 1] = self.points_list[:, 1] * h
return self.points_list
def calculate_bounding_box(self):
"""
Calculate bounding box in the **scaled** points list
The empasis on on scaled because the convexHull does not support
small values, the normalized_flattened_points_list does not work.
"""
assert self.points_list is not None, \
'the list points already need to be scaled order to correctly work'
hull = cv2.convexHull(self.points_list, returnPoints=True)
return cv2.boundingRect(hull)
def get_bounding_box_2(self):
pass
#hull = cv2.convexHull(self.points_list, returnPoints=True)
#x, y, w_slice, h_slice = cv2.boundingRect(hull)
#return cv2.boundingRect()
def get_mean(vector):
""" construct a mean from a matrix of x,y values
Args:
......@@ -80,19 +130,12 @@ def sample_from_triangles(src, points2d_src, points2d_dst, triangles, dst):
points2d_dst(ndarray, dtype=np.int32): shape array [[x, y], ... [x, y]]
triangles(ndarray, ndim=3, dtype=np.int32): shape array [[p1, p2, p3]_0, .. [p1, p2, p3]_n].
returns:
ndarray(dtype=uint8): flattened array of bounding boxes around the
given triangles(in order).
"""
triangles_pixels = []
for tri in triangles:
src_p1, src_p2, src_p3 = points2d_src[tri]
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
get_row_colors_triangle(
tx.fill_triangle_src_dst(
src, dst,
src_p1[0], src_p1[1],
src_p2[0], src_p2[1],
......@@ -102,41 +145,38 @@ def sample_from_triangles(src, points2d_src, points2d_dst, triangles, dst):
dst_p3[0], dst_p3[1]
)
#triangles_pixels.extend(dst.flatten())
#return np.asarray(triangles_pixels, dtype=np.uint8)
def build_texture_feature_vectors(files, get_image_with_shape, mean_shape, triangles):
def build_texture_feature_vectors(files, get_image_with_points, MeanPoints, triangles):
"""
Args:
files (list): list files
flattened (bool): Flatten the inner feature vectors, see
flatten_feature_vectors.
MeanPoints(AAMPoints): AAMPoints object
Returns:
list: list of feature vectors
"""
mean_texture = []
mean_shape_scaled = mean_shape.reshape((58, 2))
mean_shape_scaled[:, 0] = mean_shape_scaled[:, 0] * 640
mean_shape_scaled[:, 1] = mean_shape_scaled[:, 1] * 480
hull = cv2.convexHull(mean_shape_scaled, returnPoints=True)
x, y, w_slice, h_slice = cv2.boundingRect(hull)
x, y, w_slice, h_slice = MeanPoints.get_bounding_box()
for i, f in enumerate(files):
image, shape = get_image_with_shape(f)
h, w, c = image.shape
image, points = get_image_with_points(f)
shape[:, 0] = shape[:, 0] * w
shape[:, 1] = shape[:, 1] * h
Points = AAMPoints(
normalized_flattened_points_list=points,
actual_shape=(58, 2)
)
dst = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
# empty colored image
dst = np.full((image.shape[0], image.shape[1], 3), fill_value=0, dtype=np.uint8)
triangles_colors = sample_from_triangles(
image, shape, mean_shape_scaled, triangles, dst
sample_from_triangles(
image,
Points.get_scaled_points(image.shape),
MeanPoints(image.shape),
triangles,
dst
)
dst_flattened = dst[y: y + h_slice, x: x + w_slice].flatten()
......
src/aam_test.py
View file @ ba216925
......@@ -2,10 +2,11 @@ import numpy as np
import cv2
import pytest
import imm_points
import aam
import pca
import imm_points as imm
from utils import triangles as tri
def test_build_mean_aan():
imm_points = np.array([
......@@ -45,10 +46,39 @@ def test_zero_mean_aan():
np.testing.assert_array_equal(zero_mean, expected)
def test_build_texture_feature_vectors():
Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load('data/test_data/pca_shape_model.npy')
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load('data/test_data/pca_texture_model.npy')
InputPoints = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
input_image = InputPoints.get_image()
MeanPoints = imm.IMMPoints(points_list=mean_value_points)
mean_points = MeanPoints.get_scaled_points(input_image.shape)
input_points = InputPoints.get_scaled_points(input_image.shape)
tri.reconstruct_texture(input_image, input_image, Vt_texture, input_points, mean_points,
mean_values_texture, triangles, n_texture_components)
dst = tri.get_texture(mean_points, mean_values_texture)
assert np.mean(input_points) > 1.0, 'should be greater than 1.0, because \
it array should be scaled to the image width and height'
assert np.mean(mean_points) > 1.0, 'should be greater than 1.0, because \
it array should be scaled to the image width and height'
#cv2.imshow('original', imm_points.get_image())
#cv2.imshow('reconstructed', input_image)
#cv2.imshow('main face', dst)
#cv2.waitKey(0) & 0xFF
#cv2.destroyAllWindows()
@pytest.mark.skipif(True, reason='not suitable for pytest')
def test_get_pixel_values():
asf_file = '../data/imm_face_db/40-2m.asf'
imm = imm_points.IMMPoints(filename=asf_file)
Vt, s, n_components, mean_shape, triangles = pca.load(args.model_shape_file)
points = imm.get_points()
image = imm.get_image()
......
src/imm_points.py
View file @ ba216925
......@@ -5,28 +5,39 @@ import numpy as np
import argparse
import os
import aam
class IMMPoints():
class IMMPoints(aam.AAMPoints):
"""Accepts IMM datapoint file which can be shown or used"""
def __init__(self, filename=None, points=None):
def __init__(self, filename=None, points_list=None):
"""
Args:
filename: optional .asf file with the imm format
points: optional list of x,y points
"""
self.points = points if points is not None else []
assert filename is not None or points_list is not None, 'filename or \
a ndarray of points list should be given'
self.filename = filename
if filename:
self.import_file(filename)
points_list = self.import_file(filename)
aam.AAMPoints.__init__(
self, normalized_flattened_points_list=points_list.flatten(),
actual_shape=(58, 2)
)
def get_points(self):
return self.points
return self.points_list
def get_image(self):
return cv2.imread(self.image_file)
def import_file(self, filename):
points_list = []
with open(filename, 'r') as f:
lines = f.readlines()
data = lines[16:74]
......@@ -34,9 +45,9 @@ class IMMPoints():
self.image_file = "{}/{}".format(dir_name, lines[-1].strip())
for d in data:
self.points.append(d.split()[2:4])
points_list.append(d.split()[2:4])
self.points = np.asarray(self.points, dtype='f')
return np.asarray(points_list, dtype='f')
def draw_triangles(self, image, points, multiply=True):
if multiply:
......@@ -61,16 +72,16 @@ class IMMPoints():
cv2.circle(image, tuple(p), 3, color=(0, 255, 100))
def show_on_image(self, image, window_name='image', multiply=True):
self.draw_triangles(image, self.points, multiply=multiply)
self.draw_triangles(image, self.points_list, multiply=multiply)
def show(self, window_name='image'):
"""show the image and datapoints on the image"""
assert(len(self.points) > 0)
assert(len(self.points_list) > 0)
assert(len(self.filename) > 0)
image = self.get_image()
self.draw_triangles(image, self.points)
self.draw_triangles(image, self.points_list)
def get_imm_points(files):
......
src/main.py
View file @ ba216925
......@@ -86,8 +86,13 @@ def save_pca_model_texture(args):
Vt, s, n_components, mean_shape, triangles = pca.load(args.model_shape_file)
MeanPoints = aam.AAMPoints(
normalized_flattened_points_list=mean_value_points,
actual_shape=(58, 2)
)
textures = aam.build_texture_feature_vectors(
args.files, imm.get_imm_image_with_landmarks, mean_shape, triangles
args.files, imm.get_imm_image_with_landmarks, MeanPoints, triangles
)
mean_texture = aam.get_mean(textures)
......@@ -166,13 +171,11 @@ def show_pca_model(args):
assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
from utils.triangles import draw_shape, draw_texture
from utils.triangles import draw_shape, get_texture
Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
image = np.full((480, 640, 3), fill_value=255, dtype=np.uint8)
imm_points = imm.IMMPoints(filename='data/imm_face_db/40-1m.asf')
input_image = imm_points.get_image()
input_points = imm_points.get_points()
......@@ -186,14 +189,10 @@ def show_pca_model(args):
mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
while True:
#draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
# mean_values_texture, triangles, n_texture_components)
#draw_shape(image, mean_values_shape, triangles, multiply=False)
draw_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
mean_values_texture, triangles, n_texture_components)
dst = get_texture(mean_values_shape, mean_values_texture)
cv2.imshow('input_image', input_image)
cv2.imshow('image', image)
cv2.imshow('image', dst)
k = cv2.waitKey(0) & 0xFF
if k == 27:
......@@ -206,40 +205,26 @@ def show_reconstruction(args):
assert args.model_shape_file, '--model_texture_file needs to be provided to save the pca model'
assert args.model_texture_file, '--model_texture_file needs to be provided to save the pca model'
from utils.triangles import draw_shape, draw_texture, reconstruct_texture
from utils import triangles as tri
Vt_shape, s, n_shape_components, mean_values_shape, triangles = pca.load(args.model_shape_file)
Vt_shape, s, n_shape_components, mean_value_points, triangles = pca.load(args.model_shape_file)
Vt_texture, s_texture, n_texture_components, mean_values_texture, _ = pca.load(args.model_texture_file)
image = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
image_2 = np.full((480, 640, 3), fill_value=0, dtype=np.uint8)
InputPoints = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
input_image = InputPoints.get_image()
imm_points = imm.IMMPoints(filename='data/imm_face_db/40-3m.asf')
input_image = imm_points.get_image()
input_points = imm_points.get_points()
h, w, c = input_image.shape
input_points[:, 0] = input_points[:, 0] * w
input_points[:, 1] = input_points[:, 1] * h
mean_values_shape = mean_values_shape.reshape((58, 2))
mean_values_shape[:, 0] = mean_values_shape[:, 0] * w
mean_values_shape[:, 1] = mean_values_shape[:, 1] * h
MeanPoints = imm.IMMPoints(points_list=mean_value_points)
MeanPoints.get_scaled_points(input_image.shape)
while True:
#reconstruct_texture(input_image, image, Vt_texture, input_points, mean_values_shape,
# mean_values_texture, triangles, n_texture_components)
reconstruct_texture(input_image, input_image, Vt_texture, input_points, mean_values_shape,
mean_values_texture, triangles, n_texture_components)
draw_texture(input_image, image_2, Vt_texture, input_points, mean_values_shape,
mean_values_texture, triangles, n_texture_components)
#draw_shape(image_2, mean_values_shape, triangles, multiply=False)
#draw_shape(input_image, input_points, triangles, multiply=False)
tri.reconstruct_texture(input_image, input_image, Vt_texture,
InputPoints, MeanPoints,
mean_values_texture, triangles, n_texture_components)
dst = tri.get_texture(MeanPoints, mean_values_texture)
cv2.imshow('original', imm_points.get_image())
cv2.imshow('original', InputPoints.get_image())
cv2.imshow('reconstructed', input_image)
cv2.imshow('image_2', image_2)
cv2.imshow('main face', dst)
k = cv2.waitKey(0) & 0xFF
......
src/utils/setup.py
View file @ ba216925
......@@ -5,8 +5,8 @@ from distutils.extension import Extension
from Cython.Build import cythonize
extensions = [
Extension(
'generate_head_texture',
['generate_head_texture.pyx'],
'texture',
['texture.pyx'],
include_dirs=[np.get_include()], )
]
......
src/utils/generate_head_texture.pyx → src/utils/texture.pyx
View file @ ba216925
......@@ -57,10 +57,9 @@ cdef inline np.ndarray[double, ndim=2] barycentric2cartesian(
@cython.boundscheck(False)
@cython.wraparound(False)
def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
def fill_triangle(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int x1, int y1, int x2, int y2, int x3, int y3, int offset,
int index):
int x1, int y1, int x2, int y2, int x3, int y3):
"""
Fill a triangle by applying the Barycentric Algorithm for deciding if a
point lies inside or outside a triangle.
......@@ -96,20 +95,18 @@ def fill_triangle(np.ndarray[unsigned char, ndim=1] src,
# notice we have a soft margin of -0.00001, which makes sure there are no
# gaps due to rounding issues
if s >= -0.01 and t >= -0.01 and s + t <= 1.0:
dst[y, x, :] = src_reshaped[j, i, :]
return (w * h * 3)
dst[y, x, :] = src[y, x, :]
@cython.boundscheck(False)
@cython.wraparound(False)
def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3,
int offset_x, int offset_y):
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3,
int offset_x, int offset_y):
cdef np.ndarray triangle_x = np.array([dst_x1, dst_x2, dst_x3])
cdef np.ndarray triangle_y = np.array([dst_y1, dst_y2, dst_y3])
......@@ -151,12 +148,12 @@ def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
@cython.boundscheck(False)
@cython.wraparound(False)
def get_row_colors_triangle(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3):
def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int src_x1, int src_y1, int src_x2, int src_y2,
int src_x3, int src_y3,
int dst_x1, int dst_y1, int dst_x2, int dst_y2,
int dst_x3, int dst_y3):
"""
Fill a triangle by applying the Barycentric Algorithm for deciding if a
point lies inside or outside a triangle.
......
src/utils/triangles.py
View file @ ba216925
......@@ -71,48 +71,45 @@ def draw_shape(image, points, triangles, multiply=True):
cv2.circle(image, tuple(p), 3, color=(0, 255, 100))
def draw_texture(src, dst, Vt, points2d_src, points2d_dst, texture, triangles, n_components):
texture = np.asarray(texture, np.uint8)
offset = 0
def get_texture(Points, flattened_texture):
offset_x, offset_y, w_slice, h_slice = Points.get_bounding_box()
for t, tri in enumerate(triangles):
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
offset += fill_triangle(
texture, dst,
dst_p1[0], dst_p1[1],
dst_p2[0], dst_p2[1],
dst_p3[0], dst_p3[1],
offset,
t
)
# Make a rectangle image from the flattened texture array
return np.asarray(flattened_texture, np.uint8).reshape((h_slice, w_slice, 3))
def reconstruct_texture(src, dst, Vt, points2d_src, points2d_dst, mean_texture, triangles, n_components):
def reconstruct_texture(src, dst, Vt, SrcPoints, DstPoints,
mean_texture, triangles, n_components):
# S_mean format
h, w, c = src.shape
input_texture = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
aam.sample_from_triangles(src, points2d_src, points2d_dst, triangles, input_texture)
hull = cv2.convexHull(points2d_dst, returnPoints=True)
offset_x, offset_y, w_slice, h_slice = cv2.boundingRect(hull)
points2d_src = SrcPoints.get_scaled_points(src.shape)
points2d_dst = DstPoints.get_scaled_points(dst.shape)
aam.sample_from_triangles(
src,
points2d_src,
points2d_dst,
triangles,
input_texture
)
offset_x, offset_y, w_slice, h_slice = DstPoints.get_bounding_box()
input_texture = input_texture[offset_y: offset_y + h_slice,
offset_x: offset_x + w_slice].flatten()
# Still in S_mean format
## Still in S_mean format
r_texture = pca.reconstruct(input_texture, Vt, mean_texture)
# Make an image from the float data
r_texture = np.asarray(r_texture, np.uint8).reshape((h_slice, w_slice, 3))
offset = 0
# subtract the offset
## subtract the offset
points2d_dst[:, 0] -= offset_x
points2d_dst[:, 1] -= offset_y
for t, tri in enumerate(triangles):
for tri in triangles:
src_p1, src_p2, src_p3 = points2d_src[tri]
dst_p1, dst_p2, dst_p3 = points2d_dst[tri]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment