Skip to content

P
py-3d-face-reconstruction
Commit 25baa363 authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Fix the errors of roundings issues for now

parent 8f6cde84
Hide whitespace changes
Inline Side-by-side
Showing with 203 additions and 138 deletions
actions.mk
View file @ 25baa363
......@@ -20,6 +20,7 @@ runnit:
src/reconstruction/texture.so: src/reconstruction/texture.pyx
$(BASE_DOCKER_CMD) /bin/bash -c '(cd reconstruction; python setup.py build_ext --inplace)'
## IMM Dataset
data/pca_imm_shape_model.npy:
$(BASE_DOCKER_CMD) python main.py \
--save_pca_shape \
......@@ -27,6 +28,16 @@ data/pca_imm_shape_model.npy:
--model_shape_file /data/pca_imm_shape_model \
--shape_type imm
data/pca_imm_texture_model.npy:
$(BASE_DOCKER_CMD) python main.py \
--save_pca_texture \
--files `./scripts/imm_train_set.sh` \
--model_texture_file /data/pca_imm_texture_model \
--model_shape_file /data/pca_imm_shape_model.npy \
--shape_type imm
## END OF IMM
## IBUG using dlib landmark detector
data/pca_ibug_shape_model.npy:
$(BASE_DOCKER_CMD) python main.py \
--save_pca_shape \
......@@ -41,14 +52,8 @@ data/pca_ibug_texture_model.npy:
--model_texture_file /data/pca_ibug_texture_model \
--model_shape_file /data/pca_ibug_shape_model.npy \
--shape_type ibug
## END OF IBUG
data/pca_imm_texture_model.npy:
$(BASE_DOCKER_CMD) python main.py \
--save_pca_texture \
--files `./scripts/imm_train_set.sh` \
--model_texture_file /data/pca_imm_texture_model \
--model_shape_file /data/pca_imm_shape_model.npy \
--shape_type imm
test_model:
$(BASE_DOCKER_CMD) python main.py \
......@@ -105,7 +110,9 @@ test:
.PHONY:= server
server:
$(BASE_DOCKER_CMD) python -m tornado.autoreload server.py
docker run $(DOCKER_RUN_FLAGS) $(EXTRA_FLAGS) \
-p 6930:8888 $(IMAGE_TAG) \
python -m tornado.autoreload server.py
.PHONY:= ember
ember:
......@@ -113,4 +120,4 @@ ember:
.PHONY:= ctags
ctags:
ctags -R --python-kinds=-i src
ctags --python-kinds=-i src
makefile
View file @ 25baa363
......@@ -5,15 +5,15 @@ HERE:=$(shell pwd)
DOCKER_RUN_FLAGS:= --rm \
--volume $(HERE)/data:/data \
--volume $(HERE)/src:/src \
-e "DEBUG=$(DEBUG)" \
-p 6930:8888
--volume ~/.bash_history:/root/.bash_history \
-e "DEBUG=$(DEBUG)"
BASE_DOCKER_CMD:= docker run $(DOCKER_RUN_FLAGS) $(IMAGE_TAG)
$(info $(TARGETS))
DEPENDENCIES:= data/imm_face_db
TARGETS:= shape_predictor_68_face_landmarks.dat\
TARGETS:= data/shape_predictor_68_face_landmarks.dat\
src/reconstruction/texture.so \
data/pca_ibug_shape_model.npy \
data/pca_ibug_texture_model.npy
......@@ -31,6 +31,10 @@ build: requirements.txt
run-bash:
docker run --interactive --tty $(DOCKER_RUN_FLAGS) $(IMAGE_TAG) /bin/bash
run-bash-cmd:
docker run --interactive --tty $(DOCKER_RUN_FLAGS) $(IMAGE_TAG) \
/bin/bash -c "$(CMD)"
$(VIRTUALENV):
virtualenv -p $(PYTHON_BIN_PATH) venv
......
src/reconstruction/reconstruction.py
View file @ 25baa363
......@@ -3,45 +3,7 @@ import numpy as np
import pca
import aam
def cartesian2barycentric(r1, r2, r3, r):
"""
Given a triangle spanned by three cartesion points
r1, r2, r2, and point r, return the barycentric weights l1, l2, l3.
Returns:
ndarray (of dim 3) weights of the barycentric coordinates
"""
x, y = r
x1, y1 = r1
x2, y2 = r2
x3, y3 = r3
a = np.array([[x1, x2, x3], [y1, y2, y3], [1, 1, 1]])
b = np.array([x, y, 1])
return np.linalg.solve(a, b)
def barycentric2cartesian(r1, r2, r3, L):
"""
Given the barycentric weights in L, and cartesian r1, r2, r3 coordinates of
points that span the triangle, return the cartesian coordinate of the
points that is located at the weights of L.
Returns:
ndarray [x,y] cartesian points.
"""
x1, y1 = r1
x2, y2 = r2
x3, y3 = r3
a = np.array([[x1, x2, x3], [y1, y2, y3], [1, 1, 1]])
b = np.array(L)
return np.asarray(np.dot(a, b), dtype=np.uint32)
from utility import import_dataset_module
def draw_shape(image, points, triangles, multiply=True):
......@@ -173,3 +135,60 @@ def reconstruct_texture(src_image, dst_image, texture_model,
triangles,
dst_image
)
def reconstruct_shape_texture(dataset_name, shape_model, texture_model,
image_filename, shape_components,
shape_eigenvalues_multiplier=[],
image_as_background=False):
"""Reconstructs shape and texture"""
dataset_module = import_dataset_module(dataset_name)
input_points = dataset_module.factory(filename=image_filename)
input_image = input_points.get_image()
mean_points = dataset_module.factory(points_list=shape_model.mean_values)
mean_points.get_scaled_points(input_image.shape)
shape_eigenvalues_multiplier = np.asarray(
shape_eigenvalues_multiplier, dtype=np.float32
)
# set dst image to an empty image if value is None
if image_as_background is False:
h, w, _ = input_image.shape
dst_image = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
else:
dst_image = input_image
# get the location of the landmarks in a list of [x,y, ... x_n, y_n]
output_points = dataset_module.factory(
points_list=input_points.get_points()
)
# get the pca components (ie., V^T)
shape_Vt = shape_model.Vt
# if a eigen value multiplier array is given, scale the Vt with this.
# the chosen PCA components will have more impact then others.
if len(shape_eigenvalues_multiplier):
shape_Vt = scale_eigenvalues(shape_Vt, shape_eigenvalues_multiplier)
# recontruct the shape
reconstruct_shape(
output_points,
shape_model,
shape_Vt=shape_Vt # overwrite by scaled Vt
)
reconstruct_texture(
input_image, # src image
dst_image, # dst image
texture_model,
input_points, # shape points input
mean_points, # shape points mean
output_points
)
# output_points.draw_triangles(image=dst_image, show_points=False)
return dst_image
src/reconstruction/texture.pyx
View file @ 25baa363
......@@ -116,42 +116,6 @@ cdef inline np.ndarray[double, ndim=2] barycentric2cartesian(
output[1] = y1 * lambdas[0] + y2 * lambdas[1] + y3 * lambdas[2]
@cython.boundscheck(False)
@cython.wraparound(False)
def fill_triangle(np.ndarray[unsigned char, ndim=3] src,
np.ndarray[unsigned char, ndim=3] dst,
int x1, int y1, int x2, int y2, int x3, int y3):
"""
Fill a triangle by applying the Barycentric Algorithm for deciding if a
point lies inside or outside a triangle.
"""
# Get the bounding box of the triangle
cdef int x_min = min(x1, min(x2, x3))
cdef int x_max = max(x1, max(x2, x3))
cdef int y_min = min(y1, min(y2, y3))
cdef int y_max = max(y1, max(y2, y3))
cdef int w = x_max - x_min
cdef int h = y_max - y_min
cdef int new_offset
cdef c_array.array dst_loc = c_array.array('f', [0., 0., 0.])
for j, y in enumerate(xrange(y_min, y_max)):
for i, x in enumerate(xrange(x_min, x_max)):
cartesian2barycentric(
x1, y1, x2, y2, x3, y3, x, y, dst_loc
)
s = dst_loc[0]
t = dst_loc[1]
# In or out the triangle (with a soft margin)
if s >= -0.01 and t >= -0.01 and s + t <= 1.001:
dst[y, x, :] = src[y, x, :]
@cython.boundscheck(False)
@cython.wraparound(False)
def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
......@@ -177,10 +141,16 @@ def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
cdef int y_min = np.argmin(triangle_y)
cdef int y_max = np.argmax(triangle_y)
cdef int src_max_dim_x = src.shape[1]
cdef int src_max_dim_y = src.shape[0]
cdef int dst_max_dim_x = dst.shape[1]
cdef int dst_max_dim_y = dst.shape[0]
# walk over x and y values of this bounding box see if the
# pixel is in or out the boudning box
for y in xrange(triangle_y[y_min] + 1, triangle_y[y_max] - 1):
for x in xrange(triangle_x[x_min] + 1, triangle_x[x_max] - 1):
for y in xrange(triangle_y[y_min], triangle_y[y_max]):
for x in xrange(triangle_x[x_min], triangle_x[x_max]):
cartesian2barycentric(
triangle_x[0], triangle_y[0],
triangle_x[1], triangle_y[1],
......@@ -192,11 +162,66 @@ def fill_triangle_src_dst(np.ndarray[unsigned char, ndim=3] src,
t = dst_loc[1]
# In or out the triangle (with a soft margin)
if s >= -0.01 and t >= -0.01 and s + t <= 1.001:
if s >= -0.01 and t >= -0.01 and s + t <= 1.01:
barycentric2cartesian(
src_x1, src_x2, src_x3,
src_y1, src_y2, src_y3,
dst_loc, src_loc
)
dst[y, x, :] = src[src_loc[1], src_loc[0], :]
#max_dim_y: 188 104.592903137 316 True
#max_dim_x: 201 188.147247314 357 True
#IndexError: index 188 is out of bounds for axis 1 with size 188
#print 'max_dim_y: ', max_dim_y, src_loc[1], y, src_loc[1] < max_dim_y
#print 'max_dim_x: ', max_dim_x, src_loc[0], x, src_loc[0] < max_dim_x
#print 'together:', src_loc[1] < max_dim_y and src_loc[0] < max_dim_x
#if src_loc[1] < src_max_dim_y and src_loc[0] < src_max_dim_x:
# print 'yo'
# print src_loc[1], src_loc[0]
# print y, x
# print dst_max_dim_y
# print dst_max_dim_x
# print src_max_dim_y
# print src_max_dim_x
if src_loc[1] < src_max_dim_y and src_loc[0] < src_max_dim_x \
and y < dst_max_dim_y and x < dst_max_dim_x:
dst[y, x, :] = src[src_loc[1], src_loc[0], :]
#@cython.boundscheck(False)
#@cython.wraparound(False)
#def fill_triangle(np.ndarray[unsigned char, ndim=3] src,
# np.ndarray[unsigned char, ndim=3] dst,
# int x1, int y1, int x2, int y2, int x3, int y3):
# """
# Fill a triangle by applying the Barycentric Algorithm for deciding if a
# point lies inside or outside a triangle.
# """
#
# # Get the bounding box of the triangle
# cdef int x_min = min(x1, min(x2, x3))
# cdef int x_max = max(x1, max(x2, x3))
# cdef int y_min = min(y1, min(y2, y3))
# cdef int y_max = max(y1, max(y2, y3))
#
# cdef int w = x_max - x_min
# cdef int h = y_max - y_min
# cdef int new_offset
#
# cdef c_array.array dst_loc = c_array.array('f', [0., 0., 0.])
#
# for j, y in enumerate(xrange(y_min, y_max)):
# for i, x in enumerate(xrange(x_min, x_max)):
# cartesian2barycentric(
# x1, y1, x2, y2, x3, y3, x, y, dst_loc
# )
#
# s = dst_loc[0]
# t = dst_loc[1]
#
# # In or out the triangle (with a soft margin)
# if s >= -0.01 and t >= -0.01 and s + t <= 1.01:
# dst[y, x, :] = src[y, x, :]
#
src/server.py
View file @ 25baa363
import os
import json
import traceback
import os.path
......@@ -8,16 +9,17 @@ import cv2
import numpy as np
from tornado import websocket, web, ioloop, autoreload
import pca
from datasets import imm
from reconstruction import reconstruction
from settings import logger
from utility import import_dataset_module
FILES_DIR = '/data/'
FACE_DB_NAME = 'imm_face_db'
FACE_DB = '{}{}'.format(FILES_DIR, FACE_DB_NAME)
DATASET = os.environ.get('DATASET', 'ibug') # see src/datasets for options
class ImageWebSocketHandler(websocket.WebSocketHandler):
......@@ -31,8 +33,10 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
self.asf = glob('{}/*.asf'.format(FACE_DB))
# todo get from settings
model_texture_file = '{}/pca_ibug_texture_model.npy'.format(FILES_DIR)
model_shape_file = '{}/pca_ibug_shape_model.npy'.format(FILES_DIR)
model_texture_file = '{}/pca_{}_texture_model.npy'.format(
FILES_DIR, DATASET)
model_shape_file = '{}/pca_{}_shape_model.npy'.format(
FILES_DIR, DATASET)
self.shape_model = pca.PCAModel(model_shape_file)
self.texture_model = pca.PCAModel(model_texture_file)
......@@ -67,55 +71,23 @@ class ImageWebSocketHandler(websocket.WebSocketHandler):
#image = message.get('image')
#input_image = base64.b64decode(image)
shape_eigenvalues_multiplier = np.asarray(
shape_eigenvalues_multiplier, dtype=np.float32
)
logger.info('using %s shape_components', shape_components)
image_filename = self.images[image_index]
dataset_module = import_dataset_module('ibug')
input_points = dataset_module.factory(filename=image_filename)
input_image = input_points.get_image()
mean_points = dataset_module.factory(points_list=self.shape_model.mean_values)
mean_points.get_scaled_points(input_image.shape)
# set dst image to an empty image if value is None
if image_as_background is False:
h, w, _ = input_image.shape
dst_image = np.full((h, w, 3), fill_value=0, dtype=np.uint8)
if DATASET == 'imm':
image_filename = self.asf[image_index]
else:
dst_image = input_image
image_filename = self.images[image_index]
output_points = dataset_module.factory(
points_list=input_points.get_points()
)
shape_Vt = self.shape_model.Vt
shape_Vt = reconstruction.scale_eigenvalues(
shape_Vt, shape_eigenvalues_multiplier
)
# recontruct the shape
reconstruction.reconstruct_shape(
output_points,
dst_image = reconstruction.reconstruct_shape_texture(
DATASET,
self.shape_model,
shape_Vt=shape_Vt # overwrite by scaled Vt
)
reconstruction.reconstruct_texture(
input_image, # src image
dst_image, # dst image
self.texture_model,
input_points, # shape points input
mean_points, # shape points mean
output_points
image_filename,
shape_components,
shape_eigenvalues_multiplier,
image_as_background=image_as_background
)
output_points.draw_triangles(image=dst_image, show_points=False)
_, reconstructed = cv2.imencode('.jpg', dst_image)
reconstructed = base64.b64encode(reconstructed)
......@@ -165,16 +137,20 @@ class ApiHandler(web.RequestHandler):
class FaceHandler(ApiHandler):
@web.asynchronous
def get(self, *args):
"""
Get's all faces in the given FACE_DB and returns the url to be able
to do requests with it in the webapplication.
"""
data = []
for id, filename in enumerate(self.asf_files):
Points = imm.IMMPoints(filename)
data.append({
'type': 'faces',
'id': id,
'attributes': {
'filename': '{}/{}'.format(FACE_DB_NAME, os.path.basename(self.images[id])),
'shape': Points.get_scaled_points((480, 640)).tolist()
'filename': '{}/{}'.format(
FACE_DB_NAME, os.path.basename(self.images[id])
),
}
})
......
src/test/reconstruction_test.py 0 → 100644
View file @ 25baa363
import cv2
import pca as pca
from settings import logger
from reconstruction import reconstruction
#model_texture_file = '/data/pca_ibug_texture_model.npy'
#model_shape_file = '/data/pca_ibug_shape_model.npy'
model_texture_file = '/data/pca_imm_texture_model.npy'
model_shape_file = '/data/pca_imm_shape_model.npy'
def main():
shape_components = 58
shape_model = pca.PCAModel(model_shape_file)
texture_model = pca.PCAModel(model_texture_file)
logger.info('using %s shape_components', shape_components)
image_filename = '/data/imm_face_db/01-1m.asf'
dst_image = reconstruction.reconstruct_shape_texture(
'imm',
shape_model,
texture_model,
image_filename,
shape_components
)
cv2.imwrite('/data/reconstructed.png', dst_image)
if __name__ == '__main__':
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment