Skip to content

E
eos
Commit 224fce4e authored by Patrik Huber's avatar Patrik Huber
Browse files
Options

Parallelised texture extraction 

Wrapped the extraction of a triangle in a lambda and calling all lambdas with std::async
parent 26371d1d
Hide whitespace changes
Inline Side-by-side
Showing with 131 additions and 120 deletions
include/eos/render/texture_extraction.hpp
View file @ 224fce4e
......@@ -32,6 +32,7 @@
#include <tuple>
#include <cassert>
#include <future>
namespace eos {
namespace render {
......@@ -112,137 +113,142 @@ inline cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, cv::Mat
Mat isomap = Mat::zeros(isomap_resolution, isomap_resolution, CV_8UC4); // #Todo: We do want an alpha channel. Will be added soon-ish.
// #Todo: We should handle gray images, but output a 4-channel isomap nevertheless I think.
std::vector<std::future<void>> results;
for (const auto& triangle_indices : mesh.tvi) {
// Find out if the current triangle is visible:
// We do a second rendering-pass here. We use the depth-buffer of the final image, and then, here,
// check if each pixel in a triangle is visible. If the whole triangle is visible, we use it to extract
// the texture.
// Possible improvement: - If only part of the triangle is visible, split it
// This could be optimized in 2 ways though:
// - Use render(), or as in render(...), transfer the vertices once, not in a loop over all triangles (vertices are getting transformed multiple times)
// - We transform them later (below) a second time. Only do it once.
// Project the triangle vertices to screen coordinates, and use the depthbuffer to check whether the triangle is visible:
Vec4f v0 = Mat(affine_camera_matrix * Mat(mesh.vertices[triangle_indices[0]]));
Vec4f v1 = Mat(affine_camera_matrix * Mat(mesh.vertices[triangle_indices[1]]));
Vec4f v2 = Mat(affine_camera_matrix * Mat(mesh.vertices[triangle_indices[2]]));
if (!detail::is_triangle_visible(v0, v1, v2, depthbuffer))
{
continue;
}
// Todo: Documentation
cv::Point2f src_tri[3];
cv::Point2f dst_tri[3];
Vec4f vec(mesh.vertices[triangle_indices[0]][0], mesh.vertices[triangle_indices[0]][1], mesh.vertices[triangle_indices[0]][2], 1.0f);
Vec4f res = Mat(affine_camera_matrix * Mat(vec));
src_tri[0] = Vec2f(res[0], res[1]);
vec = Vec4f(mesh.vertices[triangle_indices[1]][0], mesh.vertices[triangle_indices[1]][1], mesh.vertices[triangle_indices[1]][2], 1.0f);
res = Mat(affine_camera_matrix * Mat(vec));
src_tri[1] = Vec2f(res[0], res[1]);
vec = Vec4f(mesh.vertices[triangle_indices[2]][0], mesh.vertices[triangle_indices[2]][1], mesh.vertices[triangle_indices[2]][2], 1.0f);
res = Mat(affine_camera_matrix * Mat(vec));
src_tri[2] = Vec2f(res[0], res[1]);
dst_tri[0] = cv::Point2f(isomap.cols*mesh.texcoords[triangle_indices[0]][0], isomap.rows*mesh.texcoords[triangle_indices[0]][1] - 1.0f);
dst_tri[1] = cv::Point2f(isomap.cols*mesh.texcoords[triangle_indices[1]][0], isomap.rows*mesh.texcoords[triangle_indices[1]][1] - 1.0f);
dst_tri[2] = cv::Point2f(isomap.cols*mesh.texcoords[triangle_indices[2]][0], isomap.rows*mesh.texcoords[triangle_indices[2]][1] - 1.0f);
// Get the inverse Affine Transform from original image: from dst to src
Mat warp_mat_org_inv = cv::getAffineTransform(dst_tri, src_tri);
warp_mat_org_inv.convertTo(warp_mat_org_inv, CV_32FC1);
// We now loop over all pixels in the triangle and select, depending on the mapping type, the corresponding texel(s) in the source image
for (int x = min(dst_tri[0].x, min(dst_tri[1].x, dst_tri[2].x)); x < max(dst_tri[0].x, max(dst_tri[1].x, dst_tri[2].x)); ++x) {
for (int y = min(dst_tri[0].y, min(dst_tri[1].y, dst_tri[2].y)); y < max(dst_tri[0].y, max(dst_tri[1].y, dst_tri[2].y)); ++y) {
if (detail::is_point_in_triangle(cv::Point2f(x, y), dst_tri[0], dst_tri[1], dst_tri[2])) {
if (mapping_type == TextureInterpolation::Area) {
// calculate positions of 4 corners of pixel in image (src)
Vec3f homogenous_dst_upper_left(x - 0.5f, y - 0.5f, 1.0f);
Vec3f homogenous_dst_upper_right(x + 0.5f, y - 0.5f, 1.0f);
Vec3f homogenous_dst_lower_left(x - 0.5f, y + 0.5f, 1.0f);
Vec3f homogenous_dst_lower_right(x + 0.5f, y + 0.5f, 1.0f);
Vec2f src_texel_upper_left = Mat(warp_mat_org_inv * Mat(homogenous_dst_upper_left));
Vec2f src_texel_upper_right = Mat(warp_mat_org_inv * Mat(homogenous_dst_upper_right));
Vec2f src_texel_lower_left = Mat(warp_mat_org_inv * Mat(homogenous_dst_lower_left));
Vec2f src_texel_lower_right = Mat(warp_mat_org_inv * Mat(homogenous_dst_lower_right));
float min_a = min(min(src_texel_upper_left[0], src_texel_upper_right[0]), min(src_texel_lower_left[0], src_texel_lower_right[0]));
float max_a = max(max(src_texel_upper_left[0], src_texel_upper_right[0]), max(src_texel_lower_left[0], src_texel_lower_right[0]));
float min_b = min(min(src_texel_upper_left[1], src_texel_upper_right[1]), min(src_texel_lower_left[1], src_texel_lower_right[1]));
float max_b = max(max(src_texel_upper_left[1], src_texel_upper_right[1]), max(src_texel_lower_left[1], src_texel_lower_right[1]));
cv::Vec3i color;
int num_texels = 0;
for (int a = ceil(min_a); a <= floor(max_a); ++a)
{
for (int b = ceil(min_b); b <= floor(max_b); ++b)
auto extract_triangle = [&mesh, &affine_camera_matrix, triangle_indices = triangle_indices, &depthbuffer, &isomap, &mapping_type, &image]() {
// Find out if the current triangle is visible:
// We do a second rendering-pass here. We use the depth-buffer of the final image, and then, here,
// check if each pixel in a triangle is visible. If the whole triangle is visible, we use it to extract
// the texture.
// Possible improvement: - If only part of the triangle is visible, split it
// This could be optimized in 2 ways though:
// - Use render(), or as in render(...), transfer the vertices once, not in a loop over all triangles (vertices are getting transformed multiple times)
// - We transform them later (below) a second time. Only do it once.
// Project the triangle vertices to screen coordinates, and use the depthbuffer to check whether the triangle is visible:
Vec4f v0 = Mat(affine_camera_matrix * Mat(mesh.vertices[triangle_indices[0]]));
Vec4f v1 = Mat(affine_camera_matrix * Mat(mesh.vertices[triangle_indices[1]]));
Vec4f v2 = Mat(affine_camera_matrix * Mat(mesh.vertices[triangle_indices[2]]));
if (!detail::is_triangle_visible(v0, v1, v2, depthbuffer))
{
//continue;
return;
}
// Todo: Documentation
cv::Point2f src_tri[3];
cv::Point2f dst_tri[3];
Vec4f vec(mesh.vertices[triangle_indices[0]][0], mesh.vertices[triangle_indices[0]][1], mesh.vertices[triangle_indices[0]][2], 1.0f);
Vec4f res = Mat(affine_camera_matrix * Mat(vec));
src_tri[0] = Vec2f(res[0], res[1]);
vec = Vec4f(mesh.vertices[triangle_indices[1]][0], mesh.vertices[triangle_indices[1]][1], mesh.vertices[triangle_indices[1]][2], 1.0f);
res = Mat(affine_camera_matrix * Mat(vec));
src_tri[1] = Vec2f(res[0], res[1]);
vec = Vec4f(mesh.vertices[triangle_indices[2]][0], mesh.vertices[triangle_indices[2]][1], mesh.vertices[triangle_indices[2]][2], 1.0f);
res = Mat(affine_camera_matrix * Mat(vec));
src_tri[2] = Vec2f(res[0], res[1]);
dst_tri[0] = cv::Point2f(isomap.cols*mesh.texcoords[triangle_indices[0]][0], isomap.rows*mesh.texcoords[triangle_indices[0]][1] - 1.0f);
dst_tri[1] = cv::Point2f(isomap.cols*mesh.texcoords[triangle_indices[1]][0], isomap.rows*mesh.texcoords[triangle_indices[1]][1] - 1.0f);
dst_tri[2] = cv::Point2f(isomap.cols*mesh.texcoords[triangle_indices[2]][0], isomap.rows*mesh.texcoords[triangle_indices[2]][1] - 1.0f);
// Get the inverse Affine Transform from original image: from dst to src
Mat warp_mat_org_inv = cv::getAffineTransform(dst_tri, src_tri);
warp_mat_org_inv.convertTo(warp_mat_org_inv, CV_32FC1);
// We now loop over all pixels in the triangle and select, depending on the mapping type, the corresponding texel(s) in the source image
for (int x = min(dst_tri[0].x, min(dst_tri[1].x, dst_tri[2].x)); x < max(dst_tri[0].x, max(dst_tri[1].x, dst_tri[2].x)); ++x) {
for (int y = min(dst_tri[0].y, min(dst_tri[1].y, dst_tri[2].y)); y < max(dst_tri[0].y, max(dst_tri[1].y, dst_tri[2].y)); ++y) {
if (detail::is_point_in_triangle(cv::Point2f(x, y), dst_tri[0], dst_tri[1], dst_tri[2])) {
if (mapping_type == TextureInterpolation::Area) {
// calculate positions of 4 corners of pixel in image (src)
Vec3f homogenous_dst_upper_left(x - 0.5f, y - 0.5f, 1.0f);
Vec3f homogenous_dst_upper_right(x + 0.5f, y - 0.5f, 1.0f);
Vec3f homogenous_dst_lower_left(x - 0.5f, y + 0.5f, 1.0f);
Vec3f homogenous_dst_lower_right(x + 0.5f, y + 0.5f, 1.0f);
Vec2f src_texel_upper_left = Mat(warp_mat_org_inv * Mat(homogenous_dst_upper_left));
Vec2f src_texel_upper_right = Mat(warp_mat_org_inv * Mat(homogenous_dst_upper_right));
Vec2f src_texel_lower_left = Mat(warp_mat_org_inv * Mat(homogenous_dst_lower_left));
Vec2f src_texel_lower_right = Mat(warp_mat_org_inv * Mat(homogenous_dst_lower_right));
float min_a = min(min(src_texel_upper_left[0], src_texel_upper_right[0]), min(src_texel_lower_left[0], src_texel_lower_right[0]));
float max_a = max(max(src_texel_upper_left[0], src_texel_upper_right[0]), max(src_texel_lower_left[0], src_texel_lower_right[0]));
float min_b = min(min(src_texel_upper_left[1], src_texel_upper_right[1]), min(src_texel_lower_left[1], src_texel_lower_right[1]));
float max_b = max(max(src_texel_upper_left[1], src_texel_upper_right[1]), max(src_texel_lower_left[1], src_texel_lower_right[1]));
cv::Vec3i color;
int num_texels = 0;
for (int a = ceil(min_a); a <= floor(max_a); ++a)
{
if (detail::is_point_in_triangle(cv::Point2f(a, b), src_texel_upper_left, src_texel_lower_left, src_texel_upper_right) || detail::is_point_in_triangle(cv::Point2f(a, b), src_texel_lower_left, src_texel_upper_right, src_texel_lower_right)) {
if (a < image.cols && b < image.rows) { // if src_texel in triangle and in image
num_texels++;
color += image.at<Vec3b>(b, a);
for (int b = ceil(min_b); b <= floor(max_b); ++b)
{
if (detail::is_point_in_triangle(cv::Point2f(a, b), src_texel_upper_left, src_texel_lower_left, src_texel_upper_right) || detail::is_point_in_triangle(cv::Point2f(a, b), src_texel_lower_left, src_texel_upper_right, src_texel_lower_right)) {
if (a < image.cols && b < image.rows) { // if src_texel in triangle and in image
num_texels++;
color += image.at<Vec3b>(b, a);
}
}
}
}
if (num_texels > 0)
color = color / num_texels;
else { // if no corresponding texel found, nearest neighbour interpolation
// calculate corresponding position of dst_coord pixel center in image (src)
Vec3f homogenous_dst_coord = Vec3f(x, y, 1.0f);
Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
if ((cvRound(src_texel[1]) < image.rows) && cvRound(src_texel[0]) < image.cols) {
color = image.at<Vec3b>(cvRound(src_texel[1]), cvRound(src_texel[0]));
}
}
isomap.at<Vec3b>(y, x) = color;
}
if (num_texels > 0)
color = color / num_texels;
else { // if no corresponding texel found, nearest neighbour interpolation
else if (mapping_type == TextureInterpolation::Bilinear) {
// calculate corresponding position of dst_coord pixel center in image (src)
Vec3f homogenous_dst_coord = Vec3f(x, y, 1.0f);
Vec3f homogenous_dst_coord(x, y, 1.0f);
Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
if ((cvRound(src_texel[1]) < image.rows) && cvRound(src_texel[0]) < image.cols) {
color = image.at<Vec3b>(cvRound(src_texel[1]), cvRound(src_texel[0]));
// calculate distances to next 4 pixels
using std::sqrt;
using std::pow;
float distance_upper_left = sqrt(pow(src_texel[0] - floor(src_texel[0]), 2) + pow(src_texel[1] - floor(src_texel[1]), 2));
float distance_upper_right = sqrt(pow(src_texel[0] - floor(src_texel[0]), 2) + pow(src_texel[1] - ceil(src_texel[1]), 2));
float distance_lower_left = sqrt(pow(src_texel[0] - ceil(src_texel[0]), 2) + pow(src_texel[1] - floor(src_texel[1]), 2));
float distance_lower_right = sqrt(pow(src_texel[0] - ceil(src_texel[0]), 2) + pow(src_texel[1] - ceil(src_texel[1]), 2));
// normalise distances
float sum_distances = distance_lower_left + distance_lower_right + distance_upper_left + distance_upper_right;
distance_lower_left /= sum_distances;
distance_lower_right /= sum_distances;
distance_upper_left /= sum_distances;
distance_upper_right /= sum_distances;
// set color depending on distance from next 4 pixels
for (int color = 0; color < 3; ++color) {
float color_upper_left = image.at<Vec3b>(floor(src_texel[1]), floor(src_texel[0]))[color] * distance_upper_left;
float color_upper_right = image.at<Vec3b>(floor(src_texel[1]), ceil(src_texel[0]))[color] * distance_upper_right;
float color_lower_left = image.at<Vec3b>(ceil(src_texel[1]), floor(src_texel[0]))[color] * distance_lower_left;
float color_lower_right = image.at<Vec3b>(ceil(src_texel[1]), ceil(src_texel[0]))[color] * distance_lower_right;
isomap.at<Vec3b>(y, x)[color] = color_upper_left + color_upper_right + color_lower_left + color_lower_right;
}
}
isomap.at<Vec3b>(y, x) = color;
}
else if (mapping_type == TextureInterpolation::Bilinear) {
// calculate corresponding position of dst_coord pixel center in image (src)
Vec3f homogenous_dst_coord(x, y, 1.0f);
Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
// calculate distances to next 4 pixels
using std::sqrt;
using std::pow;
float distance_upper_left = sqrt(pow(src_texel[0] - floor(src_texel[0]), 2) + pow(src_texel[1] - floor(src_texel[1]), 2));
float distance_upper_right = sqrt(pow(src_texel[0] - floor(src_texel[0]), 2) + pow(src_texel[1] - ceil(src_texel[1]), 2));
float distance_lower_left = sqrt(pow(src_texel[0] - ceil(src_texel[0]), 2) + pow(src_texel[1] - floor(src_texel[1]), 2));
float distance_lower_right = sqrt(pow(src_texel[0] - ceil(src_texel[0]), 2) + pow(src_texel[1] - ceil(src_texel[1]), 2));
// normalise distances
float sum_distances = distance_lower_left + distance_lower_right + distance_upper_left + distance_upper_right;
distance_lower_left /= sum_distances;
distance_lower_right /= sum_distances;
distance_upper_left /= sum_distances;
distance_upper_right /= sum_distances;
// set color depending on distance from next 4 pixels
for (int color = 0; color < 3; ++color) {
float color_upper_left = image.at<Vec3b>(floor(src_texel[1]), floor(src_texel[0]))[color] * distance_upper_left;
float color_upper_right = image.at<Vec3b>(floor(src_texel[1]), ceil(src_texel[0]))[color] * distance_upper_right;
float color_lower_left = image.at<Vec3b>(ceil(src_texel[1]), floor(src_texel[0]))[color] * distance_lower_left;
float color_lower_right = image.at<Vec3b>(ceil(src_texel[1]), ceil(src_texel[0]))[color] * distance_lower_right;
isomap.at<Vec3b>(y, x)[color] = color_upper_left + color_upper_right + color_lower_left + color_lower_right;
}
}
else if (mapping_type == TextureInterpolation::NearestNeighbour) {
else if (mapping_type == TextureInterpolation::NearestNeighbour) {
// calculate corresponding position of dst_coord pixel center in image (src)
Vec3f homogenous_dst_coord = Vec3f(x, y, 1.0f);
Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
// calculate corresponding position of dst_coord pixel center in image (src)
Vec3f homogenous_dst_coord = Vec3f(x, y, 1.0f);
Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
if ((cvRound(src_texel[1]) < image.rows) && (cvRound(src_texel[0]) < image.cols) && cvRound(src_texel[0]) > 0 && cvRound(src_texel[1]) > 0)
{
......@@ -255,7 +261,12 @@ inline cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, cv::Mat
}
}
}
}
}; // end lambda auto extract_triangle();
results.emplace_back(std::async(extract_triangle));
} // end for all mesh.tvi
// Collect all the launched tasks:
for (auto&& r : results) {
r.get();
}
return isomap;
};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment