22 #ifndef TEXTURE_EXTRACTION_HPP_ 23 #define TEXTURE_EXTRACTION_HPP_ 25 #include "eos/render/detail/texture_extraction_detail.hpp" 26 #include "eos/render/Mesh.hpp" 27 #include "eos/render/render_affine.hpp" 28 #include "eos/render/detail/render_detail.hpp" 30 #include "opencv2/core/core.hpp" 31 #include "opencv2/imgproc/imgproc.hpp" 52 namespace detail { cv::Mat interpolate_black_line(cv::Mat isomap); }
76 inline cv::Mat
extract_texture(
Mesh mesh, cv::Mat affine_camera_matrix, cv::Mat image,
bool compute_view_angle =
false,
TextureInterpolation mapping_type = TextureInterpolation::NearestNeighbour,
int isomap_resolution = 512)
80 std::tie(std::ignore, depthbuffer) =
render::render_affine(mesh, affine_camera_matrix, image.cols, image.rows);
84 return extract_texture(mesh, affine_camera_matrix, image, depthbuffer, compute_view_angle, mapping_type, isomap_resolution);
107 inline cv::Mat
extract_texture(
Mesh mesh, cv::Mat affine_camera_matrix, cv::Mat image, cv::Mat depthbuffer,
bool compute_view_angle =
false,
TextureInterpolation mapping_type = TextureInterpolation::NearestNeighbour,
int isomap_resolution = 512)
110 assert(image.type() == CV_8UC3);
122 affine_camera_matrix = detail::calculate_affine_z_direction(affine_camera_matrix);
124 Mat isomap = Mat::zeros(isomap_resolution, isomap_resolution, CV_8UC4);
127 std::vector<std::future<void>> results;
128 for (
const auto& triangle_indices : mesh.
tvi) {
131 auto extract_triangle = [&mesh, &affine_camera_matrix, &triangle_indices, &depthbuffer, &isomap, &mapping_type, &image, &compute_view_angle]() {
144 const Vec4f v0 = Mat(affine_camera_matrix * Mat(mesh.
vertices[triangle_indices[0]]));
145 const Vec4f v1 = Mat(affine_camera_matrix * Mat(mesh.
vertices[triangle_indices[1]]));
146 const Vec4f v2 = Mat(affine_camera_matrix * Mat(mesh.
vertices[triangle_indices[2]]));
148 if (!detail::is_triangle_visible(v0, v1, v2, depthbuffer))
155 if (compute_view_angle)
159 const Vec3f face_normal =
calculate_face_normal(Vec3f(Mat(mesh.
vertices[triangle_indices[0]]).rowRange(0, 3)), Vec3f(Mat(mesh.
vertices[triangle_indices[1]]).rowRange(0, 3)), Vec3f(Mat(mesh.
vertices[triangle_indices[2]]).rowRange(0, 3)));
161 Vec3f face_normal_transformed = Mat(affine_camera_matrix.rowRange(0, 3).colRange(0, 3) * Mat(face_normal));
162 face_normal_transformed /= cv::norm(face_normal_transformed, cv::NORM_L2);
170 const float angle = -face_normal_transformed[2];
171 assert(angle >= -1.f && angle <= 1.f);
181 alpha_value = angle * 255.0f;
186 alpha_value = 255.0f;
190 cv::Point2f src_tri[3];
191 cv::Point2f dst_tri[3];
193 Vec4f vec(mesh.
vertices[triangle_indices[0]][0], mesh.
vertices[triangle_indices[0]][1], mesh.
vertices[triangle_indices[0]][2], 1.0f);
194 Vec4f res = Mat(affine_camera_matrix * Mat(vec));
195 src_tri[0] = Vec2f(res[0], res[1]);
197 vec = Vec4f(mesh.
vertices[triangle_indices[1]][0], mesh.
vertices[triangle_indices[1]][1], mesh.
vertices[triangle_indices[1]][2], 1.0f);
198 res = Mat(affine_camera_matrix * Mat(vec));
199 src_tri[1] = Vec2f(res[0], res[1]);
201 vec = Vec4f(mesh.
vertices[triangle_indices[2]][0], mesh.
vertices[triangle_indices[2]][1], mesh.
vertices[triangle_indices[2]][2], 1.0f);
202 res = Mat(affine_camera_matrix * Mat(vec));
203 src_tri[2] = Vec2f(res[0], res[1]);
205 dst_tri[0] = cv::Point2f(isomap.cols*mesh.
texcoords[triangle_indices[0]][0], isomap.rows*mesh.
texcoords[triangle_indices[0]][1] - 1.0f);
206 dst_tri[1] = cv::Point2f(isomap.cols*mesh.
texcoords[triangle_indices[1]][0], isomap.rows*mesh.
texcoords[triangle_indices[1]][1] - 1.0f);
207 dst_tri[2] = cv::Point2f(isomap.cols*mesh.
texcoords[triangle_indices[2]][0], isomap.rows*mesh.
texcoords[triangle_indices[2]][1] - 1.0f);
210 Mat warp_mat_org_inv = cv::getAffineTransform(dst_tri, src_tri);
211 warp_mat_org_inv.convertTo(warp_mat_org_inv, CV_32FC1);
214 for (
int x = min(dst_tri[0].x, min(dst_tri[1].x, dst_tri[2].x)); x < max(dst_tri[0].x, max(dst_tri[1].x, dst_tri[2].x)); ++x) {
215 for (
int y = min(dst_tri[0].y, min(dst_tri[1].y, dst_tri[2].y)); y < max(dst_tri[0].y, max(dst_tri[1].y, dst_tri[2].y)); ++y) {
216 if (detail::is_point_in_triangle(cv::Point2f(x, y), dst_tri[0], dst_tri[1], dst_tri[2])) {
217 if (mapping_type == TextureInterpolation::Area) {
220 Vec3f homogenous_dst_upper_left(x - 0.5f, y - 0.5f, 1.0f);
221 Vec3f homogenous_dst_upper_right(x + 0.5f, y - 0.5f, 1.0f);
222 Vec3f homogenous_dst_lower_left(x - 0.5f, y + 0.5f, 1.0f);
223 Vec3f homogenous_dst_lower_right(x + 0.5f, y + 0.5f, 1.0f);
225 Vec2f src_texel_upper_left = Mat(warp_mat_org_inv * Mat(homogenous_dst_upper_left));
226 Vec2f src_texel_upper_right = Mat(warp_mat_org_inv * Mat(homogenous_dst_upper_right));
227 Vec2f src_texel_lower_left = Mat(warp_mat_org_inv * Mat(homogenous_dst_lower_left));
228 Vec2f src_texel_lower_right = Mat(warp_mat_org_inv * Mat(homogenous_dst_lower_right));
230 float min_a = min(min(src_texel_upper_left[0], src_texel_upper_right[0]), min(src_texel_lower_left[0], src_texel_lower_right[0]));
231 float max_a = max(max(src_texel_upper_left[0], src_texel_upper_right[0]), max(src_texel_lower_left[0], src_texel_lower_right[0]));
232 float min_b = min(min(src_texel_upper_left[1], src_texel_upper_right[1]), min(src_texel_lower_left[1], src_texel_lower_right[1]));
233 float max_b = max(max(src_texel_upper_left[1], src_texel_upper_right[1]), max(src_texel_lower_left[1], src_texel_lower_right[1]));
238 for (
int a = ceil(min_a); a <= floor(max_a); ++a)
240 for (
int b = ceil(min_b); b <= floor(max_b); ++b)
242 if (detail::is_point_in_triangle(cv::Point2f(a, b), src_texel_upper_left, src_texel_lower_left, src_texel_upper_right) || detail::is_point_in_triangle(cv::Point2f(a, b), src_texel_lower_left, src_texel_upper_right, src_texel_lower_right)) {
243 if (a < image.cols && b < image.rows) {
245 color += image.at<Vec3b>(b, a);
251 color = color / num_texels;
254 Vec3f homogenous_dst_coord = Vec3f(x, y, 1.0f);
255 Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
257 if ((cvRound(src_texel[1]) < image.rows) && cvRound(src_texel[0]) < image.cols) {
258 color = image.at<Vec3b>(cvRound(src_texel[1]), cvRound(src_texel[0]));
261 isomap.at<Vec3b>(y, x) = color;
263 else if (mapping_type == TextureInterpolation::Bilinear) {
266 Vec3f homogenous_dst_coord(x, y, 1.0f);
267 Vec2f src_texel = Mat(warp_mat_org_inv * Mat(homogenous_dst_coord));
272 float distance_upper_left = sqrt(pow(src_texel[0] - floor(src_texel[0]), 2) + pow(src_texel[1] - floor(src_texel[1]), 2));
273 float distance_upper_right = sqrt(pow(src_texel[0] - floor(src_texel[0]), 2) + pow(src_texel[1] - ceil(src_texel[1]), 2));
274 float distance_lower_left = sqrt(pow(src_texel[0] - ceil(src_texel[0]), 2) + pow(src_texel[1] - floor(src_texel[1]), 2));
275 float distance_lower_right = sqrt(pow(src_texel[0] - ceil(src_texel[0]), 2) + pow(src_texel[1] - ceil(src_texel[1]), 2));
278 float sum_distances = distance_lower_left + distance_lower_right + distance_upper_left + distance_upper_right;
279 distance_lower_left /= sum_distances;
280 distance_lower_right /= sum_distances;
281 distance_upper_left /= sum_distances;
282 distance_upper_right /= sum_distances;
285 for (
int color = 0; color < 3; ++color) {
286 float color_upper_left = image.at<Vec3b>(floor(src_texel[1]), floor(src_texel[0]))[color] * distance_upper_left;
287 float color_upper_right = image.at<Vec3b>(floor(src_texel[1]), ceil(src_texel[0]))[color] * distance_upper_right;
288 float color_lower_left = image.at<Vec3b>(ceil(src_texel[1]), floor(src_texel[0]))[color] * distance_lower_left;
289 float color_lower_right = image.at<Vec3b>(ceil(src_texel[1]), ceil(src_texel[0]))[color] * distance_lower_right;
291 isomap.at<Vec3b>(y, x)[color] = color_upper_left + color_upper_right + color_lower_left + color_lower_right;
294 else if (mapping_type == TextureInterpolation::NearestNeighbour) {
297 const Mat homogenous_dst_coord(Vec3f(x, y, 1.0f));
298 const Vec2f src_texel = Mat(warp_mat_org_inv * homogenous_dst_coord);
300 if ((cvRound(src_texel[1]) < image.rows) && (cvRound(src_texel[0]) < image.cols) && cvRound(src_texel[0]) > 0 && cvRound(src_texel[1]) > 0)
302 cv::Vec4b isomap_pixel;
303 isomap.at<cv::Vec4b>(y, x)[0] = image.at<Vec3b>(cvRound(src_texel[1]), cvRound(src_texel[0]))[0];
304 isomap.at<cv::Vec4b>(y, x)[1] = image.at<Vec3b>(cvRound(src_texel[1]), cvRound(src_texel[0]))[1];
305 isomap.at<cv::Vec4b>(y, x)[2] = image.at<Vec3b>(cvRound(src_texel[1]), cvRound(src_texel[0]))[2];
306 isomap.at<cv::Vec4b>(y, x)[3] = static_cast<uchar>(alpha_value);
313 results.emplace_back(std::async(extract_triangle));
316 for (
auto&& r : results) {
323 isomap = detail::interpolate_black_line(isomap);
337 cv::Mat interpolate_black_line(cv::Mat isomap)
339 assert(isomap.type() == CV_8UC4);
341 int col = isomap.cols / 2;
342 for (
int row = 0; row < isomap.rows; ++row)
344 if (isomap.at<cv::Vec4b>(row, col) == cv::Vec4b(0, 0, 0, 0))
346 isomap.at<cv::Vec4b>(row, col) = isomap.at<cv::Vec4b>(row, col - 1) * 0.5f + isomap.at<cv::Vec4b>(row, col + 1) * 0.5f;
352 if (isomap.rows == 512)
355 for (
int c = 206; c <= 306; ++c)
357 if (isomap.at<cv::Vec4b>(r, c) == cv::Vec4b(0, 0, 0, 0))
359 isomap.at<cv::Vec4b>(r, c) = isomap.at<cv::Vec4b>(r - 1, c) * 0.5f + isomap.at<cv::Vec4b>(r + 1, c) * 0.5f;
363 if (isomap.rows == 1024)
366 for (
int c = 437; c <= 587; ++c)
368 if (isomap.at<cv::Vec4b>(r, c) == cv::Vec4b(0, 0, 0, 0))
370 isomap.at<cv::Vec4b>(r, c) = isomap.at<cv::Vec4b>(r - 1, c) * 0.5f + isomap.at<cv::Vec4b>(r + 1, c) * 0.5f;
374 for (
int c = 411; c <= 613; ++c)
376 if (isomap.at<cv::Vec4b>(r, c) == cv::Vec4b(0, 0, 0, 0))
378 isomap.at<cv::Vec4b>(r, c) = isomap.at<cv::Vec4b>(r - 1, c) * 0.5f + isomap.at<cv::Vec4b>(r + 1, c) * 0.5f;
cv::Vec3f calculate_face_normal(const cv::Vec3f &v0, const cv::Vec3f &v1, const cv::Vec3f &v2)
Definition: utils.hpp:95
std::pair< cv::Mat, cv::Mat > render(Mesh mesh, cv::Mat model_view_matrix, cv::Mat projection_matrix, int viewport_width, int viewport_height, const boost::optional< Texture > &texture=boost::none, bool enable_backface_culling=false, bool enable_near_clipping=true, bool enable_far_clipping=true)
Definition: render.hpp:125
TextureInterpolation
Definition: texture_extraction.hpp:44
cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, cv::Mat image, cv::Mat depthbuffer, bool compute_view_angle, TextureInterpolation mapping_type, int isomap_resolution)
Definition: texture_extraction.hpp:107
std::pair< cv::Mat, cv::Mat > render_affine(Mesh mesh, cv::Mat affine_camera_matrix, int viewport_width, int viewport_height, bool do_backface_culling=true)
Definition: render_affine.hpp:52
Namespace containing all of eos's 3D model fitting functionality.
std::vector< cv::Vec4f > vertices
3D vertex positions.
Definition: Mesh.hpp:47
This class represents a 3D mesh consisting of vertices, vertex colour information and texture coordin...
Definition: Mesh.hpp:45
std::vector< std::array< int, 3 > > tvi
Triangle vertex indices.
Definition: Mesh.hpp:51
std::vector< cv::Vec2f > texcoords
Texture coordinates for each vertex.
Definition: Mesh.hpp:49