Moved the triangle visibility check of the texture extraction to a separate function

This increases the readability of extract_texture a lot.

include/eos/render/texture_extraction.hpp

@@ -77,66 +77,32 @@ enum class TextureInterpolation {
 };
 
 /**
- * Extracts the texture of the face from the given image
- * and stores it as isomap (a rectangular texture map).
+ * Checks whether all pixels in the given triangle are visible and
+ * returns true if and only if the whole triangle is visible.
+ * The vertices should be given in screen coordinates, but with their
+ * z-values preserved, so they can be compared against the depthbuffer.
  *
- * @param[in] mesh A mesh with texture coordinates.
- * @param[in] affine_camera_matrix An estimated 3x4 affine camera matrix.
- * @param[in] viewport_width Screen width.
- * @param[in] viewport_height Screen height.
- * @param[in] image The image to extract the texture from.
- * @param[in] depthbuffer A precalculated depthbuffer image.
- * @param[in] mapping_type The interpolation type to be used for the extraction.
- * @param[in] isomap_resolution The resolution of the generated isomap. Defaults to 512x512.
- * @return The extracted texture as isomap (texture map).
+ * #Todo: Move to detail namespace
+ *
+ * @param[in] v0 First vertex, in screen coordinates (but still with their z-value).
+ * @param[in] v1 Second vertex.
+ * @param[in] v2 Third vertex.
+ * @param[in] depthbuffer Pre-calculated depthbuffer.
+ * @return True if the whole triangle is visible in the image.
  */
-inline cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, int viewport_width, int viewport_height, cv::Mat image, cv::Mat depthbuffer, TextureInterpolation mapping_type, int isomap_resolution = 512)
+bool is_triangle_visible(const cv::Vec4f& v0, const cv::Vec4f& v1, const cv::Vec4f& v2, cv::Mat depthbuffer)
 {
-	assert(mesh.vertices.size() == mesh.texcoords.size());
-
-	using cv::Mat;
-	using cv::Vec2f;
-	using cv::Vec3f;
-	using cv::Vec4f;
-	using std::min;
-	using std::max;
-	using std::floor;
-	using std::ceil;
-
-	affine_camera_matrix = detail::calculate_affine_z_direction(affine_camera_matrix);
-
-	Mat texture_map = Mat::zeros(isomap_resolution, isomap_resolution, CV_8UC3); // #Todo: We do want an alpha channel. Will be added soon-ish.
-	// #Todo: We should handle gray images, but output a 3-channel isomap nevertheless I think.
-
-	for (const auto& triangle_indices : mesh.tvi) {
-		// Find out if the current triangle is visible:
-		// We do a second rendering-pass here. We use the depth-buffer of the final image, and then, here,
-		// check if each pixel in a triangle is visible. If the whole triangle is visible, we use it to extract
-		// the texture.
-		// Possible improvement: - If only part of the triangle is visible, split it
-		// - Share more code with the renderer?
-		Vec4f v0_3d = mesh.vertices[triangle_indices[0]];
-		Vec4f v1_3d = mesh.vertices[triangle_indices[1]];
-		Vec4f v2_3d = mesh.vertices[triangle_indices[2]];
+	// #Todo: Actually, only check the 3 vertex points, don't loop over the pixels - this should be enough.
 
-		Vec4f v0, v1, v2; // we don't copy the color and texcoords, we only do the visibility check here.
-		// This could be optimized in 2 ways though:
-		// - Use render(), or as in render(...), transfer the vertices once, not in a loop over all triangles (vertices are getting transformed multiple times)
-		// - We transform them later (below) a second time. Only do it once.
-		v0 = Mat(affine_camera_matrix * Mat(v0_3d));
-		v1 = Mat(affine_camera_matrix * Mat(v1_3d));
-		v2 = Mat(affine_camera_matrix * Mat(v2_3d));
+	auto viewport_width = depthbuffer.cols;
+	auto viewport_height = depthbuffer.rows;
 
 	// Well, in in principle, we'd have to do the whole stuff as in render(), like
 	// clipping against the frustums etc.
-		// But as long as our model is fully on the screen, we're fine.
+	// But as long as our model is fully on the screen, we're fine. Todo: Doublecheck that.
 
-		// Todo: This is all very similar to processProspectiveTri(...), except the other function does texturing stuff as well. Remove code duplication!
-
-		//if (doBackfaceCulling) {
 	if (!detail::are_vertices_ccw_in_screen_space(v0, v1, v2))
-			continue;
-		//}
+		return false;
 
 	cv::Rect bbox = detail::calculate_clipped_bounding_box(v0, v1, v2, viewport_width, viewport_height);
 	int minX = bbox.x;
@@ -145,7 +111,7 @@ inline cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, int view
 	int maxY = bbox.y + bbox.height;
 
 	//if (t.maxX <= t.minX || t.maxY <= t.minY) 	// Note: Can the width/height of the bbox be negative? Maybe we only need to check for equality here?
-		//	continue;
+	//	continue;									// Also, I'm not entirely sure why I commented this out
 
 	bool whole_triangle_is_visible = true;
 	for (int yi = minY; yi <= maxY; yi++)
@@ -180,6 +146,64 @@ inline cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, int view
 	}
 
 	if (!whole_triangle_is_visible) {
+		return false;
+	}
+	return true;
+};
+
+/**
+ * Extracts the texture of the face from the given image
+ * and stores it as isomap (a rectangular texture map).
+ *
+ * @param[in] mesh A mesh with texture coordinates.
+ * @param[in] affine_camera_matrix An estimated 3x4 affine camera matrix.
+ * @param[in] viewport_width Screen width.
+ * @param[in] viewport_height Screen height.
+ * @param[in] image The image to extract the texture from.
+ * @param[in] depthbuffer A precalculated depthbuffer image.
+ * @param[in] mapping_type The interpolation type to be used for the extraction.
+ * @param[in] isomap_resolution The resolution of the generated isomap. Defaults to 512x512.
+ * @return The extracted texture as isomap (texture map).
+ */
+inline cv::Mat extract_texture(Mesh mesh, cv::Mat affine_camera_matrix, int viewport_width, int viewport_height, cv::Mat image, cv::Mat depthbuffer, TextureInterpolation mapping_type, int isomap_resolution = 512)
+{
+	assert(mesh.vertices.size() == mesh.texcoords.size());
+
+	using cv::Mat;
+	using cv::Vec2f;
+	using cv::Vec3f;
+	using cv::Vec4f;
+	using std::min;
+	using std::max;
+	using std::floor;
+	using std::ceil;
+
+	affine_camera_matrix = detail::calculate_affine_z_direction(affine_camera_matrix);
+
+	Mat texture_map = Mat::zeros(isomap_resolution, isomap_resolution, CV_8UC3); // #Todo: We do want an alpha channel. Will be added soon-ish.
+	// #Todo: We should handle gray images, but output a 3-channel isomap nevertheless I think.
+
+	for (const auto& triangle_indices : mesh.tvi) {
+		// Find out if the current triangle is visible:
+		// We do a second rendering-pass here. We use the depth-buffer of the final image, and then, here,
+		// check if each pixel in a triangle is visible. If the whole triangle is visible, we use it to extract
+		// the texture.
+		// Possible improvement: - If only part of the triangle is visible, split it
+		// - Share more code with the renderer?
+		Vec4f v0_3d = mesh.vertices[triangle_indices[0]];
+		Vec4f v1_3d = mesh.vertices[triangle_indices[1]];
+		Vec4f v2_3d = mesh.vertices[triangle_indices[2]];
+
+		Vec4f v0, v1, v2; // we don't copy the color and texcoords, we only do the visibility check here.
+		// This could be optimized in 2 ways though:
+		// - Use render(), or as in render(...), transfer the vertices once, not in a loop over all triangles (vertices are getting transformed multiple times)
+		// - We transform them later (below) a second time. Only do it once.
+		v0 = Mat(affine_camera_matrix * Mat(v0_3d));
+		v1 = Mat(affine_camera_matrix * Mat(v1_3d));
+		v2 = Mat(affine_camera_matrix * Mat(v2_3d));
+
+		if (!is_triangle_visible(v0, v1, v2, depthbuffer))
+		{
 			continue;
 		}