Added experimental new texture extraction

Using the new renderer stuff.

include/eos/render/texture_extraction.hpp

@@ -3,7 +3,7 @@
  *
  * File: include/eos/render/texture_extraction.hpp
  *
- * Copyright 2014, 2015 Patrik Huber
+ * Copyright 2014-2017 Patrik Huber
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,14 @@
 #include "eos/render/detail/texture_extraction_detail.hpp"
 #include "eos/render/render_affine.hpp"
 #include "eos/render/detail/render_detail.hpp"
-
+#include "eos/render/SoftwareRenderer.hpp"
+#include "eos/render/VertexShader.hpp" // remove after adding Rasterizer
+#include "eos/render/FragmentShader.hpp"
+#include "eos/fitting/closest_edge_fitting.hpp" // for ray_triangle_intersect()
+
+#include "glm/mat4x4.hpp"
+#include "glm/vec2.hpp"
+#include "glm/vec3.hpp"
 #include "glm/vec4.hpp"
 
 #include "opencv2/core/core.hpp"
@@ -35,6 +42,7 @@
 #include <tuple>
 #include <cassert>
 #include <future>
+#include <vector>
 
 namespace eos {
 	namespace render {
@@ -345,6 +353,142 @@ inline cv::Mat extract_texture(core::Mesh mesh, cv::Mat affine_camera_matrix, cv
 	return isomap;
 };
 
+/* New texture extraction, will replace above one at some point: */
+namespace v2 {
+
+/**
+ * @brief Extracts the texture of the face from the given image and stores it as isomap (a rectangular texture map).
+ *
+ * New texture extraction, will replace above one at some point.
+ * Copy the documentation from above extract_texture function, once we replace it.
+ *
+ * Note/Todo: Add an overload that takes a vector of bool / visible vertices, for the case when we already computed the visibility? (e.g. for edge-fitting)
+ *
+ * @param[in] mesh A mesh with texture coordinates.
+ * @param[in] view_model_matrix Todo.
+ * @param[in] projection_matrix Todo.
+ * @param[in] viewport Not needed at the moment. Might be, if we change clip_to_screen_space() to take a viewport.
+ * @param[in] image The image to extract the texture from. Todo: Does it have to be 8UC3 or something, or does it not matter?
+ * @param[in] compute_view_angle Unused at the moment.
+ * @param[in] isomap_resolution The resolution of the generated isomap. Defaults to 512x512.
+ * @return The extracted texture as isomap (texture map).
+ */
+cv::Mat extract_texture(core::Mesh mesh, glm::mat4x4 view_model_matrix, glm::mat4x4 projection_matrix,
+                        glm::vec4 /*viewport, not needed at the moment */, cv::Mat image,
+                        bool /* compute_view_angle, unused atm */, int isomap_resolution = 512)
+{
+    using glm::vec2;
+    using glm::vec3;
+    using glm::vec4;
+    using std::vector;
+    // actually we only need a rasteriser for this!
+    SoftwareRenderer<VertexShader, ExtractionFragmentShader> extraction_renderer(isomap_resolution,
+                                                                                 isomap_resolution);
+    Texture image_to_extract_from_as_tex = create_mipmapped_texture(image, 1);
+    extraction_renderer.enable_depth_test = false;
+    extraction_renderer.extracting_tex = true;
+
+    vector<bool> visibility_ray;
+    std::vector<glm::vec4> rotated_vertices;
+    // In perspective case... does the perspective projection matrix not change visibility? Do we not need to
+    // apply it?
+    // (If so, then we can change the two input matrices to this function to one (mvp_matrix)).
+    std::for_each(std::begin(mesh.vertices), std::end(mesh.vertices),
+                  [&rotated_vertices, &view_model_matrix](auto&& v) {
+                      rotated_vertices.push_back(view_model_matrix * v);
+                  });
+    // This code is duplicated from the edge-fitting. I think I can put this into a function in the library.
+    for (const auto& vertex : rotated_vertices)
+    {
+        bool visible = true;
+        // For every tri of the rotated mesh:
+        for (auto&& tri : mesh.tvi)
+        {
+            auto& v0 = rotated_vertices[tri[0]];
+            auto& v1 = rotated_vertices[tri[1]];
+            auto& v2 = rotated_vertices[tri[2]];
+
+            vec3 ray_origin = vertex;
+            vec3 ray_direction(0.0f, 0.0f, 1.0f); // we shoot the ray from the vertex towards the camera
+            auto intersect = fitting::ray_triangle_intersect(ray_origin, ray_direction, vec3(v0), vec3(v1),
+                                                             vec3(v2), false);
+            // first is bool intersect, second is the distance t
+            if (intersect.first == true)
+            {
+                // We've hit a triangle. Ray hit its own triangle. If it's behind the ray origin, ignore the
+                // intersection:
+                // Check if in front or behind?
+                if (intersect.second.get() <= 1e-4)
+                {
+                    continue; // the intersection is behind the vertex, we don't care about it
+                }
+                // Otherwise, we've hit a genuine triangle, and the vertex is not visible:
+                visible = false;
+                break;
+            }
+        }
+        visibility_ray.push_back(visible);
+    }
+
+    vector<vec4> wnd_coords; // will contain [x_wnd, y_wnd, z_ndc, 1/w_clip]
+    for (auto&& vtx : mesh.vertices)
+    {
+        auto clip_coords = projection_matrix * view_model_matrix * vtx;
+        clip_coords = divide_by_w(clip_coords);
+        const vec2 screen_coords = clip_to_screen_space(clip_coords.x, clip_coords.y, image.cols, image.rows);
+        clip_coords.x = screen_coords.x;
+        clip_coords.y = screen_coords.y;
+        wnd_coords.push_back(clip_coords);
+    }
+
+    // Go on with extracting: This only needs the rasteriser/FS, not the whole Renderer.
+    const int tex_width = isomap_resolution;
+    const int tex_height =
+        isomap_resolution; // keeping this in case we need non-square texture maps at some point
+    for (const auto& tvi : mesh.tvi)
+    {
+        if (visibility_ray[tvi[0]] && visibility_ray[tvi[1]] &&
+            visibility_ray[tvi[2]]) // can also try using ||, but...
+        {
+            // Test with a rendered & re-extracted texture shows that we're off by a pixel or more,
+            // definitely need to correct this. Probably here.
+            // It looks like it is 1-2 pixels off. Definitely a bit more than 1.
+            detail::v2::Vertex<double> pa{
+                vec4(mesh.texcoords[tvi[0]][0] * tex_width, mesh.texcoords[tvi[0]][1] * tex_height,
+                     wnd_coords[tvi[0]].z /* z_ndc */, wnd_coords[tvi[0]].w /* 1/w_clip */),
+                vec3(/* empty */),
+                vec2(
+                    wnd_coords[tvi[0]].x / image.cols,
+                    /* maybe 1 - ... ? */ wnd_coords[tvi[0]].y /
+                        image
+                            .rows /* wndcoords of the projected/rendered model triangle (in the input img). Normalised to 0,1. */)};
+            detail::v2::Vertex<double> pb{
+                vec4(mesh.texcoords[tvi[1]][0] * tex_width, mesh.texcoords[tvi[1]][1] * tex_height,
+                     wnd_coords[tvi[1]].z /* z_ndc */, wnd_coords[tvi[1]].w /* 1/w_clip */),
+                vec3(/* empty */),
+                vec2(
+                    wnd_coords[tvi[1]].x / image.cols,
+                    /* maybe 1 - ... ? */ wnd_coords[tvi[1]].y /
+                        image
+                            .rows /* wndcoords of the projected/rendered model triangle (in the input img). Normalised to 0,1. */)};
+            detail::v2::Vertex<double> pc{
+                vec4(mesh.texcoords[tvi[2]][0] * tex_width, mesh.texcoords[tvi[2]][1] * tex_height,
+                     wnd_coords[tvi[2]].z /* z_ndc */, wnd_coords[tvi[2]].w /* 1/w_clip */),
+                vec3(/* empty */),
+                vec2(
+                    wnd_coords[tvi[2]].x / image.cols,
+                    /* maybe 1 - ... ? */ wnd_coords[tvi[2]].y /
+                        image
+                            .rows /* wndcoords of the projected/rendered model triangle (in the input img). Normalised to 0,1. */)};
+            extraction_renderer.raster_triangle(pa, pb, pc, image_to_extract_from_as_tex);
+        }
+    }
+
+    return extraction_renderer.colorbuffer;
+};
+
+} /* namespace v2 */
+
 namespace detail {
 
 // Workaround for the pixels that don't get filled in extract_texture().