Commit 240d166a authored by Patrik Huber's avatar Patrik Huber

Merge branch 'devel' into python-bindings

parents 7e9a977d 66a712dd
......@@ -25,7 +25,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
endif()
endif()
# Note: gcc is fine without -pthreads.
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread")
elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # the quotes are needed here, maybe because "MSVC" seems to be a keyword
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19)
message(FATAL_ERROR "Visual Studio 2015 or newer is required.")
......@@ -54,6 +54,8 @@ endif()
message(STATUS "Options:")
option(BUILD_EXAMPLES "Build the example applications." ON)
message(STATUS "BUILD_EXAMPLES: ${BUILD_EXAMPLES}")
option(BUILD_CERES_EXAMPLE "Build the fit-model-ceres example (requires Ceres)." OFF)
message(STATUS "BUILD_CERES_EXAMPLE: ${BUILD_CERES_EXAMPLE}")
option(BUILD_UTILS "Build utility applications." OFF)
message(STATUS "BUILD_UTILS: ${BUILD_UTILS}")
option(BUILD_DOCUMENTATION "Build the library documentation." OFF)
......@@ -106,14 +108,17 @@ set(HEADERS
include/eos/morphablemodel/io/cvssp.hpp
include/eos/morphablemodel/io/mat_cerealisation.hpp
include/eos/fitting/affine_camera_estimation.hpp
include/eos/fitting/orthographic_camera_estimation_linear.hpp
include/eos/fitting/nonlinear_camera_estimation.hpp
include/eos/fitting/detail/nonlinear_camera_estimation_detail.hpp
include/eos/fitting/detail/optional_cerealisation.hpp
include/eos/fitting/detail/glm_cerealisation.hpp
include/eos/fitting/linear_shape_fitting.hpp
include/eos/fitting/contour_correspondence.hpp
include/eos/fitting/blendshape_fitting.hpp
include/eos/fitting/fitting.hpp
include/eos/fitting/ceres_nonlinear.hpp
include/eos/fitting/RenderingParameters.hpp
include/eos/render/Mesh.hpp
include/eos/render/utils.hpp
include/eos/render/render.hpp
......
......@@ -15,7 +15,7 @@
/**
* @namespace eos::morphablemodel
* @brief Functionality to represent a Morphable Model, containing it's PCA models and loading functions.
* @brief Functionality to represent a Morphable Model, its PCA models, and functions to load models and blendshapes.
*/
/**
......
......@@ -33,18 +33,21 @@ if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
endif()
# Find Ceres, for the fit-model-ceres app:
find_package(Ceres REQUIRED)
message(STATUS "Ceres locations: Headers: ${CERES_INCLUDE_DIRS} Library: ${CERES_LIBRARIES}")
include_directories(${CERES_INCLUDE_DIRS})
# Model fitting (affine cam & shape to landmarks) example:
add_executable(fit-model fit-model.cpp)
target_link_libraries(fit-model ${OpenCV_LIBS} ${Boost_LIBRARIES})
# Single and multi-image non-linear model fitting with Ceres example:
add_executable(fit-model-ceres fit-model-ceres.cpp)
target_link_libraries(fit-model-ceres ${CERES_LIBRARIES} ${OpenCV_LIBS} ${Boost_LIBRARIES})
if(BUILD_CERES_EXAMPLE)
# Find Ceres, for the fit-model-ceres app:
find_package(Ceres REQUIRED)
message(STATUS "Ceres locations: Headers: ${CERES_INCLUDE_DIRS} Library: ${CERES_LIBRARIES}")
include_directories(${CERES_INCLUDE_DIRS})
# Single and multi-image non-linear model fitting with Ceres example:
add_executable(fit-model-ceres fit-model-ceres.cpp)
target_link_libraries(fit-model-ceres ${CERES_LIBRARIES} ${OpenCV_LIBS} ${Boost_LIBRARIES})
install(TARGETS fit-model-ceres DESTINATION bin)
endif()
# Generate random samples from the model:
add_executable(generate-obj generate-obj.cpp)
......
......@@ -19,7 +19,8 @@
*/
#include "eos/core/Landmark.hpp"
#include "eos/core/LandmarkMapper.hpp"
#include "eos/fitting/nonlinear_camera_estimation.hpp"
#include "eos/fitting/orthographic_camera_estimation_linear.hpp"
#include "eos/fitting/RenderingParameters.hpp"
#include "eos/fitting/linear_shape_fitting.hpp"
#include "eos/render/utils.hpp"
#include "eos/render/texture_extraction.hpp"
......@@ -187,14 +188,15 @@ int main(int argc, char *argv[])
}
// Estimate the camera (pose) from the 2D - 3D point correspondences
fitting::RenderingParameters rendering_params = fitting::estimate_orthographic_camera(image_points, model_points, image.cols, image.rows);
Mat affine_from_ortho = get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows);
fitting::RenderingParameters rendering_params(pose, image.cols, image.rows);
// The 3D head pose can be recovered as follows:
float yaw_angle = glm::degrees(rendering_params.r_y);
// and similarly for pitch (r_x) and roll (r_z).
float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation()));
// and similarly for pitch and roll.
// Estimate the shape coefficients by fitting the shape to the landmarks:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows);
vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices);
// Obtain the full mesh with the estimated coefficients:
......
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: include/eos/fitting/RenderingParameters.hpp
*
* Copyright 2016 Patrik Huber
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifndef RENDERINGPARAMETERS_HPP_
#define RENDERINGPARAMETERS_HPP_
#include "eos/fitting/orthographic_camera_estimation_linear.hpp"
#include "glm/gtc/matrix_transform.hpp"
#include "glm/gtc/quaternion.hpp"
#include "eos/fitting/detail/optional_cerealisation.hpp"
#include "eos/fitting/detail/glm_cerealisation.hpp"
#include "cereal/cereal.hpp"
#include "cereal/archives/json.hpp"
#include "opencv2/core/core.hpp"
#include "boost/optional.hpp"
#include <string>
#include <fstream>
#include <stdexcept>
namespace eos {
namespace fitting {
/**
* @brief A class representing a camera viewing frustum. At the moment only
* fully tested with orthographic camera.
*/
struct Frustum
{
// float l, r, b, t;
Frustum() {};
Frustum(float l, float r, float b, float t) : l(l), r(r), b(b), t(t) {};
//Frustum(float l, float r, float b, float t, float n, float f) : l(l), r(r), b(b), t(t), n(n), f(f) {};
float l = -1.0f;
float r = 1.0f;
float b = -1.0f;
float t = 1.0f;
//boost::optional<float> n; // These are not needed yet but probably will in the future,
//boost::optional<float> f; // and then it's good if the old serialised files stay compatible.
friend class cereal::access;
/**
* Serialises this class using cereal.
*
* @param[in] ar The archive to serialise to (or to serialise from).
*/
template<class Archive>
void serialize(Archive& archive)
{
archive(CEREAL_NVP(l), CEREAL_NVP(r), CEREAL_NVP(b), CEREAL_NVP(t));
};
};
enum class CameraType
{
Orthographic,
Perspective
};
/**
* @brief Represents a set of estimated model parameters (rotation, translation) and
* camera parameters (viewing frustum).
*
* The estimated rotation and translation transform the model from model-space to camera-space,
* and, if one wishes to use OpenGL, can be used to build the model-view matrix.
* The parameters are the inverse of the camera position in 3D space.
*
* The camera frustum describes the size of the viewing plane of the camera, and
* can be used to build an OpenGL-conformant orthographic projection matrix.
*
* Together, these parameters fully describe the imaging process of a given model instance
* (under an orthographic projection).
*
* The rotation values are given in radians and estimated using the RPY convention.
* Yaw is applied first to the model, then pitch, then roll (R * P * Y * vertex).
* In general, the convention is as follows:
* r_x = Pitch
* r_y = Yaw. Positive means subject is looking left (we see her right cheek).
* r_z = Roll. Positive means the subject's right eye is further down than the other one (he tilts his head to the right).
* However, we're using a quaternion now to represent the rotation, and glm::eulerAngles() will give
* slightly different angles (according to a different (undocumented)) convention. However, the
* rotation is exactly the same! (i.e. they are represented by the same quaternion / rotation matrix).
*
* This should always represent all parameters necessary to render the model to an image, and be completely OpenGL compliant.
*/
class RenderingParameters
{
public:
// Creates with default frustum...
RenderingParameters() {};
// Initialisation for Eigen::LevMarq
// This creates the correct rotation quaternion in the case the angles were estimated/given by R*P*Y*v.
// Angles given in radian.
// Note: If you subsequently use RP::get_rotation() and glm::eulerAngles() on it, the angles you get out will be slightly different from the ones you put in here.
// But they will describe the same rotation! Just in a different order. (i.e. the rotation matrix or quaternion for both of these two sets of angles is identical.)
RenderingParameters(CameraType camera_type, Frustum camera_frustum, float r_x, float r_y, float r_z, float tx, float ty, int screen_width, int screen_height) : camera_type(camera_type), frustum(camera_frustum), t_x(tx), t_y(ty), screen_width(screen_width), screen_height(screen_height) {
auto rot_mtx_x = glm::rotate(glm::mat4(1.0f), r_x, glm::vec3{ 1.0f, 0.0f, 0.0f });
auto rot_mtx_y = glm::rotate(glm::mat4(1.0f), r_y, glm::vec3{ 0.0f, 1.0f, 0.0f });
auto rot_mtx_z = glm::rotate(glm::mat4(1.0f), r_z, glm::vec3{ 0.0f, 0.0f, 1.0f });
auto zxy_rotation_matrix = rot_mtx_z * rot_mtx_x * rot_mtx_y;
rotation = zxy_rotation_matrix;
};
// This assumes estimate_sop was run on points with OpenCV viewport! I.e. y flipped.
RenderingParameters(ScaledOrthoProjectionParameters ortho_params, int image_width, int image_height) {
camera_type = CameraType::Orthographic;
rotation = ortho_params.R; // convert the rotation matrix to a quaternion
t_x = ortho_params.tx;
t_y = ortho_params.ty;
const auto l = 0.0;
const auto r = image_width / ortho_params.s;
const auto b = 0.0; // The b and t values are not tested for what happens if the SOP parameters
const auto t = image_height / ortho_params.s; // were estimated on a non-flipped viewport.
frustum = Frustum(l, r, b, t);
};
glm::quat get_rotation() const {
return rotation;
};
glm::mat4x4 get_modelview() const {
glm::mat4x4 modelview = glm::mat4_cast(rotation);
modelview[3][0] = t_x;
modelview[3][1] = t_y;
return modelview;
};
glm::mat4x4 get_projection() const {
if (camera_type == CameraType::Orthographic)
{
return glm::ortho<float>(frustum.l, frustum.r, frustum.b, frustum.t);
}
else {
throw std::runtime_error("get_projection() for CameraType::Perspective is not implemented yet.");
}
};
private:
CameraType camera_type = CameraType::Orthographic;
Frustum frustum; // Can construct a glm::ortho or glm::perspective matrix from this.
glm::quat rotation;
float t_x;
float t_y;
//boost::optional<float> t_z;
//boost::optional<float> focal_length; // only for certain camera types. Should it go into Frustum?
int screen_width; // (why) do we need these?
int screen_height;
friend class cereal::access;
/**
* Serialises this class using cereal.
*
* @param[in] ar The archive to serialise to (or to serialise from).
*/
template<class Archive>
void serialize(Archive& archive)
{
archive(CEREAL_NVP(camera_type), CEREAL_NVP(frustum), CEREAL_NVP(rotation), CEREAL_NVP(t_x), CEREAL_NVP(t_y), CEREAL_NVP(screen_width), CEREAL_NVP(screen_height));
};
};
/**
* Saves the rendering parameters for an image to a json file.
*
* @param[in] rendering_parameters An instance of class RenderingParameters.
* @param[in] filename The file to write.
* @throws std::runtime_error if unable to open the given file for writing.
*/
void save_rendering_parameters(RenderingParameters rendering_parameters, std::string filename)
{
std::ofstream file(filename);
if (file.fail()) {
throw std::runtime_error("Error opening file for writing: " + filename);
}
cereal::JSONOutputArchive output_archive(file);
output_archive(cereal::make_nvp("rendering_parameters", rendering_parameters));
};
/**
* @brief Returns a glm/OpenGL compatible viewport vector that flips y and
* has the origin on the top-left, like in OpenCV.
*/
glm::vec4 get_opencv_viewport(int width, int height)
{
return glm::vec4(0, height, width, -height);
};
/**
* @brief Converts a glm::mat4x4 to a cv::Mat.
*
* Note: I think the last use of this function is below in
* get_3x4_affine_camera_matrix().
*/
cv::Mat to_mat(const glm::mat4x4& glm_matrix)
{
// glm stores its matrices in col-major order in memory, OpenCV in row-major order.
// Hence we transpose the glm matrix to flip the memory layout, and then point OpenCV
// to that location.
auto glm_matrix_t = glm::transpose(glm_matrix);
cv::Mat opencv_mat(4, 4, CV_32FC1, &glm_matrix_t[0]);
// we need to clone because the underlying data of the original goes out of scope
return opencv_mat.clone();
};
/**
* @brief Creates a 3x4 affine camera matrix from given fitting parameters. The
* matrix transforms points directly from model-space to screen-space.
*
* This function is mainly used since the linear shape fitting fitting::fit_shape_to_landmarks_linear
* expects one of these 3x4 affine camera matrices, as well as render::extract_texture.
*/
cv::Mat get_3x4_affine_camera_matrix(fitting::RenderingParameters params, int width, int height)
{
auto view_model = to_mat(params.get_modelview());
auto ortho_projection = to_mat(params.get_projection());
cv::Mat mvp = ortho_projection * view_model;
glm::vec4 viewport(0, height, width, -height); // flips y, origin top-left, like in OpenCV
// equivalent to what glm::project's viewport does, but we don't change z and w:
cv::Mat viewport_mat = (cv::Mat_<float>(4, 4) << viewport[2] / 2.0f, 0.0f, 0.0f, viewport[2] / 2.0f + viewport[0],
0.0f, viewport[3] / 2.0f, 0.0f, viewport[3] / 2.0f + viewport[1],
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
cv::Mat full_projection_4x4 = viewport_mat * mvp;
cv::Mat full_projection_3x4 = full_projection_4x4.rowRange(0, 3); // we take the first 3 rows, but then set the last one to [0 0 0 1]
full_projection_3x4.at<float>(2, 0) = 0.0f;
full_projection_3x4.at<float>(2, 1) = 0.0f;
full_projection_3x4.at<float>(2, 2) = 0.0f;
full_projection_3x4.at<float>(2, 3) = 1.0f;
return full_projection_3x4;
};
} /* namespace fitting */
} /* namespace eos */
#endif /* RENDERINGPARAMETERS_HPP_ */
......@@ -26,6 +26,9 @@
#include "eos/morphablemodel/Blendshape.hpp"
#include "glm/gtc/quaternion.hpp"
#include "glm/gtx/transform.hpp"
#include "ceres/cubic_interpolation.h"
#include "opencv2/core/core.hpp" // for Vec2f
......
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: include/eos/fitting/detail/glm_cerealisation.hpp
*
* Copyright 2016 Patrik Huber
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifndef GLMCEREALISATION_HPP_
#define GLMCEREALISATION_HPP_
#include "cereal/cereal.hpp"
#include "glm/gtc/quaternion.hpp"
/**
* @brief Serialisation of GLM \c glm::quat quaternion for the serialisation
* library cereal (http://uscilab.github.io/cereal/index.html).
*
* Contains serialisation for \c glm::quat.
*/
namespace glm {
/**
* @brief Serialisation of a glm::quat using cereal.
*
* @param[in] ar The archive to (de)serialise.
* @param[in] vec The vector to (de)serialise.
*/
template <class Archive>
void serialize(Archive& ar, glm::quat& q)
{
ar(q.w, q.x, q.y, q.z);
};
} /* namespace glm */
#endif /* GLMCEREALISATION_HPP_ */
......@@ -23,205 +23,19 @@
#define NONLINEARCAMERAESTIMATION_HPP_
#include "eos/fitting/detail/nonlinear_camera_estimation_detail.hpp"
#include "glm/gtc/matrix_transform.hpp"
#include "eos/fitting/detail/optional_cerealisation.hpp"
#include "cereal/cereal.hpp"
#include "cereal/archives/json.hpp"
#include "eos/fitting/RenderingParameters.hpp"
#include "Eigen/Geometry"
#include "unsupported/Eigen/NonLinearOptimization"
#include "opencv2/core/core.hpp"
#include "boost/optional.hpp"
#include <vector>
#include <cassert>
namespace eos {
namespace fitting {
/**
* @brief A class representing a camera viewing frustum. At the
* moment used as orthographic camera only.
*/
struct Frustum
{
float l, r, b, t;
// optional<float> n, f;
boost::optional<float> n;
boost::optional<float> f;
friend class cereal::access;
/**
* Serialises this class using cereal.
*
* @param[in] ar The archive to serialise to (or to serialise from).
*/
template<class Archive>
void serialize(Archive& archive)
{
archive(CEREAL_NVP(l), CEREAL_NVP(r), CEREAL_NVP(b), CEREAL_NVP(t), CEREAL_NVP(n), CEREAL_NVP(f));
};
};
enum class CameraType
{
Orthographic,
Perspective
};
/**
* @brief Represents a set of estimated model parameters (rotation, translation) and
* camera parameters (viewing frustum).
*
* The estimated rotation and translation transform the model from model-space to camera-space,
* and, if one wishes to use OpenGL, can be used to build the model-view matrix.
* The parameters are the inverse of the camera position in 3D space.
*
* The camera frustum describes the size of the viewing plane of the camera, and
* can be used to build an OpenGL-conformant orthographic projection matrix.
*
* Together, these parameters fully describe the imaging process of a given model instance
* (under an orthographic projection).
*
* The rotation values are given in radians and estimated using the RPY convention.
* Yaw is applied first to the model, then pitch, then roll (R * P * Y * vertex).
*/
struct RenderingParameters
{
CameraType camera_type; // what's the default?
Frustum frustum;
float r_x; // Pitch.
float r_y; // Yaw. Positive means subject is looking left (we see her right cheek).
float r_z; // Roll. Positive means the subject's right eye is further down than the other one (he tilts his head to the right).
float t_x; // Todo: define whether it's the camera translation/rotation or the model's.
float t_y;
int screen_width;
int screen_height;
boost::optional<float> focal_length; // only for certain camera types
friend class cereal::access;
/**
* Serialises this class using cereal.
*
* @param[in] ar The archive to serialise to (or to serialise from).
*/
template<class Archive>
void serialize(Archive& archive)
{
archive(CEREAL_NVP(camera_type), CEREAL_NVP(frustum), CEREAL_NVP(r_x), CEREAL_NVP(r_y), CEREAL_NVP(r_z), CEREAL_NVP(t_x), CEREAL_NVP(t_y), CEREAL_NVP(screen_width), CEREAL_NVP(screen_height), CEREAL_NVP(focal_length));
};
};
/**
* Saves the rendering parameters for an image to a json file.
*
* @param[in] rendering_parameters An instance of class RenderingParameters.
* @param[in] filename The file to write.
* @throws std::runtime_error if unable to open the given file for writing.
*/
void save_rendering_parameters(RenderingParameters rendering_parameters, std::string filename)
{
std::ofstream file(filename);
if (file.fail()) {
throw std::runtime_error("Error opening file for writing: " + filename);
}
cereal::JSONOutputArchive output_archive(file);
output_archive(cereal::make_nvp("rendering_parameters", rendering_parameters));
};
/**
* @brief Converts a glm::mat4x4 to a cv::Mat.
*
* Note: move to render namespace
*/
cv::Mat to_mat(const glm::mat4x4& glm_matrix)
{
// glm stores its matrices in col-major order in memory, OpenCV in row-major order.
// Hence we transpose the glm matrix to flip the memory layout, and then point OpenCV
// to that location.
auto glm_matrix_t = glm::transpose(glm_matrix);
cv::Mat opencv_mat(4, 4, CV_32FC1, &glm_matrix_t[0]);
// we need to clone because the underlying data of the original goes out of scope
return opencv_mat.clone();
};
/**
* @brief Creates a 4x4 model-view matrix from given fitting parameters.
*
* Together with the Frustum information, this describes the full
* orthographic rendering parameters of the OpenGL pipeline.
* Example:
*
* @code
* fitting::OrthographicRenderingParameters rendering_params = ...;
* glm::mat4x4 view_model = get_4x4_modelview_matrix(rendering_params);
* glm::mat4x4 ortho_projection = glm::ortho(rendering_params.frustum.l, rendering_params.frustum.r, rendering_params.frustum.b, rendering_params.frustum.t);
* glm::vec4 viewport(0, image.rows, image.cols, -image.rows); // flips y, origin top-left, like in OpenCV
*
* // project a point from 3D to 2D:
* glm::vec3 point_3d = ...; // from a mesh for example
* glm::vec3 point_2d = glm::project(point_3d, view_model, ortho_projection, viewport);
* @endcode
*/
glm::mat4x4 get_4x4_modelview_matrix(fitting::RenderingParameters params)
{
// rotation order: RPY * P
auto rot_mtx_x = glm::rotate(glm::mat4(1.0f), params.r_x, glm::vec3{ 1.0f, 0.0f, 0.0f });
auto rot_mtx_y = glm::rotate(glm::mat4(1.0f), params.r_y, glm::vec3{ 0.0f, 1.0f, 0.0f });
auto rot_mtx_z = glm::rotate(glm::mat4(1.0f), params.r_z, glm::vec3{ 0.0f, 0.0f, 1.0f });
auto t_mtx = glm::translate(glm::mat4(1.0f), glm::vec3{ params.t_x, params.t_y, 0.0f });
auto modelview = t_mtx * rot_mtx_z * rot_mtx_x * rot_mtx_y;
return modelview;
};
/**
* @brief Creates a 3x4 affine camera matrix from given fitting parameters. The
* matrix transforms points directly from model-space to screen-space.
*
* This function is mainly used since the linear shape fitting fitting::fit_shape_to_landmarks_linear
* expects one of these 3x4 affine camera matrices, as well as render::extract_texture.
*/
cv::Mat get_3x4_affine_camera_matrix(fitting::RenderingParameters params, int width, int height)
{
auto view_model = to_mat(get_4x4_modelview_matrix(params));
auto ortho_projection = to_mat(glm::ortho(params.frustum.l, params.frustum.r, params.frustum.b, params.frustum.t));
cv::Mat mvp = ortho_projection * view_model;
glm::vec4 viewport(0, height, width, -height); // flips y, origin top-left, like in OpenCV
// equivalent to what glm::project's viewport does, but we don't change z and w:
cv::Mat viewport_mat = (cv::Mat_<float>(4, 4) << viewport[2] / 2.0f, 0.0f, 0.0f, viewport[2] / 2.0f + viewport[0],
0.0f, viewport[3] / 2.0f, 0.0f, viewport[3] / 2.0f + viewport[1],
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
cv::Mat full_projection_4x4 = viewport_mat * mvp;
cv::Mat full_projection_3x4 = full_projection_4x4.rowRange(0, 3); // we take the first 3 rows, but then set the last one to [0 0 0 1]
full_projection_3x4.at<float>(2, 0) = 0.0f;
full_projection_3x4.at<float>(2, 1) = 0.0f;
full_projection_3x4.at<float>(2, 2) = 0.0f;
full_projection_3x4.at<float>(2, 3) = 1.0f;
return full_projection_3x4;
};
/**
* @brief Returns a glm/OpenGL compatible viewport vector that flips y and
* has the origin on the top-left, like in OpenCV.
*
* Note: Move to detail namespace / not used at the moment.
*/
glm::vec4 get_opencv_viewport(int width, int height)
{
return glm::vec4(0, height, width, -height);
};
/**
* @brief This algorithm estimates the rotation angles and translation of the model, as
* well as the viewing frustum of the camera, given a set of corresponding 2D-3D points.
......@@ -241,6 +55,8 @@ glm::vec4 get_opencv_viewport(int width, int height)
* - A better initial guess (see e.g. Prince)
* - Using the analytic derivatives instead of Eigen::NumericalDiff - they're easy to calculate.
*
* Note/Todo: Could add a parameter: \c OrthographicRenderingParameters initial_guess = {}
*
* @param[in] image_points A list of 2D image points.
* @param[in] model_points Corresponding points of a 3D model.
* @param[in] width Width of the image (or viewport).
......@@ -271,7 +87,7 @@ RenderingParameters estimate_orthographic_camera(std::vector<cv::Vec2f> image_po
// 'parameters' contains the solution now.
Frustum camera_frustum{ -1.0f * aspect * static_cast<float>(parameters[5]), 1.0f * aspect * static_cast<float>(parameters[5]), -1.0f * static_cast<float>(parameters[5]), 1.0f * static_cast<float>(parameters[5]) };
return RenderingParameters{ CameraType::Orthographic, camera_frustum, static_cast<float>(parameters[0]), static_cast<float>(parameters[1]), static_cast<float>(parameters[2]), static_cast<float>(parameters[3]), static_cast<float>(parameters[4]), width, height };
return RenderingParameters(CameraType::Orthographic, camera_frustum, static_cast<float>(parameters[0]), static_cast<float>(parameters[1]), static_cast<float>(parameters[2]), static_cast<float>(parameters[3]), static_cast<float>(parameters[4]), width, height);
};
} /* namespace fitting */
......
/*
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
*
* File: include/eos/fitting/orthographic_camera_estimation_linear.hpp
*
* Copyright 2016 Patrik Huber
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#ifndef ORTHOGRAPHICCAMERAESTIMATIONLINEAR_HPP_
#define ORTHOGRAPHICCAMERAESTIMATIONLINEAR_HPP_
#include "glm/mat3x3.hpp"
#include "opencv2/core/core.hpp"
#include "boost/optional.hpp"
#include <vector>
#include <cassert>
namespace eos {
namespace fitting {
/**
* Parameters of an estimated scaled orthographic projection.
*/
struct ScaledOrthoProjectionParameters {
glm::mat3x3 R;
double tx, ty;
double s;
};
/**
* Estimates the parameters of a scaled orthographic projection.
*
* Given a set of 2D-3D correspondences, this algorithm estimates rotation,
* translation (in x and y) and a scaling parameters of the scaled orthographic
* projection model using a closed-form solution. It does so by first computing
* an affine camera matrix using algorithm [1], and then finds the closest
* orthonormal matrix to the estimated affine transform using SVD.
* This algorithm follows the original implementation [2] of William Smith,
* University of York.
*
* Requires >= 4 corresponding points.
*
* [1]: Gold Standard Algorithm for estimating an affine camera matrix from
* world to image correspondences, Algorithm 7.2 in Multiple View Geometry,
* Hartley & Zisserman, 2nd Edition, 2003.
* [2]: https://github.com/waps101/3DMM_edges/blob/master/utils/POS.m
*
* @param[in] image_points A list of 2D image points.
* @param[in] model_points Corresponding points of a 3D model.
* @param[in] is_viewport_upsidedown Flag to set whether the viewport of the image points is upside-down (e.g. as in OpenCV).
* @param[in] viewport_height Height of the viewport of the image points (needs to be given if is_viewport_upsidedown == true).
* @return Rotation, translation and scaling of the estimated scaled orthographic projection.
*/
ScaledOrthoProjectionParameters estimate_orthographic_projection_linear(std::vector<cv::Vec2f> image_points, std::vector<cv::Vec4f> model_points, bool is_viewport_upsidedown, boost::optional<int> viewport_height = boost::none)
{
using cv::Mat;
assert(image_points.size() == model_points.size());
assert(image_points.size() >= 4); // Number of correspondence points given needs to be equal to or larger than 4
const int num_correspondences = static_cast<int>(image_points.size());
if (is_viewport_upsidedown)
{
if (viewport_height == boost::none)
{
throw std::runtime_error("Error: If is_viewport_upsidedown is set to true, viewport_height needs to be given.");
}
for (auto&& ip : image_points)
{
ip[1] = viewport_height.get() - ip[1];
}
}
Mat A = Mat::zeros(2 * num_correspondences, 8, CV_32FC1);
int row_index = 0;
for (int i = 0; i < model_points.size(); ++i)
{
Mat p = Mat(model_points[i]).t();
p.copyTo(A.row(row_index++).colRange(0, 4)); // even row - copy to left side (first row is row 0)
p.copyTo(A.row(row_index++).colRange(4, 8)); // odd row - copy to right side
} // 4th coord (homogeneous) is already 1
Mat b(2 * num_correspondences, 1, CV_32FC1);
row_index = 0;
for (int i = 0; i < image_points.size(); ++i)
{
b.at<float>(row_index++) = image_points[i][0];
b.at<float>(row_index++) = image_points[i][1];
}
Mat k; // resulting affine matrix (8x1)
bool solved = cv::solve(A, b, k, cv::DECOMP_SVD);
const Mat R1 = k.rowRange(0, 3);
const Mat R2 = k.rowRange(4, 7);
const float sTx = k.at<float>(3);
const float sTy = k.at<float>(7);
const auto s = (cv::norm(R1) + cv::norm(R2)) / 2.0;
Mat r1 = R1 / cv::norm(R1);
Mat r2 = R2 / cv::norm(R2);
Mat r3 = r1.cross(r2);
Mat R;
r1 = r1.t();
r2 = r2.t();
r3 = r3.t();
R.push_back(r1);
R.push_back(r2);
R.push_back(r3);
// Set R to the closest orthonormal matrix to the estimated affine transform:
Mat S, U, Vt;
cv::SVDecomp(R, S, U, Vt);
Mat R_ortho = U * Vt;
// The determinant of R must be 1 for it to be a valid rotation matrix
if (cv::determinant(R_ortho) < 0)
{
U.row(2) = -U.row(2); // not sure this works...
R_ortho = U * Vt;
}
// Remove the scale from the translations:
const auto t1 = sTx / s;
const auto t2 = sTy / s;
// Convert to a glm::mat4x4:
glm::mat3x3 R_glm; // identity
for (int r = 0; r < 3; ++r) {
for (int c = 0; c < 3; ++c) {
R_glm[c][r] = R_ortho.at<float>(r, c);
}
}
return ScaledOrthoProjectionParameters{ R_glm, t1, t2, s };
};
} /* namespace fitting */
} /* namespace eos */
#endif /* ORTHOGRAPHICCAMERAESTIMATIONLINEAR_HPP_ */
......@@ -174,6 +174,16 @@ public:
return model_sample;
};
/**
* @copydoc PcaModel::draw_sample(std::vector<double>)
*/
cv::Mat draw_sample(std::vector<double> coefficients) const
{
// We have to convert the vector of doubles to float:
std::vector<float> coeffs_float(std::begin(coefficients), std::end(coefficients));
return draw_sample(coeffs_float);
};
/**
* Returns the PCA basis matrix, i.e. the eigenvectors.
* Each column of the matrix is an eigenvector.
......
......@@ -300,6 +300,14 @@ cv::Vec3f tex2d(const cv::Vec2f& texcoords, const Texture& texture, float dudx,
return (1.0f / 255.0f) * tex2d_linear_mipmap_linear(texcoords, texture, dudx, dudy, dvdx, dvdy);
};
template<typename T, glm::precision P = glm::defaultp>
glm::tvec3<T, P> tex2d(const glm::tvec2<T, P>& texcoords, const Texture& texture, float dudx, float dudy, float dvdx, float dvdy)
{
// Todo: Change everything to GLM.
cv::Vec3f ret = (1.0f / 255.0f) * tex2d_linear_mipmap_linear(cv::Vec2f(texcoords[0], texcoords[1]), texture, dudx, dudy, dvdx, dvdy);
return glm::tvec3<T, P>(ret[0], ret[1], ret[2]);
};
cv::Vec3f tex2d_linear_mipmap_linear(const cv::Vec2f& texcoords, const Texture& texture, float dudx, float dudy, float dvdx, float dvdy)
{
using cv::Vec2f;
......
......@@ -23,6 +23,7 @@
# Configuration options
# ==============================
set(BUILD_EXAMPLES ON CACHE BOOL "Build the example applications." FORCE)
set(BUILD_CERES_EXAMPLE OFF CACHE BOOL "Build the fit-model-ceres example (requires Ceres)." FORCE)
set(BUILD_UTILS OFF CACHE BOOL "Build utility applications." FORCE)
set(BUILD_DOCUMENTATION OFF CACHE BOOL "Build the library documentation." FORCE)
set(GENERATE_PYTHON_BINDINGS OFF CACHE BOOL "Build python bindings. Needs BUILD_UTILS enabled too." FORCE)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment