Skip to content

E
eos
Commit e94b6716 authored by Richard Torenvliet's avatar Richard Torenvliet
Browse files
Options

Add capability to read a settings file (.ini format) and fix bug with... 

Add capability to read a settings file (.ini format) and fix bug with unaligned landmarks and frames
parent 3b7d34d3
Hide whitespace changes
Inline Side-by-side
Showing with 96 additions and 34 deletions
include/eos/video/Keyframe.hpp
View file @ e94b6716
......@@ -28,6 +28,8 @@
#include "eos/morphablemodel/MorphableModel.hpp"
#include "eos/render/texture_extraction.hpp"
#include <boost/property_tree/ptree.hpp>
#include "opencv2/core/core.hpp"
#include <deque>
......@@ -40,7 +42,7 @@ namespace video {
*
* Contains the original frame, all necessary fitting parameters, and a score.
*/
struct Keyframe {
class Keyframe {
public:
Keyframe() {}
/**
......@@ -76,10 +78,11 @@ public:
this->fitting_result = fitting_result;
}
int frame_number;
float score = 0.0f;
cv::Mat frame;
fitting::FittingResult fitting_result;
int frame_number;
};
/**
......@@ -318,8 +321,7 @@ public:
BufferedVideoIterator() {};
// TODO: build support for setting the amount of max_frames in the buffer.
BufferedVideoIterator(std::string videoFilePath,
uint max_frames = 5, uint min_frames = 4) {
BufferedVideoIterator(std::string videoFilePath, boost::property_tree::ptree settings) {
std::ifstream file(videoFilePath);
std::cout << "video file path: " << videoFilePath << std::endl;
......@@ -334,8 +336,13 @@ public:
}
this->cap = tmp_cap;
this->max_frames = max_frames;
this->min_frames = min_frames;
this->max_frames = settings.get<int>("video.max_frames", 5);
this->min_frames = settings.get<int>("video.min_frames", 5);
this->drop_frames = settings.get<int>("video.drop_frames", 0);
this->laplacian_threshold = settings.get<int>("video.blur_threshold", 1000);
// TODO: Implement this.
this->skip_frames = settings.get<int>("video.skip_frames", 0);
}
/**
......@@ -352,6 +359,7 @@ public:
cv::Mat frame;
cap >> frame;
// Pop if we exceeded max_frames.
if (n_frames > max_frames) {
frame_buffer.pop_front();
......@@ -362,13 +370,20 @@ public:
if (frame_laplacian_score < laplacian_threshold && frame_laplacian_score > 0) {
frame_buffer.push_back(Keyframe(frame_laplacian_score, frame, total_frames_processed));
total_frames_processed++;
n_frames++;
std::cout << total_frames_processed << ": laplacian score " << frame_laplacian_score << std::endl;
} else {
std::cout << "skipping frame, too blurry or total black" << std::endl;
std::cout << total_frames_processed << ": skipping frame " << std::endl;
if (frame_laplacian_score == 0) {
std::cout << "total black" << std::endl;
} else {
std::cout << "too blurry" << std::endl;
}
}
total_frames_processed++;
// fill up the buffer until we hit the minimum frames we want in the buffer.
if(n_frames < min_frames) {
frame_buffer = next();
......@@ -386,21 +401,23 @@ private:
std::deque<Keyframe> frame_buffer;
// TODO: make set-able
uint total_frames_processed = 0;
uint skip_frames = 0;
// total frames in processed, not persee in buffer (skipped frames)
int total_frames_processed = 0;
// total frames in buffer
uint n_frames = 0;
int n_frames = 0;
// number of frames to skip at before starting
int skip_frames = 0;
// min frames to load at the start.
uint min_frames = 5;
// minimum amount of frames to keep in buffer.
int min_frames = 5;
// keep max_frames into the buffer.
uint max_frames = 5;
int max_frames = 5;
// Note: these settings are for future use
uint drop_frames = 0;
int drop_frames = 0;
// laplacian threshold
double laplacian_threshold = 10000000;
......
utils/accuracy-evaluation.cpp
View file @ e94b6716
......@@ -31,6 +31,8 @@
#include "boost/program_options.hpp"
#include "boost/filesystem.hpp"
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include "glm/gtx/string_cast.hpp"
#include <vector>
......@@ -185,17 +187,17 @@ void evaluate_results(
// and similarly for pitch and roll.
// Extract the texture from the image using given mesh and camera parameters:
Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(
rendering_paramss[i], frame_width, frame_height
rendering_paramss[i], frame_width, frame_height
);
// Draw the loaded landmarks:
Mat isomap = render::extract_texture(meshs[i], affine_from_ortho, frame);
Mat outimg = frame.clone();
for (auto&& lm : landmark_list[frame_number]) {
for (auto &&lm : landmark_list[i]) {
cv::rectangle(
outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f),
cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), { 255, 0, 0 }
cv::Point2f(lm.coordinates[0], lm.coordinates[1] + 2.0f), {255, 0, 0}
);
}
......@@ -208,7 +210,8 @@ void evaluate_results(
fitting::get_opencv_viewport(frame_width, frame_height)
);
std::string outputfile = fs::path(annotations[frame_number]).replace_extension("").string();
fs::path path = (fs::path(annotations[frame_number]).parent_path() / "eval");
std::string outputfile = (path / fs::path(annotations[frame_number]).replace_extension("").filename()).string();
std::string iter = "_" + std::to_string(n_iter) + "_" + std::to_string(i);
cv::imwrite(outputfile + iter + ".annotated.png", outimg);
......@@ -234,16 +237,17 @@ void evaluate_results(
merged_isomap = isomap_averaging.add_and_merge(isomap);
evaluate_accuracy(
landmark_list[frame_number],
landmark_mapper,
meshs[i],
affine_from_ortho
landmark_list[i],
landmark_mapper,
meshs[i],
affine_from_ortho
);
}
// save the merged isomap:
std::string iter = "_" + std::to_string(n_iter);
std::string outputfile = fs::path(annotations[n_iter]).replace_extension("").string();
fs::path path = (fs::path(annotations[n_iter]).parent_path() / "eval");
std::string outputfile = (path / fs::path(annotations[n_iter]).replace_extension("").filename()).string();
cv::imwrite(outputfile + iter + ".isomap.png", merged_isomap);
......@@ -264,6 +268,44 @@ void evaluate_results(
std::cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile + iter + ".obj" << std::endl;
}
/**
* Parse config file
* @param filename
*/
boost::property_tree::ptree get_reconstruction_config(std::string filename) {
boost::property_tree::ptree pt;
boost::property_tree::ini_parser::read_ini(filename, pt);
std::cout << pt.get<std::string>("video.max_frames") << std::endl;
std::cout << pt.get<std::string>("video.drop_frames") << std::endl;
std::cout << pt.get<std::string>("video.min_frames") << std::endl;
std::cout << pt.get<std::string>("video.skip_frames") << std::endl;
std::cout << pt.get<std::string>("video.blur_threshold") << std::endl;
return pt;
}
/**
*
* Return a list of landmarks based on the keyframe's frame_number. Such that the frame and the landmarks
* are aligned. The VideoIterator is able to skip frames based on certain conditions, skipping frames
* causes un-alignment of the total landmarks list and the list of frames. This samples the correct landmark
* annotations with the based on a given keyframe list.
*
* @param key_frames
* @param landmarks
* @return
*/
vector<core::LandmarkCollection<cv::Vec2f>> sample_landmarks(std::deque<eos::video::Keyframe> key_frames, vector<core::LandmarkCollection<cv::Vec2f>> landmarks) {
vector<core::LandmarkCollection<cv::Vec2f>> sublist;
for (auto& f : key_frames) {
sublist.push_back(landmarks[f.frame_number]);
}
return sublist;
}
/**
* This app demonstrates estimation of the camera and fitting of the shape
* model of a 3D Morphable Model from an ibug LFPW image with its landmarks.
......@@ -274,7 +316,7 @@ void evaluate_results(
* to vertex indices using the LandmarkMapper.
*/
int main(int argc, char *argv[]) {
fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile;
fs::path modelfile, isomapfile, videofile, landmarksfile, mappingsfile, contourfile, edgetopologyfile, blendshapesfile, outputfile, reconstruction_config;
std::vector<std::string> annotations;
// get annotaitions from one file
......@@ -287,10 +329,12 @@ int main(int argc, char *argv[]) {
"display the help message")
("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"),
"a Morphable Model stored as cereal BinaryArchive")
("video,i", po::value<fs::path>(&videofile)->required(),
("video,v", po::value<fs::path>(&videofile)->required(),
"an input image")
("config,c", po::value<fs::path>(&reconstruction_config)->default_value("../share/default_reconstruction_config.ini"),
"configuration file for the reconstruction")
("get_annotations,g", po::bool_switch(&get_annotations)->default_value(false),
"read .pts annotation file locations from one file, one file path per line")
"read .pts annotation file locations from one file, put one file path on each line")
("annotations,l", po::value<vector<std::string>>(&annotations)->multitoken(),
".pts annotation files per frame of video")
("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"),
......@@ -323,6 +367,7 @@ int main(int argc, char *argv[]) {
// start loading prerequisites
morphablemodel::MorphableModel morphable_model;
try {
morphable_model = morphablemodel::load_model(modelfile.string());
} catch (const std::runtime_error &e) {
......@@ -359,9 +404,10 @@ int main(int argc, char *argv[]) {
vector<fitting::RenderingParameters> rendering_paramss;
BufferedVideoIterator vid_iterator;
boost::property_tree::ptree settings = get_reconstruction_config(reconstruction_config.string());
try {
vid_iterator = BufferedVideoIterator(videofile.string(), 5, 5);
vid_iterator = BufferedVideoIterator(videofile.string(), settings);
} catch(std::runtime_error &e) {
std::cout << e.what() << std::endl;
return EXIT_FAILURE;
......@@ -378,17 +424,16 @@ int main(int argc, char *argv[]) {
std::vector<std::vector<float>> blend_shape_coefficients;
std::vector<std::vector<cv::Vec2f>> fitted_image_points;
int n_iter = 0;
while(!(key_frames.empty())) {
if (n_iter == 10) {
break;
}
// load all annotation files into lists of landmarks
vector<core::LandmarkCollection<cv::Vec2f>> landmark_sublist(landmark_list.begin() + n_iter, landmark_list.end());
vector<core::LandmarkCollection<cv::Vec2f>> landmark_sublist = sample_landmarks(
key_frames, landmark_list
);
std::tie(meshs, rendering_paramss) = fitting::fit_shape_and_pose_multi(
morphable_model,
......@@ -413,7 +458,7 @@ int main(int argc, char *argv[]) {
evaluate_results(
key_frames,
rendering_paramss,
landmark_list,
landmark_sublist,
morphable_model,
meshs,
pca_shape_coefficients,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment