OpenShot Library | libopenshot  0.3.3
CVTracker.cpp
Go to the documentation of this file.
1 
10 // Copyright (c) 2008-2019 OpenShot Studios, LLC
11 //
12 // SPDX-License-Identifier: LGPL-3.0-or-later
13 
14 #include <fstream>
15 #include <iomanip>
16 #include <iostream>
17 
18 #include <google/protobuf/util/time_util.h>
19 
20 #include "OpenCVUtilities.h"
21 #include "CVTracker.h"
22 #include "trackerdata.pb.h"
23 #include "Exceptions.h"
24 
25 using namespace openshot;
26 using google::protobuf::util::TimeUtil;
27 
28 // Constructor
29 CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController)
30 : processingController(&processingController), json_interval(false){
31  SetJson(processInfoJson);
32  start = 1;
33  end = 1;
34 }
35 
36 // Set desirable tracker method
37 cv::Ptr<OPENCV_TRACKER_TYPE> CVTracker::selectTracker(std::string trackerType){
38 
39  if (trackerType == "BOOSTING")
40  return OPENCV_TRACKER_NS::TrackerBoosting::create();
41  if (trackerType == "MIL")
42  return OPENCV_TRACKER_NS::TrackerMIL::create();
43  if (trackerType == "KCF")
44  return OPENCV_TRACKER_NS::TrackerKCF::create();
45  if (trackerType == "TLD")
46  return OPENCV_TRACKER_NS::TrackerTLD::create();
47  if (trackerType == "MEDIANFLOW")
48  return OPENCV_TRACKER_NS::TrackerMedianFlow::create();
49  if (trackerType == "MOSSE")
50  return OPENCV_TRACKER_NS::TrackerMOSSE::create();
51  if (trackerType == "CSRT")
52  return OPENCV_TRACKER_NS::TrackerCSRT::create();
53 
54  return nullptr;
55 }
56 
57 // Track object in the hole clip or in a given interval
58 void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
59 
60  video.Open();
61  if(!json_interval){
62  start = _start; end = _end;
63 
64  if(!process_interval || end <= 1 || end-start == 0){
65  // Get total number of frames in video
66  start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
67  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
68  }
69  }
70  else{
71  start = (int)(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1;
72  end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1;
73  }
74 
75  if(error){
76  return;
77  }
78 
79  processingController->SetError(false, "");
80  bool trackerInit = false;
81 
82  size_t frame;
83  // Loop through video
84  for (frame = start; frame <= end; frame++)
85  {
86 
87  // Stop the feature tracker process
88  if(processingController->ShouldStop()){
89  return;
90  }
91 
92  size_t frame_number = frame;
93  // Get current frame
94  std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
95 
96  // Grab OpenCV Mat image
97  cv::Mat cvimage = f->GetImageCV();
98 
99  if(frame == start){
100  // Take the normalized inital bounding box and multiply to the current video shape
101  bbox = cv::Rect2d(int(bbox.x*cvimage.cols), int(bbox.y*cvimage.rows),
102  int(bbox.width*cvimage.cols), int(bbox.height*cvimage.rows));
103  }
104 
105  // Pass the first frame to initialize the tracker
106  if(!trackerInit){
107 
108  // Initialize the tracker
109  initTracker(cvimage, frame_number);
110 
111  trackerInit = true;
112  }
113  else{
114  // Update the object tracker according to frame
115  trackerInit = trackFrame(cvimage, frame_number);
116 
117  // Draw box on image
118  FrameData fd = GetTrackedData(frame_number);
119 
120  }
121  // Update progress
122  processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
123  }
124 }
125 
126 // Initialize the tracker
127 bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){
128 
129  // Create new tracker object
130  tracker = selectTracker(trackerType);
131 
132  // Correct if bounding box contains negative proportions (width and/or height < 0)
133  if(bbox.width < 0){
134  bbox.x = bbox.x - abs(bbox.width);
135  bbox.width = abs(bbox.width);
136  }
137  if(bbox.height < 0){
138  bbox.y = bbox.y - abs(bbox.height);
139  bbox.height = abs(bbox.height);
140  }
141 
142  // Initialize tracker
143  tracker->init(frame, bbox);
144 
145  float fw = frame.size().width;
146  float fh = frame.size().height;
147 
148  // Add new frame data
149  trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw,
150  (bbox.y)/fh,
151  (bbox.x+bbox.width)/fw,
152  (bbox.y+bbox.height)/fh);
153 
154  return true;
155 }
156 
157 // Update the object tracker according to frame
158 bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
159  // Update the tracking result
160  bool ok = tracker->update(frame, bbox);
161 
162  // Add frame number and box coords if tracker finds the object
163  // Otherwise add only frame number
164  if (ok)
165  {
166  float fw = frame.size().width;
167  float fh = frame.size().height;
168 
169  cv::Rect2d filtered_box = filter_box_jitter(frameId);
170  // Add new frame data
171  trackedDataById[frameId] = FrameData(frameId, 0, (filtered_box.x)/fw,
172  (filtered_box.y)/fh,
173  (filtered_box.x+filtered_box.width)/fw,
174  (filtered_box.y+filtered_box.height)/fh);
175  }
176  else
177  {
178  // Copy the last frame data if the tracker get lost
179  trackedDataById[frameId] = trackedDataById[frameId-1];
180  }
181 
182  return ok;
183 }
184 
185 cv::Rect2d CVTracker::filter_box_jitter(size_t frameId){
186  // get tracked data for the previous frame
187  float last_box_width = trackedDataById[frameId-1].x2 - trackedDataById[frameId-1].x1;
188  float last_box_height = trackedDataById[frameId-1].y2 - trackedDataById[frameId-1].y1;
189 
190  float curr_box_width = bbox.width;
191  float curr_box_height = bbox.height;
192  // keep the last width and height if the difference is less than 1%
193  float threshold = 0.01;
194 
195  cv::Rect2d filtered_box = bbox;
196  if(std::abs(1-(curr_box_width/last_box_width)) <= threshold){
197  filtered_box.width = last_box_width;
198  }
199  if(std::abs(1-(curr_box_height/last_box_height)) <= threshold){
200  filtered_box.height = last_box_height;
201  }
202  return filtered_box;
203 }
204 
206  using std::ios;
207 
208  // Create tracker message
209  pb_tracker::Tracker trackerMessage;
210 
211  // Iterate over all frames data and save in protobuf message
212  for(std::map<size_t,FrameData>::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){
213  FrameData fData = it->second;
214  pb_tracker::Frame* pbFrameData;
215  AddFrameDataToProto(trackerMessage.add_frame(), fData);
216  }
217 
218  // Add timestamp
219  *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
220 
221  {
222  // Write the new message to disk.
223  std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
224  if (!trackerMessage.SerializeToOstream(&output)) {
225  std::cerr << "Failed to write protobuf message." << std::endl;
226  return false;
227  }
228  }
229 
230  // Delete all global objects allocated by libprotobuf.
231  google::protobuf::ShutdownProtobufLibrary();
232 
233  return true;
234 
235 }
236 
237 // Add frame tracked data into protobuf message.
238 void CVTracker::AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData) {
239 
240  // Save frame number and rotation
241  pbFrameData->set_id(fData.frame_id);
242  pbFrameData->set_rotation(0);
243 
244  pb_tracker::Frame::Box* box = pbFrameData->mutable_bounding_box();
245  // Save bounding box data
246  box->set_x1(fData.x1);
247  box->set_y1(fData.y1);
248  box->set_x2(fData.x2);
249  box->set_y2(fData.y2);
250 }
251 
252 // Get tracker info for the desired frame
254 
255  // Check if the tracker info for the requested frame exists
256  if ( trackedDataById.find(frameId) == trackedDataById.end() ) {
257 
258  return FrameData();
259  } else {
260 
261  return trackedDataById[frameId];
262  }
263 
264 }
265 
266 // Load JSON string into this object
267 void CVTracker::SetJson(const std::string value) {
268  // Parse JSON string into JSON objects
269  try
270  {
271  const Json::Value root = openshot::stringToJson(value);
272  // Set all values that match
273 
274  SetJsonValue(root);
275  }
276  catch (const std::exception& e)
277  {
278  // Error parsing JSON (or missing keys)
279  throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)");
280  }
281 }
282 
283 // Load Json::Value into this object
284 void CVTracker::SetJsonValue(const Json::Value root) {
285 
286  // Set data from Json (if key is found)
287  if (!root["protobuf_data_path"].isNull()){
288  protobuf_data_path = (root["protobuf_data_path"].asString());
289  }
290  if (!root["tracker-type"].isNull()){
291  trackerType = (root["tracker-type"].asString());
292  }
293 
294  if (!root["region"].isNull()){
295  double x = root["region"]["normalized_x"].asDouble();
296  double y = root["region"]["normalized_y"].asDouble();
297  double w = root["region"]["normalized_width"].asDouble();
298  double h = root["region"]["normalized_height"].asDouble();
299  cv::Rect2d prev_bbox(x,y,w,h);
300  bbox = prev_bbox;
301 
302  if (!root["region"]["first-frame"].isNull()){
303  start = root["region"]["first-frame"].asInt64();
304  json_interval = true;
305  }
306  else{
307  processingController->SetError(true, "No first-frame");
308  error = true;
309  }
310 
311  }
312  else{
313  processingController->SetError(true, "No initial bounding box selected");
314  error = true;
315  }
316 
317 }
318 
319 /*
320 ||||||||||||||||||||||||||||||||||||||||||||||||||
321  ONLY FOR MAKE TEST
322 ||||||||||||||||||||||||||||||||||||||||||||||||||
323 */
324 
325 // Load protobuf data file
327  using std::ios;
328 
329  // Create tracker message
330  pb_tracker::Tracker trackerMessage;
331 
332  {
333  // Read the existing tracker message.
334  std::fstream input(protobuf_data_path, ios::in | ios::binary);
335  if (!trackerMessage.ParseFromIstream(&input)) {
336  std::cerr << "Failed to parse protobuf message." << std::endl;
337  return false;
338  }
339  }
340 
341  // Make sure the trackedData is empty
342  trackedDataById.clear();
343 
344  // Iterate over all frames of the saved message
345  for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
346  const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i);
347 
348  // Load frame and rotation data
349  size_t id = pbFrameData.id();
350  float rotation = pbFrameData.rotation();
351 
352  // Load bounding box data
353  const pb_tracker::Frame::Box& box = pbFrameData.bounding_box();
354  float x1 = box.x1();
355  float y1 = box.y1();
356  float x2 = box.x2();
357  float y2 = box.y2();
358 
359  // Assign data to tracker map
360  trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
361  }
362 
363  // Delete all global objects allocated by libprotobuf.
364  google::protobuf::ShutdownProtobufLibrary();
365 
366  return true;
367 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::Clip::Open
void Open() override
Open the internal reader.
Definition: Clip.cpp:320
openshot::CVTracker::GetTrackedData
FrameData GetTrackedData(size_t frameId)
Get tracked data for a given frame.
Definition: CVTracker.cpp:253
CVTracker.h
Track an object selected by the user.
openshot::FrameData::x2
float x2
Definition: CVTracker.h:50
ProcessingController::ShouldStop
bool ShouldStop()
Definition: ProcessingController.h:68
ProcessingController::SetError
void SetError(bool err, std::string message)
Definition: ProcessingController.h:74
openshot::FrameData::frame_id
size_t frame_id
Definition: CVTracker.h:46
openshot::CVTracker::AddFrameDataToProto
void AddFrameDataToProto(pb_tracker::Frame *pbFrameData, FrameData &fData)
Add frame tracked data into protobuf message.
Definition: CVTracker.cpp:238
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
openshot::Clip
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:89
openshot::Clip::End
float End() const override
Get end position (in seconds) of clip (trim end of video), which can be affected by the time curve.
Definition: Clip.cpp:356
openshot::Clip::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:391
openshot::CVTracker::SetJson
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CVTracker.cpp:267
openshot::CVTracker::SaveTrackedData
bool SaveTrackedData()
Save protobuf file.
Definition: CVTracker.cpp:205
openshot::FrameData
Definition: CVTracker.h:45
openshot::CVTracker::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CVTracker.cpp:284
openshot::FrameData::y2
float y2
Definition: CVTracker.h:51
openshot::CVTracker::CVTracker
CVTracker(std::string processInfoJson, ProcessingController &processingController)
Definition: CVTracker.cpp:29
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:217
openshot::FrameData::x1
float x1
Definition: CVTracker.h:48
openshot::ClipBase::Start
void Start(float value)
Set start position (in seconds) of clip (trim start of video)
Definition: ClipBase.cpp:42
openshot::FrameData::y1
float y1
Definition: CVTracker.h:49
openshot::CVTracker::_LoadTrackedData
bool _LoadTrackedData()
Definition: CVTracker.cpp:326
openshot::CVTracker::filter_box_jitter
cv::Rect2d filter_box_jitter(size_t frameId)
Filter current bounding box jitter.
Definition: CVTracker.cpp:185
openshot::CVTracker::selectTracker
cv::Ptr< OPENCV_TRACKER_TYPE > selectTracker(std::string trackerType)
Definition: CVTracker.cpp:37
ProcessingController
Definition: ProcessingController.h:20
openshot::CVTracker::trackClip
void trackClip(openshot::Clip &video, size_t _start=0, size_t _end=0, bool process_interval=false)
Definition: CVTracker.cpp:58
openshot::Clip::Reader
void Reader(openshot::ReaderBase *new_reader)
Set the current reader.
Definition: Clip.cpp:274
ProcessingController::SetProgress
void SetProgress(uint p)
Definition: ProcessingController.h:52
Exceptions.h
Header file for all Exception classes.
OpenCVUtilities.h
Header file for OpenCVUtilities (set some common macros)