OpenShot Library | libopenshot 0.2.7
Tracker.cpp
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Source file for Tracker effect class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 * @author Brenno Caldato <brenno.caldato@outlook.com>
6 *
7 * @ref License
8 */
9
10/* LICENSE
11 *
12 * Copyright (c) 2008-2019 OpenShot Studios, LLC
13 * <http://www.openshotstudios.com/>. This file is part of
14 * OpenShot Library (libopenshot), an open-source project dedicated to
15 * delivering high quality video editing and animation solutions to the
16 * world. For more information visit <http://www.openshot.org/>.
17 *
18 * OpenShot Library (libopenshot) is free software: you can redistribute it
19 * and/or modify it under the terms of the GNU Lesser General Public License
20 * as published by the Free Software Foundation, either version 3 of the
21 * License, or (at your option) any later version.
22 *
23 * OpenShot Library (libopenshot) is distributed in the hope that it will be
24 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 * GNU Lesser General Public License for more details.
27 *
28 * You should have received a copy of the GNU Lesser General Public License
29 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30 */
31
32#include <string>
33#include <memory>
34#include <fstream>
35#include <iostream>
36
37#include "effects/Tracker.h"
38#include "Exceptions.h"
39#include "Timeline.h"
40
41#include <google/protobuf/util/time_util.h>
42
43#include <QImage>
44#include <QPainter>
45#include <QRectF>
46
47using namespace std;
48using namespace openshot;
49using google::protobuf::util::TimeUtil;
50
51/// Blank constructor, useful when using Json to load the effect properties
52Tracker::Tracker(std::string clipTrackerDataPath)
53{
54 // Init effect properties
55 init_effect_details();
56 // Instantiate a TrackedObjectBBox object and point to it
57 TrackedObjectBBox trackedDataObject;
58 trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59 // Tries to load the tracked object's data from protobuf file
60 trackedData->LoadBoxData(clipTrackerDataPath);
61 ClipBase* parentClip = this->ParentClip();
62 trackedData->ParentClip(parentClip);
63 trackedData->Id(std::to_string(0));
64 // Insert TrackedObject with index 0 to the trackedObjects map
65 trackedObjects.insert({0, trackedData});
66}
67
68// Default constructor
70{
71 // Init effect properties
72 init_effect_details();
73 // Instantiate a TrackedObjectBBox object and point to it
74 TrackedObjectBBox trackedDataObject;
75 trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
76 ClipBase* parentClip = this->ParentClip();
77 trackedData->ParentClip(parentClip);
78 trackedData->Id(std::to_string(0));
79 // Insert TrackedObject with index 0 to the trackedObjects map
80 trackedObjects.insert({0, trackedData});
81}
82
83
84// Init effect settings
85void Tracker::init_effect_details()
86{
87 /// Initialize the values of the EffectInfo struct.
89
90 /// Set the effect info
91 info.class_name = "Tracker";
92 info.name = "Tracker";
93 info.description = "Track the selected bounding box through the video.";
94 info.has_audio = false;
95 info.has_video = true;
97
98 this->TimeScale = 1.0;
99}
100
101// This method is required for all derived classes of EffectBase, and returns a
102// modified openshot::Frame object
103std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
104{
105 // Get the frame's image
106 cv::Mat frame_image = frame->GetImageCV();
107
108 // Initialize the Qt rectangle that will hold the positions of the bounding-box
109 QRectF boxRect;
110 // Initialize the image of the TrackedObject child clip
111 std::shared_ptr<QImage> childClipImage = nullptr;
112
113 // Check if frame isn't NULL
114 if(!frame_image.empty() &&
115 trackedData->Contains(frame_number) &&
116 trackedData->visible.GetValue(frame_number) == 1)
117 {
118 // Get the width and height of the image
119 float fw = frame_image.size().width;
120 float fh = frame_image.size().height;
121
122 // Get the bounding-box of given frame
123 BBox fd = trackedData->GetBox(frame_number);
124
125 // Check if track data exists for the requested frame
126 if (trackedData->draw_box.GetValue(frame_number) == 1)
127 {
128 std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
129 int stroke_width = trackedData->stroke_width.GetValue(frame_number);
130 float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
131 std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
132 float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
133
134 // Create a rotated rectangle object that holds the bounding box
135 cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
136 cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
137 (int) (fd.angle) );
138
139 DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
140 DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
141 }
142
143 // Get the image of the Tracked Object' child clip
144 if (trackedData->ChildClipId() != ""){
145 // Cast the parent timeline of this effect
146 Timeline* parentTimeline = (Timeline *) ParentTimeline();
147 if (parentTimeline){
148 // Get the Tracked Object's child clip
149 Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
150 if (childClip){
151 // Get the image of the child clip for this frame
152 std::shared_ptr<Frame> f(new Frame(1, frame->GetWidth(), frame->GetHeight(), "#00000000"));
153 std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(f, frame_number);
154 childClipImage = childClipFrame->GetImage();
155
156 // Set the Qt rectangle with the bounding-box properties
157 boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
158 (int)((fd.cy - fd.height/2)*fh),
159 (int)(fd.width*fw),
160 (int)(fd.height*fh) );
161 }
162 }
163 }
164
165 }
166
167 // Set image with drawn box to frame
168 // If the input image is NULL or doesn't have tracking data, it's returned as it came
169 frame->SetImageCV(frame_image);
170
171 // Set the bounding-box image with the Tracked Object's child clip image
172 if (childClipImage){
173 // Get the frame image
174 QImage frameImage = *(frame->GetImage());
175
176 // Set a Qt painter to the frame image
177 QPainter painter(&frameImage);
178
179 // Draw the child clip image inside the bounding-box
180 painter.drawImage(boxRect, *childClipImage, QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
181
182 // Set the frame image as the composed image
183 frame->AddImage(std::make_shared<QImage>(frameImage));
184 }
185
186 return frame;
187}
188
189void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
190 // Get the bouding box vertices
191 cv::Point2f vertices2f[4];
192 box.points(vertices2f);
193
194 // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
195 // select min enclosing rectangle to draw on a small portion of the image
196 // cv::Rect rect = box.boundingRect();
197 // cv::Mat image = frame_image(rect)
198
199 if(is_background){
200 cv::Mat overlayFrame;
201 frame_image.copyTo(overlayFrame);
202
203 // draw bounding box background
204 cv::Point vertices[4];
205 for(int i = 0; i < 4; ++i){
206 vertices[i] = vertices2f[i];}
207
208 cv::Rect rect = box.boundingRect();
209 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
210 // add opacity
211 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
212 }
213 else{
214 cv::Mat overlayFrame;
215 frame_image.copyTo(overlayFrame);
216
217 // Draw bounding box
218 for (int i = 0; i < 4; i++)
219 {
220 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
221 thickness, cv::LINE_AA);
222 }
223
224 // add opacity
225 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
226 }
227}
228
229// Get the indexes and IDs of all visible objects in the given frame
230std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
231
232 // Initialize the JSON objects
233 Json::Value root;
234 root["visible_objects_index"] = Json::Value(Json::arrayValue);
235 root["visible_objects_id"] = Json::Value(Json::arrayValue);
236
237 // Iterate through the tracked objects
238 for (const auto& trackedObject : trackedObjects){
239 // Get the tracked object JSON properties for this frame
240 Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
241 if (trackedObjectJSON["visible"]["value"].asBool()){
242 // Save the object's index and ID if it's visible in this frame
243 root["visible_objects_index"].append(trackedObject.first);
244 root["visible_objects_id"].append(trackedObject.second->Id());
245 }
246 }
247
248 return root.toStyledString();
249}
250
251// Generate JSON string of this object
252std::string Tracker::Json() const {
253
254 // Return formatted string
255 return JsonValue().toStyledString();
256}
257
258// Generate Json::Value for this object
259Json::Value Tracker::JsonValue() const {
260
261 // Create root json object
262 Json::Value root = EffectBase::JsonValue(); // get parent properties
263
264 // Save the effect's properties on root
265 root["type"] = info.class_name;
266 root["protobuf_data_path"] = protobuf_data_path;
267 root["BaseFPS"]["num"] = BaseFPS.num;
268 root["BaseFPS"]["den"] = BaseFPS.den;
269 root["TimeScale"] = this->TimeScale;
270
271 // Add trackedObjects IDs to JSON
272 Json::Value objects;
273 for (auto const& trackedObject : trackedObjects){
274 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
275 // add object json
276 objects[trackedObject.second->Id()] = trackedObjectJSON;
277 }
278 root["objects"] = objects;
279
280 // return JsonValue
281 return root;
282}
283
284// Load JSON string into this object
285void Tracker::SetJson(const std::string value) {
286
287 // Parse JSON string into JSON objects
288 try
289 {
290 const Json::Value root = openshot::stringToJson(value);
291 // Set all values that match
292 SetJsonValue(root);
293 }
294 catch (const std::exception& e)
295 {
296 // Error parsing JSON (or missing keys)
297 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
298 }
299 return;
300}
301
302// Load Json::Value into this object
303void Tracker::SetJsonValue(const Json::Value root) {
304
305 // Set parent data
307
308 if(!root["type"].isNull())
309 info.class_name = root["type"].asString();
310
311 if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
312 {
313 if (!root["BaseFPS"]["num"].isNull())
314 {
315 BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
316 }
317 if (!root["BaseFPS"]["den"].isNull())
318 {
319 BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
320 }
321 }
322
323 if (!root["TimeScale"].isNull())
324 TimeScale = (double) root["TimeScale"].asDouble();
325
326 // Set data from Json (if key is found)
327 if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
328 {
329 protobuf_data_path = root["protobuf_data_path"].asString();
330 if(!trackedData->LoadBoxData(protobuf_data_path))
331 {
332 std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
334 }
335 }
336
337 if (!root["objects"].isNull()){
338 for (auto const& trackedObject : trackedObjects){
339 std::string obj_id = std::to_string(trackedObject.first);
340 if(!root["objects"][obj_id].isNull()){
341 trackedObject.second->SetJsonValue(root["objects"][obj_id]);
342 }
343 }
344 }
345
346 // Set the tracked object's ids
347 if (!root["objects_id"].isNull()){
348 for (auto const& trackedObject : trackedObjects){
349 Json::Value trackedObjectJSON;
350 trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
351 trackedObject.second->SetJsonValue(trackedObjectJSON);
352 }
353 }
354
355 return;
356}
357
358// Get all properties for a specific frame
359std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
360
361 // Generate JSON properties list
362 Json::Value root;
363
364 // Add trackedObject properties to JSON
365 Json::Value objects;
366 for (auto const& trackedObject : trackedObjects){
367 Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
368 // add object json
369 objects[trackedObject.second->Id()] = trackedObjectJSON;
370 }
371 root["objects"] = objects;
372
373 // Append effect's properties
374 root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
375 root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
376 root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
377 root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
378 root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
379 root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
380
381 // Return formatted string
382 return root.toStyledString();
383}
Header file for all Exception classes.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:51
openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
Definition: ClipBase.h:113
float End() const
Get end position (in seconds) of clip (trim end of video)
Definition: ClipBase.h:111
float Start() const
Get start position (in seconds) of clip (trim start of video)
Definition: ClipBase.h:110
float Duration() const
Get the length of this clip (in seconds)
Definition: ClipBase.h:112
std::string Id() const
Get the Id of this clip object.
Definition: ClipBase.h:107
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
Definition: ClipBase.h:109
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:108
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
Definition: ClipBase.cpp:68
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
Definition: EffectBase.cpp:92
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
Definition: EffectBase.cpp:188
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: EffectBase.cpp:127
EffectInfoStruct info
Information about the current effect.
Definition: EffectBase.h:87
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
Definition: EffectBase.h:84
int num
Numerator for the fraction.
Definition: Fraction.h:50
int den
Denominator for the fraction.
Definition: Fraction.h:51
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:108
Exception for invalid JSON.
Definition: Exceptions.h:206
This class represents a timeline.
Definition: Timeline.h:168
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:404
This class contains the properties of a tracked object and functions to manipulate it.
std::string Json() const override
Generate JSON string of this object.
Definition: Tracker.cpp:252
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
Definition: Tracker.cpp:230
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: Tracker.cpp:259
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: Tracker.cpp:285
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
This method is required for all derived classes of EffectBase, and returns a modified openshot::Frame...
Definition: Tracker.cpp:103
std::string PropertiesJSON(int64_t requested_frame) const override
Definition: Tracker.cpp:359
std::shared_ptr< TrackedObjectBBox > trackedData
Pointer to an object that holds the bounding-box data and it's Keyframes.
Definition: Tracker.h:68
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: Tracker.cpp:303
Tracker()
Default constructor.
Definition: Tracker.cpp:69
void DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector< int > color, float alpha, int thickness, bool is_background)
Definition: Tracker.cpp:189
std::string protobuf_data_path
Path to the protobuf file that holds the bounding-box data.
Definition: Tracker.h:67
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
float angle
bounding box rotation angle [degrees]
bool has_video
Determines if this effect manipulates the image of a frame.
Definition: EffectBase.h:58
bool has_audio
Determines if this effect manipulates the audio of a frame.
Definition: EffectBase.h:59
std::string class_name
The class name of the effect.
Definition: EffectBase.h:54
std::string name
The name of the effect.
Definition: EffectBase.h:55
std::string description
The description of this effect and what it does.
Definition: EffectBase.h:56
bool has_tracked_object
Determines if this effect track objects through the clip.
Definition: EffectBase.h:60