48ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
51 init_effect_details();
64 init_effect_details();
71void ObjectDetection::init_effect_details()
90 cv::Mat cv_image = frame->GetImageCV();
98 std::vector<QRectF> boxRects;
100 std::vector<std::shared_ptr<QImage>> childClipImages;
103 if (detectionsData.find(frame_number) != detectionsData.end()) {
104 float fw = cv_image.size().width;
105 float fh = cv_image.size().height;
108 for(
int i = 0; i<detections.
boxes.size(); i++){
111 if(detections.
confidences.at(i) < confidence_threshold){
115 if( display_classes.size() > 0 &&
116 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
121 int objectId = detections.
objectIds.at(i);
127 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<TrackedObjectBBox>(trackedObject_it->second);
130 if (trackedObject->Contains(frame_number) &&
131 trackedObject->visible.GetValue(frame_number) == 1)
134 BBox trackedBox = trackedObject->GetBox(frame_number);
135 bool draw_text = !display_box_text.
GetValue(frame_number);
136 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
137 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
138 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
139 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
140 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
152 (
int)( (trackedBox.
cx-trackedBox.
width/2)*fw),
153 (
int)( (trackedBox.
cy-trackedBox.
height/2)*fh),
154 (
int)( trackedBox.
width*fw),
155 (
int)( trackedBox.
height*fh)
159 if (trackedObject->draw_box.GetValue(frame_number) == 0)
166 box, cv_image, detections.
objectIds.at(i), bg_rgba, bg_alpha, 1,
true, draw_text);
168 box, cv_image, detections.
objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width,
false, draw_text);
172 if (trackedObject->ChildClipId() !=
""){
177 Clip* childClip = parentTimeline->
GetClip(trackedObject->ChildClipId());
180 std::shared_ptr<Frame> f(
new Frame(1, frame->GetWidth(), frame->GetHeight(),
"#00000000"));
182 std::shared_ptr<Frame> childClipFrame = childClip->
GetFrame(f, frame_number);
183 childClipImages.push_back(childClipFrame->GetImage());
187 boxRect.setRect((
int)((trackedBox.
cx-trackedBox.
width/2)*fw),
188 (
int)((trackedBox.
cy - trackedBox.
height/2)*fh),
189 (
int)(trackedBox.
width*fw),
190 (
int)(trackedBox.
height*fh));
191 boxRects.push_back(boxRect);
200 frame->SetImageCV(cv_image);
203 if(boxRects.size() > 0){
205 QImage frameImage = *(frame->GetImage());
206 for(
int i; i < boxRects.size();i++){
208 QPainter painter(&frameImage);
210 painter.drawImage(boxRects[i], *childClipImages[i], QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
213 frame->AddImage(std::make_shared<QImage>(frameImage));
219void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color,
float alpha,
220 int thickness,
bool is_background){
222 cv::Point2f vertices2f[4];
223 box.points(vertices2f);
231 cv::Mat overlayFrame;
232 frame_image.copyTo(overlayFrame);
235 cv::Point vertices[4];
236 for(
int i = 0; i < 4; ++i){
237 vertices[i] = vertices2f[i];}
239 cv::Rect rect = box.boundingRect();
240 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
242 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
245 cv::Mat overlayFrame;
246 frame_image.copyTo(overlayFrame);
249 for (
int i = 0; i < 4; i++)
251 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
252 thickness, cv::LINE_AA);
256 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
260void ObjectDetection::drawPred(
int classId,
float conf, cv::Rect2d box, cv::Mat& frame,
int objectNumber, std::vector<int> color,
261 float alpha,
int thickness,
bool is_background,
bool display_text)
265 cv::Mat overlayFrame;
266 frame.copyTo(overlayFrame);
269 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
272 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
275 cv::Mat overlayFrame;
276 frame.copyTo(overlayFrame);
279 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness);
283 std::string label = cv::format(
"%.2f", conf);
284 if (!classNames.empty())
286 CV_Assert(classId < (
int)classNames.size());
287 label = classNames[classId] +
":" + label;
292 cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
295 double top = std::max((
int)box.y, labelSize.height);
297 cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine),
298 cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
299 putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
302 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
309 pb_objdetect::ObjDetect objMessage;
312 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
313 if (!objMessage.ParseFromIstream(&input)) {
314 std::cerr <<
"Failed to parse protobuf message." << std::endl;
320 detectionsData.clear();
326 for(
int i = 0; i < objMessage.classnames_size(); i++)
328 classNames.push_back(objMessage.classnames(i));
329 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
333 for (
size_t i = 0; i < objMessage.frame_size(); i++)
336 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
339 size_t id = pbFrameData.id();
342 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
345 std::vector<int> classIds;
346 std::vector<float> confidences;
347 std::vector<cv::Rect_<float>> boxes;
348 std::vector<int> objectIds;
351 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
354 float x = pBox.Get(i).x();
355 float y = pBox.Get(i).y();
356 float w = pBox.Get(i).w();
357 float h = pBox.Get(i).h();
359 int classId = pBox.Get(i).classid();
361 float confidence = pBox.Get(i).confidence();
364 int objectId = pBox.Get(i).objectid();
372 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
377 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
378 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
380 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
382 trackedObjPtr->ParentClip(parentClip);
386 trackedObjPtr->Id(std::to_string(objectId));
391 cv::Rect_<float> box(x, y, w, h);
394 boxes.push_back(box);
395 classIds.push_back(classId);
396 confidences.push_back(confidence);
397 objectIds.push_back(objectId);
401 detectionsData[
id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
405 google::protobuf::ShutdownProtobufLibrary();
415 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
416 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
419 if (detectionsData.find(frame_number) == detectionsData.end()){
420 return root.toStyledString();
425 for(
int i = 0; i<detections.
boxes.size(); i++){
427 if(detections.
confidences.at(i) < confidence_threshold){
432 if( display_classes.size() > 0 &&
433 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
437 int objectId = detections.
objectIds.at(i);
442 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
444 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
445 trackedObject->second->ExactlyContains(frame_number)){
447 root[
"visible_objects_index"].append(trackedObject->first);
448 root[
"visible_objects_id"].append(trackedObject->second->Id());
452 return root.toStyledString();
468 root[
"protobuf_data_path"] = protobuf_data_path;
470 root[
"confidence_threshold"] = confidence_threshold;
471 root[
"display_box_text"] = display_box_text.
JsonValue();
476 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
478 objects[trackedObject.second->Id()] = trackedObjectJSON;
480 root[
"objects"] = objects;
496 catch (
const std::exception& e)
499 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
509 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
510 protobuf_data_path = root[
"protobuf_data_path"].asString();
513 throw InvalidFile(
"Invalid protobuf data path",
"");
514 protobuf_data_path =
"";
519 if (!root[
"selected_object_index"].isNull())
522 if (!root[
"confidence_threshold"].isNull())
523 confidence_threshold = root[
"confidence_threshold"].asFloat();
525 if (!root[
"display_box_text"].isNull())
526 display_box_text.
SetJsonValue(root[
"display_box_text"]);
528 if (!root[
"class_filter"].isNull()){
529 class_filter = root[
"class_filter"].asString();
530 std::stringstream ss(class_filter);
531 display_classes.clear();
536 std::getline( ss, substr,
',' );
537 display_classes.push_back( substr );
541 if (!root[
"objects"].isNull()){
543 std::string obj_id = std::to_string(trackedObject.first);
544 if(!root[
"objects"][obj_id].isNull()){
545 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
551 if (!root[
"objects_id"].isNull()){
553 Json::Value trackedObjectJSON;
554 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
555 trackedObject.second->SetJsonValue(trackedObjectJSON);
570 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
572 objects[selectedObject->Id()] = trackedObjectJSON;
575 root[
"objects"] = objects;
578 root[
"id"] =
add_property_json(
"ID", 0.0,
"string",
Id(), NULL, -1, -1,
true, requested_frame);
579 root[
"position"] =
add_property_json(
"Position",
Position(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
581 root[
"start"] =
add_property_json(
"Start",
Start(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
582 root[
"end"] =
add_property_json(
"End",
End(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
583 root[
"duration"] =
add_property_json(
"Duration",
Duration(),
"float",
"", NULL, 0, 1000 * 60 * 30,
true, requested_frame);
584 root[
"confidence_threshold"] =
add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
585 root[
"class_filter"] =
add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
587 root[
"display_box_text"] =
add_property_json(
"Draw Box Text", display_box_text.
GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1.0,
false, requested_frame);
592 return root.toStyledString();
Header file for all Exception classes.
Header file for Object Detection effect class.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
float End() const
Get end position (in seconds) of clip (trim end of video)
float Start() const
Get start position (in seconds) of clip (trim start of video)
float Duration() const
Get the length of this clip (in seconds)
std::string Id() const
Get the Id of this clip object.
Json::Value add_property_choice_json(std::string name, int value, int selected_value) const
Generate JSON choice for a property (dropdown properties)
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
float Position() const
Get position on timeline (in seconds)
std::string id
ID Property for all derived Clip and Effect classes.
Json::Value add_property_json(std::string name, float value, std::string type, std::string memo, const Keyframe *keyframe, float min_value, float max_value, bool readonly, int64_t requested_frame) const
Generate JSON for a property.
This class represents a clip (used to arrange readers on the timeline)
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
virtual Json::Value JsonValue() const
Generate Json::Value for this object.
openshot::ClipBase * ParentClip()
Parent clip object of this effect (which can be unparented and NULL)
virtual void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
EffectInfoStruct info
Information about the current effect.
std::map< int, std::shared_ptr< openshot::TrackedObjectBase > > trackedObjects
Map of Tracked Object's by their indices (used by Effects that track objects on clips)
This class represents a single frame of video (i.e. image & audio data)
Exception for files that can not be found or opened.
Exception for invalid JSON.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
double GetValue(int64_t index) const
Get the value at a specific index.
Json::Value JsonValue() const
Generate Json::Value for this object.
Json::Value JsonValue() const override
Generate Json::Value for this object.
int selectedObjectIndex
Index of the Tracked Object that was selected to modify it's properties.
std::shared_ptr< Frame > GetFrame(std::shared_ptr< Frame > frame, int64_t frame_number) override
This method is required for all derived classes of EffectBase, and returns a modified openshot::Frame...
ObjectDetection()
Default constructor.
bool LoadObjDetectdData(std::string inputFilePath)
Load protobuf data file.
std::string GetVisibleObjects(int64_t frame_number) const override
Get the indexes and IDs of all visible objects in the given frame.
std::string Json() const override
Generate JSON string of this object.
std::string PropertiesJSON(int64_t requested_frame) const override
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
void SetJson(const std::string value) override
Load JSON string into this object.
This class represents a timeline.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
This class contains the properties of a tracked object and functions to manipulate it.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
This namespace is the default namespace for all code in the openshot library.
const Json::Value stringToJson(const std::string value)
std::vector< cv::Rect_< float > > boxes
std::vector< float > confidences
std::vector< int > classIds
std::vector< int > objectIds
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
bool has_video
Determines if this effect manipulates the image of a frame.
bool has_audio
Determines if this effect manipulates the audio of a frame.
std::string class_name
The class name of the effect.
std::string name
The name of the effect.
std::string description
The description of this effect and what it does.
bool has_tracked_object
Determines if this effect track objects through the clip.