OpenShot Library | libopenshot 0.2.7
CVObjectDetection.h
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Header file for CVObjectDetection class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 * @author Brenno Caldato <brenno.caldato@outlook.com>
6 *
7 * @ref License
8 */
9
10/* LICENSE
11 *
12 * Copyright (c) 2008-2019 OpenShot Studios, LLC
13 * <http://www.openshotstudios.com/>. This file is part of
14 * OpenShot Library (libopenshot), an open-source project dedicated to
15 * delivering high quality video editing and animation solutions to the
16 * world. For more information visit <http://www.openshot.org/>.
17 *
18 * OpenShot Library (libopenshot) is free software: you can redistribute it
19 * and/or modify it under the terms of the GNU Lesser General Public License
20 * as published by the Free Software Foundation, either version 3 of the
21 * License, or (at your option) any later version.
22 *
23 * OpenShot Library (libopenshot) is distributed in the hope that it will be
24 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 * GNU Lesser General Public License for more details.
27 *
28 * You should have received a copy of the GNU Lesser General Public License
29 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30 */
31
32#pragma once
33
34#define int64 opencv_broken_int
35#define uint64 opencv_broken_uint
36#include <opencv2/dnn.hpp>
37#include <opencv2/opencv.hpp>
38#include <opencv2/core.hpp>
39#undef uint64
40#undef int64
41#include "Json.h"
43#include "Clip.h"
44#include "protobuf_messages/objdetectdata.pb.h"
45
46#include "sort_filter/sort.hpp"
47
48namespace openshot
49{
50 // Stores the detected object bounding boxes and its properties.
54 std::vector<int> _classIds,
55 std::vector<float> _confidences,
56 std::vector<cv::Rect_<float>> _boxes,
57 size_t _frameId,
58 std::vector<int> _objectIds)
59 {
60 classIds = _classIds;
61 confidences = _confidences;
62 boxes = _boxes;
63 frameId = _frameId;
64 objectIds = _objectIds;
65 }
66 size_t frameId;
67 std::vector<int> classIds;
68 std::vector<float> confidences;
69 std::vector<cv::Rect_<float>> boxes;
70 std::vector<int> objectIds;
71 };
72
73 /**
74 * @brief This class runs trought a clip to detect objects and returns the bounding boxes and its properties.
75 *
76 * Object detection is performed using YoloV3 model with OpenCV DNN module
77 */
79
80 private:
81
82 cv::dnn::Net net;
83 std::vector<std::string> classNames;
84 float confThreshold, nmsThreshold;
85
86 std::string classesFile;
87 std::string modelConfiguration;
88 std::string modelWeights;
89 std::string processingDevice;
90 std::string protobuf_data_path;
91
92 SortTracker sort;
93
94 uint progress;
95
96 size_t start;
97 size_t end;
98
99 bool error = false;
100
101 /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes
102 ProcessingController *processingController;
103
104 void setProcessingDevice();
105
106 // Detect onbects on a single frame
107 void DetectObjects(const cv::Mat &frame, size_t frame_number);
108
109 bool iou(cv::Rect pred_box, cv::Rect sort_box);
110
111 // Remove the bounding boxes with low confidence using non-maxima suppression
112 void postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& out, size_t frame_number);
113
114 // Get the names of the output layers
115 std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net);
116
117 public:
118
119 std::map<size_t, CVDetectionData> detectionsData;
120
121 CVObjectDetection(std::string processInfoJson, ProcessingController &processingController);
122
123 // Iterate over a clip object and run inference for each video frame
124 void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false);
125
126 CVDetectionData GetDetectionData(size_t frameId);
127
128 /// Protobuf Save and Load methods
129 // Save protobuf file
130 bool SaveObjDetectedData();
131 // Add frame object detection data into protobuf message.
132 void AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CVDetectionData& dData);
133
134 // Get and Set JSON methods
135 void SetJson(const std::string value); ///< Load JSON string into this object
136 void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
137
138 // Load protobuf file (ONLY FOR MAKE TEST)
139 bool _LoadObjDetectdData();
140 };
141
142}
Header file for Clip class.
Header file for JSON class.
This is a message class for thread safe comunication between ClipProcessingJobs and OpenCV classes.
This class runs trought a clip to detect objects and returns the bounding boxes and its properties.
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
CVObjectDetection(std::string processInfoJson, ProcessingController &processingController)
void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false)
void AddFrameDataToProto(pb_objdetect::Frame *pbFrameData, CVDetectionData &dData)
CVDetectionData GetDetectionData(size_t frameId)
std::map< size_t, CVDetectionData > detectionsData
void SetJson(const std::string value)
Load JSON string into this object.
bool SaveObjDetectedData()
Protobuf Save and Load methods.
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
std::vector< int > objectIds
std::vector< cv::Rect_< float > > boxes
std::vector< int > classIds
CVDetectionData(std::vector< int > _classIds, std::vector< float > _confidences, std::vector< cv::Rect_< float > > _boxes, size_t _frameId, std::vector< int > _objectIds)
std::vector< float > confidences