OpenShot Library | libopenshot 0.2.7
Timeline.h
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Header file for Timeline class
4 * @author Jonathan Thomas <jonathan@openshot.org>
5 *
6 * @ref License
7 */
8
9/* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC
12 * <http://www.openshotstudios.com/>. This file is part of
13 * OpenShot Library (libopenshot), an open-source project dedicated to
14 * delivering high quality video editing and animation solutions to the
15 * world. For more information visit <http://www.openshot.org/>.
16 *
17 * OpenShot Library (libopenshot) is free software: you can redistribute it
18 * and/or modify it under the terms of the GNU Lesser General Public License
19 * as published by the Free Software Foundation, either version 3 of the
20 * License, or (at your option) any later version.
21 *
22 * OpenShot Library (libopenshot) is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU Lesser General Public License for more details.
26 *
27 * You should have received a copy of the GNU Lesser General Public License
28 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29 */
30
31#ifndef OPENSHOT_TIMELINE_H
32#define OPENSHOT_TIMELINE_H
33
34#include <list>
35#include <memory>
36#include <mutex>
37#include <set>
38#include <QtGui/QImage>
39#include <QtGui/QPainter>
40#include <QtCore/QRegularExpression>
41
42#include "TimelineBase.h"
43#include "ReaderBase.h"
44
45#include "Color.h"
46#include "Clip.h"
47#include "EffectBase.h"
48#include "Fraction.h"
49#include "Frame.h"
50#include "KeyFrame.h"
51#ifdef USE_OPENCV
52#include "TrackedObjectBBox.h"
53#endif
54#include "TrackedObjectBase.h"
55
56
57
58namespace openshot {
59
60 // Forward decls
61 class FrameMapper;
62 class CacheBase;
63
64 /// Comparison method for sorting clip pointers (by Layer and then Position). Clips are sorted
65 /// from lowest layer to top layer (since that is the sequence they need to be combined), and then
66 /// by position (left to right).
69 if( lhs->Layer() < rhs->Layer() ) return true;
70 if( lhs->Layer() == rhs->Layer() && lhs->Position() <= rhs->Position() ) return true;
71 return false;
72 }};
73
74 /// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted
75 /// from lowest layer to top layer (since that is sequence clips are combined), and then by
76 /// position, and then by effect order.
79 if( lhs->Layer() < rhs->Layer() ) return true;
80 if( lhs->Layer() == rhs->Layer() && lhs->Position() < rhs->Position() ) return true;
81 if( lhs->Layer() == rhs->Layer() && lhs->Position() == rhs->Position() && lhs->Order() > rhs->Order() ) return true;
82 return false;
83 }};
84
85 /// Comparison method for finding the far end of the timeline, by locating
86 /// the Clip with the highest end-frame number using std::max_element
88 bool operator()(const openshot::Clip* lhs, const openshot::Clip* rhs) {
89 return (lhs->Position() + lhs->Duration())
90 <= (rhs->Position() + rhs->Duration());
91 }};
92
93 /// Like CompareClipEndFrames, but for effects
96 return (lhs->Position() + lhs->Duration())
97 <= (rhs->Position() + rhs->Duration());
98 }};
99
100 /**
101 * @brief This class represents a timeline
102 *
103 * The timeline is one of the <b>most important</b> features of a video editor, and controls all
104 * aspects of how video, image, and audio clips are combined together, and how the final
105 * video output will be rendered. It has a collection of layers and clips, that arrange,
106 * sequence, and generate the final video output.
107 *
108 * The <b>following graphic</b> displays a timeline, and how clips can be arranged, scaled, and layered together. It
109 * also demonstrates how the viewport can be scaled smaller than the canvas, which can be used to zoom and pan around the
110 * canvas (i.e. pan & scan).
111 * \image html /doc/images/Timeline_Layers.png
112 *
113 * The <b>following graphic</b> displays how the playhead determines which frames to combine and layer.
114 * \image html /doc/images/Playhead.png
115 *
116 * Lets take a look at what the code looks like:
117 * @code
118 * // Create a Timeline
119 * Timeline t(1280, // width
120 * 720, // height
121 * Fraction(25,1), // framerate
122 * 44100, // sample rate
123 * 2 // channels
124 * ChannelLayout::LAYOUT_STEREO,
125 * );
126 *
127 * // Create some clips
128 * Clip c1(new ImageReader("MyAwesomeLogo.jpeg"));
129 * Clip c2(new FFmpegReader("BackgroundVideo.webm"));
130 *
131 * // CLIP 1 (logo) - Set some clip properties (with Keyframes)
132 * c1.Position(0.0); // Set the position or location (in seconds) on the timeline
133 * c1.gravity = GRAVITY_LEFT; // Set the alignment / gravity of the clip (position on the screen)
134 * c1.scale = SCALE_CROP; // Set the scale mode (how the image is resized to fill the screen)
135 * c1.Layer(1); // Set the layer of the timeline (higher layers cover up images of lower layers)
136 * c1.Start(0.0); // Set the starting position of the video (trim the left side of the video)
137 * c1.End(16.0); // Set the ending position of the video (trim the right side of the video)
138 * c1.alpha.AddPoint(1, 0.0); // Set the alpha to transparent on frame #1
139 * c1.alpha.AddPoint(500, 0.0); // Keep the alpha transparent until frame #500
140 * c1.alpha.AddPoint(565, 1.0); // Animate the alpha from transparent to visible (between frame #501 and #565)
141 *
142 * // CLIP 2 (background video) - Set some clip properties (with Keyframes)
143 * c2.Position(0.0); // Set the position or location (in seconds) on the timeline
144 * c2.Start(10.0); // Set the starting position of the video (trim the left side of the video)
145 * c2.Layer(0); // Set the layer of the timeline (higher layers cover up images of lower layers)
146 * c2.alpha.AddPoint(1, 1.0); // Set the alpha to visible on frame #1
147 * c2.alpha.AddPoint(150, 0.0); // Animate the alpha to transparent (between frame 2 and frame #150)
148 * c2.alpha.AddPoint(360, 0.0, LINEAR); // Keep the alpha transparent until frame #360
149 * c2.alpha.AddPoint(384, 1.0); // Animate the alpha to visible (between frame #360 and frame #384)
150 *
151 * // Add clips to timeline
152 * t.AddClip(&c1);
153 * t.AddClip(&c2);
154 *
155 * // Open the timeline reader
156 * t.Open();
157 *
158 * // Get frame number 1 from the timeline (This will generate a new frame, made up from the previous clips and settings)
159 * std::shared_ptr<Frame> f = t.GetFrame(1);
160 *
161 * // Now that we have an openshot::Frame object, lets have some fun!
162 * f->Display(); // Display the frame on the screen
163 *
164 * // Close the timeline reader
165 * t.Close();
166 * @endcode
167 */
169 private:
170 bool is_open; ///<Is Timeline Open?
171 bool auto_map_clips; ///< Auto map framerates and sample rates to all clips
172 std::list<openshot::Clip*> clips; ///<List of clips on this timeline
173 std::list<openshot::Clip*> closing_clips; ///<List of clips that need to be closed
174 std::map<openshot::Clip*, openshot::Clip*> open_clips; ///<List of 'opened' clips on this timeline
175 std::list<openshot::EffectBase*> effects; ///<List of clips on this timeline
176 openshot::CacheBase *final_cache; ///<Final cache of timeline frames
177 std::set<openshot::FrameMapper*> allocated_frame_mappers; ///< all the frame mappers we allocated and must free
178 bool managed_cache; ///< Does this timeline instance manage the cache object
179 std::string path; ///< Optional path of loaded UTF-8 OpenShot JSON project file
180 std::mutex get_frame_mutex; ///< Mutex to protect GetFrame method from different threads calling it
181 int max_concurrent_frames; ///< Max concurrent frames to process at one time
182
183 std::map<std::string, std::shared_ptr<openshot::TrackedObjectBase>> tracked_objects; ///< map of TrackedObjectBBoxes and their IDs
184
185 /// Process a new layer of video or audio
186 void add_layer(std::shared_ptr<openshot::Frame> new_frame, openshot::Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume);
187
188 /// Apply a FrameMapper to a clip which matches the settings of this timeline
189 void apply_mapper_to_clip(openshot::Clip* clip);
190
191 // Apply JSON Diffs to various objects contained in this timeline
192 void apply_json_to_clips(Json::Value change); ///<Apply JSON diff to clips
193 void apply_json_to_effects(Json::Value change); ///< Apply JSON diff to effects
194 void apply_json_to_effects(Json::Value change, openshot::EffectBase* existing_effect); ///<Apply JSON diff to a specific effect
195 void apply_json_to_timeline(Json::Value change); ///<Apply JSON diff to timeline properties
196
197 /// Calculate time of a frame number, based on a framerate
198 double calculate_time(int64_t number, openshot::Fraction rate);
199
200 /// Find intersecting (or non-intersecting) openshot::Clip objects
201 ///
202 /// @returns A list of openshot::Clip objects
203 /// @param requested_frame The frame number that is requested.
204 /// @param number_of_frames The number of frames to check
205 /// @param include Include or Exclude intersecting clips
206 std::vector<openshot::Clip*> find_intersecting_clips(int64_t requested_frame, int number_of_frames, bool include);
207
208 /// Get a clip's frame or generate a blank frame
209 std::shared_ptr<openshot::Frame> GetOrCreateFrame(std::shared_ptr<Frame> background_frame, openshot::Clip* clip, int64_t number, openshot::TimelineInfoStruct* options);
210
211 /// Compare 2 floating point numbers for equality
212 bool isEqual(double a, double b);
213
214 /// Sort clips by position on the timeline
215 void sort_clips();
216
217 /// Sort effects by position on the timeline
218 void sort_effects();
219
220 /// Update the list of 'opened' clips
221 void update_open_clips(openshot::Clip *clip, bool does_clip_intersect);
222
223 public:
224
225 /// @brief Constructor for the timeline (which configures the default frame properties)
226 /// @param width The image width of generated openshot::Frame objects
227 /// @param height The image height of generated openshot::Frame objects
228 /// @param fps The frame rate of the generated video
229 /// @param sample_rate The audio sample rate
230 /// @param channels The number of audio channels
231 /// @param channel_layout The channel layout (i.e. mono, stereo, 3 point surround, etc...)
232 Timeline(int width, int height, openshot::Fraction fps, int sample_rate, int channels, openshot::ChannelLayout channel_layout);
233
234 /// @brief Constructor which takes a ReaderInfo struct to configure parameters
235 /// @param info The reader parameters to configure the new timeline with
237
238 /// @brief Project-file constructor for the timeline
239 ///
240 /// Loads a JSON structure from a file path, and
241 /// initializes the timeline described within.
242 ///
243 /// @param projectPath The path of the UTF-8 *.osp project file (JSON contents). Contents will be loaded automatically.
244 /// @param convert_absolute_paths Should all paths be converted to absolute paths (relative to the location of projectPath)
245 Timeline(const std::string& projectPath, bool convert_absolute_paths);
246
247 virtual ~Timeline();
248
249 /// Add to the tracked_objects map a pointer to a tracked object (TrackedObjectBBox)
250 void AddTrackedObject(std::shared_ptr<openshot::TrackedObjectBase> trackedObject);
251 /// Return tracked object pointer by it's id
252 std::shared_ptr<openshot::TrackedObjectBase> GetTrackedObject(std::string id) const;
253 /// Return the ID's of the tracked objects as a list of strings
254 std::list<std::string> GetTrackedObjectsIds() const;
255 /// Return the trackedObject's properties as a JSON string
256 #ifdef USE_OPENCV
257 std::string GetTrackedObjectValues(std::string id, int64_t frame_number) const;
258 #endif
259
260 /// @brief Add an openshot::Clip to the timeline
261 /// @param clip Add an openshot::Clip to the timeline. A clip can contain any type of Reader.
263
264 /// @brief Add an effect to the timeline
265 /// @param effect Add an effect to the timeline. An effect can modify the audio or video of an openshot::Frame.
266 void AddEffect(openshot::EffectBase* effect);
267
268 /// Apply global/timeline effects to the source frame (if any)
269 std::shared_ptr<openshot::Frame> apply_effects(std::shared_ptr<openshot::Frame> frame, int64_t timeline_frame_number, int layer);
270
271 /// Apply the timeline's framerate and samplerate to all clips
272 void ApplyMapperToClips();
273
274 /// Determine if clips are automatically mapped to the timeline's framerate and samplerate
275 bool AutoMapClips() { return auto_map_clips; };
276
277 /// @brief Automatically map all clips to the timeline's framerate and samplerate
278 void AutoMapClips(bool auto_map) { auto_map_clips = auto_map; };
279
280 /// Clear all cache for this timeline instance, and all clips, mappers, and readers under it
281 void ClearAllCache();
282
283 /// Return a list of clips on the timeline
284 std::list<openshot::Clip*> Clips() { return clips; };
285
286 /// Look up a single clip by ID
287 openshot::Clip* GetClip(const std::string& id);
288
289 /// Look up a clip effect by ID
290 openshot::EffectBase* GetClipEffect(const std::string& id);
291
292 /// Look up a timeline effect by ID
293 openshot::EffectBase* GetEffect(const std::string& id);
294
295 /// Look up the end time of the latest timeline element
296 double GetMaxTime();
297 /// Look up the end frame number of the latest element on the timeline
298 int64_t GetMaxFrame();
299
300 /// Close the timeline reader (and any resources it was consuming)
301 void Close() override;
302
303 /// Return the list of effects on the timeline
304 std::list<openshot::EffectBase*> Effects() { return effects; };
305
306 /// Return the list of effects on all clips
307 std::list<openshot::EffectBase*> ClipEffects() const;
308
309 /// Get the cache object used by this reader
310 openshot::CacheBase* GetCache() override { return final_cache; };
311
312 /// Set the cache object used by this reader. You must now manage the lifecycle
313 /// of this cache object though (Timeline will not delete it for you).
314 void SetCache(openshot::CacheBase* new_cache);
315
316 /// Get an openshot::Frame object for a specific frame number of this timeline.
317 ///
318 /// @returns The requested frame (containing the image)
319 /// @param requested_frame The frame number that is requested.
320 std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame) override;
321
322 // Curves for the viewport
323 openshot::Keyframe viewport_scale; ///<Curve representing the scale of the viewport (0 to 100)
324 openshot::Keyframe viewport_x; ///<Curve representing the x coordinate for the viewport
325 openshot::Keyframe viewport_y; ///<Curve representing the y coordinate for the viewport
326
327 // Background color
328 openshot::Color color; ///<Background color of timeline canvas
329
330 /// Determine if reader is open or closed
331 bool IsOpen() override { return is_open; };
332
333 /// Return the type name of the class
334 std::string Name() override { return "Timeline"; };
335
336 // Get and Set JSON methods
337 std::string Json() const override; ///< Generate JSON string of this object
338 void SetJson(const std::string value) override; ///< Load JSON string into this object
339 Json::Value JsonValue() const override; ///< Generate Json::Value for this object
340 void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object
341
342 /// Set Max Image Size (used for performance optimization). Convenience function for setting
343 /// Settings::Instance()->MAX_WIDTH and Settings::Instance()->MAX_HEIGHT.
344 void SetMaxSize(int width, int height);
345
346 /// @brief Apply a special formatted JSON object, which represents a change to the timeline (add, update, delete)
347 /// This is primarily designed to keep the timeline (and its child objects... such as clips and effects) in sync
348 /// with another application... such as OpenShot Video Editor (http://www.openshot.org).
349 /// @param value A JSON string containing a key, value, and type of change.
350 void ApplyJsonDiff(std::string value);
351
352 /// Open the reader (and start consuming resources)
353 void Open() override;
354
355 /// @brief Remove an openshot::Clip from the timeline
356 /// @param clip Remove an openshot::Clip from the timeline.
358
359 /// @brief Remove an effect from the timeline
360 /// @param effect Remove an effect from the timeline.
362 };
363
364}
365
366#endif // OPENSHOT_TIMELINE_H
Header file for Clip class.
Header file for Color class.
Header file for EffectBase class.
Header file for Fraction class.
Header file for Frame class.
Header file for the Keyframe class.
Header file for ReaderBase class.
Header file for Timeline class.
Header file for the TrackedObjectBBox class.
Header file for the TrackedObjectBase class.
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:49
float Duration() const
Get the length of this clip (in seconds)
Definition: ClipBase.h:112
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
Definition: ClipBase.h:109
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:108
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
This class represents a color (used on the timeline and clips)
Definition: Color.h:45
This abstract class is the base class, used by all effects in libopenshot.
Definition: EffectBase.h:71
int Order() const
Get the order that this effect should be executed.
Definition: EffectBase.h:130
This class represents a fraction.
Definition: Fraction.h:48
A Keyframe is a collection of Point instances, which is used to vary a number or property over time.
Definition: KeyFrame.h:72
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:98
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
openshot::ClipBase * clip
Pointer to the parent clip instance (if any)
Definition: ReaderBase.h:103
This class represents a timeline (used for building generic timeline implementations)
Definition: TimelineBase.h:54
This class represents a timeline.
Definition: Timeline.h:168
void AddTrackedObject(std::shared_ptr< openshot::TrackedObjectBase > trackedObject)
Add to the tracked_objects map a pointer to a tracked object (TrackedObjectBBox)
Definition: Timeline.cpp:247
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: Timeline.cpp:987
std::string Name() override
Return the type name of the class.
Definition: Timeline.h:334
openshot::Keyframe viewport_scale
Curve representing the scale of the viewport (0 to 100)
Definition: Timeline.h:323
void ApplyJsonDiff(std::string value)
Apply a special formatted JSON object, which represents a change to the timeline (add,...
Definition: Timeline.cpp:1117
openshot::EffectBase * GetClipEffect(const std::string &id)
Look up a clip effect by ID.
Definition: Timeline.cpp:427
void AddClip(openshot::Clip *clip)
Add an openshot::Clip to the timeline.
Definition: Timeline.cpp:356
virtual ~Timeline()
Definition: Timeline.cpp:222
std::list< openshot::Clip * > Clips()
Return a list of clips on the timeline.
Definition: Timeline.h:284
std::list< openshot::EffectBase * > ClipEffects() const
Return the list of effects on all clips.
Definition: Timeline.cpp:440
void AutoMapClips(bool auto_map)
Automatically map all clips to the timeline's framerate and samplerate.
Definition: Timeline.h:278
std::list< std::string > GetTrackedObjectsIds() const
Return the ID's of the tracked objects as a list of strings.
Definition: Timeline.cpp:282
std::string Json() const override
Generate JSON string of this object.
Definition: Timeline.cpp:980
std::list< openshot::EffectBase * > Effects()
Return the list of effects on the timeline.
Definition: Timeline.h:304
int64_t GetMaxFrame()
Look up the end frame number of the latest element on the timeline.
Definition: Timeline.cpp:477
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: Timeline.cpp:771
bool AutoMapClips()
Determine if clips are automatically mapped to the timeline's framerate and samplerate.
Definition: Timeline.h:275
void ClearAllCache()
Clear all cache for this timeline instance, and all clips, mappers, and readers under it.
Definition: Timeline.cpp:1496
void ApplyMapperToClips()
Apply the timeline's framerate and samplerate to all clips.
Definition: Timeline.cpp:513
openshot::Color color
Background color of timeline canvas.
Definition: Timeline.h:328
std::string GetTrackedObjectValues(std::string id, int64_t frame_number) const
Return the trackedObject's properties as a JSON string.
Definition: Timeline.cpp:298
Timeline(int width, int height, openshot::Fraction fps, int sample_rate, int channels, openshot::ChannelLayout channel_layout)
Constructor for the timeline (which configures the default frame properties)
Definition: Timeline.cpp:46
std::shared_ptr< openshot::TrackedObjectBase > GetTrackedObject(std::string id) const
Return tracked object pointer by it's id.
Definition: Timeline.cpp:265
openshot::EffectBase * GetEffect(const std::string &id)
Look up a timeline effect by ID.
Definition: Timeline.cpp:416
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: Timeline.cpp:1041
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:404
void AddEffect(openshot::EffectBase *effect)
Add an effect to the timeline.
Definition: Timeline.cpp:379
std::shared_ptr< openshot::Frame > apply_effects(std::shared_ptr< openshot::Frame > frame, int64_t timeline_frame_number, int layer)
Apply global/timeline effects to the source frame (if any)
Definition: Timeline.cpp:537
void SetCache(openshot::CacheBase *new_cache)
Definition: Timeline.cpp:967
bool IsOpen() override
Determine if reader is open or closed.
Definition: Timeline.h:331
openshot::CacheBase * GetCache() override
Get the cache object used by this reader.
Definition: Timeline.h:310
openshot::Keyframe viewport_x
Curve representing the x coordinate for the viewport.
Definition: Timeline.h:324
void RemoveClip(openshot::Clip *clip)
Remove an openshot::Clip from the timeline.
Definition: Timeline.cpp:398
void SetMaxSize(int width, int height)
Definition: Timeline.cpp:1522
double GetMaxTime()
Look up the end time of the latest timeline element.
Definition: Timeline.cpp:459
void RemoveEffect(openshot::EffectBase *effect)
Remove an effect from the timeline.
Definition: Timeline.cpp:392
void Open() override
Open the reader (and start consuming resources)
Definition: Timeline.cpp:759
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: Timeline.cpp:1021
openshot::Keyframe viewport_y
Curve representing the y coordinate for the viewport.
Definition: Timeline.h:325
void Close() override
Close the timeline reader (and any resources it was consuming)
Definition: Timeline.cpp:739
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
bool operator()(const openshot::Clip *lhs, const openshot::Clip *rhs)
Definition: Timeline.h:88
bool operator()(openshot::Clip *lhs, openshot::Clip *rhs)
Definition: Timeline.h:68
Like CompareClipEndFrames, but for effects.
Definition: Timeline.h:94
bool operator()(const openshot::EffectBase *lhs, const openshot::EffectBase *rhs)
Definition: Timeline.h:95
bool operator()(openshot::EffectBase *lhs, openshot::EffectBase *rhs)
Definition: Timeline.h:78
This struct contains info about a media file, such as height, width, frames per second,...
Definition: ReaderBase.h:61
This struct contains info about the current Timeline clip instance.
Definition: TimelineBase.h:47