OpenShot Library | libopenshot 0.2.7
FFmpegReader.cpp
Go to the documentation of this file.
1/**
2 * @file
3 * @brief Source file for FFmpegReader class
4 * @author Jonathan Thomas <jonathan@openshot.org>, Fabrice Bellard
5 *
6 * @ref License
7 */
8
9/* LICENSE
10 *
11 * Copyright (c) 2008-2019 OpenShot Studios, LLC, Fabrice Bellard
12 * (http://www.openshotstudios.com). This file is part of
13 * OpenShot Library (http://www.openshot.org), an open-source project
14 * dedicated to delivering high quality video editing and animation solutions
15 * to the world.
16 *
17 * This file is originally based on the Libavformat API example, and then modified
18 * by the libopenshot project.
19 *
20 * OpenShot Library (libopenshot) is free software: you can redistribute it
21 * and/or modify it under the terms of the GNU Lesser General Public License
22 * as published by the Free Software Foundation, either version 3 of the
23 * License, or (at your option) any later version.
24 *
25 * OpenShot Library (libopenshot) is distributed in the hope that it will be
26 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU Lesser General Public License for more details.
29 *
30 * You should have received a copy of the GNU Lesser General Public License
31 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
32 */
33
34#include "FFmpegReader.h"
35#include "Exceptions.h"
36#include "Timeline.h"
37
38#include <thread> // for std::this_thread::sleep_for
39#include <chrono> // for std::chrono::milliseconds
40
41#define ENABLE_VAAPI 0
42
43#if USE_HW_ACCEL
44#define MAX_SUPPORTED_WIDTH 1950
45#define MAX_SUPPORTED_HEIGHT 1100
46
47#if ENABLE_VAAPI
48#include "libavutil/hwcontext_vaapi.h"
49
50typedef struct VAAPIDecodeContext {
51 VAProfile va_profile;
52 VAEntrypoint va_entrypoint;
53 VAConfigID va_config;
54 VAContextID va_context;
55
56#if FF_API_STRUCT_VAAPI_CONTEXT
57 // FF_DISABLE_DEPRECATION_WARNINGS
58 int have_old_context;
59 struct vaapi_context *old_context;
60 AVBufferRef *device_ref;
61 // FF_ENABLE_DEPRECATION_WARNINGS
62#endif
63
64 AVHWDeviceContext *device;
65 AVVAAPIDeviceContext *hwctx;
66
67 AVHWFramesContext *frames;
68 AVVAAPIFramesContext *hwfc;
69
70 enum AVPixelFormat surface_format;
71 int surface_count;
72 } VAAPIDecodeContext;
73#endif // ENABLE_VAAPI
74#endif // USE_HW_ACCEL
75
76
77using namespace openshot;
78
79int hw_de_on = 0;
80#if USE_HW_ACCEL
81 AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
82 AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
83#endif
84
85FFmpegReader::FFmpegReader(const std::string& path, bool inspect_reader)
86 : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0),
87 audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false),
88 check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),
89 prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0),
90 current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0),
91 packet(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS) {
92
93 // Initialize FFMpeg, and register all formats and codecs
96
97 // Init cache
98 working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
99 missing_frames.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
101
102 // Open and Close the reader, to populate its attributes (such as height, width, etc...)
103 if (inspect_reader) {
104 Open();
105 Close();
106 }
107}
108
110 if (is_open)
111 // Auto close reader if not already done
112 Close();
113}
114
115// This struct holds the associated video frame and starting sample # for an audio packet.
116bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) {
117 // Is frame even close to this one?
118 if (abs(location.frame - frame) >= 2)
119 // This is too far away to be considered
120 return false;
121
122 // Note that samples_per_frame can vary slightly frame to frame when the
123 // audio sampling rate is not an integer multiple of the video fps.
124 int64_t diff = samples_per_frame * (location.frame - frame) + location.sample_start - sample_start;
125 if (abs(diff) <= amount)
126 // close
127 return true;
128
129 // not close
130 return false;
131}
132
133#if USE_HW_ACCEL
134
135// Get hardware pix format
136static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
137{
138 const enum AVPixelFormat *p;
139
140 for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
141 switch (*p) {
142#if defined(__linux__)
143 // Linux pix formats
144 case AV_PIX_FMT_VAAPI:
145 hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI;
146 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI;
147 return *p;
148 break;
149 case AV_PIX_FMT_VDPAU:
150 hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU;
151 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU;
152 return *p;
153 break;
154#endif
155#if defined(_WIN32)
156 // Windows pix formats
157 case AV_PIX_FMT_DXVA2_VLD:
158 hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD;
159 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2;
160 return *p;
161 break;
162 case AV_PIX_FMT_D3D11:
163 hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11;
164 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA;
165 return *p;
166 break;
167#endif
168#if defined(__APPLE__)
169 // Apple pix formats
170 case AV_PIX_FMT_VIDEOTOOLBOX:
171 hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX;
172 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
173 return *p;
174 break;
175#endif
176 // Cross-platform pix formats
177 case AV_PIX_FMT_CUDA:
178 hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA;
179 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA;
180 return *p;
181 break;
182 case AV_PIX_FMT_QSV:
183 hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV;
184 hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV;
185 return *p;
186 break;
187 default:
188 // This is only here to silence unused-enum warnings
189 break;
190 }
191 }
192 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)");
193 return AV_PIX_FMT_NONE;
194}
195
196int FFmpegReader::IsHardwareDecodeSupported(int codecid)
197{
198 int ret;
199 switch (codecid) {
200 case AV_CODEC_ID_H264:
201 case AV_CODEC_ID_MPEG2VIDEO:
202 case AV_CODEC_ID_VC1:
203 case AV_CODEC_ID_WMV1:
204 case AV_CODEC_ID_WMV2:
205 case AV_CODEC_ID_WMV3:
206 ret = 1;
207 break;
208 default :
209 ret = 0;
210 break;
211 }
212 return ret;
213}
214#endif // USE_HW_ACCEL
215
217 // Open reader if not already open
218 if (!is_open) {
219 // Initialize format context
220 pFormatCtx = NULL;
221 {
223 }
224
225 // Open video file
226 if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0)
227 throw InvalidFile("File could not be opened.", path);
228
229 // Retrieve stream information
230 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
231 throw NoStreamsFound("No streams found in file.", path);
232
233 videoStream = -1;
234 audioStream = -1;
235 // Loop through each stream, and identify the video and audio stream index
236 for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
237 // Is this a video stream?
238 if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) {
239 videoStream = i;
240 }
241 // Is this an audio stream?
242 if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_AUDIO && audioStream < 0) {
243 audioStream = i;
244 }
245 }
246 if (videoStream == -1 && audioStream == -1)
247 throw NoStreamsFound("No video or audio streams found in this file.", path);
248
249 // Is there a video stream?
250 if (videoStream != -1) {
251 // Set the stream index
252 info.video_stream_index = videoStream;
253
254 // Set the codec and codec context pointers
255 pStream = pFormatCtx->streams[videoStream];
256
257 // Find the codec ID from stream
258 const AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(pStream);
259
260 // Get codec and codec context from stream
261 const AVCodec *pCodec = avcodec_find_decoder(codecId);
262 AVDictionary *opts = NULL;
263 int retry_decode_open = 2;
264 // If hw accel is selected but hardware cannot handle repeat with software decoding
265 do {
266 pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
267#if USE_HW_ACCEL
268 if (hw_de_on && (retry_decode_open==2)) {
269 // Up to here no decision is made if hardware or software decode
270 hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
271 }
272#endif
273 retry_decode_open = 0;
274
275 // Set number of threads equal to number of processors (not to exceed 16)
276 pCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
277
278 if (pCodec == NULL) {
279 throw InvalidCodec("A valid video codec could not be found for this file.", path);
280 }
281
282 // Init options
283 av_dict_set(&opts, "strict", "experimental", 0);
284#if USE_HW_ACCEL
285 if (hw_de_on && hw_de_supported) {
286 // Open Hardware Acceleration
287 int i_decoder_hw = 0;
288 char adapter[256];
289 char *adapter_ptr = NULL;
290 int adapter_num;
292 fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num);
293
294 // Set hardware pix format (callback)
295 pCodecCtx->get_format = get_hw_dec_format;
296
297 if (adapter_num < 3 && adapter_num >=0) {
298#if defined(__linux__)
299 snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128);
300 adapter_ptr = adapter;
302 switch (i_decoder_hw) {
303 case 1:
304 hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
305 break;
306 case 2:
307 hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
308 break;
309 case 6:
310 hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU;
311 break;
312 case 7:
313 hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
314 break;
315 default:
316 hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
317 break;
318 }
319
320#elif defined(_WIN32)
321 adapter_ptr = NULL;
323 switch (i_decoder_hw) {
324 case 2:
325 hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA;
326 break;
327 case 3:
328 hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
329 break;
330 case 4:
331 hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA;
332 break;
333 case 7:
334 hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
335 break;
336 default:
337 hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2;
338 break;
339 }
340#elif defined(__APPLE__)
341 adapter_ptr = NULL;
343 switch (i_decoder_hw) {
344 case 5:
345 hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
346 break;
347 case 7:
348 hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV;
349 break;
350 default:
351 hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
352 break;
353 }
354#endif
355
356 } else {
357 adapter_ptr = NULL; // Just to be sure
358 }
359
360 // Check if it is there and writable
361#if defined(__linux__)
362 if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) {
363#elif defined(_WIN32)
364 if( adapter_ptr != NULL ) {
365#elif defined(__APPLE__)
366 if( adapter_ptr != NULL ) {
367#endif
368 ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device");
369 }
370 else {
371 adapter_ptr = NULL; // use default
372 ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default");
373 }
374
375 hw_device_ctx = NULL;
376 // Here the first hardware initialisations are made
377 if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
378 if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) {
379 throw InvalidCodec("Hardware device reference create failed.", path);
380 }
381
382 /*
383 av_buffer_unref(&ist->hw_frames_ctx);
384 ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
385 if (!ist->hw_frames_ctx) {
386 av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
387 return AVERROR(ENOMEM);
388 }
389
390 frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
391
392 frames_ctx->format = AV_PIX_FMT_CUDA;
393 frames_ctx->sw_format = avctx->sw_pix_fmt;
394 frames_ctx->width = avctx->width;
395 frames_ctx->height = avctx->height;
396
397 av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
398 av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
399
400
401 ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx);
402 ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
403 if (ret < 0) {
404 av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
405 return ret;
406 }
407 */
408 }
409 else {
410 throw InvalidCodec("Hardware device create failed.", path);
411 }
412 }
413#endif // USE_HW_ACCEL
414
415 // Disable per-frame threading for album arts
416 // Using FF_THREAD_FRAME adds one frame decoding delay per thread,
417 // but there's only one frame in this case.
418 if (HasAlbumArt())
419 {
420 pCodecCtx->thread_type &= ~FF_THREAD_FRAME;
421 }
422
423 // Open video codec
424 if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
425 throw InvalidCodec("A video codec was found, but could not be opened.", path);
426
427#if USE_HW_ACCEL
428 if (hw_de_on && hw_de_supported) {
429 AVHWFramesConstraints *constraints = NULL;
430 void *hwconfig = NULL;
431 hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx);
432
433// TODO: needs va_config!
434#if ENABLE_VAAPI
435 ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
436 constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
437#endif // ENABLE_VAAPI
438 if (constraints) {
439 if (pCodecCtx->coded_width < constraints->min_width ||
440 pCodecCtx->coded_height < constraints->min_height ||
441 pCodecCtx->coded_width > constraints->max_width ||
442 pCodecCtx->coded_height > constraints->max_height) {
443 ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n");
444 hw_de_supported = 0;
445 retry_decode_open = 1;
446 AV_FREE_CONTEXT(pCodecCtx);
447 if (hw_device_ctx) {
448 av_buffer_unref(&hw_device_ctx);
449 hw_device_ctx = NULL;
450 }
451 }
452 else {
453 // All is just peachy
454 ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
455 retry_decode_open = 0;
456 }
457 av_hwframe_constraints_free(&constraints);
458 if (hwconfig) {
459 av_freep(&hwconfig);
460 }
461 }
462 else {
463 int max_h, max_w;
464 //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" )));
466 //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" )));
468 ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n");
469 //cerr << "Constraints could not be found using default limit\n";
470 if (pCodecCtx->coded_width < 0 ||
471 pCodecCtx->coded_height < 0 ||
472 pCodecCtx->coded_width > max_w ||
473 pCodecCtx->coded_height > max_h ) {
474 ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
475 hw_de_supported = 0;
476 retry_decode_open = 1;
477 AV_FREE_CONTEXT(pCodecCtx);
478 if (hw_device_ctx) {
479 av_buffer_unref(&hw_device_ctx);
480 hw_device_ctx = NULL;
481 }
482 }
483 else {
484 ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height);
485 retry_decode_open = 0;
486 }
487 }
488 } // if hw_de_on && hw_de_supported
489 else {
490 ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n");
491 }
492#else
493 retry_decode_open = 0;
494#endif // USE_HW_ACCEL
495 } while (retry_decode_open); // retry_decode_open
496 // Free options
497 av_dict_free(&opts);
498
499 // Update the File Info struct with video details (if a video stream is found)
500 UpdateVideoInfo();
501 }
502
503 // Is there an audio stream?
504 if (audioStream != -1) {
505 // Set the stream index
506 info.audio_stream_index = audioStream;
507
508 // Get a pointer to the codec context for the audio stream
509 aStream = pFormatCtx->streams[audioStream];
510
511 // Find the codec ID from stream
512 AVCodecID codecId = AV_FIND_DECODER_CODEC_ID(aStream);
513
514 // Get codec and codec context from stream
515 const AVCodec *aCodec = avcodec_find_decoder(codecId);
516 aCodecCtx = AV_GET_CODEC_CONTEXT(aStream, aCodec);
517
518 // Set number of threads equal to number of processors (not to exceed 16)
519 aCodecCtx->thread_count = std::min(FF_NUM_PROCESSORS, 16);
520
521 if (aCodec == NULL) {
522 throw InvalidCodec("A valid audio codec could not be found for this file.", path);
523 }
524
525 // Init options
526 AVDictionary *opts = NULL;
527 av_dict_set(&opts, "strict", "experimental", 0);
528
529 // Open audio codec
530 if (avcodec_open2(aCodecCtx, aCodec, &opts) < 0)
531 throw InvalidCodec("An audio codec was found, but could not be opened.", path);
532
533 // Free options
534 av_dict_free(&opts);
535
536 // Update the File Info struct with audio details (if an audio stream is found)
537 UpdateAudioInfo();
538 }
539
540 // Add format metadata (if any)
541 AVDictionaryEntry *tag = NULL;
542 while ((tag = av_dict_get(pFormatCtx->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
543 QString str_key = tag->key;
544 QString str_value = tag->value;
545 info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
546 }
547
548 // Init previous audio location to zero
549 previous_packet_location.frame = -1;
550 previous_packet_location.sample_start = 0;
551
552 // Adjust cache size based on size of frame and audio
553 working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels);
554 missing_frames.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels);
556
557 // Mark as "open"
558 is_open = true;
559 }
560}
561
563 // Close all objects, if reader is 'open'
564 if (is_open) {
565 // Mark as "closed"
566 is_open = false;
567
568 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close");
569
570 if (packet) {
571 // Remove previous packet before getting next one
572 RemoveAVPacket(packet);
573 packet = NULL;
574 }
575
576 // Close the codec
577 if (info.has_video) {
578 avcodec_flush_buffers(pCodecCtx);
579 AV_FREE_CONTEXT(pCodecCtx);
580#if USE_HW_ACCEL
581 if (hw_de_on) {
582 if (hw_device_ctx) {
583 av_buffer_unref(&hw_device_ctx);
584 hw_device_ctx = NULL;
585 }
586 }
587#endif // USE_HW_ACCEL
588 }
589 if (info.has_audio) {
590 avcodec_flush_buffers(aCodecCtx);
591 AV_FREE_CONTEXT(aCodecCtx);
592 }
593
594 // Clear final cache
596 working_cache.Clear();
597 missing_frames.Clear();
598
599 // Clear processed lists
600 {
601 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
602 processed_video_frames.clear();
603 processed_audio_frames.clear();
604 processing_video_frames.clear();
605 processing_audio_frames.clear();
606 missing_audio_frames.clear();
607 missing_video_frames.clear();
608 missing_audio_frames_source.clear();
609 missing_video_frames_source.clear();
610 checked_frames.clear();
611 }
612
613 // Close the video file
614 avformat_close_input(&pFormatCtx);
615 av_freep(&pFormatCtx);
616
617 // Reset some variables
618 last_frame = 0;
619 largest_frame_processed = 0;
620 seek_audio_frame_found = 0;
621 seek_video_frame_found = 0;
622 current_video_frame = 0;
623 has_missing_frames = false;
624
625 last_video_frame.reset();
626 }
627}
628
629bool FFmpegReader::HasAlbumArt() {
630 // Check if the video stream we use is an attached picture
631 // This won't return true if the file has a cover image as a secondary stream
632 // like an MKV file with an attached image file
633 return pFormatCtx && videoStream >= 0 && pFormatCtx->streams[videoStream]
634 && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC);
635}
636
637void FFmpegReader::UpdateAudioInfo() {
638 // Set values of FileInfo struct
639 info.has_audio = true;
640 info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
641 info.acodec = aCodecCtx->codec->name;
642 info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
643 if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0)
644 AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels);
645 info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout;
646 info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate;
647 info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate;
648
649 // Set audio timebase
650 info.audio_timebase.num = aStream->time_base.num;
651 info.audio_timebase.den = aStream->time_base.den;
652
653 // Get timebase of audio stream (if valid) and greater than the current duration
654 if (aStream->duration > 0.0f && aStream->duration > info.duration)
655 info.duration = aStream->duration * info.audio_timebase.ToDouble();
656
657 // Check for an invalid video length
658 if (info.has_video && info.video_length <= 0) {
659 // Calculate the video length from the audio duration
661 }
662
663 // Set video timebase (if no video stream was found)
664 if (!info.has_video) {
665 // Set a few important default video settings (so audio can be divided into frames)
666 info.fps.num = 24;
667 info.fps.den = 1;
671 info.width = 720;
672 info.height = 480;
673 }
674
675 // Fix invalid video lengths for certain types of files (MP3 for example)
676 if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) {
678 }
679
680 // Add audio metadata (if any found)
681 AVDictionaryEntry *tag = NULL;
682 while ((tag = av_dict_get(aStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
683 QString str_key = tag->key;
684 QString str_value = tag->value;
685 info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
686 }
687}
688
689void FFmpegReader::UpdateVideoInfo() {
690 if (check_fps)
691 // Already initialized all the video metadata, no reason to do it again
692 return;
693
694 // Set values of FileInfo struct
695 info.has_video = true;
696 info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1;
697 info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height;
698 info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width;
699 info.vcodec = pCodecCtx->codec->name;
700 info.video_bit_rate = (pFormatCtx->bit_rate / 8);
701
702 // Frame rate from the container and codec
703 AVRational framerate = av_guess_frame_rate(pFormatCtx, pStream, NULL);
704 info.fps.num = framerate.num;
705 info.fps.den = framerate.den;
706
707 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo", "info.fps.num", info.fps.num, "info.fps.den", info.fps.den);
708
709 // TODO: remove excessive debug info in the next releases
710 // The debug info below is just for comparison and troubleshooting on users side during the transition period
711 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdateVideoInfo (pStream->avg_frame_rate)", "num", pStream->avg_frame_rate.num, "den", pStream->avg_frame_rate.den);
712
713 if (pStream->sample_aspect_ratio.num != 0) {
714 info.pixel_ratio.num = pStream->sample_aspect_ratio.num;
715 info.pixel_ratio.den = pStream->sample_aspect_ratio.den;
716 } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) {
717 info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num;
718 info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den;
719 } else {
720 info.pixel_ratio.num = 1;
721 info.pixel_ratio.den = 1;
722 }
723 info.pixel_format = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
724
725 // Calculate the DAR (display aspect ratio)
727
728 // Reduce size fraction
729 size.Reduce();
730
731 // Set the ratio based on the reduced fraction
732 info.display_ratio.num = size.num;
733 info.display_ratio.den = size.den;
734
735 // Get scan type and order from codec context/params
736 if (!check_interlace) {
737 check_interlace = true;
738 AVFieldOrder field_order = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->field_order;
739 switch(field_order) {
740 case AV_FIELD_PROGRESSIVE:
741 info.interlaced_frame = false;
742 break;
743 case AV_FIELD_TT:
744 case AV_FIELD_TB:
745 info.interlaced_frame = true;
746 info.top_field_first = true;
747 break;
748 case AV_FIELD_BT:
749 case AV_FIELD_BB:
750 info.interlaced_frame = true;
751 info.top_field_first = false;
752 break;
753 case AV_FIELD_UNKNOWN:
754 // Check again later?
755 check_interlace = false;
756 break;
757 }
758 // check_interlace will prevent these checks being repeated,
759 // unless it was cleared because we got an AV_FIELD_UNKNOWN response.
760 }
761
762 // Set the video timebase
763 info.video_timebase.num = pStream->time_base.num;
764 info.video_timebase.den = pStream->time_base.den;
765
766 // Set the duration in seconds, and video length (# of frames)
767 info.duration = pStream->duration * info.video_timebase.ToDouble();
768
769 // Check for valid duration (if found)
770 if (info.duration <= 0.0f && pFormatCtx->duration >= 0)
771 // Use the format's duration
772 info.duration = float(pFormatCtx->duration) / AV_TIME_BASE;
773
774 // Calculate duration from filesize and bitrate (if any)
775 if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0)
776 // Estimate from bitrate, total bytes, and framerate
778
779 // No duration found in stream of file
780 if (info.duration <= 0.0f) {
781 // No duration is found in the video stream
782 info.duration = -1;
783 info.video_length = -1;
784 is_duration_known = false;
785 } else {
786 // Yes, a duration was found
787 is_duration_known = true;
788
789 // Calculate number of frames
791 }
792
793 // Override an invalid framerate
794 if (info.fps.ToFloat() > 240.0f || (info.fps.num <= 0 || info.fps.den <= 0) || info.video_length <= 0) {
795 // Calculate FPS, duration, video bit rate, and video length manually
796 // by scanning through all the video stream packets
797 CheckFPS();
798 }
799
800 // Add video metadata (if any)
801 AVDictionaryEntry *tag = NULL;
802 while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) {
803 QString str_key = tag->key;
804 QString str_value = tag->value;
805 info.metadata[str_key.toStdString()] = str_value.trimmed().toStdString();
806 }
807}
808
810 return this->is_duration_known;
811}
812
813std::shared_ptr<Frame> FFmpegReader::GetFrame(int64_t requested_frame) {
814 // Check for open reader (or throw exception)
815 if (!is_open)
816 throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path);
817
818 // Adjust for a requested frame that is too small or too large
819 if (requested_frame < 1)
820 requested_frame = 1;
821 if (requested_frame > info.video_length && is_duration_known)
822 requested_frame = info.video_length;
823 if (info.has_video && info.video_length == 0)
824 // Invalid duration of video file
825 throw InvalidFile("Could not detect the duration of the video or audio stream.", path);
826
827 // Debug output
828 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "requested_frame", requested_frame, "last_frame", last_frame);
829
830 // Check the cache for this frame
831 std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
832 if (frame) {
833 // Debug output
834 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame);
835
836 // Return the cached frame
837 return frame;
838 } else {
839 // Check the cache a 2nd time (due to a potential previous lock)
840 frame = final_cache.GetFrame(requested_frame);
841 if (frame) {
842 // Debug output
843 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame);
844
845 // Return the cached frame
846 } else {
847 // Frame is not in cache
848 // Reset seek count
849 seek_count = 0;
850
851 // Check for first frame (always need to get frame 1 before other frames, to correctly calculate offsets)
852 if (last_frame == 0 && requested_frame != 1)
853 // Get first frame
854 ReadStream(1);
855
856 // Are we within X frames of the requested frame?
857 int64_t diff = requested_frame - last_frame;
858 if (diff >= 1 && diff <= 20) {
859 // Continue walking the stream
860 frame = ReadStream(requested_frame);
861 } else {
862 // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame
863 if (enable_seek)
864 // Only seek if enabled
865 Seek(requested_frame);
866
867 else if (!enable_seek && diff < 0) {
868 // Start over, since we can't seek, and the requested frame is smaller than our position
869 Close();
870 Open();
871 }
872
873 // Then continue walking the stream
874 frame = ReadStream(requested_frame);
875 }
876 }
877 return frame;
878 }
879}
880
881// Read the stream until we find the requested Frame
882std::shared_ptr<Frame> FFmpegReader::ReadStream(int64_t requested_frame) {
883 // Allocate video frame
884 bool end_of_stream = false;
885 bool check_seek = false;
886 bool frame_finished = false;
887 int packet_error = -1;
888
889 // Minimum number of packets to process (for performance reasons)
890 int packets_processed = 0;
891 int minimum_packets = 1;
892 int max_packets = 4096;
893
894 // Debug output
895 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "max_concurrent_frames", max_concurrent_frames);
896
897 // Loop through the stream until the correct frame is found
898 while (true) {
899 // Get the next packet into a local variable called packet
900 packet_error = GetNextPacket();
901
902 int processing_video_frames_size = 0;
903 int processing_audio_frames_size = 0;
904 {
905 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
906 processing_video_frames_size = processing_video_frames.size();
907 processing_audio_frames_size = processing_audio_frames.size();
908 }
909
910 // Wait if too many frames are being processed
911 while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) {
912 std::this_thread::sleep_for(std::chrono::milliseconds(3));
913 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
914 processing_video_frames_size = processing_video_frames.size();
915 processing_audio_frames_size = processing_audio_frames.size();
916 }
917
918 // Get the next packet (if any)
919 if (packet_error < 0) {
920 // Break loop when no more packets found
921 end_of_stream = true;
922 break;
923 }
924
925 // Debug output
926 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking);
927
928 // Video packet
929 if (info.has_video && packet->stream_index == videoStream) {
930 // Reset this counter, since we have a video packet
931 num_packets_since_video_frame = 0;
932
933 // Check the status of a seek (if any)
934 if (is_seeking) {
935 check_seek = CheckSeek(true);
936 } else {
937 check_seek = false;
938 }
939
940 if (check_seek) {
941 // Jump to the next iteration of this loop
942 continue;
943 }
944
945 // Packet may become NULL on Close inside Seek if CheckSeek returns false
946 if (!packet) {
947 // Jump to the next iteration of this loop
948 continue;
949 }
950
951 // Get the AVFrame from the current packet
952 frame_finished = GetAVFrame();
953
954 // Check if the AVFrame is finished and set it
955 if (frame_finished) {
956 // Update PTS / Frame Offset (if any)
957 UpdatePTSOffset(true);
958
959 // Process Video Packet
960 ProcessVideoPacket(requested_frame);
961 }
962
963 }
964 // Audio packet
965 else if (info.has_audio && packet->stream_index == audioStream) {
966 // Increment this (to track # of packets since the last video packet)
967 num_packets_since_video_frame++;
968
969 // Check the status of a seek (if any)
970 if (is_seeking) {
971 check_seek = CheckSeek(false);
972 } else {
973 check_seek = false;
974 }
975
976 if (check_seek) {
977 // Jump to the next iteration of this loop
978 continue;
979 }
980
981 // Packet may become NULL on Close inside Seek if CheckSeek returns false
982 if (!packet) {
983 // Jump to the next iteration of this loop
984 continue;
985 }
986
987 // Update PTS / Frame Offset (if any)
988 UpdatePTSOffset(false);
989
990 // Determine related video frame and starting sample # from audio PTS
991 AudioLocation location = GetAudioPTSLocation(packet->pts);
992
993 // Process Audio Packet
994 ProcessAudioPacket(requested_frame, location.frame, location.sample_start);
995 }
996
997 // Check if working frames are 'finished'
998 if (!is_seeking) {
999 // Check for final frames
1000 CheckWorkingFrames(false, requested_frame);
1001 }
1002
1003 // Check if requested 'final' frame is available
1004 bool is_cache_found = (final_cache.GetFrame(requested_frame) != NULL);
1005
1006 // Increment frames processed
1007 packets_processed++;
1008
1009 // Break once the frame is found
1010 if ((is_cache_found && packets_processed >= minimum_packets) || packets_processed > max_packets)
1011 break;
1012
1013 } // end while
1014
1015 // Debug output
1016 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (Completed)", "packets_processed", packets_processed, "end_of_stream", end_of_stream, "largest_frame_processed", largest_frame_processed, "Working Cache Count", working_cache.Count());
1017
1018 // End of stream?
1019 if (end_of_stream)
1020 // Mark the any other working frames as 'finished'
1021 CheckWorkingFrames(end_of_stream, requested_frame);
1022
1023 // Return requested frame (if found)
1024 std::shared_ptr<Frame> frame = final_cache.GetFrame(requested_frame);
1025 if (frame)
1026 // Return prepared frame
1027 return frame;
1028 else {
1029
1030 // Check if largest frame is still cached
1031 frame = final_cache.GetFrame(largest_frame_processed);
1032 if (frame) {
1033 // return the largest processed frame (assuming it was the last in the video file)
1034 return frame;
1035 } else {
1036 // The largest processed frame is no longer in cache, return a blank frame
1037 std::shared_ptr<Frame> f = CreateFrame(largest_frame_processed);
1038 f->AddColor(info.width, info.height, "#000");
1039 return f;
1040 }
1041 }
1042
1043}
1044
1045// Get the next packet (if any)
1046int FFmpegReader::GetNextPacket() {
1047 int found_packet = 0;
1048 AVPacket *next_packet;
1049 next_packet = new AVPacket();
1050 found_packet = av_read_frame(pFormatCtx, next_packet);
1051
1052 if (packet) {
1053 // Remove previous packet before getting next one
1054 RemoveAVPacket(packet);
1055 packet = NULL;
1056 }
1057 if (found_packet >= 0) {
1058 // Update current packet pointer
1059 packet = next_packet;
1060 } else {
1061 delete next_packet;
1062 }
1063 // Return if packet was found (or error number)
1064 return found_packet;
1065}
1066
1067// Get an AVFrame (if any)
1068bool FFmpegReader::GetAVFrame() {
1069 int frameFinished = -1;
1070 int ret = 0;
1071
1072 // Decode video frame
1073 AVFrame *next_frame = AV_ALLOCATE_FRAME();
1074
1075#if IS_FFMPEG_3_2
1076 frameFinished = 0;
1077 ret = avcodec_send_packet(pCodecCtx, packet);
1078
1079 #if USE_HW_ACCEL
1080 // Get the format from the variables set in get_hw_dec_format
1081 hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
1082 hw_de_av_device_type = hw_de_av_device_type_global;
1083 #endif // USE_HW_ACCEL
1084 if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1085 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)");
1086 }
1087 else {
1088 AVFrame *next_frame2;
1089 #if USE_HW_ACCEL
1090 if (hw_de_on && hw_de_supported) {
1091 next_frame2 = AV_ALLOCATE_FRAME();
1092 }
1093 else
1094 #endif // USE_HW_ACCEL
1095 {
1096 next_frame2 = next_frame;
1097 }
1098 pFrame = AV_ALLOCATE_FRAME();
1099 while (ret >= 0) {
1100 ret = avcodec_receive_frame(pCodecCtx, next_frame2);
1101 if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
1102 break;
1103 }
1104 if (ret != 0) {
1105 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (invalid return frame received)");
1106 }
1107 #if USE_HW_ACCEL
1108 if (hw_de_on && hw_de_supported) {
1109 int err;
1110 if (next_frame2->format == hw_de_av_pix_fmt) {
1111 next_frame->format = AV_PIX_FMT_YUV420P;
1112 if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) {
1113 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)");
1114 }
1115 if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) {
1116 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)");
1117 }
1118 }
1119 }
1120 else
1121 #endif // USE_HW_ACCEL
1122 { // No hardware acceleration used -> no copy from GPU memory needed
1123 next_frame = next_frame2;
1124 }
1125
1126 // TODO also handle possible further frames
1127 // Use only the first frame like avcodec_decode_video2
1128 if (frameFinished == 0 ) {
1129 frameFinished = 1;
1130 av_image_alloc(pFrame->data, pFrame->linesize, info.width, info.height, (AVPixelFormat)(pStream->codecpar->format), 1);
1131 av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize,
1132 (AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
1133 }
1134 }
1135 #if USE_HW_ACCEL
1136 if (hw_de_on && hw_de_supported) {
1137 AV_FREE_FRAME(&next_frame2);
1138 }
1139 #endif // USE_HW_ACCEL
1140 }
1141#else
1142 avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
1143
1144 // always allocate pFrame (because we do that in the ffmpeg >= 3.2 as well); it will always be freed later
1145 pFrame = AV_ALLOCATE_FRAME();
1146
1147 // is frame finished
1148 if (frameFinished) {
1149 // AVFrames are clobbered on the each call to avcodec_decode_video, so we
1150 // must make a copy of the image data before this method is called again.
1151 avpicture_alloc((AVPicture *) pFrame, pCodecCtx->pix_fmt, info.width, info.height);
1152 av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
1153 info.height);
1154 }
1155#endif // IS_FFMPEG_3_2
1156
1157 // deallocate the frame
1158 AV_FREE_FRAME(&next_frame);
1159
1160 // Did we get a video frame?
1161 return frameFinished;
1162}
1163
1164// Check the current seek position and determine if we need to seek again
1165bool FFmpegReader::CheckSeek(bool is_video) {
1166 // Are we seeking for a specific frame?
1167 if (is_seeking) {
1168 // Determine if both an audio and video packet have been decoded since the seek happened.
1169 // If not, allow the ReadStream method to keep looping
1170 if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found))
1171 return false;
1172
1173 // Check for both streams
1174 if ((info.has_video && !seek_video_frame_found) || (info.has_audio && !seek_audio_frame_found))
1175 return false;
1176
1177 // Determine max seeked frame
1178 int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
1179 if (seek_video_frame_found > max_seeked_frame)
1180 max_seeked_frame = seek_video_frame_found;
1181
1182 // determine if we are "before" the requested frame
1183 if (max_seeked_frame >= seeking_frame) {
1184 // SEEKED TOO FAR
1185 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1186
1187 // Seek again... to the nearest Keyframe
1188 Seek(seeking_frame - (10 * seek_count * seek_count));
1189 } else {
1190 // SEEK WORKED
1191 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found);
1192
1193 // Seek worked, and we are "before" the requested frame
1194 is_seeking = false;
1195 seeking_frame = 0;
1196 seeking_pts = -1;
1197 }
1198 }
1199
1200 // return the pts to seek to (if any)
1201 return is_seeking;
1202}
1203
1204// Process a video packet
1205void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
1206 // Calculate current frame #
1207 int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS());
1208
1209 // Track 1st video packet after a successful seek
1210 if (!seek_video_frame_found && is_seeking)
1211 seek_video_frame_found = current_frame;
1212
1213 // Are we close enough to decode the frame? and is this frame # valid?
1214 if ((current_frame < (requested_frame - 20)) or (current_frame == -1)) {
1215 // Remove frame and packet
1216 RemoveAVFrame(pFrame);
1217
1218 // Debug output
1219 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Skipped)", "requested_frame", requested_frame, "current_frame", current_frame);
1220
1221 // Skip to next frame without decoding or caching
1222 return;
1223 }
1224
1225 // Debug output
1226 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame);
1227
1228 // Init some things local (for OpenMP)
1229 PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx);
1230 int height = info.height;
1231 int width = info.width;
1232 int64_t video_length = info.video_length;
1233 AVFrame *my_frame = pFrame;
1234 pFrame = NULL;
1235
1236 // Add video frame to list of processing video frames
1237 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1238 processing_video_frames[current_frame] = current_frame;
1239
1240 // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it)
1241 AVFrame *pFrameRGB = nullptr;
1242 uint8_t *buffer = nullptr;
1243
1244 // Allocate an AVFrame structure
1245 pFrameRGB = AV_ALLOCATE_FRAME();
1246 if (pFrameRGB == nullptr)
1247 throw OutOfMemory("Failed to allocate frame buffer", path);
1248
1249 // Determine the max size of this source image (based on the timeline's size, the scaling mode,
1250 // and the scaling keyframes). This is a performance improvement, to keep the images as small as possible,
1251 // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline
1252 // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in
1253 // the future.
1254 int max_width = info.width;
1255 int max_height = info.height;
1256
1257 Clip *parent = (Clip *) ParentClip();
1258 if (parent) {
1259 if (parent->ParentTimeline()) {
1260 // Set max width/height based on parent clip's timeline (if attached to a timeline)
1261 max_width = parent->ParentTimeline()->preview_width;
1262 max_height = parent->ParentTimeline()->preview_height;
1263 }
1264 if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) {
1265 // Best fit or Stretch scaling (based on max timeline size * scaling keyframes)
1266 float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1267 float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1268 max_width = std::max(float(max_width), max_width * max_scale_x);
1269 max_height = std::max(float(max_height), max_height * max_scale_y);
1270
1271 } else if (parent->scale == SCALE_CROP) {
1272 // Cropping scale mode (based on max timeline size * cropped size * scaling keyframes)
1273 float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1274 float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1275 QSize width_size(max_width * max_scale_x,
1276 round(max_width / (float(info.width) / float(info.height))));
1277 QSize height_size(round(max_height / (float(info.height) / float(info.width))),
1278 max_height * max_scale_y);
1279 // respect aspect ratio
1280 if (width_size.width() >= max_width && width_size.height() >= max_height) {
1281 max_width = std::max(max_width, width_size.width());
1282 max_height = std::max(max_height, width_size.height());
1283 } else {
1284 max_width = std::max(max_width, height_size.width());
1285 max_height = std::max(max_height, height_size.height());
1286 }
1287
1288 } else {
1289 // Scale video to equivalent unscaled size
1290 // Since the preview window can change sizes, we want to always
1291 // scale against the ratio of original video size to timeline size
1292 float preview_ratio = 1.0;
1293 if (parent->ParentTimeline()) {
1294 Timeline *t = (Timeline *) parent->ParentTimeline();
1295 preview_ratio = t->preview_width / float(t->info.width);
1296 }
1297 float max_scale_x = parent->scale_x.GetMaxPoint().co.Y;
1298 float max_scale_y = parent->scale_y.GetMaxPoint().co.Y;
1299 max_width = info.width * max_scale_x * preview_ratio;
1300 max_height = info.height * max_scale_y * preview_ratio;
1301 }
1302 }
1303
1304 // Determine if image needs to be scaled (for performance reasons)
1305 int original_height = height;
1306 if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) {
1307 // Override width and height (but maintain aspect ratio)
1308 float ratio = float(width) / float(height);
1309 int possible_width = round(max_height * ratio);
1310 int possible_height = round(max_width / ratio);
1311
1312 if (possible_width <= max_width) {
1313 // use calculated width, and max_height
1314 width = possible_width;
1315 height = max_height;
1316 } else {
1317 // use max_width, and calculated height
1318 width = max_width;
1319 height = possible_height;
1320 }
1321 }
1322
1323 // Determine required buffer size and allocate buffer
1324 const int bytes_per_pixel = 4;
1325 int buffer_size = width * height * bytes_per_pixel;
1326 buffer = new unsigned char[buffer_size]();
1327
1328 // Copy picture data from one AVFrame (or AVPicture) to another one.
1329 AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
1330
1331 int scale_mode = SWS_FAST_BILINEAR;
1332 if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
1333 scale_mode = SWS_BICUBIC;
1334 }
1335 SwsContext *img_convert_ctx = sws_getContext(info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width,
1336 height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
1337
1338 // Resize / Convert to RGB
1339 sws_scale(img_convert_ctx, my_frame->data, my_frame->linesize, 0,
1340 original_height, pFrameRGB->data, pFrameRGB->linesize);
1341
1342 // Create or get the existing frame object
1343 std::shared_ptr<Frame> f = CreateFrame(current_frame);
1344
1345 // Add Image data to frame
1346 if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) {
1347 // Add image with no alpha channel, Speed optimization
1348 f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer);
1349 } else {
1350 // Add image with alpha channel (this will be converted to premultipled when needed, but is slower)
1351 f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer);
1352 }
1353
1354 // Update working cache
1355 working_cache.Add(f);
1356
1357 // Keep track of last last_video_frame
1358 last_video_frame = f;
1359
1360 // Free the RGB image
1361 AV_FREE_FRAME(&pFrameRGB);
1362
1363 // Remove frame and packet
1364 RemoveAVFrame(my_frame);
1365 sws_freeContext(img_convert_ctx);
1366
1367 // Remove video frame from list of processing video frames
1368 {
1369 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1370 processing_video_frames.erase(current_frame);
1371 processed_video_frames[current_frame] = current_frame;
1372 }
1373
1374 // Debug output
1375 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (After)", "requested_frame", requested_frame, "current_frame", current_frame, "f->number", f->number);
1376}
1377
1378// Process an audio packet
1379void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample) {
1380 // Track 1st audio packet after a successful seek
1381 if (!seek_audio_frame_found && is_seeking)
1382 seek_audio_frame_found = target_frame;
1383
1384 // Are we close enough to decode the frame's audio?
1385 if (target_frame < (requested_frame - 20)) {
1386 // Debug output
1387 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1388
1389 // Skip to next frame without decoding or caching
1390 return;
1391 }
1392
1393 // Debug output
1394 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Before)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample);
1395
1396 // Init an AVFrame to hold the decoded audio samples
1397 int frame_finished = 0;
1398 AVFrame *audio_frame = AV_ALLOCATE_FRAME();
1399 AV_RESET_FRAME(audio_frame);
1400
1401 int packet_samples = 0;
1402 int data_size = 0;
1403
1404#if IS_FFMPEG_3_2
1405 int ret = 0;
1406 frame_finished = 1;
1407 while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) {
1408 frame_finished = 0;
1409 ret = avcodec_send_packet(aCodecCtx, packet);
1410 if (ret < 0 && ret != AVERROR(EINVAL) && ret != AVERROR_EOF) {
1411 avcodec_send_packet(aCodecCtx, NULL);
1412 break;
1413 }
1414 if (ret >= 0)
1415 packet->size = 0;
1416 ret = avcodec_receive_frame(aCodecCtx, audio_frame);
1417 if (ret >= 0)
1418 frame_finished = 1;
1419 if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) {
1420 avcodec_flush_buffers(aCodecCtx);
1421 ret = 0;
1422 }
1423 if (ret >= 0) {
1424 ret = frame_finished;
1425 }
1426 }
1427 if (!packet->data && !frame_finished)
1428 {
1429 ret = -1;
1430 }
1431#else
1432 int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet);
1433#endif
1434
1435 if (frame_finished) {
1436
1437 // determine how many samples were decoded
1438 int plane_size = -1;
1439 data_size = av_samples_get_buffer_size(&plane_size,
1440 AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels,
1441 audio_frame->nb_samples,
1442 (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1);
1443
1444 // Calculate total number of samples
1445 packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels;
1446 }
1447
1448 // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp)
1449 int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array
1450
1451 // DEBUG (FOR AUDIO ISSUES) - Get the audio packet start time (in seconds)
1452 int64_t adjusted_pts = packet->pts + audio_pts_offset;
1453 double audio_seconds = double(adjusted_pts) * info.audio_timebase.ToDouble();
1454 double sample_seconds = double(pts_total) / info.sample_rate;
1455
1456 // Debug output
1457 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info A)", "pts_counter", pts_counter, "PTS", adjusted_pts, "Offset", audio_pts_offset, "PTS Diff", adjusted_pts - prev_pts, "Samples", pts_remaining_samples, "Sample PTS ratio", float(adjusted_pts - prev_pts) / pts_remaining_samples);
1458 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Decode Info B)", "Sample Diff", pts_remaining_samples - prev_samples - prev_pts, "Total", pts_total, "PTS Seconds", audio_seconds, "Sample Seconds", sample_seconds, "Seconds Diff", audio_seconds - sample_seconds, "raw samples", packet_samples);
1459
1460 // DEBUG (FOR AUDIO ISSUES)
1461 prev_pts = adjusted_pts;
1462 pts_total += pts_remaining_samples;
1463 pts_counter++;
1464 prev_samples = pts_remaining_samples;
1465
1466 // Add audio frame to list of processing audio frames
1467 {
1468 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1469 processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1470 }
1471
1472 while (pts_remaining_samples) {
1473 // Get Samples per frame (for this frame number)
1474 int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels);
1475
1476 // Calculate # of samples to add to this frame
1477 int samples = samples_per_frame - previous_packet_location.sample_start;
1478 if (samples > pts_remaining_samples)
1479 samples = pts_remaining_samples;
1480
1481 // Decrement remaining samples
1482 pts_remaining_samples -= samples;
1483
1484 if (pts_remaining_samples > 0) {
1485 // next frame
1486 previous_packet_location.frame++;
1487 previous_packet_location.sample_start = 0;
1488
1489 // Add audio frame to list of processing audio frames
1490 {
1491 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1492 processing_audio_frames.insert(std::pair<int, int>(previous_packet_location.frame, previous_packet_location.frame));
1493 }
1494
1495 } else {
1496 // Increment sample start
1497 previous_packet_location.sample_start += samples;
1498 }
1499 }
1500
1501
1502 // Allocate audio buffer
1503 int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE];
1504
1505 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16);
1506
1507 // Create output frame
1508 AVFrame *audio_converted = AV_ALLOCATE_FRAME();
1509 AV_RESET_FRAME(audio_converted);
1510 audio_converted->nb_samples = audio_frame->nb_samples;
1511 av_samples_alloc(audio_converted->data, audio_converted->linesize, info.channels, audio_frame->nb_samples, AV_SAMPLE_FMT_S16, 0);
1512
1513 SWRCONTEXT *avr = NULL;
1514 int nb_samples = 0;
1515
1516 // setup resample context
1517 avr = SWR_ALLOC();
1518 av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1519 av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0);
1520 av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0);
1521 av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);
1522 av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0);
1523 av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0);
1524 av_opt_set_int(avr, "in_channels", info.channels, 0);
1525 av_opt_set_int(avr, "out_channels", info.channels, 0);
1526 SWR_INIT(avr);
1527
1528 // Convert audio samples
1529 nb_samples = SWR_CONVERT(avr, // audio resample context
1530 audio_converted->data, // output data pointers
1531 audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown)
1532 audio_converted->nb_samples, // maximum number of samples that the output buffer can hold
1533 audio_frame->data, // input data pointers
1534 audio_frame->linesize[0], // input plane size, in bytes (0 if unknown)
1535 audio_frame->nb_samples); // number of input samples to convert
1536
1537 // Copy audio samples over original samples
1538 memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels);
1539
1540 // Deallocate resample buffer
1541 SWR_CLOSE(avr);
1542 SWR_FREE(&avr);
1543 avr = NULL;
1544
1545 // Free AVFrames
1546 av_free(audio_converted->data[0]);
1547 AV_FREE_FRAME(&audio_converted);
1548
1549 int64_t starting_frame_number = -1;
1550 bool partial_frame = true;
1551 for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) {
1552 // Array of floats (to hold samples for each channel)
1553 starting_frame_number = target_frame;
1554 int channel_buffer_size = packet_samples / info.channels;
1555 float *channel_buffer = new float[channel_buffer_size];
1556
1557 // Init buffer array
1558 for (int z = 0; z < channel_buffer_size; z++)
1559 channel_buffer[z] = 0.0f;
1560
1561 // Loop through all samples and add them to our Frame based on channel.
1562 // Toggle through each channel number, since channel data is stored like (left right left right)
1563 int channel = 0;
1564 int position = 0;
1565 for (int sample = 0; sample < packet_samples; sample++) {
1566 // Only add samples for current channel
1567 if (channel_filter == channel) {
1568 // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0))
1569 channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15));
1570
1571 // Increment audio position
1572 position++;
1573 }
1574
1575 // increment channel (if needed)
1576 if ((channel + 1) < info.channels)
1577 // move to next channel
1578 channel++;
1579 else
1580 // reset channel
1581 channel = 0;
1582 }
1583
1584 // Loop through samples, and add them to the correct frames
1585 int start = starting_sample;
1586 int remaining_samples = channel_buffer_size;
1587 float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer
1588 while (remaining_samples > 0) {
1589 // Get Samples per frame (for this frame number)
1590 int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels);
1591
1592 // Calculate # of samples to add to this frame
1593 int samples = samples_per_frame - start;
1594 if (samples > remaining_samples)
1595 samples = remaining_samples;
1596
1597 // Create or get the existing frame object
1598 std::shared_ptr<Frame> f = CreateFrame(starting_frame_number);
1599
1600 // Determine if this frame was "partially" filled in
1601 if (samples_per_frame == start + samples)
1602 partial_frame = false;
1603 else
1604 partial_frame = true;
1605
1606 // Add samples for current channel to the frame.
1607 f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 1.0f);
1608
1609 // Debug output
1610 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame);
1611
1612 // Add or update cache
1613 working_cache.Add(f);
1614
1615 // Decrement remaining samples
1616 remaining_samples -= samples;
1617
1618 // Increment buffer (to next set of samples)
1619 if (remaining_samples > 0)
1620 iterate_channel_buffer += samples;
1621
1622 // Increment frame number
1623 starting_frame_number++;
1624
1625 // Reset starting sample #
1626 start = 0;
1627 }
1628
1629 // clear channel buffer
1630 delete[] channel_buffer;
1631 channel_buffer = NULL;
1632 iterate_channel_buffer = NULL;
1633 }
1634
1635 // Clean up some arrays
1636 delete[] audio_buf;
1637 audio_buf = NULL;
1638
1639 // Remove audio frame from list of processing audio frames
1640 {
1641 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1642 // Update all frames as completed
1643 for (int64_t f = target_frame; f < starting_frame_number; f++) {
1644 // Remove the frame # from the processing list. NOTE: If more than one thread is
1645 // processing this frame, the frame # will be in this list multiple times. We are only
1646 // removing a single instance of it here.
1647 processing_audio_frames.erase(processing_audio_frames.find(f));
1648
1649 // Check and see if this frame is also being processed by another thread
1650 if (processing_audio_frames.count(f) == 0)
1651 // No other thread is processing it. Mark the audio as processed (final)
1652 processed_audio_frames[f] = f;
1653 }
1654
1655 if (target_frame == starting_frame_number) {
1656 // This typically never happens, but just in case, remove the currently processing number
1657 processing_audio_frames.erase(processing_audio_frames.find(target_frame));
1658 }
1659 }
1660
1661 // Free audio frame
1662 AV_FREE_FRAME(&audio_frame);
1663
1664 // Debug output
1665 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1);
1666
1667}
1668
1669
1670// Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs.
1671void FFmpegReader::Seek(int64_t requested_frame) {
1672 // Adjust for a requested frame that is too small or too large
1673 if (requested_frame < 1)
1674 requested_frame = 1;
1675 if (requested_frame > info.video_length)
1676 requested_frame = info.video_length;
1677
1678 int processing_video_frames_size = 0;
1679 int processing_audio_frames_size = 0;
1680 {
1681 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1682 processing_video_frames_size = processing_video_frames.size();
1683 processing_audio_frames_size = processing_audio_frames.size();
1684 }
1685
1686 // Debug output
1687 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Seek", "requested_frame", requested_frame, "seek_count", seek_count, "last_frame", last_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "video_pts_offset", video_pts_offset);
1688
1689 // Wait for any processing frames to complete
1690 while (processing_video_frames_size + processing_audio_frames_size > 0) {
1691 std::this_thread::sleep_for(std::chrono::milliseconds(3));
1692 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1693 processing_video_frames_size = processing_video_frames.size();
1694 processing_audio_frames_size = processing_audio_frames.size();
1695 }
1696
1697 // Clear working cache (since we are seeking to another location in the file)
1698 working_cache.Clear();
1699 missing_frames.Clear();
1700
1701 // Clear processed lists
1702 {
1703 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1704 processing_audio_frames.clear();
1705 processing_video_frames.clear();
1706 processed_video_frames.clear();
1707 processed_audio_frames.clear();
1708 missing_audio_frames.clear();
1709 missing_video_frames.clear();
1710 missing_audio_frames_source.clear();
1711 missing_video_frames_source.clear();
1712 checked_frames.clear();
1713 }
1714
1715 // Reset the last frame variable
1716 last_frame = 0;
1717 current_video_frame = 0;
1718 largest_frame_processed = 0;
1719 num_checks_since_final = 0;
1720 num_packets_since_video_frame = 0;
1721 has_missing_frames = false;
1722 bool has_audio_override = info.has_audio;
1723 bool has_video_override = info.has_video;
1724
1725 // Increment seek count
1726 seek_count++;
1727
1728 // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking)
1729 int buffer_amount = std::max(max_concurrent_frames, 8);
1730 if (requested_frame - buffer_amount < 20) {
1731 // Close and re-open file (basically seeking to frame 1)
1732 Close();
1733 Open();
1734
1735 // Update overrides (since closing and re-opening might update these)
1736 info.has_audio = has_audio_override;
1737 info.has_video = has_video_override;
1738
1739 // Not actually seeking, so clear these flags
1740 is_seeking = false;
1741 if (seek_count == 1) {
1742 // Don't redefine this on multiple seek attempts for a specific frame
1743 seeking_frame = 1;
1744 seeking_pts = ConvertFrameToVideoPTS(1);
1745 }
1746 seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1747 seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1748
1749 } else {
1750 // Seek to nearest key-frame (aka, i-frame)
1751 bool seek_worked = false;
1752 int64_t seek_target = 0;
1753
1754 // Seek video stream (if any), except album arts
1755 if (!seek_worked && info.has_video && !HasAlbumArt()) {
1756 seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount);
1757 if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1758 fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME);
1759 } else {
1760 // VIDEO SEEK
1761 is_video_seek = true;
1762 seek_worked = true;
1763 }
1764 }
1765
1766 // Seek audio stream (if not already seeked... and if an audio stream is found)
1767 if (!seek_worked && info.has_audio) {
1768 seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount);
1769 if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) {
1770 fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME);
1771 } else {
1772 // AUDIO SEEK
1773 is_video_seek = false;
1774 seek_worked = true;
1775 }
1776 }
1777
1778 // Was the seek successful?
1779 if (seek_worked) {
1780 // Flush audio buffer
1781 if (info.has_audio)
1782 avcodec_flush_buffers(aCodecCtx);
1783
1784 // Flush video buffer
1785 if (info.has_video)
1786 avcodec_flush_buffers(pCodecCtx);
1787
1788 // Reset previous audio location to zero
1789 previous_packet_location.frame = -1;
1790 previous_packet_location.sample_start = 0;
1791
1792 // init seek flags
1793 is_seeking = true;
1794 if (seek_count == 1) {
1795 // Don't redefine this on multiple seek attempts for a specific frame
1796 seeking_pts = seek_target;
1797 seeking_frame = requested_frame;
1798 }
1799 seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek
1800 seek_video_frame_found = 0; // used to detect which frames to throw away after a seek
1801
1802 } else {
1803 // seek failed
1804 is_seeking = false;
1805 seeking_pts = 0;
1806 seeking_frame = 0;
1807
1808 // dislable seeking for this reader (since it failed)
1809 // TODO: Find a safer way to do this... not sure how common it is for a seek to fail.
1810 enable_seek = false;
1811
1812 // Close and re-open file (basically seeking to frame 1)
1813 Close();
1814 Open();
1815
1816 // Update overrides (since closing and re-opening might update these)
1817 info.has_audio = has_audio_override;
1818 info.has_video = has_video_override;
1819 }
1820 }
1821}
1822
1823// Get the PTS for the current video packet
1824int64_t FFmpegReader::GetVideoPTS() {
1825 int64_t current_pts = 0;
1826 if (packet->dts != AV_NOPTS_VALUE)
1827 current_pts = packet->dts;
1828
1829 // Return adjusted PTS
1830 return current_pts;
1831}
1832
1833// Update PTS Offset (if any)
1834void FFmpegReader::UpdatePTSOffset(bool is_video) {
1835 // Determine the offset between the PTS and Frame number (only for 1st frame)
1836 if (is_video) {
1837 // VIDEO PACKET
1838 if (video_pts_offset == 99999) // Has the offset been set yet?
1839 {
1840 // Find the difference between PTS and frame number
1841 video_pts_offset = 0 - GetVideoPTS();
1842
1843 // Find the difference between PTS and frame number
1844 // Also, determine if PTS is invalid (too far away from zero)
1845 // We compare the PTS to the timebase value equal to 1 second (which means the PTS
1846 // must be within the -1 second to +1 second of zero, otherwise we ignore it)
1847 // TODO: Please see https://github.com/OpenShot/libopenshot/pull/565#issuecomment-690985272
1848 // for ideas to improve this logic.
1849 int64_t max_offset = info.video_timebase.Reciprocal().ToFloat();
1850 if (video_pts_offset < -max_offset || video_pts_offset > max_offset) {
1851 // Ignore PTS, it seems invalid
1852 video_pts_offset = 0;
1853 }
1854
1855 // debug output
1856 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video);
1857 }
1858 } else {
1859 // AUDIO PACKET
1860 if (audio_pts_offset == 99999) // Has the offset been set yet?
1861 {
1862 // Find the difference between PTS and frame number
1863 // Also, determine if PTS is invalid (too far away from zero)
1864 // We compare the PTS to the timebase value equal to 1 second (which means the PTS
1865 // must be within the -1 second to +1 second of zero, otherwise we ignore it)
1866 // TODO: Please see https://github.com/OpenShot/libopenshot/pull/565#issuecomment-690985272
1867 // for ideas to improve this logic.
1868 audio_pts_offset = 0 - packet->pts;
1869 int64_t max_offset = info.audio_timebase.Reciprocal().ToFloat();
1870 if (audio_pts_offset < -max_offset || audio_pts_offset > max_offset) {
1871 // Ignore PTS, it seems invalid
1872 audio_pts_offset = 0;
1873 }
1874
1875 // debug output
1876 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Audio)", "audio_pts_offset", audio_pts_offset, "is_video", is_video);
1877 }
1878 }
1879}
1880
1881// Convert PTS into Frame Number
1882int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) {
1883 // Apply PTS offset
1884 pts = pts + video_pts_offset;
1885 int64_t previous_video_frame = current_video_frame;
1886
1887 // Get the video packet start time (in seconds)
1888 double video_seconds = double(pts) * info.video_timebase.ToDouble();
1889
1890 // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1891 int64_t frame = round(video_seconds * info.fps.ToDouble()) + 1;
1892
1893 // Keep track of the expected video frame #
1894 if (current_video_frame == 0)
1895 current_video_frame = frame;
1896 else {
1897
1898 // Sometimes frames are duplicated due to identical (or similar) timestamps
1899 if (frame == previous_video_frame) {
1900 // return -1 frame number
1901 frame = -1;
1902 } else {
1903 // Increment expected frame
1904 current_video_frame++;
1905 }
1906
1907 if (current_video_frame < frame)
1908 // has missing frames
1909 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (detected missing frame)", "calculated frame", frame, "previous_video_frame", previous_video_frame, "current_video_frame", current_video_frame);
1910
1911 // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine
1912 // if we are missing a video frame.
1913 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
1914 while (current_video_frame < frame) {
1915 if (!missing_video_frames.count(current_video_frame)) {
1916 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame);
1917 missing_video_frames.insert(std::pair<int64_t, int64_t>(current_video_frame, previous_video_frame));
1918 missing_video_frames_source.insert(std::pair<int64_t, int64_t>(previous_video_frame, current_video_frame));
1919 }
1920
1921 // Mark this reader as containing missing frames
1922 has_missing_frames = true;
1923
1924 // Increment current frame
1925 current_video_frame++;
1926 }
1927 }
1928
1929 // Return frame #
1930 return frame;
1931}
1932
1933// Convert Frame Number into Video PTS
1934int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) {
1935 // Get timestamp of this frame (in seconds)
1936 double seconds = double(frame_number) / info.fps.ToDouble();
1937
1938 // Calculate the # of video packets in this timestamp
1939 int64_t video_pts = round(seconds / info.video_timebase.ToDouble());
1940
1941 // Apply PTS offset (opposite)
1942 return video_pts - video_pts_offset;
1943}
1944
1945// Convert Frame Number into Video PTS
1946int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) {
1947 // Get timestamp of this frame (in seconds)
1948 double seconds = double(frame_number) / info.fps.ToDouble();
1949
1950 // Calculate the # of audio packets in this timestamp
1951 int64_t audio_pts = round(seconds / info.audio_timebase.ToDouble());
1952
1953 // Apply PTS offset (opposite)
1954 return audio_pts - audio_pts_offset;
1955}
1956
1957// Calculate Starting video frame and sample # for an audio PTS
1958AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) {
1959 // Apply PTS offset
1960 pts = pts + audio_pts_offset;
1961
1962 // Get the audio packet start time (in seconds)
1963 double audio_seconds = double(pts) * info.audio_timebase.ToDouble();
1964
1965 // Divide by the video timebase, to get the video frame number (frame # is decimal at this point)
1966 double frame = (audio_seconds * info.fps.ToDouble()) + 1;
1967
1968 // Frame # as a whole number (no more decimals)
1969 int64_t whole_frame = int64_t(frame);
1970
1971 // Remove the whole number, and only get the decimal of the frame
1972 double sample_start_percentage = frame - double(whole_frame);
1973
1974 // Get Samples per frame
1975 int samples_per_frame = Frame::GetSamplesPerFrame(whole_frame, info.fps, info.sample_rate, info.channels);
1976
1977 // Calculate the sample # to start on
1978 int sample_start = round(double(samples_per_frame) * sample_start_percentage);
1979
1980 // Protect against broken (i.e. negative) timestamps
1981 if (whole_frame < 1)
1982 whole_frame = 1;
1983 if (sample_start < 0)
1984 sample_start = 0;
1985
1986 // Prepare final audio packet location
1987 AudioLocation location = {whole_frame, sample_start};
1988
1989 // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps)
1990 if (previous_packet_location.frame != -1) {
1991 if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) {
1992 int64_t orig_frame = location.frame;
1993 int orig_start = location.sample_start;
1994
1995 // Update sample start, to prevent gaps in audio
1996 location.sample_start = previous_packet_location.sample_start;
1997 location.frame = previous_packet_location.frame;
1998
1999 // Debug output
2000 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Detected)", "Source Frame", orig_frame, "Source Audio Sample", orig_start, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2001
2002 } else {
2003 // Debug output
2004 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts);
2005
2006 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2007 for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) {
2008 if (!missing_audio_frames.count(audio_frame)) {
2009 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame);
2010 missing_audio_frames.insert(std::pair<int64_t, int64_t>(audio_frame, previous_packet_location.frame - 1));
2011 }
2012 }
2013 }
2014 }
2015
2016 // Set previous location
2017 previous_packet_location = location;
2018
2019 // Return the associated video frame and starting sample #
2020 return location;
2021}
2022
2023// Create a new Frame (or return an existing one) and add it to the working queue.
2024std::shared_ptr<Frame> FFmpegReader::CreateFrame(int64_t requested_frame) {
2025 // Check working cache
2026 std::shared_ptr<Frame> output = working_cache.GetFrame(requested_frame);
2027
2028 if (!output) {
2029 // Lock
2030 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2031
2032 // (re-)Check working cache
2033 output = working_cache.GetFrame(requested_frame);
2034 if(output) return output;
2035
2036 // Create a new frame on the working cache
2037 output = std::make_shared<Frame>(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels);
2038 output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio
2039 output->ChannelsLayout(info.channel_layout); // update audio channel layout from the parent reader
2040 output->SampleRate(info.sample_rate); // update the frame's sample rate of the parent reader
2041
2042 working_cache.Add(output);
2043
2044 // Set the largest processed frame (if this is larger)
2045 if (requested_frame > largest_frame_processed)
2046 largest_frame_processed = requested_frame;
2047 }
2048 // Return frame
2049 return output;
2050}
2051
2052// Determine if frame is partial due to seek
2053bool FFmpegReader::IsPartialFrame(int64_t requested_frame) {
2054
2055 // Sometimes a seek gets partial frames, and we need to remove them
2056 bool seek_trash = false;
2057 int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame
2058 if (seek_video_frame_found > max_seeked_frame) {
2059 max_seeked_frame = seek_video_frame_found;
2060 }
2061 if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) ||
2062 (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) {
2063 seek_trash = true;
2064 }
2065
2066 return seek_trash;
2067}
2068
2069// Check if a frame is missing and attempt to replace its frame image (and
2070bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) {
2071 // Lock
2072 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2073
2074 // Increment check count for this frame (or init to 1)
2075 ++checked_frames[requested_frame];
2076
2077 // Debug output
2078 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame", "requested_frame", requested_frame, "has_missing_frames", has_missing_frames, "missing_video_frames.size()", missing_video_frames.size(), "checked_count", checked_frames[requested_frame]);
2079
2080 // Missing frames (sometimes frame #'s are skipped due to invalid or missing timestamps)
2081 std::map<int64_t, int64_t>::iterator itr;
2082 bool found_missing_frame = false;
2083
2084 // Special MP3 Handling (ignore more than 1 video frame)
2085 if (info.has_audio and info.has_video) {
2086 // If MP3 with single video frame, handle this special case by copying the previously
2087 // decoded image to the new frame. Otherwise, it will spend a huge amount of
2088 // CPU time looking for missing images for all the audio-only frames.
2089 if (checked_frames[requested_frame] > 8 && !missing_video_frames.count(requested_frame) &&
2090 !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) &&
2091 last_video_frame && last_video_frame->has_image_data && HasAlbumArt()) {
2092 missing_video_frames.insert(std::pair<int64_t, int64_t>(requested_frame, last_video_frame->number));
2093 missing_video_frames_source.insert(std::pair<int64_t, int64_t>(last_video_frame->number, requested_frame));
2094 missing_frames.Add(last_video_frame);
2095 }
2096 }
2097
2098 // Check if requested video frame is a missing
2099 if (missing_video_frames.count(requested_frame)) {
2100 int64_t missing_source_frame = missing_video_frames.find(requested_frame)->second;
2101
2102 // Increment missing source frame check count (or init to 1)
2103 ++checked_frames[missing_source_frame];
2104
2105 // Get the previous frame of this missing frame (if it's available in missing cache)
2106 std::shared_ptr<Frame> parent_frame = missing_frames.GetFrame(missing_source_frame);
2107 if (parent_frame == NULL) {
2108 parent_frame = final_cache.GetFrame(missing_source_frame);
2109 if (parent_frame != NULL) {
2110 // Add missing final frame to missing cache
2111 missing_frames.Add(parent_frame);
2112 }
2113 }
2114
2115 // Create blank missing frame
2116 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2117
2118 // Debug output
2119 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Is Previous Video Frame Final)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2120
2121 // If previous frame found, copy image from previous to missing frame (else we'll just wait a bit and try again later)
2122 if (parent_frame != NULL) {
2123 // Debug output
2124 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (AddImage from Previous Video Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "missing_source_frame", missing_source_frame);
2125
2126 // Add this frame to the processed map (since it's already done)
2127 std::shared_ptr<QImage> parent_image = parent_frame->GetImage();
2128 if (parent_image) {
2129 missing_frame->AddImage(std::make_shared<QImage>(*parent_image));
2130 processed_video_frames[missing_frame->number] = missing_frame->number;
2131 }
2132 }
2133 }
2134
2135 // Check if requested audio frame is a missing
2136 if (missing_audio_frames.count(requested_frame)) {
2137
2138 // Create blank missing frame
2139 std::shared_ptr<Frame> missing_frame = CreateFrame(requested_frame);
2140
2141 // Get Samples per frame (for this frame number)
2142 int samples_per_frame = Frame::GetSamplesPerFrame(missing_frame->number, info.fps, info.sample_rate, info.channels);
2143
2144 // Debug output
2145 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckMissingFrame (Add Silence for Missing Audio Frame)", "requested_frame", requested_frame, "missing_frame->number", missing_frame->number, "samples_per_frame", samples_per_frame);
2146
2147 // Add this frame to the processed map (since it's already done)
2148 missing_frame->AddAudioSilence(samples_per_frame);
2149 processed_audio_frames[missing_frame->number] = missing_frame->number;
2150 }
2151
2152 return found_missing_frame;
2153}
2154
2155// Check the working queue, and move finished frames to the finished queue
2156void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame) {
2157 // Loop through all working queue frames
2158 bool checked_count_tripped = false;
2159 int max_checked_count = 80;
2160
2161 // Check if requested frame is 'missing'
2162 CheckMissingFrame(requested_frame);
2163
2164 while (true) {
2165 // Get the front frame of working cache
2166 std::shared_ptr<Frame> f(working_cache.GetSmallestFrame());
2167
2168 // Was a frame found?
2169 if (!f)
2170 // No frames found
2171 break;
2172
2173 // Remove frames which are too old
2174 if (f->number < (requested_frame - (max_concurrent_frames * 2))) {
2175 working_cache.Remove(f->number);
2176 }
2177
2178 // Check if this frame is 'missing'
2179 CheckMissingFrame(f->number);
2180
2181 // Init # of times this frame has been checked so far
2182 int checked_count = 0;
2183 int checked_frames_size = 0;
2184
2185 bool is_video_ready = false;
2186 bool is_audio_ready = false;
2187 { // limit scope of next few lines
2188 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2189 is_video_ready = processed_video_frames.count(f->number);
2190 is_audio_ready = processed_audio_frames.count(f->number);
2191
2192 // Get check count for this frame
2193 checked_frames_size = checked_frames.size();
2194 if (!checked_count_tripped || f->number >= requested_frame)
2195 checked_count = checked_frames[f->number];
2196 else
2197 // Force checked count over the limit
2198 checked_count = max_checked_count;
2199 }
2200
2201 if (previous_packet_location.frame == f->number && !end_of_stream)
2202 is_audio_ready = false; // don't finalize the last processed audio frame
2203 bool is_seek_trash = IsPartialFrame(f->number);
2204
2205 // Adjust for available streams
2206 if (!info.has_video) is_video_ready = true;
2207 if (!info.has_audio) is_audio_ready = true;
2208
2209 // Make final any frames that get stuck (for whatever reason)
2210 if (checked_count >= max_checked_count && (!is_video_ready || !is_audio_ready)) {
2211 // Debug output
2212 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2213
2214 // Trigger checked count tripped mode (clear out all frames before requested frame)
2215 checked_count_tripped = true;
2216
2217 if (info.has_video && !is_video_ready && last_video_frame) {
2218 // Copy image from last frame
2219 f->AddImage(std::make_shared<QImage>(*last_video_frame->GetImage()));
2220 is_video_ready = true;
2221 }
2222
2223 if (info.has_audio && !is_audio_ready) {
2224 // Mark audio as processed, and indicate the frame has audio data
2225 is_audio_ready = true;
2226 }
2227 }
2228
2229 // Debug output
2230 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size);
2231
2232 // Check if working frame is final
2233 if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash) {
2234 // Debug output
2235 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream);
2236
2237 if (!is_seek_trash) {
2238 // Add missing image (if needed - sometimes end_of_stream causes frames with only audio)
2239 if (info.has_video && !is_video_ready && last_video_frame)
2240 // Copy image from last frame
2241 f->AddImage(std::make_shared<QImage>(*last_video_frame->GetImage()));
2242
2243 // Reset counter since last 'final' frame
2244 num_checks_since_final = 0;
2245
2246 // Move frame to final cache
2247 final_cache.Add(f);
2248
2249 // Add to missing cache (if another frame depends on it)
2250 {
2251 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2252 if (missing_video_frames_source.count(f->number)) {
2253 // Debug output
2254 ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count());
2255 missing_frames.Add(f);
2256 }
2257
2258 // Remove from 'checked' count
2259 checked_frames.erase(f->number);
2260 }
2261
2262 // Remove frame from working cache
2263 working_cache.Remove(f->number);
2264
2265 // Update last frame processed
2266 last_frame = f->number;
2267
2268 } else {
2269 // Seek trash, so delete the frame from the working cache, and never add it to the final cache.
2270 working_cache.Remove(f->number);
2271 }
2272
2273 } else {
2274 // Stop looping
2275 break;
2276 }
2277 }
2278}
2279
2280// Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets.
2281void FFmpegReader::CheckFPS() {
2282 check_fps = true;
2283
2284
2285 int first_second_counter = 0;
2286 int second_second_counter = 0;
2287 int third_second_counter = 0;
2288 int forth_second_counter = 0;
2289 int fifth_second_counter = 0;
2290 int frames_detected = 0;
2291 int64_t pts = 0;
2292
2293 // Loop through the stream
2294 while (true) {
2295 // Get the next packet (if any)
2296 if (GetNextPacket() < 0)
2297 // Break loop when no more packets found
2298 break;
2299
2300 // Video packet
2301 if (packet->stream_index == videoStream) {
2302 // Check if the AVFrame is finished and set it
2303 if (GetAVFrame()) {
2304 // Update PTS / Frame Offset (if any)
2305 UpdatePTSOffset(true);
2306
2307 // Get PTS of this packet
2308 pts = GetVideoPTS();
2309
2310 // Remove pFrame
2311 RemoveAVFrame(pFrame);
2312
2313 // Apply PTS offset
2314 pts += video_pts_offset;
2315
2316 // Get the video packet start time (in seconds)
2317 double video_seconds = double(pts) * info.video_timebase.ToDouble();
2318
2319 // Increment the correct counter
2320 if (video_seconds <= 1.0)
2321 first_second_counter++;
2322 else if (video_seconds > 1.0 && video_seconds <= 2.0)
2323 second_second_counter++;
2324 else if (video_seconds > 2.0 && video_seconds <= 3.0)
2325 third_second_counter++;
2326 else if (video_seconds > 3.0 && video_seconds <= 4.0)
2327 forth_second_counter++;
2328 else if (video_seconds > 4.0 && video_seconds <= 5.0)
2329 fifth_second_counter++;
2330
2331 // Increment counters
2332 frames_detected++;
2333 }
2334 }
2335 }
2336
2337 // Double check that all counters have greater than zero (or give up)
2338 if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) {
2339 // Calculate average FPS (average of first few seconds)
2340 int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter;
2341 int avg_fps = round(sum_fps / 4.0f);
2342
2343 // Update FPS
2344 info.fps = Fraction(avg_fps, 1);
2345
2346 // Update Duration and Length
2347 info.video_length = frames_detected;
2348 info.duration = frames_detected / (sum_fps / 4.0f);
2349
2350 // Update video bit rate
2352 } else if (second_second_counter != 0 && third_second_counter != 0) {
2353 // Calculate average FPS (only on second 2)
2354 int sum_fps = second_second_counter;
2355
2356 // Update FPS
2357 info.fps = Fraction(sum_fps, 1);
2358
2359 // Update Duration and Length
2360 info.video_length = frames_detected;
2361 info.duration = frames_detected / float(sum_fps);
2362
2363 // Update video bit rate
2365 } else {
2366 // Too short to determine framerate, just default FPS
2367 // Set a few important default video settings (so audio can be divided into frames)
2368 info.fps.num = 30;
2369 info.fps.den = 1;
2370
2371 // Calculate number of frames
2372 info.video_length = frames_detected;
2373 info.duration = frames_detected / info.fps.ToFloat();
2374 }
2375}
2376
2377// Remove AVFrame from cache (and deallocate its memory)
2378void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
2379 // Remove pFrame (if exists)
2380 if (remove_frame) {
2381 // Free memory
2382 av_freep(&remove_frame->data[0]);
2383#ifndef WIN32
2384 AV_FREE_FRAME(&remove_frame);
2385#endif
2386 }
2387}
2388
2389// Remove AVPacket from cache (and deallocate its memory)
2390void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) {
2391 // deallocate memory for packet
2392 AV_FREE_PACKET(remove_packet);
2393
2394 // Delete the object
2395 delete remove_packet;
2396}
2397
2398/// Get the smallest video frame that is still being processed
2399int64_t FFmpegReader::GetSmallestVideoFrame() {
2400 // Loop through frame numbers
2401 std::map<int64_t, int64_t>::iterator itr;
2402 int64_t smallest_frame = -1;
2403 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2404 for (itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr) {
2405 if (itr->first < smallest_frame || smallest_frame == -1)
2406 smallest_frame = itr->first;
2407 }
2408
2409 // Return frame number
2410 return smallest_frame;
2411}
2412
2413/// Get the smallest audio frame that is still being processed
2414int64_t FFmpegReader::GetSmallestAudioFrame() {
2415 // Loop through frame numbers
2416 std::map<int64_t, int64_t>::iterator itr;
2417 int64_t smallest_frame = -1;
2418 const GenericScopedLock <CriticalSection> lock(processingCriticalSection);
2419 for (itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr) {
2420 if (itr->first < smallest_frame || smallest_frame == -1)
2421 smallest_frame = itr->first;
2422 }
2423
2424 // Return frame number
2425 return smallest_frame;
2426}
2427
2428// Generate JSON string of this object
2429std::string FFmpegReader::Json() const {
2430
2431 // Return formatted string
2432 return JsonValue().toStyledString();
2433}
2434
2435// Generate Json::Value for this object
2436Json::Value FFmpegReader::JsonValue() const {
2437
2438 // Create root json object
2439 Json::Value root = ReaderBase::JsonValue(); // get parent properties
2440 root["type"] = "FFmpegReader";
2441 root["path"] = path;
2442
2443 // return JsonValue
2444 return root;
2445}
2446
2447// Load JSON string into this object
2448void FFmpegReader::SetJson(const std::string value) {
2449
2450 // Parse JSON string into JSON objects
2451 try {
2452 const Json::Value root = openshot::stringToJson(value);
2453 // Set all values that match
2454 SetJsonValue(root);
2455 }
2456 catch (const std::exception& e) {
2457 // Error parsing JSON (or missing keys)
2458 throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
2459 }
2460}
2461
2462// Load Json::Value into this object
2463void FFmpegReader::SetJsonValue(const Json::Value root) {
2464
2465 // Set parent data
2467
2468 // Set data from Json (if key is found)
2469 if (!root["path"].isNull())
2470 path = root["path"].asString();
2471
2472 // Re-Open path, and re-init everything (if needed)
2473 if (is_open) {
2474 Close();
2475 Open();
2476 }
2477}
Header file for all Exception classes.
AVPixelFormat hw_de_av_pix_fmt_global
AVHWDeviceType hw_de_av_device_type_global
int hw_de_on
Header file for FFmpegReader class.
#define AV_FREE_CONTEXT(av_context)
#define SWR_INIT(ctx)
#define AV_FREE_FRAME(av_frame)
#define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count)
#define SWR_ALLOC()
#define SWR_CLOSE(ctx)
#define AV_GET_CODEC_TYPE(av_stream)
#define PixelFormat
#define AV_GET_CODEC_PIXEL_FORMAT(av_stream, av_context)
#define AV_GET_CODEC_CONTEXT(av_stream, av_codec)
#define AV_FIND_DECODER_CODEC_ID(av_stream)
#define AV_ALLOCATE_FRAME()
#define AV_REGISTER_ALL
#define PIX_FMT_RGBA
#define SWR_FREE(ctx)
#define AV_COPY_PICTURE_DATA(av_frame, buffer, pix_fmt, width, height)
#define AV_FREE_PACKET(av_packet)
#define SWRCONTEXT
#define AVCODEC_REGISTER_ALL
#define AVCODEC_MAX_AUDIO_FRAME_SIZE
#define AV_GET_CODEC_ATTRIBUTES(av_stream, av_context)
#define MY_INPUT_BUFFER_PADDING_SIZE
#define AV_GET_SAMPLE_FORMAT(av_stream, av_context)
#define AV_RESET_FRAME(av_frame)
AVDictionary * opts
if(!codec) codec
#define FF_NUM_PROCESSORS
#define OPEN_MP_NUM_PROCESSORS
Header file for Timeline class.
void SetMaxBytesFromInfo(int64_t number_of_frames, int width, int height, int sample_rate, int channels)
Set maximum bytes to a different amount based on a ReaderInfo struct.
Definition: CacheBase.cpp:49
int64_t Count()
Count the frames in the queue.
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
void Remove(int64_t frame_number)
Remove a specific frame.
void Clear()
Clear the cache of all frames.
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
openshot::TimelineBase * ParentTimeline()
Get the associated Timeline pointer (if any)
Definition: ClipBase.h:113
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
openshot::Keyframe scale_x
Curve representing the horizontal scaling in percent (0 to 1)
Definition: Clip.h:305
openshot::Keyframe scale_y
Curve representing the vertical scaling in percent (0 to 1)
Definition: Clip.h:306
openshot::ScaleType scale
The scale determines how a clip should be resized to fit its parent.
Definition: Clip.h:175
double Y
The Y value of the coordinate (usually representing the value of the property being animated)
Definition: Coordinate.h:57
void Open() override
Open File - which is called by the constructor automatically.
Json::Value JsonValue() const override
Generate Json::Value for this object.
bool GetIsDurationKnown()
Return true if frame can be read with GetFrame()
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
CacheMemory final_cache
Final cache object used to hold final frames.
Definition: FFmpegReader.h:233
virtual ~FFmpegReader()
Destructor.
std::string Json() const override
Generate JSON string of this object.
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
void Close() override
Close File.
void SetJson(const std::string value) override
Load JSON string into this object.
This class represents a fraction.
Definition: Fraction.h:48
int num
Numerator for the fraction.
Definition: Fraction.h:50
float ToFloat()
Return this fraction as a float (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:54
double ToDouble() const
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:59
Fraction Reciprocal() const
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:94
int den
Denominator for the fraction.
Definition: Fraction.h:51
int GetSamplesPerFrame(openshot::Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:536
Exception when no valid codec is found for a file.
Definition: Exceptions.h:158
Exception for files that can not be found or opened.
Definition: Exceptions.h:174
Exception for invalid JSON.
Definition: Exceptions.h:206
Point GetMaxPoint() const
Get max point (by Y coordinate)
Definition: KeyFrame.cpp:255
Exception when no streams are found in the file.
Definition: Exceptions.h:270
Exception when memory could not be allocated.
Definition: Exceptions.h:322
Coordinate co
This is the primary coordinate.
Definition: Point.h:84
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
juce::CriticalSection processingCriticalSection
Definition: ReaderBase.h:102
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: ReaderBase.cpp:171
virtual Json::Value JsonValue() const =0
Generate Json::Value for this object.
Definition: ReaderBase.cpp:116
openshot::ClipBase * ParentClip()
Parent clip object of this reader (which can be unparented and NULL)
Definition: ReaderBase.cpp:254
Exception when a reader is closed, and a frame is requested.
Definition: Exceptions.h:338
int DE_LIMIT_WIDTH_MAX
Maximum columns that hardware decode can handle.
Definition: Settings.h:95
int HW_DE_DEVICE_SET
Which GPU to use to decode (0 is the first)
Definition: Settings.h:98
int DE_LIMIT_HEIGHT_MAX
Maximum rows that hardware decode can handle.
Definition: Settings.h:92
static Settings * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: Settings.cpp:41
int HARDWARE_DECODER
Use video codec for faster video decoding (if supported)
Definition: Settings.h:80
int preview_height
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:58
int preview_width
Optional preview width of timeline image. If your preview window is smaller than the timeline,...
Definition: TimelineBase.h:57
This class represents a timeline.
Definition: Timeline.h:168
void AppendDebugMethod(std::string method_name, std::string arg1_name="", float arg1_value=-1.0, std::string arg2_name="", float arg2_value=-1.0, std::string arg3_name="", float arg3_value=-1.0, std::string arg4_name="", float arg4_value=-1.0, std::string arg5_name="", float arg5_value=-1.0, std::string arg6_name="", float arg6_value=-1.0)
Append debug information.
Definition: ZmqLogger.cpp:190
static ZmqLogger * Instance()
Create or get an instance of this logger singleton (invoke the class with this method)
Definition: ZmqLogger.cpp:52
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
@ SCALE_FIT
Scale the clip until either height or width fills the canvas (with no cropping)
Definition: Enums.h:55
@ SCALE_STRETCH
Scale the clip until both height and width fill the canvas (distort to fit)
Definition: Enums.h:56
@ SCALE_CROP
Scale the clip until both height and width fill the canvas (cropping the overlap)
Definition: Enums.h:54
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
This struct holds the associated video frame and starting sample # for an audio packet.
Definition: FFmpegReader.h:60
bool is_near(AudioLocation location, int samples_per_frame, int64_t amount)
int audio_bit_rate
The bit rate of the audio stream (in bytes)
Definition: ReaderBase.h:81
int video_bit_rate
The bit rate of the video stream (in bytes)
Definition: ReaderBase.h:71
float duration
Length of time (in seconds)
Definition: ReaderBase.h:65
openshot::Fraction audio_timebase
The audio timebase determines how long each audio packet should be played.
Definition: ReaderBase.h:86
int width
The width of the video (in pixesl)
Definition: ReaderBase.h:68
int channels
The number of audio channels used in the audio stream.
Definition: ReaderBase.h:83
openshot::Fraction fps
Frames per second, as a fraction (i.e. 24/1 = 24 fps)
Definition: ReaderBase.h:70
openshot::Fraction display_ratio
The ratio of width to height of the video stream (i.e. 640x480 has a ratio of 4/3)
Definition: ReaderBase.h:73
int height
The height of the video (in pixels)
Definition: ReaderBase.h:67
int pixel_format
The pixel format (i.e. YUV420P, RGB24, etc...)
Definition: ReaderBase.h:69
int64_t video_length
The number of frames in the video stream.
Definition: ReaderBase.h:75
std::string acodec
The name of the audio codec used to encode / decode the video stream.
Definition: ReaderBase.h:80
std::map< std::string, std::string > metadata
An optional map/dictionary of metadata for this reader.
Definition: ReaderBase.h:87
std::string vcodec
The name of the video codec used to encode / decode the video stream.
Definition: ReaderBase.h:74
openshot::Fraction pixel_ratio
The pixel ratio of the video stream as a fraction (i.e. some pixels are not square)
Definition: ReaderBase.h:72
openshot::ChannelLayout channel_layout
The channel layout (mono, stereo, 5 point surround, etc...)
Definition: ReaderBase.h:84
bool has_video
Determines if this file has a video stream.
Definition: ReaderBase.h:62
bool has_audio
Determines if this file has an audio stream.
Definition: ReaderBase.h:63
openshot::Fraction video_timebase
The video timebase determines how long each frame stays on the screen.
Definition: ReaderBase.h:77
int video_stream_index
The index of the video stream.
Definition: ReaderBase.h:76
int sample_rate
The number of audio samples per second (44100 is a common sample rate)
Definition: ReaderBase.h:82
int audio_stream_index
The index of the audio stream.
Definition: ReaderBase.h:85
int64_t file_size
Size of file (in bytes)
Definition: ReaderBase.h:66