From 6715acdef8b0543966f8656cf63c1891db983e87 Mon Sep 17 00:00:00 2001 From: evgeny Date: Sun, 18 Oct 2026 16:50:33 +0200 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20GPU-=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D0=B9=D0=B5=D1=80?= =?UTF-8?q?=20=D0=B7=D0=B2=D0=BE=D0=BD=D0=BA=D0=B0=20=D1=81=20=D0=B8=D1=81?= =?UTF-8?q?=D1=85=D0=BE=D0=B4=D0=BD=D1=8B=D0=BC=D0=B8=20PTS=20=D0=B8=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=B4=D1=82=D0=B2=D0=B5=D1=80=D0=B6=D0=B4=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=D0=BC=20=D1=81=D0=BC=D0=B5=D0=BD=D1=8B=20?= =?UTF-8?q?=D0=BF=D0=BE=20IDR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/chatgui/src/call_video_encoder.cpp | 107 +++++++++++++---------- tools/chatgui/src/call_video_encoder.h | 15 ++-- tools/chatgui/src/live_video_encoder.cpp | 72 ++++++++++++--- 3 files changed, 130 insertions(+), 64 deletions(-) diff --git a/tools/chatgui/src/call_video_encoder.cpp b/tools/chatgui/src/call_video_encoder.cpp index 68efba95..81777736 100644 --- a/tools/chatgui/src/call_video_encoder.cpp +++ b/tools/chatgui/src/call_video_encoder.cpp @@ -17,7 +17,7 @@ bool CallVideoEncoder::open(const call_video_format& format, uint32_t bitrate, c config.bitrate = bitrate; config.references = 1; config.backend = backend; config.profile = format.codec == CALL_VIDEO_H264 ? LiveVideoProfile::H264ConstrainedBaseline : LiveVideoProfile::H265Main; auto candidate = std::make_unique(); - if (!candidate->open(config)) { + if (!candidate->open(config, backend.isEmpty() ? LiveVideoEncoder::Backend::HardwareOnly : LiveVideoEncoder::Backend::Automatic)) { m_error = candidate->error(); DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video open failed: %s", qPrintable(m_error)); return false; } close(); m_encoder = std::move(candidate); m_config = config; m_config.backend = m_encoder->name(); m_format = format; @@ -29,9 +29,10 @@ void CallVideoEncoder::close() { if (m_encoder) DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "call video encoder close generation=%llu pending_request=%llu", (unsigned long long)m_format.generation, (unsigned long long)m_request); m_prepared = {}; // Завершение фоновой подготовки ждём только при остановке/смене режима. - m_encoder.reset(); m_config = {}; m_config.bitrate = 0; m_error.clear(); + m_replacement.reset(); m_encoder.reset(); m_config = {}; m_config.bitrate = 0; m_error.clear(); m_format = {}; m_request = m_lastRequest = m_lastPts = m_lastKey = 0; - m_pendingBitrate = 0; m_haveFrame = false; + m_lastOutputPts = m_switchPts = 0; m_backgroundMs = 0; m_switchInputs = 0; + m_pendingBitrate = 0; m_haveFrame = m_haveInput = m_keyPending = false; } // Ядро сериализует команды; повторные/чужие requests не заменяют уже ожидающую команду. @@ -64,71 +65,83 @@ bool CallVideoEncoder::requestBitrate(uint64_t generation, uint64_t request, uin } // Capture PTS продолжает идти при backpressure, поэтому запрет отправки не задерживает применение. -bool CallVideoEncoder::encode(const QImage& image, uint64_t pts, bool forceKey, LiveEncodedFrame& output, +bool CallVideoEncoder::encode(const QImage& image, uint64_t pts, bool forceKey, QList& output, CallVideoRateCompletion& completion) { - completion = {}; - if (!m_encoder || image.isNull() || pts > INT64_MAX || (m_haveFrame && pts <= m_lastPts)) { + completion = {}; output.clear(); + if (!m_encoder || image.isNull() || pts > INT64_MAX || (m_haveInput && pts <= m_lastPts)) { m_error = "Invalid call video frame or non-monotonic PTS"; DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video encode invalid generation=%llu pts=%llu last=%llu active=%d image_null=%d", (unsigned long long)m_format.generation, (unsigned long long)pts, (unsigned long long)m_lastPts, m_encoder != nullptr, image.isNull()); return false; } - bool key = forceKey || !m_haveFrame || pts - m_lastKey >= 1000000; + bool key = !m_keyPending && (forceKey || !m_haveInput || (pts >= m_lastKey && pts - m_lastKey >= 1000000)); QList frames; bool prepared = !m_prepared.valid() || m_prepared.wait_for(std::chrono::seconds(0)) == std::future_status::ready; if (key && m_request && !prepared) DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "call video preparation pending request=%llu; keep current encoder", (unsigned long long)m_request); - if (key && m_request && prepared) { - completion = {m_format.generation, m_request, 0}; - if (m_pendingBitrate == (uint32_t)m_config.bitrate) completion.bitrate = m_pendingBitrate; + if (key && m_request && prepared && !m_replacement) { + if (m_pendingBitrate == (uint32_t)m_config.bitrate) { + completion = {m_format.generation, m_request, m_pendingBitrate}; m_request = 0; m_pendingBitrate = 0; + } else { - LiveVideoConfig next = m_config; next.bitrate = m_pendingBitrate; - auto replacement = m_prepared.get(); auto& candidate = replacement.encoder; - QElapsedTimer timer; timer.start(); - bool ready = candidate && candidate->encode(image, pts, true, frames) - && frames.size() == 1 && frames.front().key && frames.front().pts == pts; - const qint64 encodedUs = timer.nsecsElapsed() / 1000; - if (ready) ready = !call_video_check_key(&m_format, - reinterpret_cast(frames.front().data.constData()), frames.front().data.size()); - const qint64 verifiedUs = timer.nsecsElapsed() / 1000; - if (ready) { - m_encoder = std::move(candidate); m_config = next; completion.bitrate = m_pendingBitrate; - const qint64 committedUs = timer.nsecsElapsed() / 1000; + auto preparedEncoder = m_prepared.get(); m_backgroundMs = preparedEncoder.elapsed; + m_replacement = std::move(preparedEncoder.encoder); m_switchPts = pts; m_switchInputs = 0; + if (!m_replacement) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU replacement open failed request=%llu: %s", + (unsigned long long)m_request, qPrintable(preparedEncoder.error)); + completion = {m_format.generation, m_request, 0}; m_request = 0; m_pendingBitrate = 0; + } + } + } + QElapsedTimer timer; timer.start(); + bool encoded = false; + if (m_replacement) { + ++m_switchInputs; + encoded = m_replacement->encode(image, pts, m_switchInputs == 1, frames); + if (encoded && !frames.isEmpty()) { + const auto& first = frames.front(); + encoded = first.key && first.pts == m_switchPts && !call_video_check_key(&m_format, + reinterpret_cast(first.data.constData()), first.data.size()); + if (encoded) { + const qint64 encodeUs = timer.nsecsElapsed() / 1000; + m_encoder = std::move(m_replacement); m_config.bitrate = m_pendingBitrate; + completion = {m_format.generation, m_request, m_pendingBitrate}; DEBUG_INFO(DEBUG_CATEGORY_CALL, "call video bitrate committed generation=%llu request=%llu bitrate=%u pts=%llu background_ms=%lld " - "commit_ms=%lld encode_us=%lld verify_us=%lld retire_us=%lld bytes=%lld", + "encode_us=%lld retire_us=%lld pipeline_frames=%u pipeline_pts_us=%llu bytes=%lld", (unsigned long long)m_format.generation, (unsigned long long)m_request, completion.bitrate, - (unsigned long long)pts, (long long)replacement.elapsed, (long long)(committedUs / 1000), - (long long)encodedUs, (long long)(verifiedUs - encodedUs), (long long)(committedUs - verifiedUs), - (long long)frames.front().data.size()); - } else { - m_error = candidate ? candidate->error() : replacement.error; - if (m_error.isEmpty()) m_error = "Replacement encoder did not produce a compatible immediate IDR"; - DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video replacement failed generation=%llu request=%llu target=%u: %s", - (unsigned long long)m_format.generation, (unsigned long long)m_request, m_pendingBitrate, qPrintable(m_error)); - frames.clear(); + (unsigned long long)first.pts, (long long)m_backgroundMs, (long long)encodeUs, + (long long)(timer.nsecsElapsed() / 1000 - encodeUs), m_switchInputs, + (unsigned long long)(pts - first.pts), (long long)first.data.size()); + m_request = 0; m_pendingBitrate = 0; } } - m_request = 0; m_pendingBitrate = 0; + if (!encoded) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU replacement failed request=%llu inputs=%u frames=%lld: %s", + (unsigned long long)m_request, m_switchInputs, (long long)frames.size(), qPrintable(m_replacement->error())); + completion = {m_format.generation, m_request, 0}; m_request = 0; m_pendingBitrate = 0; + m_replacement.reset(); frames.clear(); key = true; m_keyPending = false; + } } - if (frames.isEmpty() && !m_encoder->encode(image, pts, key, frames)) { + if (!encoded) encoded = m_encoder->encode(image, pts, key, frames); + m_haveInput = true; m_lastPts = pts; if (key) m_keyPending = true; + if (!encoded) { m_error = m_encoder->error(); if (completion.request) completion.bitrate = 0; DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video encoding failed generation=%llu: %s", (unsigned long long)m_format.generation, qPrintable(m_error)); return false; } - if (frames.size() != 1 || frames.front().pts != pts || (key && !frames.front().key)) { - m_error = "Call encoder delayed output or failed to produce IDR"; - if (completion.request) completion.bitrate = 0; - DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video output contract failed generation=%llu pts=%llu frames=%lld key_requested=%d", - (unsigned long long)m_format.generation, (unsigned long long)pts, (long long)frames.size(), key); return false; - } - if (frames.front().key && call_video_check_key(&m_format, - reinterpret_cast(frames.front().data.constData()), frames.front().data.size())) { - m_error = "Encoder produced an incompatible IDR"; - if (completion.request) completion.bitrate = 0; - return false; + for (const auto& frame : frames) { + if ((m_haveFrame && frame.pts <= m_lastOutputPts) || frame.pts > pts || (!m_haveFrame && !frame.key) + || (frame.key && call_video_check_key(&m_format, + reinterpret_cast(frame.data.constData()), frame.data.size()))) { + m_error = "GPU produced an incompatible video frame"; + if (completion.request) completion.bitrate = 0; + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU output rejected pts=%llu last=%llu capture=%llu key=%d", + (unsigned long long)frame.pts, (unsigned long long)m_lastOutputPts, (unsigned long long)pts, frame.key); return false; + } + m_haveFrame = true; m_lastOutputPts = frame.pts; + if (frame.key) { m_lastKey = frame.pts; m_keyPending = false; } } - m_haveFrame = true; m_lastPts = pts; if (frames.front().key) m_lastKey = pts; - output = std::move(frames.front()); return true; + output = std::move(frames); return true; } diff --git a/tools/chatgui/src/call_video_encoder.h b/tools/chatgui/src/call_video_encoder.h index c29375f3..5bbf3ce7 100644 --- a/tools/chatgui/src/call_video_encoder.h +++ b/tools/chatgui/src/call_video_encoder.h @@ -1,6 +1,7 @@ /* Кодер видеотрека звонка; только video worker, без камеры, транспорта и таймеров. * Смена bitrate ожидает ближайший IDR (период <= 1с по capture PTS). Новый encoder - * заменяет старый только после полного первого IDR. Completion передаётся в uasync + * заменяет старый только после полного первого IDR. По умолчанию выбирается GPU; + * явный backend позволяет отдельно проверять software в тестах. Completion передаётся в uasync * независимо от admission видео. PTS, generation и frame_id принадлежат владельцу. * open получает согласованный call_video_format; соответствие level/размера/FPS * проверяет общий модуль, фактический bitstream — LiveVideoEncoder. CAPS проверяет владелец. */ @@ -19,8 +20,9 @@ public: bool open(const call_video_format& format, uint32_t bitrate, const QString& backend = {}); void close(); bool requestBitrate(uint64_t generation, uint64_t request, uint32_t bitrate); - // Один вход -> один выход. На false output не меняется; completion нужно обработать и на ошибке. - bool encode(const QImage& image, uint64_t pts, bool forceKey, LiveEncodedFrame& output, CallVideoRateCompletion& completion); + // Готовые кадры выдаются сразу пакетом; пустой output означает работу GPU, без ожидания следующего capture. + // На false output пуст; completion нужно обработать и на ошибке. PTS готового выхода сохраняет capture PTS. + bool encode(const QImage& image, uint64_t pts, bool forceKey, QList& output, CallVideoRateCompletion& completion); uint32_t bitrate() const { return m_encoder ? m_config.bitrate : 0; } QString error() const { return m_error; } private: @@ -30,12 +32,15 @@ private: qint64 elapsed = 0; }; std::unique_ptr m_encoder; + std::unique_ptr m_replacement; // Новый GPU работает до первого IDR; старый сохранён для отказа замены. LiveVideoConfig m_config; call_video_format m_format{}; QString m_error; - uint64_t m_request = 0, m_lastRequest = 0, m_lastPts = 0, m_lastKey = 0; + uint64_t m_request = 0, m_lastRequest = 0, m_lastPts = 0, m_lastKey = 0, m_lastOutputPts = 0, m_switchPts = 0; + qint64 m_backgroundMs = 0; + unsigned m_switchInputs = 0; uint32_t m_pendingBitrate = 0; - bool m_haveFrame = false; + bool m_haveFrame = false, m_haveInput = false, m_keyPending = false; // Отдельный контекст готовится в фоне; worker забирает его только после завершения open. std::future m_prepared; }; diff --git a/tools/chatgui/src/live_video_encoder.cpp b/tools/chatgui/src/live_video_encoder.cpp index dfa91a79..3103a9be 100644 --- a/tools/chatgui/src/live_video_encoder.cpp +++ b/tools/chatgui/src/live_video_encoder.cpp @@ -4,6 +4,7 @@ #include #include #include +#include extern "C" { #include "../../../src/stream/stream_media.h" #ifdef HAVE_FFMPEG @@ -20,6 +21,8 @@ struct LiveVideoEncoder::Impl { QString error, name; LiveVideoConfig config; int64_t submittedFrames = 0; + bool forcedKeys = true; + std::deque> timestamps; // Codec PTS -> исходный capture PTS; максимум восемь входов звонка. QElapsedTimer reportTimer; unsigned inputs = 0, outputs = 0, keys = 0; bool slowReported = false; @@ -64,14 +67,15 @@ bool matchesBackend(const QByteArray& name, LiveVideoEncoder::Backend backend) { bool hardware = name.endsWith("_nvenc") || name.endsWith("_qsv") || name.endsWith("_amf") || name.endsWith("_vaapi"); return backend == LiveVideoEncoder::Backend::Automatic || hardware == (backend == LiveVideoEncoder::Backend::HardwareOnly); } -AVPixelFormat softwareFormat(const AVCodec* codec) { +AVPixelFormat softwareFormat(const AVCodec* codec, bool hardware = false) { const AVPixelFormat* formats = nullptr; #if LIBAVCODEC_VERSION_MAJOR >= 61 avcodec_get_supported_config(nullptr, codec, AV_CODEC_CONFIG_PIX_FORMAT, 0, reinterpret_cast(&formats), nullptr); #else formats = codec->pix_fmts; #endif - for (auto preferred : {AV_PIX_FMT_YUV420P, AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P10LE}) { + for (auto preferred : {hardware ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUV420P, + hardware ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P10LE}) { if (!formats) return preferred; for (auto f = formats; *f != AV_PIX_FMT_NONE; f++) if (*f == preferred) return preferred; } @@ -133,7 +137,6 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) { for (const auto& name : encoders(config.codec)) { if (!matchesBackend(name, backend)) continue; if (!config.backend.isEmpty() && name != config.backend.toLatin1()) continue; - if (constrained && name != "libx264" && name != "libx265") continue; const AVCodec* encoder = avcodec_find_encoder_by_name(name.constData()); if (!encoder) continue; // Проба не оставляет references/PTS в live-контексте: после двух KEY encoder открывается заново. @@ -143,14 +146,19 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) { if (!m->codec) { m->error = "Video encoder allocation failed"; break; } auto* c = m->codec; c->width = config.width; c->height = config.height; c->time_base = {1, 1000000}; c->framerate = {config.fps, 1}; + if (constrained && name.endsWith("_qsv")) c->time_base = {1, 90000}; c->bit_rate = config.bitrate; c->rc_max_rate = config.bitrate; c->rc_buffer_size = config.vbvBits ? config.vbvBits : config.bitrate / 4; c->gop_size = config.fps; c->max_b_frames = 0; c->thread_count = 2; if (config.references) c->refs = config.references; c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER | AV_CODEC_FLAG_CLOSED_GOP; + if (constrained) { + c->flags |= AV_CODEC_FLAG_LOW_DELAY; + c->level = name.endsWith("_qsv") && config.codec == STREAM_MEDIA_H265 ? config.level / 3 : config.level; + } c->color_range = AVCOL_RANGE_MPEG; c->colorspace = AVCOL_SPC_BT709; c->color_primaries = AVCOL_PRI_BT709; c->color_trc = AVCOL_TRC_BT709; - AVPixelFormat sw = constrained ? AV_PIX_FMT_YUV420P : softwareFormat(encoder); + AVPixelFormat sw = softwareFormat(encoder, !matchesBackend(name, Backend::SoftwareOnly)); int result = 0; if (name.endsWith("_vaapi")) { sw = AV_PIX_FMT_NV12; c->pix_fmt = AV_PIX_FMT_VAAPI; @@ -192,22 +200,43 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) { av_dict_set(&options, "preset", "p1", 0); av_dict_set(&options, "tune", "ull", 0); av_dict_set(&options, "zerolatency", "1", 0); av_dict_set(&options, "rc-lookahead", "0", 0); av_dict_set(&options, "forced-idr", "1", 0); + if (constrained) { + av_dict_set(&options, "delay", "0", 0); av_dict_set(&options, "rc", "cbr", 0); + av_dict_set(&options, "profile", config.codec == STREAM_MEDIA_H264 ? "baseline" : "main", 0); + av_dict_set_int(&options, "level", config.level, 0); + if (config.codec == STREAM_MEDIA_H265) av_dict_set(&options, "tier", "main", 0); + } } else if (name.endsWith("_amf")) { av_dict_set(&options, "usage", "ultralowlatency", 0); av_dict_set(&options, "quality", "speed", 0); av_dict_set(&options, "async_depth", "1", 0); bool forceIdr = config.codec == STREAM_MEDIA_H264 || config.codec == STREAM_MEDIA_H265; - if (forceIdr) av_dict_set(&options, "forced_idr", "1", 0); + m->forcedKeys = forceIdr && av_opt_find(c->priv_data, "forced_idr", nullptr, 0, 0); + if (m->forcedKeys) av_dict_set(&options, "forced_idr", "1", 0); + if (constrained) { + av_dict_set(&options, "rc", "cbr", 0); + av_dict_set(&options, "profile", config.codec == STREAM_MEDIA_H264 ? "constrained_baseline" : "main", 0); + if (config.codec == STREAM_MEDIA_H265) av_dict_set(&options, "profile_tier", "main", 0); + } DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop AMF low latency settings name=%s usage=ultralowlatency quality=speed async_depth=1 forced_idr=%d", - name.constData(), forceIdr); + name.constData(), m->forcedKeys); } else if (name == "libvpx-vp9" || name == "libaom-av1") { av_dict_set(&options, "deadline", "realtime", 0); av_dict_set(&options, "cpu-used", "8", 0); av_dict_set(&options, "lag-in-frames", "0", 0); av_dict_set(&options, "row-mt", "1", 0); } else if (name.endsWith("_qsv")) { av_dict_set(&options, "async_depth", "1", 0); av_dict_set(&options, "look_ahead", "0", 0); + if (constrained) { + av_dict_set(&options, "forced_idr", "1", 0); av_dict_set(&options, "preset", "veryfast", 0); + av_dict_set(&options, "profile", config.codec == STREAM_MEDIA_H264 ? "baseline" : "main", 0); + if (config.codec == STREAM_MEDIA_H265) av_dict_set(&options, "tier", "main", 0); + } } else if (name.endsWith("_vaapi")) { av_dict_set(&options, "async_depth", "1", 0); + if (constrained) { + c->profile = config.codec == STREAM_MEDIA_H264 ? AV_PROFILE_H264_CONSTRAINED_BASELINE : AV_PROFILE_HEVC_MAIN; + av_dict_set(&options, "rc_mode", "CBR", 0); + } } else if (name == "libvvenc") av_dict_set(&options, "preset", "fast", 0); if (result >= 0) result = avcodec_open2(c, encoder, &options); av_dict_free(&options); @@ -269,7 +298,7 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) { name.constData(), qPrintable(m->error)); break; } - if (name.endsWith("_amf")) { + if (!m->forcedKeys) { DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop AMF uses periodic IDR name=%s gop_frames=%d forced_key_requests=disabled", name.constData(), c->gop_size); @@ -288,9 +317,9 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) { continue; } DEBUG_INFO(DEBUG_CATEGORY_VIDEO, - "desktop encoder ready name=%s codec=%d size=%dx%d fps=%d bitrate=%d level=%d refs=%d vbv=%d profile_verified=%d", + "desktop encoder ready name=%s codec=%d size=%dx%d fps=%d bitrate=%d level=%d refs=%d vbv=%d profile_verified=%d forced_keys=%d", name.constData(), config.codec, config.width, config.height, config.fps, config.bitrate, - config.level, config.references, c->rc_buffer_size, constrained); + config.level, config.references, c->rc_buffer_size, constrained, m->forcedKeys); return true; } } @@ -325,11 +354,18 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList int stride[4] = {static_cast(canvas.bytesPerLine()), 0, 0, 0}; result = sws_scale(m->scale, input, stride, 0, canvas.height(), m->frame->data, m->frame->linesize); } - bool forceKey = key && (!m->name.endsWith("_amf") || m->submittedFrames == 0); + bool constrained = m->config.profile != LiveVideoProfile::Automatic; + if (constrained && m->timestamps.size() >= 8) { + m->error = "GPU encoder exceeded the bounded input pipeline"; + DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "call encoder stalled name=%s pending=%zu pts=%llu", + qPrintable(m->name), m->timestamps.size(), (unsigned long long)pts); return false; + } + bool forceKey = key && (m->forcedKeys || m->submittedFrames == 0); if (key && !forceKey) DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop AMF ignores forced key request; periodic GOP IDR remains active name=%s submitted=%lld", qPrintable(m->name), (long long)m->submittedFrames); - m->frame->pts = static_cast(pts); m->frame->pict_type = forceKey ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; + m->frame->pts = av_rescale_q(static_cast(pts), AVRational{1, 1000000}, m->codec->time_base); + m->frame->pict_type = forceKey ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; if (forceKey) m->frame->flags |= AV_FRAME_FLAG_KEY; else m->frame->flags &= ~AV_FRAME_FLAG_KEY; AVFrame* hardware = nullptr; if (result >= 0 && m->codec->hw_frames_ctx) { @@ -341,7 +377,10 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList const qint64 preparedUs = timer.nsecsElapsed() / 1000; if (result >= 0) result = avcodec_send_frame(m->codec, hardware ? hardware : m->frame); const qint64 submittedUs = timer.nsecsElapsed() / 1000; - if (result >= 0) m->submittedFrames++; + if (result >= 0) { + m->submittedFrames++; + if (constrained) m->timestamps.emplace_back(m->frame->pts, pts); + } av_frame_free(&hardware); AVPacket* packet = av_packet_alloc(); if (!packet) result = AVERROR(ENOMEM); @@ -355,6 +394,15 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList } LiveEncodedFrame encoded; encoded.key = (packet->flags & AV_PKT_FLAG_KEY) != 0; encoded.pts = packet->pts >= 0 ? static_cast(packet->pts) : pts; + if (constrained) { + if (m->timestamps.empty() || packet->pts != m->timestamps.front().first) { + DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "call GPU output PTS mismatch name=%s packet=%lld expected=%lld pending=%zu", + qPrintable(m->name), (long long)packet->pts, + (long long)(m->timestamps.empty() ? -1 : m->timestamps.front().first), m->timestamps.size()); + result = AVERROR_INVALIDDATA; break; + } + encoded.pts = m->timestamps.front().second; m->timestamps.pop_front(); + } if (key) DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop requested KEY name=%s input_PTS=%llu output_PTS=%llu key=%d flags=%x bytes=%d", qPrintable(m->name), (unsigned long long)pts, (unsigned long long)encoded.pts, encoded.key, packet->flags, packet->size); if (encoded.key) {