jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Добавить GPU-конвейер звонка с исходными PTS и подтверждением смены по IDR

Browse Source
This commit is contained in:
evgeny committed 2026-10-18 16:50:33 +02:00
1 parent 3ed9acb384
commit 6715acdef8
3 files changed
+133 -67
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/chatgui/src/live_video_encoder.cpp
+60 -12
View File
@@ -4,6 +4,7 @@
#include <QPainter>
#include <algorithm>
#include <cstring>
#include <deque>
extern "C" {
#include "../../../src/stream/stream_media.h"
#ifdef HAVE_FFMPEG
@@ -20,6 +21,8 @@ struct LiveVideoEncoder::Impl {
QString error, name;
LiveVideoConfig config;
int64_t submittedFrames = 0;
bool forcedKeys = true;
std::deque<std::pair<int64_t, uint64_t>> timestamps; // Codec PTS -> исходный capture PTS; максимум восемь входов звонка.
QElapsedTimer reportTimer;
unsigned inputs = 0, outputs = 0, keys = 0;
bool slowReported = false;
@@ -64,14 +67,15 @@ bool matchesBackend(const QByteArray& name, LiveVideoEncoder::Backend backend) {
bool hardware = name.endsWith("_nvenc") || name.endsWith("_qsv") || name.endsWith("_amf") || name.endsWith("_vaapi");
return backend == LiveVideoEncoder::Backend::Automatic || hardware == (backend == LiveVideoEncoder::Backend::HardwareOnly);
}
AVPixelFormat softwareFormat(const AVCodec* codec) {
AVPixelFormat softwareFormat(const AVCodec* codec, bool hardware = false) {
const AVPixelFormat* formats = nullptr;
#if LIBAVCODEC_VERSION_MAJOR >= 61
avcodec_get_supported_config(nullptr, codec, AV_CODEC_CONFIG_PIX_FORMAT, 0, reinterpret_cast<const void**>(&formats), nullptr);
#else
formats = codec->pix_fmts;
#endif
for (auto preferred : {AV_PIX_FMT_YUV420P, AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P10LE}) {
for (auto preferred : {hardware ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUV420P,
hardware ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P10LE}) {
if (!formats) return preferred;
for (auto f = formats; *f != AV_PIX_FMT_NONE; f++) if (*f == preferred) return preferred;
}
@@ -133,7 +137,6 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
for (const auto& name : encoders(config.codec)) {
if (!matchesBackend(name, backend)) continue;
if (!config.backend.isEmpty() && name != config.backend.toLatin1()) continue;
if (constrained && name != "libx264" && name != "libx265") continue;
const AVCodec* encoder = avcodec_find_encoder_by_name(name.constData());
if (!encoder) continue;
// Проба не оставляет references/PTS в live-контексте: после двух KEY encoder открывается заново.
@@ -143,14 +146,19 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
if (!m->codec) { m->error = "Video encoder allocation failed"; break; }
auto* c = m->codec;
c->width = config.width; c->height = config.height; c->time_base = {1, 1000000}; c->framerate = {config.fps, 1};
if (constrained && name.endsWith("_qsv")) c->time_base = {1, 90000};
c->bit_rate = config.bitrate; c->rc_max_rate = config.bitrate;
c->rc_buffer_size = config.vbvBits ? config.vbvBits : config.bitrate / 4;
c->gop_size = config.fps; c->max_b_frames = 0; c->thread_count = 2;
if (config.references) c->refs = config.references;
c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER | AV_CODEC_FLAG_CLOSED_GOP;
if (constrained) {
c->flags |= AV_CODEC_FLAG_LOW_DELAY;
c->level = name.endsWith("_qsv") && config.codec == STREAM_MEDIA_H265 ? config.level / 3 : config.level;
}
c->color_range = AVCOL_RANGE_MPEG; c->colorspace = AVCOL_SPC_BT709;
c->color_primaries = AVCOL_PRI_BT709; c->color_trc = AVCOL_TRC_BT709;
AVPixelFormat sw = constrained ? AV_PIX_FMT_YUV420P : softwareFormat(encoder);
AVPixelFormat sw = softwareFormat(encoder, !matchesBackend(name, Backend::SoftwareOnly));
int result = 0;
if (name.endsWith("_vaapi")) {
sw = AV_PIX_FMT_NV12; c->pix_fmt = AV_PIX_FMT_VAAPI;
@@ -192,22 +200,43 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
av_dict_set(&options, "preset", "p1", 0); av_dict_set(&options, "tune", "ull", 0);
av_dict_set(&options, "zerolatency", "1", 0); av_dict_set(&options, "rc-lookahead", "0", 0);
av_dict_set(&options, "forced-idr", "1", 0);
if (constrained) {
av_dict_set(&options, "delay", "0", 0); av_dict_set(&options, "rc", "cbr", 0);
av_dict_set(&options, "profile", config.codec == STREAM_MEDIA_H264 ? "baseline" : "main", 0);
av_dict_set_int(&options, "level", config.level, 0);
if (config.codec == STREAM_MEDIA_H265) av_dict_set(&options, "tier", "main", 0);
}
} else if (name.endsWith("_amf")) {
av_dict_set(&options, "usage", "ultralowlatency", 0);
av_dict_set(&options, "quality", "speed", 0);
av_dict_set(&options, "async_depth", "1", 0);
bool forceIdr = config.codec == STREAM_MEDIA_H264 || config.codec == STREAM_MEDIA_H265;
if (forceIdr) av_dict_set(&options, "forced_idr", "1", 0);
m->forcedKeys = forceIdr && av_opt_find(c->priv_data, "forced_idr", nullptr, 0, 0);
if (m->forcedKeys) av_dict_set(&options, "forced_idr", "1", 0);
if (constrained) {
av_dict_set(&options, "rc", "cbr", 0);
av_dict_set(&options, "profile", config.codec == STREAM_MEDIA_H264 ? "constrained_baseline" : "main", 0);
if (config.codec == STREAM_MEDIA_H265) av_dict_set(&options, "profile_tier", "main", 0);
}
DEBUG_INFO(DEBUG_CATEGORY_VIDEO,
"desktop AMF low latency settings name=%s usage=ultralowlatency quality=speed async_depth=1 forced_idr=%d",
name.constData(), forceIdr);
name.constData(), m->forcedKeys);
} else if (name == "libvpx-vp9" || name == "libaom-av1") {
av_dict_set(&options, "deadline", "realtime", 0); av_dict_set(&options, "cpu-used", "8", 0);
av_dict_set(&options, "lag-in-frames", "0", 0); av_dict_set(&options, "row-mt", "1", 0);
} else if (name.endsWith("_qsv")) {
av_dict_set(&options, "async_depth", "1", 0); av_dict_set(&options, "look_ahead", "0", 0);
if (constrained) {
av_dict_set(&options, "forced_idr", "1", 0); av_dict_set(&options, "preset", "veryfast", 0);
av_dict_set(&options, "profile", config.codec == STREAM_MEDIA_H264 ? "baseline" : "main", 0);
if (config.codec == STREAM_MEDIA_H265) av_dict_set(&options, "tier", "main", 0);
}
} else if (name.endsWith("_vaapi")) {
av_dict_set(&options, "async_depth", "1", 0);
if (constrained) {
c->profile = config.codec == STREAM_MEDIA_H264 ? AV_PROFILE_H264_CONSTRAINED_BASELINE : AV_PROFILE_HEVC_MAIN;
av_dict_set(&options, "rc_mode", "CBR", 0);
}
} else if (name == "libvvenc") av_dict_set(&options, "preset", "fast", 0);
if (result >= 0) result = avcodec_open2(c, encoder, &options);
av_dict_free(&options);
@@ -269,7 +298,7 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
name.constData(), qPrintable(m->error));
break;
}
if (name.endsWith("_amf")) {
if (!m->forcedKeys) {
DEBUG_INFO(DEBUG_CATEGORY_VIDEO,
"desktop AMF uses periodic IDR name=%s gop_frames=%d forced_key_requests=disabled",
name.constData(), c->gop_size);
@@ -288,9 +317,9 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
continue;
}
DEBUG_INFO(DEBUG_CATEGORY_VIDEO,
"desktop encoder ready name=%s codec=%d size=%dx%d fps=%d bitrate=%d level=%d refs=%d vbv=%d profile_verified=%d",
"desktop encoder ready name=%s codec=%d size=%dx%d fps=%d bitrate=%d level=%d refs=%d vbv=%d profile_verified=%d forced_keys=%d",
name.constData(), config.codec, config.width, config.height, config.fps, config.bitrate,
config.level, config.references, c->rc_buffer_size, constrained);
config.level, config.references, c->rc_buffer_size, constrained, m->forcedKeys);
return true;
}
}
@@ -325,11 +354,18 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList
int stride[4] = {static_cast<int>(canvas.bytesPerLine()), 0, 0, 0};
result = sws_scale(m->scale, input, stride, 0, canvas.height(), m->frame->data, m->frame->linesize);
}
bool forceKey = key && (!m->name.endsWith("_amf") || m->submittedFrames == 0);
bool constrained = m->config.profile != LiveVideoProfile::Automatic;
if (constrained && m->timestamps.size() >= 8) {
m->error = "GPU encoder exceeded the bounded input pipeline";
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "call encoder stalled name=%s pending=%zu pts=%llu",
qPrintable(m->name), m->timestamps.size(), (unsigned long long)pts); return false;
}
bool forceKey = key && (m->forcedKeys || m->submittedFrames == 0);
if (key && !forceKey) DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO,
"desktop AMF ignores forced key request; periodic GOP IDR remains active name=%s submitted=%lld",
qPrintable(m->name), (long long)m->submittedFrames);
m->frame->pts = static_cast<int64_t>(pts); m->frame->pict_type = forceKey ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
m->frame->pts = av_rescale_q(static_cast<int64_t>(pts), AVRational{1, 1000000}, m->codec->time_base);
m->frame->pict_type = forceKey ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
if (forceKey) m->frame->flags |= AV_FRAME_FLAG_KEY; else m->frame->flags &= ~AV_FRAME_FLAG_KEY;
AVFrame* hardware = nullptr;
if (result >= 0 && m->codec->hw_frames_ctx) {
@@ -341,7 +377,10 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList
const qint64 preparedUs = timer.nsecsElapsed() / 1000;
if (result >= 0) result = avcodec_send_frame(m->codec, hardware ? hardware : m->frame);
const qint64 submittedUs = timer.nsecsElapsed() / 1000;
if (result >= 0) m->submittedFrames++;
if (result >= 0) {
m->submittedFrames++;
if (constrained) m->timestamps.emplace_back(m->frame->pts, pts);
}
av_frame_free(&hardware);
AVPacket* packet = av_packet_alloc();
if (!packet) result = AVERROR(ENOMEM);
@@ -355,6 +394,15 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList
}
LiveEncodedFrame encoded; encoded.key = (packet->flags & AV_PKT_FLAG_KEY) != 0;
encoded.pts = packet->pts >= 0 ? static_cast<uint64_t>(packet->pts) : pts;
if (constrained) {
if (m->timestamps.empty() || packet->pts != m->timestamps.front().first) {
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "call GPU output PTS mismatch name=%s packet=%lld expected=%lld pending=%zu",
qPrintable(m->name), (long long)packet->pts,
(long long)(m->timestamps.empty() ? -1 : m->timestamps.front().first), m->timestamps.size());
result = AVERROR_INVALIDDATA; break;
}
encoded.pts = m->timestamps.front().second; m->timestamps.pop_front();
}
if (key) DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop requested KEY name=%s input_PTS=%llu output_PTS=%llu key=%d flags=%x bytes=%d",
qPrintable(m->name), (unsigned long long)pts, (unsigned long long)encoded.pts, encoded.key, packet->flags, packet->size);
if (encoded.key) {