From 98d0040b27ae4a982ff75e2b7578b17d3cdb901f Mon Sep 17 00:00:00 2001
From: evgeny <evgeny@adaptto.ru>
Date: Fri, 16 Oct 2026 21:26:58 +0200
Subject: [PATCH] Fix AMF live encoder startup probe

---
 tools/chatgui/src/live_video_encoder.cpp | 50 +++++++++++++++++++++---
 1 file changed, 44 insertions(+), 6 deletions(-)

diff --git a/tools/chatgui/src/live_video_encoder.cpp b/tools/chatgui/src/live_video_encoder.cpp
index 80c20740..b30d9483 100644
--- a/tools/chatgui/src/live_video_encoder.cpp
+++ b/tools/chatgui/src/live_video_encoder.cpp
@@ -181,6 +181,15 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
                 av_dict_set(&options, "preset", "p1", 0); av_dict_set(&options, "tune", "ull", 0);
                 av_dict_set(&options, "zerolatency", "1", 0); av_dict_set(&options, "rc-lookahead", "0", 0);
                 av_dict_set(&options, "forced-idr", "1", 0);
+            } else if (name.endsWith("_amf")) {
+                av_dict_set(&options, "usage", "ultralowlatency", 0);
+                av_dict_set(&options, "quality", "speed", 0);
+                av_dict_set(&options, "async_depth", "1", 0);
+                bool forceIdr = config.codec == STREAM_MEDIA_H264 || config.codec == STREAM_MEDIA_H265;
+                if (forceIdr) av_dict_set(&options, "forced_idr", "1", 0);
+                DEBUG_INFO(DEBUG_CATEGORY_VIDEO,
+                    "desktop AMF low latency settings name=%s usage=ultralowlatency quality=speed async_depth=1 forced_idr=%d",
+                    name.constData(), forceIdr);
             } else if (name == "libvpx-vp9" || name == "libaom-av1") {
                 av_dict_set(&options, "deadline", "realtime", 0); av_dict_set(&options, "cpu-used", "8", 0);
                 av_dict_set(&options, "lag-in-frames", "0", 0); av_dict_set(&options, "row-mt", "1", 0);
@@ -215,16 +224,45 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
             }
             if (!pass) {
                 QImage probe(config.width, config.height, QImage::Format_RGB32); probe.fill(Qt::black);
+                constexpr int maxDelayFrames = 4;
+                auto probeKey = [&](uint64_t pts, QList<LiveEncodedFrame>& verified) {
+                    const uint64_t frameDuration = 1000000 / config.fps;
+                    QList<LiveEncodedFrame> output;
+                    int received = 0;
+                    for (int attempt = 0; attempt <= maxDelayFrames; attempt++) {
+                        if (!encode(probe, pts + frameDuration * attempt, attempt == 0, output)) return false;
+                        received += static_cast<int>(output.size());
+                        for (const auto& frame : output) {
+                            if (frame.pts != pts) continue;
+                            if (!frame.key) {
+                                m->error = "Encoder did not honor the requested key frame";
+                                DEBUG_WARN(DEBUG_CATEGORY_VIDEO,
+                                    "desktop encoder probe returned non-key name=%s pts=%llu attempt=%d received=%d",
+                                    name.constData(), (unsigned long long)pts, attempt, received);
+                                return false;
+                            }
+                            verified = {frame};
+                            return true;
+                        }
+                        output.clear();
+                    }
+                    m->error = "Encoder did not produce a key frame within the startup probe limit";
+                    DEBUG_WARN(DEBUG_CATEGORY_VIDEO,
+                        "desktop encoder probe key delayed beyond limit name=%s pts=%llu max_delay_frames=%d received=%d",
+                        name.constData(), (unsigned long long)pts, maxDelayFrames, received);
+                    return false;
+                };
                 QList<LiveEncodedFrame> output;
-                if (!encode(probe, 0, true, output) || output.isEmpty() || !output.front().key) {
-                    if (m->error.isEmpty()) m->error = "Encoder did not produce an immediate key frame";
-                    DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop encoder probe lacks immediate KEY name=%s error=%s", name.constData(), qPrintable(m->error));
+                if (!probeKey(0, output)) {
+                    DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop encoder initial KEY probe failed name=%s error=%s",
+                        name.constData(), qPrintable(m->error));
                     break;
                 }
                 output.clear();
-                if (!encode(probe, 1000000 / config.fps, true, output) || output.isEmpty() || !output.front().key) {
-                    if (m->error.isEmpty()) m->error = "Encoder cannot produce an independent key frame on request";
-                    DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop encoder cannot force an independent KEY name=%s error=%s", name.constData(), qPrintable(m->error));
+                const uint64_t forcedKeyPts = (1000000 / config.fps) * (maxDelayFrames + 2);
+                if (!probeKey(forcedKeyPts, output)) {
+                    DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop encoder forced KEY probe failed name=%s error=%s",
+                        name.constData(), qPrintable(m->error));
                     break;
                 }
                 if (constrained && !verifiedProfile(config, output)) {