Drain old GPU frames before bitrate replacement and poll outputs while blocked

tools/chatgui/src/call_video_encoder.cpp

@@ -29,10 +29,10 @@ void CallVideoEncoder::close() {
     if (m_encoder) DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "call video encoder close generation=%llu pending_request=%llu",
                               (unsigned long long)m_format.generation, (unsigned long long)m_request);
     m_prepared = {}; // Завершение фоновой подготовки ждём только при остановке/смене режима.
-    m_replacement.reset(); m_encoder.reset(); m_config = {}; m_config.bitrate = 0; m_error.clear();
+    m_encoder.reset(); m_config = {}; m_config.bitrate = 0; m_error.clear();
     m_format = {}; m_request = m_lastRequest = m_lastPts = m_lastKey = 0;
-    m_lastOutputPts = m_switchPts = 0; m_backgroundMs = 0; m_switchInputs = 0;
-    m_pendingBitrate = 0; m_haveFrame = m_haveInput = m_keyPending = false;
+    m_lastOutputPts = m_switchPts = 0; m_backgroundMs = 0;
+    m_pendingBitrate = 0; m_haveFrame = m_haveInput = m_keyPending = m_switching = false;
 }
 
 // Ядро сериализует команды; повторные/чужие requests не заменяют уже ожидающую команду.
@@ -64,90 +64,72 @@ bool CallVideoEncoder::requestBitrate(uint64_t generation, uint64_t request, uin
                 (unsigned long long)generation, (unsigned long long)request, m_config.bitrate, bitrate, (unsigned long long)m_lastKey); return true;
 }
 
-// Capture PTS продолжает идти при backpressure, поэтому запрет отправки не задерживает применение.
+// При backpressure забираем готовые output без новых raw входов. Замена сначала
+// полностью завершает старый encoder; уже принятые им PTS не исчезают на границе IDR.
 bool CallVideoEncoder::encode(const QImage& image, uint64_t pts, bool forceKey, QList<LiveEncodedFrame>& output,
                               CallVideoRateCompletion& completion) {
     completion = {}; output.clear();
-    if (!m_encoder || image.isNull() || pts > INT64_MAX || (m_haveInput && pts <= m_lastPts)) {
+    const bool input = !image.isNull();
+    if (!input) pts = m_lastPts;
+    if (!m_encoder || pts > INT64_MAX || (input && m_haveInput && pts <= m_lastPts)) {
         m_error = "Invalid call video frame or non-monotonic PTS";
-        DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video encode invalid generation=%llu pts=%llu last=%llu active=%d image_null=%d",
-                    (unsigned long long)m_format.generation, (unsigned long long)pts, (unsigned long long)m_lastPts,
-                    m_encoder != nullptr, image.isNull()); return false;
+        DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video encode invalid generation=%llu pts=%llu last=%llu active=%d input=%d",
+            (unsigned long long)m_format.generation, (unsigned long long)pts, (unsigned long long)m_lastPts,
+            m_encoder != nullptr, input); return false;
     }
-    bool key = !m_keyPending && (forceKey || !m_haveInput || (pts >= m_lastKey && pts - m_lastKey >= 1000000));
+    if (!input && !m_haveInput) return true;
+    bool key = input && !m_keyPending && (forceKey || !m_haveInput || pts - m_lastKey >= 1000000);
     QList<LiveEncodedFrame> frames;
     bool prepared = !m_prepared.valid() || m_prepared.wait_for(std::chrono::seconds(0)) == std::future_status::ready;
-    if (key && m_request && !prepared)
-        DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "call video preparation pending request=%llu; keep current encoder", (unsigned long long)m_request);
-    if (key && m_request && prepared && !m_replacement) {
+    if (key && m_request && prepared && !m_switching) {
         if (m_pendingBitrate == (uint32_t)m_config.bitrate) {
             completion = {m_format.generation, m_request, m_pendingBitrate}; m_request = 0; m_pendingBitrate = 0;
-        }
-        else {
-            auto preparedEncoder = m_prepared.get(); m_backgroundMs = preparedEncoder.elapsed;
-            m_replacement = std::move(preparedEncoder.encoder); m_switchPts = pts; m_switchInputs = 0;
-            if (!m_replacement) {
+        } else {
+            auto candidate = m_prepared.get(); m_backgroundMs = candidate.elapsed;
+            if (!candidate.encoder) {
                 DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU replacement open failed request=%llu: %s",
-                    (unsigned long long)m_request, qPrintable(preparedEncoder.error));
+                    (unsigned long long)m_request, qPrintable(candidate.error));
                 completion = {m_format.generation, m_request, 0}; m_request = 0; m_pendingBitrate = 0;
+            } else {
+                QElapsedTimer timer; timer.start();
+                if (!m_encoder->encode({}, m_lastPts, false, frames, true)) {
+                    m_error = m_encoder->error(); completion = {m_format.generation, m_request, 0};
+                    DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU old encoder drain failed request=%llu: %s",
+                        (unsigned long long)m_request, qPrintable(m_error)); return false;
+                }
+                DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "call GPU replacement drained old request=%llu frames=%lld drain_us=%lld",
+                    (unsigned long long)m_request, (long long)frames.size(), (long long)(timer.nsecsElapsed() / 1000));
+                m_encoder = std::move(candidate.encoder); m_config.bitrate = m_pendingBitrate;
+                m_switchPts = pts; m_switching = true;
             }
         }
     }
-    QElapsedTimer timer; timer.start();
-    bool encoded = false;
-    if (m_replacement) {
-        ++m_switchInputs;
-        encoded = m_replacement->encode(image, pts, m_switchInputs == 1, frames);
-        if (encoded && !frames.isEmpty()) {
-            const auto& first = frames.front();
-            encoded = first.key && first.pts == m_switchPts && !call_video_check_key(&m_format,
-                reinterpret_cast<const uint8_t*>(first.data.constData()), first.data.size());
-            if (encoded) {
-                const qint64 encodeUs = timer.nsecsElapsed() / 1000;
-                m_encoder = std::move(m_replacement); m_config.bitrate = m_pendingBitrate;
-                completion = {m_format.generation, m_request, m_pendingBitrate};
-                DEBUG_INFO(DEBUG_CATEGORY_CALL,
-                    "call video bitrate committed generation=%llu request=%llu bitrate=%u pts=%llu background_ms=%lld "
-                    "encode_us=%lld retire_us=%lld pipeline_frames=%u pipeline_pts_us=%llu bytes=%lld",
-                    (unsigned long long)m_format.generation, (unsigned long long)m_request, completion.bitrate,
-                    (unsigned long long)first.pts, (long long)m_backgroundMs, (long long)encodeUs,
-                    (long long)(timer.nsecsElapsed() / 1000 - encodeUs), m_switchInputs,
-                    (unsigned long long)(pts - first.pts), (long long)first.data.size());
-                m_request = 0; m_pendingBitrate = 0;
-            }
-        }
-        if (!encoded) {
-            DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU replacement failed request=%llu inputs=%u frames=%lld: %s",
-                (unsigned long long)m_request, m_switchInputs, (long long)frames.size(), qPrintable(m_replacement->error()));
-            completion = {m_format.generation, m_request, 0}; m_request = 0; m_pendingBitrate = 0;
-            m_replacement.reset(); frames.clear(); key = true; m_keyPending = false;
-        } else if (m_replacement && frames.isEmpty()) {
-            // До первого нового IDR продолжаем выдавать только старые кадры с PTS раньше границы замены.
-            // Уже выданный PTS не может обогнать ожидаемый IDR и сделать его устаревшим.
-            QList<LiveEncodedFrame> previous;
-            if (m_encoder->encode(image, pts, false, previous)) {
-                for (auto& frame : previous) if (frame.pts < m_switchPts) frames.push_back(std::move(frame));
-            } else DEBUG_WARN(DEBUG_CATEGORY_CALL, "old GPU failed while replacement is pending request=%llu: %s",
-                (unsigned long long)m_request, qPrintable(m_encoder->error()));
-        }
-    }
-    if (!encoded) encoded = m_encoder->encode(image, pts, key, frames);
-    m_haveInput = true; m_lastPts = pts; if (key) m_keyPending = true;
-    if (!encoded) {
+    if (!m_encoder->encode(image, pts, key, frames)) {
         m_error = m_encoder->error();
-        if (completion.request) completion.bitrate = 0;
+        if (m_request) completion = {m_format.generation, m_request, 0};
         DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call video encoding failed generation=%llu: %s",
-                    (unsigned long long)m_format.generation, qPrintable(m_error)); return false;
+            (unsigned long long)m_format.generation, qPrintable(m_error)); return false;
     }
+    if (input) { m_haveInput = true; m_lastPts = pts; if (key) m_keyPending = true; }
     for (const auto& frame : frames) {
         if ((m_haveFrame && frame.pts <= m_lastOutputPts) || frame.pts > pts || (!m_haveFrame && !frame.key)
+            || (m_switching && frame.pts >= m_switchPts && frame.pts != m_switchPts)
             || (frame.key && call_video_check_key(&m_format,
                 reinterpret_cast<const uint8_t*>(frame.data.constData()), frame.data.size()))) {
             m_error = "GPU produced an incompatible video frame";
-            if (completion.request) completion.bitrate = 0;
+            if (m_request) completion = {m_format.generation, m_request, 0};
             DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call GPU output rejected pts=%llu last=%llu capture=%llu key=%d",
                 (unsigned long long)frame.pts, (unsigned long long)m_lastOutputPts, (unsigned long long)pts, frame.key); return false;
         }
+        if (m_switching && frame.pts == m_switchPts) {
+            if (!frame.key) { m_error = "Replacement encoder did not start with KEY";
+                DEBUG_ERROR(DEBUG_CATEGORY_CALL, "%s", qPrintable(m_error)); return false; }
+            completion = {m_format.generation, m_request, m_pendingBitrate};
+            DEBUG_INFO(DEBUG_CATEGORY_CALL, "call video bitrate committed generation=%llu request=%llu bitrate=%u pts=%llu background_ms=%lld",
+                (unsigned long long)m_format.generation, (unsigned long long)m_request, m_pendingBitrate,
+                (unsigned long long)frame.pts, (long long)m_backgroundMs);
+            m_request = 0; m_pendingBitrate = 0; m_switching = false;
+        }
         m_haveFrame = true; m_lastOutputPts = frame.pts;
         if (frame.key) { m_lastKey = frame.pts; m_keyPending = false; }
     }

tools/chatgui/src/call_video_encoder.h

@@ -20,10 +20,11 @@ public:
     bool open(const call_video_format& format, uint32_t bitrate, const QString& backend = {});
     void close();
     bool requestBitrate(uint64_t generation, uint64_t request, uint32_t bitrate);
-    // Готовые кадры выдаются сразу пакетом; пустой output означает работу GPU, без ожидания следующего capture.
+    // Пустое image забирает готовые кадры без нового input (pts игнорируется).
     // На false output пуст; completion нужно обработать и на ошибке. PTS готового выхода сохраняет capture PTS.
     bool encode(const QImage& image, uint64_t pts, bool forceKey, QList<LiveEncodedFrame>& output, CallVideoRateCompletion& completion);
     uint32_t bitrate() const { return m_encoder ? m_config.bitrate : 0; }
+    unsigned pendingFrames() const { return m_encoder ? m_encoder->pendingFrames() : 0; }
     QString error() const { return m_error; }
 private:
     struct Prepared {
@@ -32,15 +33,13 @@ private:
         qint64 elapsed = 0;
     };
     std::unique_ptr<LiveVideoEncoder> m_encoder;
-    std::unique_ptr<LiveVideoEncoder> m_replacement; // Новый GPU работает до первого IDR; старый сохранён для отказа замены.
     LiveVideoConfig m_config;
     call_video_format m_format{};
     QString m_error;
     uint64_t m_request = 0, m_lastRequest = 0, m_lastPts = 0, m_lastKey = 0, m_lastOutputPts = 0, m_switchPts = 0;
     qint64 m_backgroundMs = 0;
-    unsigned m_switchInputs = 0;
     uint32_t m_pendingBitrate = 0;
-    bool m_haveFrame = false, m_haveInput = false, m_keyPending = false;
+    bool m_haveFrame = false, m_haveInput = false, m_keyPending = false, m_switching = false;
     // Отдельный контекст готовится в фоне; worker забирает его только после завершения open.
     std::future<Prepared> m_prepared;
 };

tools/chatgui/src/call_video_engine.cpp

@@ -262,7 +262,7 @@ void CallVideoEngine::transmit() {
             }
         }
         if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0;
-        if (!enabled || !state.have_peer || pts <= lastPts) { idle(); continue; }
+        if (!enabled || !state.have_peer || pts < 0) { idle(); continue; }
         if (revision != appliedRevision) {
             call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0};
             bool opened = false;
@@ -291,15 +291,16 @@ void CallVideoEngine::transmit() {
                 call_video_port_applied(m->id, format.generation, request, 0);
         }
 #ifdef HAVE_STREAM_CAPTURE
-        if (!state.can_encode) { idle(); continue; }
-        if (image.isNull() || nowUs() - pts > CallCameraMaxFrameAgeUs) {
+        bool submit = state.can_encode && pts > lastPts;
+        if (submit && (image.isNull() || nowUs() - pts > CallCameraMaxFrameAgeUs)) {
             DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX skips old capture age_us=%lld", (long long)(nowUs() - pts));
             lastPts = pts; ++expired; continue;
         }
         QList<LiveEncodedFrame> outputs; CallVideoRateCompletion completion;
         int64_t encodeStarted = nowUs();
         inputAgeMax = std::max(inputAgeMax, encodeStarted - pts);
-        bool encoded = encoder.encode(image, pts, forceKey, outputs, completion); lastPts = pts;
+        bool encoded = encoder.encode(submit ? image : QImage{}, pts, forceKey, outputs, completion);
+        if (submit) { lastPts = pts; if (encoded) forceKey = false; }
         encodeMax = std::max(encodeMax, nowUs() - encodeStarted);
         if (completion.request) {
             call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate);
@@ -308,7 +309,7 @@ void CallVideoEngine::transmit() {
         }
         if (!encoded) { forceKey = true; m->report(encoder.error()); continue; }
         // GPU может вернуть несколько уже готовых кадров: отправляем все, не добавляя очередь в TX worker.
-        if (outputs.isEmpty()) continue;
+        if (outputs.isEmpty()) { idle(); continue; }
         if (outputs.size() > CALL_VIDEO_PORT_TX_FRAMES) {
             DEBUG_ERROR(DEBUG_CATEGORY_CALL, "GPU output batch exceeds mailbox frames=%lld", (long long)outputs.size());
             forceKey = true; continue;
@@ -329,7 +330,7 @@ void CallVideoEngine::transmit() {
                   (long long)(nowUs() - packets[0].pts_us), count);
               expired += count; forceKey = true; pendingOutputs.clear(); continue;
           }
-          pendingCount = count; forceKey = false; }
+          pendingCount = count; }
         for (unsigned i = 0; i < count; ++i) {
             const auto& output = pendingOutputs[i];
             // Одинаковое окно исходного PTS на всех этапах: примерно один кадр в секунду, без трассы каждого кадра.