Preserve video frames until audio clock reaches their PTS

src/stream/stream_buffer.c

@@ -125,7 +125,7 @@ static void buffer_evict_oldest(struct stream_buffer *b) {
     b->stats.blocks -= batch->count;
     b->stats.stored_bytes -= batch->count * b->block_bytes;
     b->stats.evicted_batches++;
-    DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "buffer evicted seq=%llu blocks=%u retained=%u bytes=%zu low=%llu",
+    DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "buffer evicted seq=%llu blocks=%u retained=%u bytes=%zu low=%llu",
                 (unsigned long long)batch->seq, batch->count, b->stats.blocks, b->stats.stored_bytes,
                 (unsigned long long)b->stats.expired_before);
     u_free(batch);

tools/chatgui/src/streamplayer_window.cpp

@@ -46,6 +46,7 @@ struct StreamPlayerWindow::Playback {
     std::thread worker;
     std::mutex lock;
     std::deque<Video> videos;
+    size_t videoBytes = 0;
     std::deque<Audio> audios;
     int64_t prefillUntil = 0, videoBase = -1, videoAt = 0, audioPts = -1, audioAt = 0;
     uint64_t shown = 0, underruns = 0;
@@ -130,19 +131,21 @@ void StreamPlayerWindow::openAudio() {
 void StreamPlayerWindow::tick() {
     auto& p = *m_playback;
     if (m_audio.needsRecovery()) { m_audio.close(); p.audioAvailable = false; openAudio(); }
-    QImage image; bool finished = false;
+    QImage image; bool finished = false, first = false;
     {
         std::lock_guard<std::mutex> guard(p.lock);
         int64_t now = nowUs(), clock = p.clock(now);
         if (now >= p.prefillUntil) while (!p.videos.empty() && p.videos.front().pts <= clock) {
+            p.videoBytes -= static_cast<size_t>(p.videos.front().image.sizeInBytes());
             image = std::move(p.videos.front().image); p.videos.pop_front();
         }
+        if (!image.isNull()) first = p.shown++ == 0;
         finished = p.ended && p.videos.empty() && (p.audios.empty() || !p.audioAvailable);
     }
     if (!image.isNull()) {
         m_render->setFrame(image);
         m_status->setText(text("Live"));
-        if (!p.shown++) {
+        if (first) {
             DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop first frame id=%llu %dx%d", (unsigned long long)p.id, image.width(), image.height());
         }
         emit framePresented();
@@ -178,13 +181,13 @@ void StreamPlayerWindow::decode() {
     opus_codec_decoder_t* opus = nullptr;
     bool waitKey = true, opened = false;
     int64_t deadline = nowUs() + 10000000, reportAt = nowUs(), expectedAudio = -1;
-    uint64_t decoded = 0, discarded = 0;
+    uint64_t decoded = 0, discarded = 0, late = 0, pressureDropped = 0, previousDecoded = 0, previousShown = 0;
     uint32_t videoRevision = 0;
     auto reset = [&] {
         avcodec_free_context(&decoder);
         opus_codec_decoder_destroy(opus); opus = nullptr; expectedAudio = -1; waitKey = true;
         std::lock_guard<std::mutex> guard(p.lock);
-        p.videos.clear(); p.audios.clear(); p.prefillUntil = 0; p.videoBase = p.audioPts = -1;
+        p.videos.clear(); p.videoBytes = 0; p.audios.clear(); p.prefillUntil = 0; p.videoBase = p.audioPts = -1;
         emit statusChanged(text("Waiting for a key frame…"));
     };
     auto fail = [&](const QString& reason) {
@@ -244,7 +247,7 @@ void StreamPlayerWindow::decode() {
             DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop video revision=%u->%u dimensions=%ux%u rotation=%u; preserve audio",
                        videoRevision, record.format.revision, record.format.width, record.format.height, record.format.rotation);
             avcodec_free_context(&decoder); av_frame_unref(frame); waitKey = true; videoRevision = record.format.revision;
-            { std::lock_guard<std::mutex> guard(p.lock); p.videos.clear(); }
+            { std::lock_guard<std::mutex> guard(p.lock); p.videos.clear(); p.videoBytes = 0; }
             deadline = nowUs() + 10000000;
         }
         if (waitKey && !(record.flags & STREAM_MEDIA_KEY)) continue;
@@ -277,7 +280,7 @@ void StreamPlayerWindow::decode() {
                 std::lock_guard<std::mutex> guard(p.lock);
                 if (!p.prefillUntil) p.prefillUntil = nowUs() + 150000;
                 if (p.videoBase < 0) { p.videoBase = pts; p.videoAt = p.prefillUntil; }
-                if (pts < p.clock(nowUs()) - 100000) { discarded++; continue; }
+                if (pts < p.clock(nowUs()) - 100000) { discarded++; late++; continue; }
             }
             converter = sws_getCachedContext(converter, frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),
                 frame->width, frame->height, AV_PIX_FMT_RGB24, SWS_BILINEAR, nullptr, nullptr, nullptr);
@@ -290,7 +293,12 @@ void StreamPlayerWindow::decode() {
             }
             if (record.format.rotation) image = image.transformed(QTransform().rotate(record.format.rotation));
             std::lock_guard<std::mutex> guard(p.lock);
-            if (p.videos.size() == 6) { p.videos.pop_front(); discarded++; }
+            size_t bytes = static_cast<size_t>(image.sizeInBytes());
+            // Будущие кадры ждут аудиочасы; при пределе памяти сохраняем ближайшие к показу.
+            if (p.videos.size() >= 128 || p.videoBytes + bytes > 256u * 1024u * 1024u) {
+                discarded++; pressureDropped++; continue;
+            }
+            p.videoBytes += bytes;
             p.videos.push_back({std::move(image), pts});
         }
         if (received != AVERROR(EAGAIN) && received != AVERROR_EOF && p.running) {
@@ -298,10 +306,20 @@ void StreamPlayerWindow::decode() {
             reset(); deadline = nowUs() + 10000000;
         }
         if (nowUs() - reportAt >= 1000000) {
-            reportAt = nowUs(); std::lock_guard<std::mutex> guard(p.lock);
+            int64_t now = nowUs(); double seconds = (now - reportAt) / 1000000.0; reportAt = now;
-            DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "desktop viewer id=%llu decoded=%llu discarded=%llu videoQ=%zu audioQ=%zu underruns=%llu",
+            std::lock_guard<std::mutex> guard(p.lock); int64_t clock = p.clock(now);
+            DEBUG_DEBUG(DEBUG_CATEGORY_STREAM,
+                       "desktop viewer id=%llu decoded=%llu discarded=%llu late=%llu pressure=%llu shown=%llu decodeFps=%.1f showFps=%.1f "
+                       "videoQ=%zu videoBytes=%zu firstAheadUs=%lld lastAheadUs=%lld audioQ=%zu audioAtDeltaUs=%lld underruns=%llu",
                        (unsigned long long)p.id, (unsigned long long)decoded, (unsigned long long)discarded,
-                       p.videos.size(), p.audios.size(), (unsigned long long)p.underruns);
+                       (unsigned long long)late, (unsigned long long)pressureDropped, (unsigned long long)p.shown,
+                       (decoded - previousDecoded) / seconds, (p.shown - previousShown) / seconds, p.videos.size(), p.videoBytes,
+                       (long long)(p.videos.empty() ? 0 : p.videos.front().pts - clock),
+                       (long long)(p.videos.empty() ? 0 : p.videos.back().pts - clock), p.audios.size(),
+                       (long long)(p.audioPts < 0 ? 0 : now - p.audioAt), (unsigned long long)p.underruns);
+            if (pressureDropped) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop video queue pressure id=%llu dropped=%llu; preserve due frames",
+                                           (unsigned long long)p.id, (unsigned long long)pressureDropped);
+            previousDecoded = decoded; previousShown = p.shown;
         }
     }
     avcodec_free_context(&decoder); av_frame_free(&frame); av_packet_free(&packet);