Align desktop audio startup with the first decoded video frame

tools/chatgui/src/streamplayer_window.cpp

@@ -182,6 +182,7 @@ void StreamPlayerWindow::decode() {
     bool waitKey = true, opened = false;
     int64_t deadline = nowUs() + 10000000, reportAt = nowUs(), expectedAudio = -1;
     uint64_t decoded = 0, discarded = 0, late = 0, pressureDropped = 0, previousPressure = 0, previousDecoded = 0, previousShown = 0;
+    uint64_t prerollSamples = 0;
     uint32_t videoRevision = 0;
     auto reset = [&] {
         avcodec_free_context(&decoder);
@@ -232,8 +233,10 @@ void StreamPlayerWindow::decode() {
                                              i == missing ? record.data.size() : 0, audio.pcm.data(), 960);
                 if (count != 960) { fail(text("Stream audio decoder failed")); failed = true; break; }
                 std::lock_guard<std::mutex> guard(p.lock);
-                if (!p.prefillUntil) p.prefillUntil = nowUs() + 150000;
-                if (p.audios.size() >= 25) { pressure = true; break; }
+                if (p.audios.size() >= 25) {
+                    if (p.videoBase >= 0) { pressure = true; break; }
+                    prerollSamples += 960; p.audios.pop_front();
+                }
                 p.audios.push_back(std::move(audio));
             }
             if (failed) break;
@@ -278,8 +281,19 @@ void StreamPlayerWindow::decode() {
             int64_t pts = frame->best_effort_timestamp == AV_NOPTS_VALUE ? static_cast<int64_t>(record.pts) : frame->best_effort_timestamp;
             {
                 std::lock_guard<std::mutex> guard(p.lock);
-                if (!p.prefillUntil) p.prefillUntil = nowUs() + 150000;
-                if (p.videoBase < 0) { p.videoBase = pts; p.videoAt = p.prefillUntil; }
+                if (p.videoBase < 0) {
+                    while (!p.audios.empty() && p.audios.front().pts + 20000 <= pts) {
+                        prerollSamples += 960; p.audios.pop_front();
+                    }
+                    if (!p.audios.empty() && p.audios.front().pts < pts) {
+                        p.audios.front().offset = static_cast<unsigned>((pts - p.audios.front().pts) * 48000 + 999999) / 1000000;
+                        prerollSamples += p.audios.front().offset;
+                    }
+                    p.prefillUntil = nowUs() + 150000; p.videoBase = pts; p.videoAt = p.prefillUntil;
+                    DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop playback start id=%llu pts=%lld prefill_ms=150 preroll_frames=%llu audioQ=%zu",
+                               (unsigned long long)p.id, (long long)pts, (unsigned long long)prerollSamples, p.audios.size());
+                    prerollSamples = 0;
+                }
                 if (pts < p.clock(nowUs()) - 100000) { discarded++; late++; continue; }
             }
             converter = sws_getCachedContext(converter, frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),

tools/chatgui/tests/test_stream_view.cpp

@@ -187,8 +187,7 @@ private slots:
 #endif
         StreamPlayerWindow window(group, inst->node_id, epoch, "Source");
         QSignalSpy shown(&window, &StreamPlayerWindow::framePresented); window.show();
-        QTimer sender;
-        connect(&sender, &QTimer::timeout, this, [this, video, &opus, &sampleFrame] {
+        auto audioPackets = [&opus, &sampleFrame] {
             QList<QByteArray> packets;
             if (opus) for (unsigned frame = 0; frame < 5; frame++) {
                 int16_t pcm[1920]; uint8_t encoded[1275];
@@ -200,7 +199,18 @@ private slots:
                 int length = opus_codec_encode(opus.get(), pcm, 960, encoded, sizeof(encoded)); QVERIFY(length > 0);
                 packets.append(QByteArray(reinterpret_cast<const char*>(encoded), length));
             }
-            publish(video, packets, 3);
+            return packets;
+        };
+#ifdef TEST_NULL_AUDIO
+        QTest::qWait(20);
+        for (unsigned batch = 0; batch < 8; batch++) { publish({}, audioPackets(), 0); uasync_poll(ua, 0); }
+        QTest::qWait(200);
+        QCOMPARE(shown.count(), 0);
+        QCOMPARE(nonzeroSamples.load(), uint64_t(0));
+#endif
+        QTimer sender;
+        connect(&sender, &QTimer::timeout, this, [this, video, &audioPackets] {
+            publish(video, audioPackets(), 3);
         });
         sender.start(100);
         QTRY_VERIFY_WITH_TIMEOUT(shown.count() > 0, 5000);