Reproduce live video starvation behind audio output latency

tools/chatgui/tests/test_stream_view.cpp

@@ -9,6 +9,7 @@
 #include <QSlider>
 #include <atomic>
 #include <cmath>
+#include <chrono>
 
 extern "C" {
 #include "utun_instance.h"
@@ -29,6 +30,14 @@ static QList<QByteArray> snapshots;
 #ifdef TEST_NULL_AUDIO
 static std::atomic<ma_device_data_proc> originalAudioCallback{nullptr};
 static std::atomic<uint64_t> nonzeroSamples{0};
+static std::atomic<int64_t> playbackDelayUs{0};
+extern "C" int64_t __real_audio_stream_frame_time(ma_device*, unsigned int, int*);
+extern "C" int64_t __wrap_audio_stream_frame_time(ma_device* device, unsigned int frames, int* valid) {
+    if (!playbackDelayUs) return __real_audio_stream_frame_time(device, frames, valid);
+    *valid = 1;
+    return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count()
+        + playbackDelayUs.load();
+}
 extern "C" ma_result __real_ma_context_init(const ma_backend*, ma_uint32, const ma_context_config*, ma_context*);
 extern "C" ma_result __real_ma_device_init(ma_context*, const ma_device_config*, ma_device*);
 extern "C" ma_result __wrap_ma_context_init(const ma_backend*, ma_uint32, const ma_context_config*, ma_context* context) {
@@ -56,7 +65,7 @@ class StreamViewTest : public QObject {
     UASYNC* ua = nullptr;
     UTUN_INSTANCE* inst = nullptr;
     STREAM_HANDLE* publisher = nullptr;
-    uint64_t group = 0, epoch = 0, frameId = 0;
+    uint64_t group = 0, epoch = 0, frameId = 0, mediaPts = 0;
     QTimer poll;
     size_t allocations = 0;
     uint32_t revision = 1;
@@ -71,15 +80,17 @@ class StreamViewTest : public QObject {
         DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop test missing record id=%llu", (unsigned long long)id); return false;
     }
 
-    void publish(const QByteArray& video, const QByteArray& audio = {}) {
+    void publish(const QByteArray& video, const QByteArray& audio = {}, unsigned videoFrames = 1) {
         stream_media_block media{}; media.format = {160, 90, rotation, STREAM_MEDIA_H264, STREAM_MEDIA_OPUS, revision};
-        media.first_frame_offset = 0; media.video_count = 1;
-        media.video[0] = {++frameId, frameId * 100000, static_cast<uint32_t>(video.size()), 0,
-            static_cast<uint32_t>(video.size()), STREAM_MEDIA_KEY, reinterpret_cast<const uint8_t*>(video.constData())};
+        mediaPts += 100000;
+        media.first_frame_offset = 0; media.video_count = videoFrames;
+        for (unsigned i = 0; i < videoFrames; i++) media.video[i] = {++frameId, mediaPts + i * 100000 / videoFrames,
+            static_cast<uint32_t>(video.size()), 0, static_cast<uint32_t>(video.size()), STREAM_MEDIA_KEY,
+            reinterpret_cast<const uint8_t*>(video.constData())};
         if (!audio.isEmpty()) {
-            media.audio_count = 1;
-            media.audio[0] = {frameId * 100000, static_cast<uint32_t>(audio.size()), 960,
-                             reinterpret_cast<const uint8_t*>(audio.constData())};
+            media.audio_count = videoFrames > 1 ? 5 : 1;
+            for (unsigned i = 0; i < media.audio_count; i++) media.audio[i] = {mediaPts + i * 20000,
+                static_cast<uint32_t>(audio.size()), 960, reinterpret_cast<const uint8_t*>(audio.constData())};
         }
         uint8_t wire[7080]; size_t length = 0;
         QCOMPARE(stream_media_encode(&media, wire, sizeof(wire), &length), STREAM_OK);
@@ -165,12 +176,14 @@ private slots:
         int16_t pcm[960]; for (unsigned i = 0; i < 960; i++) pcm[i] = static_cast<int16_t>(6000 * std::sin(i * 0.05));
         uint8_t encoded[1275]; int length = opus_codec_encode(opus, pcm, 960, encoded, sizeof(encoded)); QVERIFY(length > 0);
         audio = QByteArray(reinterpret_cast<const char*>(encoded), length); opus_codec_encoder_destroy(opus);
-        nonzeroSamples = 0;
+        nonzeroSamples = 0; playbackDelayUs = 200000;
 #endif
         StreamPlayerWindow window(group, inst->node_id, epoch, "Source");
         QSignalSpy shown(&window, &StreamPlayerWindow::framePresented); window.show();
-        QTimer sender; connect(&sender, &QTimer::timeout, this, [this, video, audio] { publish(video, audio); }); sender.start(100);
+        QTimer sender; connect(&sender, &QTimer::timeout, this, [this, video, audio] { publish(video, audio, 3); }); sender.start(100);
         QTRY_VERIFY_WITH_TIMEOUT(shown.count() > 0, 5000);
+        int beforeSteady = shown.count(); QTest::qWait(1500);
+        QVERIFY2(shown.count() >= beforeSteady + 30, "30fps video must keep advancing with 200ms physical audio latency");
         int beforeSwitch = shown.count(); revision++; rotation = 270;
         QTRY_VERIFY_WITH_TIMEOUT(shown.count() > beforeSwitch + 2, 5000);
         stream_info switched{}; QCOMPARE(stream_get_info(publisher, &switched), STREAM_OK);
@@ -181,6 +194,9 @@ private slots:
         QTest::qWait(50); uint64_t played = nonzeroSamples.load(); QTest::qWait(100); QCOMPARE(nonzeroSamples.load(), played);
 #endif
         sender.stop(); window.close(); QTest::qWait(20);
+#ifdef TEST_NULL_AUDIO
+        playbackDelayUs = 0;
+#endif
         stream_info info{}; QCOMPARE(stream_get_info(publisher, &info), STREAM_OK); QCOMPARE(info.owners, 1u);
 #else
         QSKIP("FFmpeg is disabled");