Track desktop video frame flow and update call preview geometry

tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallScreen.kt

@@ -127,9 +127,9 @@ fun CallScreen(
     val video by CallController.video.collectAsState()
     val videoActive by CallController.videoActive.collectAsState()
     val remote = video?.display?.collectAsState()?.value
-    val local = video?.preview?.collectAsState()?.value
+    val local = video?.preview?.state?.collectAsState()?.value
     val receiving = phase == CallPhase.ACTIVE && remote != null
-    val sending = phase == CallPhase.ACTIVE && local != null
+    val sending = phase == CallPhase.ACTIVE && local?.active == true
     var selectedVideo by rememberSaveable(st?.callId) { mutableStateOf<CallVideoSource?>(CallVideoSource.REMOTE) }
     val expandedVideo = selectedVideo?.takeIf { if (it == CallVideoSource.REMOTE) receiving else sending }
     val viewingVideo = expandedVideo != null
@@ -170,9 +170,15 @@ fun CallScreen(
         }
     }
     val rotated = remote?.rotation?.rem(180) != 0
-    val videoWidth = if (expandedVideo == CallVideoSource.LOCAL) local?.width
+    val localFormat = local?.format
+    val localRotated = (localFormat?.camera?.sensor ?: 0) % 180 != 0
+    val videoWidth = if (expandedVideo == CallVideoSource.LOCAL) {
+        if (localRotated) localFormat?.height else localFormat?.width
+    }
         else if (rotated) remote?.height else remote?.width
-    val videoHeight = if (expandedVideo == CallVideoSource.LOCAL) local?.height
+    val videoHeight = if (expandedVideo == CallVideoSource.LOCAL) {
+        if (localRotated) localFormat?.width else localFormat?.height
+    }
         else if (rotated) remote?.width else remote?.height
     LaunchedEffect(video, phase, expandedVideo, videoWidth, videoHeight) {
         val activity = context.findActivity()

tools/chatgui/src/call_video_engine.cpp

@@ -31,6 +31,7 @@ struct CallVideoEngine::Impl {
     bool enabled = false;
     int width = 640, height = 480, codec = 0;
     int64_t pts = -1, backend = -1, anchor = -1;
+    uint64_t captured = 0;
     QImage shown;
     uint64_t shownSerial = 0, deliveredSerial = 0;
     int64_t previewPts = -1, previewAt = 0;
@@ -138,7 +139,7 @@ void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int wid
             }
             pts = m->anchor + frame.startTime() - m->backend;
         }
-        if (pts > m->pts) { m->raw = frame; m->pts = pts; }
+        if (pts > m->pts) { m->raw = frame; m->pts = pts; ++m->captured; }
     }, Qt::DirectConnection);
     connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) {
         DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error);
@@ -170,20 +171,33 @@ void CallVideoEngine::transmit() {
     unsigned excluded = 0;
     int64_t lastPts = -1, lastKey = -1;
     bool forceKey = true;
+    int64_t reportAt = nowUs(), encodeMax = 0;
+    uint64_t captureBase = 0;
+    unsigned encodedCount = 0, keyCount = 0, sendBusy = 0, paced = 0;
+    int64_t pacedPts = -1;
     while (m->running) {
         struct call_video_port_status state{};
         if (call_video_port_status(m->id, &state) < 0) break;
         if (state.error) { m->report(tr("Core rejected video format")); break; }
-        uint64_t revision; bool enabled; int width, height, codec; int64_t pts;
+        uint64_t revision, captured; bool enabled; int width, height, codec; int64_t pts;
 #ifdef HAVE_STREAM_CAPTURE
         QVideoFrame raw;
 #endif
         { std::lock_guard<std::mutex> guard(m->lock);
           revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts;
+          captured = m->captured;
 #ifdef HAVE_STREAM_CAPTURE
           raw = m->raw;
 #endif
         }
+        int64_t now = nowUs();
+        if (now - reportAt >= 1000000) {
+            DEBUG_DEBUG(DEBUG_CATEGORY_CALL,
+                "video capture flow call=%llu interval_us=%lld captured=%llu encoded=%u keys=%u send_busy=%u paced=%u encode_max_us=%lld",
+                (unsigned long long)m->id, (long long)(now - reportAt), (unsigned long long)(captured - captureBase),
+                encodedCount, keyCount, sendBusy, paced, (long long)encodeMax);
+            reportAt = now; captureBase = captured; encodedCount = keyCount = sendBusy = paced = 0; encodeMax = 0;
+        }
         if (revision != userRevision) { userRevision = revision; excluded = 0; }
         if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted
             && rejectedGeneration != format.generation) {
@@ -195,7 +209,11 @@ void CallVideoEngine::transmit() {
             }
         }
         if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0;
-        if (!enabled || !state.have_peer || pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) { idle(); continue; }
+        if (!enabled || !state.have_peer || pts <= lastPts) { idle(); continue; }
+        if (lastPts >= 0 && pts - lastPts < 33333) {
+            if (pacedPts != pts) { pacedPts = pts; ++paced; }
+            idle(); continue;
+        }
         if (revision != appliedRevision) {
             if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; }
             call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0};
@@ -228,19 +246,23 @@ void CallVideoEngine::transmit() {
         QImage image = raw.toImage();
         if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; }
         LiveEncodedFrame output; CallVideoRateCompletion completion;
+        int64_t encodeStarted = nowUs();
         bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts;
+        encodeMax = std::max(encodeMax, nowUs() - encodeStarted);
         if (completion.request) {
             call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate);
             m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264")
                 .arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000));
         }
         if (!encoded) { forceKey = true; m->report(encoder.error()); continue; }
+        ++encodedCount; if (output.key) ++keyCount;
         if (output.key) lastKey = pts;
         call_video_frame packet{};
         packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format;
         packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast<const uint8_t*>(output.data.constData());
         packet.flags = output.key ? CALL_VIDEO_KEY : 0;
         forceKey = call_video_port_send(&packet) != 1;
+        if (forceKey) ++sendBusy;
 #else
         idle();
 #endif