From 95ab911caf31e0ad7eea0784e1869db226b842c8 Mon Sep 17 00:00:00 2001 From: evgeny Date: Thu, 15 Oct 2026 01:59:59 +0200 Subject: [PATCH] Track desktop video frame flow and update call preview geometry --- .../com/utun/chat/ui/screens/CallScreen.kt | 14 +++++++--- tools/chatgui/src/call_video_engine.cpp | 28 +++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallScreen.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallScreen.kt index eb555b94..6d7764db 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallScreen.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallScreen.kt @@ -127,9 +127,9 @@ fun CallScreen( val video by CallController.video.collectAsState() val videoActive by CallController.videoActive.collectAsState() val remote = video?.display?.collectAsState()?.value - val local = video?.preview?.collectAsState()?.value + val local = video?.preview?.state?.collectAsState()?.value val receiving = phase == CallPhase.ACTIVE && remote != null - val sending = phase == CallPhase.ACTIVE && local != null + val sending = phase == CallPhase.ACTIVE && local?.active == true var selectedVideo by rememberSaveable(st?.callId) { mutableStateOf(CallVideoSource.REMOTE) } val expandedVideo = selectedVideo?.takeIf { if (it == CallVideoSource.REMOTE) receiving else sending } val viewingVideo = expandedVideo != null @@ -170,9 +170,15 @@ fun CallScreen( } } val rotated = remote?.rotation?.rem(180) != 0 - val videoWidth = if (expandedVideo == CallVideoSource.LOCAL) local?.width + val localFormat = local?.format + val localRotated = (localFormat?.camera?.sensor ?: 0) % 180 != 0 + val videoWidth = if (expandedVideo == CallVideoSource.LOCAL) { + if (localRotated) localFormat?.height else localFormat?.width + } else if (rotated) remote?.height else remote?.width - val videoHeight = if (expandedVideo == CallVideoSource.LOCAL) local?.height + val videoHeight = if (expandedVideo == CallVideoSource.LOCAL) { + if (localRotated) localFormat?.width else localFormat?.height + } else if (rotated) remote?.width else remote?.height LaunchedEffect(video, phase, expandedVideo, videoWidth, videoHeight) { val activity = context.findActivity() diff --git a/tools/chatgui/src/call_video_engine.cpp b/tools/chatgui/src/call_video_engine.cpp index 16485217..d9e1c4ae 100644 --- a/tools/chatgui/src/call_video_engine.cpp +++ b/tools/chatgui/src/call_video_engine.cpp @@ -31,6 +31,7 @@ struct CallVideoEngine::Impl { bool enabled = false; int width = 640, height = 480, codec = 0; int64_t pts = -1, backend = -1, anchor = -1; + uint64_t captured = 0; QImage shown; uint64_t shownSerial = 0, deliveredSerial = 0; int64_t previewPts = -1, previewAt = 0; @@ -138,7 +139,7 @@ void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int wid } pts = m->anchor + frame.startTime() - m->backend; } - if (pts > m->pts) { m->raw = frame; m->pts = pts; } + if (pts > m->pts) { m->raw = frame; m->pts = pts; ++m->captured; } }, Qt::DirectConnection); connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error); @@ -170,20 +171,33 @@ void CallVideoEngine::transmit() { unsigned excluded = 0; int64_t lastPts = -1, lastKey = -1; bool forceKey = true; + int64_t reportAt = nowUs(), encodeMax = 0; + uint64_t captureBase = 0; + unsigned encodedCount = 0, keyCount = 0, sendBusy = 0, paced = 0; + int64_t pacedPts = -1; while (m->running) { struct call_video_port_status state{}; if (call_video_port_status(m->id, &state) < 0) break; if (state.error) { m->report(tr("Core rejected video format")); break; } - uint64_t revision; bool enabled; int width, height, codec; int64_t pts; + uint64_t revision, captured; bool enabled; int width, height, codec; int64_t pts; #ifdef HAVE_STREAM_CAPTURE QVideoFrame raw; #endif { std::lock_guard guard(m->lock); revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts; + captured = m->captured; #ifdef HAVE_STREAM_CAPTURE raw = m->raw; #endif } + int64_t now = nowUs(); + if (now - reportAt >= 1000000) { + DEBUG_DEBUG(DEBUG_CATEGORY_CALL, + "video capture flow call=%llu interval_us=%lld captured=%llu encoded=%u keys=%u send_busy=%u paced=%u encode_max_us=%lld", + (unsigned long long)m->id, (long long)(now - reportAt), (unsigned long long)(captured - captureBase), + encodedCount, keyCount, sendBusy, paced, (long long)encodeMax); + reportAt = now; captureBase = captured; encodedCount = keyCount = sendBusy = paced = 0; encodeMax = 0; + } if (revision != userRevision) { userRevision = revision; excluded = 0; } if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted && rejectedGeneration != format.generation) { @@ -195,7 +209,11 @@ void CallVideoEngine::transmit() { } } if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0; - if (!enabled || !state.have_peer || pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) { idle(); continue; } + if (!enabled || !state.have_peer || pts <= lastPts) { idle(); continue; } + if (lastPts >= 0 && pts - lastPts < 33333) { + if (pacedPts != pts) { pacedPts = pts; ++paced; } + idle(); continue; + } if (revision != appliedRevision) { if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; } call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0}; @@ -228,19 +246,23 @@ void CallVideoEngine::transmit() { QImage image = raw.toImage(); if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; } LiveEncodedFrame output; CallVideoRateCompletion completion; + int64_t encodeStarted = nowUs(); bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts; + encodeMax = std::max(encodeMax, nowUs() - encodeStarted); if (completion.request) { call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate); m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264") .arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000)); } if (!encoded) { forceKey = true; m->report(encoder.error()); continue; } + ++encodedCount; if (output.key) ++keyCount; if (output.key) lastKey = pts; call_video_frame packet{}; packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format; packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast(output.data.constData()); packet.flags = output.key ? CALL_VIDEO_KEY : 0; forceKey = call_video_port_send(&packet) != 1; + if (forceKey) ++sendBusy; #else idle(); #endif