From 21c22bd911437b16ebd6684250fe31474c62c48d Mon Sep 17 00:00:00 2001 From: evgeny Date: Thu, 15 Oct 2026 00:44:45 +0200 Subject: [PATCH] Preserve Android video decoder across audio clock corrections and expose previews --- .../com/utun/chat/data/CallVideoEngine.kt | 41 ++++++++++++++++++- .../com/utun/chat/data/CallVideoPlayer.kt | 39 ++++++++++++++---- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt index 3b31163a..ef1170ea 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -3,6 +3,9 @@ package com.utun.chat.data import android.annotation.SuppressLint import android.content.Context import android.graphics.ImageFormat +import android.graphics.Bitmap +import android.graphics.Matrix +import android.media.Image import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice @@ -34,6 +37,10 @@ class CallVideoEngine(private val context: Context, private val id: Long) { private val handler = Handler(thread.looper) private val manager = context.getSystemService(CameraManager::class.java) private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } } + val display: StateFlow = player.display + private val previewFrames = MutableStateFlow(null) + val preview: StateFlow = previewFrames + private var previewAt = 0L @Volatile private var stopped = false private var encoders = emptyMap() private var caps = ByteArray(64) @@ -97,7 +104,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { } } - fun surface(surface: Surface?) = player.surface(surface) + fun surface(surface: Surface?, release: () -> Unit = {}) = player.surface(surface, release) fun select(settings: CallVideoSettings) { handler.post { if (!stopped) { wanted = settings; excluded = emptySet(); mutable.value = mutable.value.copy(settings = settings) } } } @@ -195,6 +202,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (stopped || !wanted.enabled || opening || closing) return val settings = active ?: return; val f = format ?: return; val token = ++cameraToken val c = manager.getCameraCharacteristics(settings.camera) + val mirror = c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME sensorOffset = null val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader @@ -206,6 +214,11 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener val now = System.nanoTime() lastCaptureAt = now + if (previewFrames.subscriptionCount.value > 0 && now - previewAt >= 200_000_000) { + previewAt = now + try { previewFrames.value = cameraPreview(image, f.rotation, mirror) } + catch (e: Exception) { LogManager.addLog("WARN", "call", "camera preview failed: ${e.message}") } + } val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also { sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it") } @@ -290,10 +303,36 @@ class CallVideoEngine(private val context: Context, private val id: Long) { } private fun closeCamera() { cameraToken++ + previewFrames.value = null; previewAt = 0 try { session?.close(); session = null; reader?.close(); reader = null if (camera != null) { closing = true; camera!!.close() } } catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") } } + + /** Миниатюра из уже захваченного YUV: второй output Camera2 и отдельная камера не нужны. */ + private fun cameraPreview(image: Image, rotation: Int, mirror: Boolean): Bitmap { + val crop = image.cropRect; val width = 192; val height = (width * crop.height() / crop.width()).coerceAtLeast(1) + val pixels = IntArray(width * height) + val planes = image.planes; val buffers = planes.map { it.buffer.duplicate() } + fun sample(plane: Int, x: Int, y: Int): Int { + val p = planes[plane]; val b = buffers[plane] + return b.get(b.position() + y * p.rowStride + x * p.pixelStride).toInt() and 255 + } + for (y in 0 until height) for (x in 0 until width) { + val sx = crop.left + x * crop.width() / width; val sy = crop.top + y * crop.height() / height + val luma = (sample(0, sx, sy) - 16).coerceAtLeast(0) + val u = sample(1, sx / 2, sy / 2) - 128; val v = sample(2, sx / 2, sy / 2) - 128 + val r = ((298 * luma + 409 * v + 128) shr 8).coerceIn(0, 255) + val g = ((298 * luma - 100 * u - 208 * v + 128) shr 8).coerceIn(0, 255) + val b = ((298 * luma + 516 * u + 128) shr 8).coerceIn(0, 255) + pixels[y * width + x] = (255 shl 24) or (r shl 16) or (g shl 8) or b + } + val bitmap = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888) + val transform = Matrix().apply { postRotate(rotation.toFloat()); if (mirror) postScale(-1f, 1f) } + val result = Bitmap.createBitmap(bitmap, 0, 0, width, height, transform, true) + if (result !== bitmap) bitmap.recycle() + return result + } fun stop() { if (stopped) return stopped = true; player.stop(); CallVideoNative.close(id) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt index 23c2f644..7b60af47 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt @@ -2,19 +2,28 @@ package com.utun.chat.data import android.media.MediaCodec import android.view.Surface +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import java.util.concurrent.ConcurrentLinkedQueue /** Один RX worker; удерживаем output buffer до фактического audio PTS. Новое поколение ждёт drain старого. */ internal class CallVideoPlayer(private val id: Long, private val failure: (String) -> Unit) { @Volatile private var running = true - @Volatile private var target: Surface? = null + private data class Output(val surface: Surface?, val release: () -> Unit) + private val outputs = ConcurrentLinkedQueue() + private val displayed = MutableStateFlow(null) + val display: StateFlow = displayed @Volatile private var verified = emptyMap() private val worker = Thread(::loop, "call-video-rx").apply { start() } - fun surface(surface: Surface?) { target = surface } + @Synchronized fun surface(surface: Surface?, release: () -> Unit = {}) { + if (running) outputs.add(Output(surface, release)) else release() + } fun decoders(value: Map) { verified = value } - fun stop() { running = false; worker.join() } + fun stop() { synchronized(this) { running = false }; worker.join() } private fun loop() { var decoder: MediaCodec? = null; var surface: Surface? = null; var format: CallVideoFormat? = null + var output: Output? = null; var lastRendered = 0L var pending: ByteArray? = null; var meta = LongArray(12) var held = -1; var heldPts = -1L; var inflight = 0; var draining = false var sourceEpoch = 0L; var lastOutput = -1L; var lastProgress = System.nanoTime() @@ -28,16 +37,21 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin while (running) { try { val now = System.nanoTime() - if (surface !== target || surface?.isValid == false) { - close(); surface = target; pending = null; CallVideoNative.requestKey(id) + while (true) { + val next = outputs.poll() ?: break + close(); output?.release?.invoke(); output = next; surface = next.surface + pending = null; displayed.value = null; CallVideoNative.requestKey(id) } val valid = NativeLib.callAudioClock(id, now / 1000, clock) && clock[5] != 0L if (valid && sourceEpoch != clock[1]) { - if (sourceEpoch != 0L) { close(); pending = null; CallVideoNative.requestKey(id) } + if (sourceEpoch != 0L) LogManager.addLog("DEBUG", "call", + "video audio clock epoch $sourceEpoch -> ${clock[1]}; decoder retained generation=${format?.generation}") sourceEpoch = clock[1] } if (held >= 0 && valid && heldPts <= clock[4]) { - decoder!!.releaseOutputBuffer(held, heldPts >= clock[4] - 500_000); held = -1 + val render = heldPts >= clock[4] - 500_000 + decoder!!.releaseOutputBuffer(held, render); held = -1 + if (render) { displayed.value = format; lastRendered = now } lastProgress = now } if (decoder != null && held < 0) { @@ -92,14 +106,21 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin } } else if (data != null && surface == null) { pending = null; CallVideoNative.requestKey(id) } if (decoder != null && held < 0 && inflight > 0 && now - lastProgress > 2_000_000_000) error("Video decoder stalled") + if (lastRendered != 0L && now - lastRendered >= 5_000_000_000) { + displayed.value = null; lastRendered = 0 + LogManager.addLog("INFO", "call", "video hidden: no rendered frames for 5s call=$id") + } } catch (e: Exception) { LogManager.addLog("ERROR", "call", "video receive call=$id: ${e.message}") val generation = if (pending != null) meta[0] else format?.generation ?: 0 if (generation != 0L) { CallVideoNative.decoded(id, generation, false); rejectedGeneration = generation } - close(); pending = null; failure(e.message ?: "Video decoder failed") + close(); pending = null; displayed.value = null; failure(e.message ?: "Video decoder failed") } Thread.sleep(5) } - } finally { close() } + } finally { + close(); displayed.value = null; output?.release?.invoke() + while (true) { val next = outputs.poll() ?: break; next.release() } + } } }