Preserve Android video decoder across audio clock corrections and expose previews

tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt

@@ -3,6 +3,9 @@ package com.utun.chat.data
 import android.annotation.SuppressLint
 import android.content.Context
 import android.graphics.ImageFormat
+import android.graphics.Bitmap
+import android.graphics.Matrix
+import android.media.Image
 import android.hardware.camera2.CameraCaptureSession
 import android.hardware.camera2.CameraCharacteristics
 import android.hardware.camera2.CameraDevice
@@ -34,6 +37,10 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
     private val handler = Handler(thread.looper)
     private val manager = context.getSystemService(CameraManager::class.java)
     private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } }
+    val display: StateFlow<CallVideoFormat?> = player.display
+    private val previewFrames = MutableStateFlow<Bitmap?>(null)
+    val preview: StateFlow<Bitmap?> = previewFrames
+    private var previewAt = 0L
     @Volatile private var stopped = false
     private var encoders = emptyMap<Int, CallVideoCodec.Companion.Encoder>()
     private var caps = ByteArray(64)
@@ -97,7 +104,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
         }
     }
 
-    fun surface(surface: Surface?) = player.surface(surface)
+    fun surface(surface: Surface?, release: () -> Unit = {}) = player.surface(surface, release)
     fun select(settings: CallVideoSettings) { handler.post {
         if (!stopped) { wanted = settings; excluded = emptySet(); mutable.value = mutable.value.copy(settings = settings) }
     } }
@@ -195,6 +202,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
         if (stopped || !wanted.enabled || opening || closing) return
         val settings = active ?: return; val f = format ?: return; val token = ++cameraToken
         val c = manager.getCameraCharacteristics(settings.camera)
+        val mirror = c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT
         realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME
         sensorOffset = null
         val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader
@@ -206,6 +214,11 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
                 if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener
                 val now = System.nanoTime()
                 lastCaptureAt = now
+                if (previewFrames.subscriptionCount.value > 0 && now - previewAt >= 200_000_000) {
+                    previewAt = now
+                    try { previewFrames.value = cameraPreview(image, f.rotation, mirror) }
+                    catch (e: Exception) { LogManager.addLog("WARN", "call", "camera preview failed: ${e.message}") }
+                }
                 val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also {
                     sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it")
                 }
@@ -290,10 +303,36 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
     }
     private fun closeCamera() {
         cameraToken++
+        previewFrames.value = null; previewAt = 0
         try { session?.close(); session = null; reader?.close(); reader = null
             if (camera != null) { closing = true; camera!!.close() }
         } catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") }
     }
+
+    /** Миниатюра из уже захваченного YUV: второй output Camera2 и отдельная камера не нужны. */
+    private fun cameraPreview(image: Image, rotation: Int, mirror: Boolean): Bitmap {
+        val crop = image.cropRect; val width = 192; val height = (width * crop.height() / crop.width()).coerceAtLeast(1)
+        val pixels = IntArray(width * height)
+        val planes = image.planes; val buffers = planes.map { it.buffer.duplicate() }
+        fun sample(plane: Int, x: Int, y: Int): Int {
+            val p = planes[plane]; val b = buffers[plane]
+            return b.get(b.position() + y * p.rowStride + x * p.pixelStride).toInt() and 255
+        }
+        for (y in 0 until height) for (x in 0 until width) {
+            val sx = crop.left + x * crop.width() / width; val sy = crop.top + y * crop.height() / height
+            val luma = (sample(0, sx, sy) - 16).coerceAtLeast(0)
+            val u = sample(1, sx / 2, sy / 2) - 128; val v = sample(2, sx / 2, sy / 2) - 128
+            val r = ((298 * luma + 409 * v + 128) shr 8).coerceIn(0, 255)
+            val g = ((298 * luma - 100 * u - 208 * v + 128) shr 8).coerceIn(0, 255)
+            val b = ((298 * luma + 516 * u + 128) shr 8).coerceIn(0, 255)
+            pixels[y * width + x] = (255 shl 24) or (r shl 16) or (g shl 8) or b
+        }
+        val bitmap = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888)
+        val transform = Matrix().apply { postRotate(rotation.toFloat()); if (mirror) postScale(-1f, 1f) }
+        val result = Bitmap.createBitmap(bitmap, 0, 0, width, height, transform, true)
+        if (result !== bitmap) bitmap.recycle()
+        return result
+    }
     fun stop() {
         if (stopped) return
         stopped = true; player.stop(); CallVideoNative.close(id)

tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt

@@ -2,19 +2,28 @@ package com.utun.chat.data
 
 import android.media.MediaCodec
 import android.view.Surface
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import java.util.concurrent.ConcurrentLinkedQueue
 
 /** Один RX worker; удерживаем output buffer до фактического audio PTS. Новое поколение ждёт drain старого. */
 internal class CallVideoPlayer(private val id: Long, private val failure: (String) -> Unit) {
     @Volatile private var running = true
-    @Volatile private var target: Surface? = null
+    private data class Output(val surface: Surface?, val release: () -> Unit)
+    private val outputs = ConcurrentLinkedQueue<Output>()
+    private val displayed = MutableStateFlow<CallVideoFormat?>(null)
+    val display: StateFlow<CallVideoFormat?> = displayed
     @Volatile private var verified = emptyMap<Int, CallVideoCodec.Companion.Decoder>()
     private val worker = Thread(::loop, "call-video-rx").apply { start() }
-    fun surface(surface: Surface?) { target = surface }
+    @Synchronized fun surface(surface: Surface?, release: () -> Unit = {}) {
+        if (running) outputs.add(Output(surface, release)) else release()
+    }
     fun decoders(value: Map<Int, CallVideoCodec.Companion.Decoder>) { verified = value }
-    fun stop() { running = false; worker.join() }
+    fun stop() { synchronized(this) { running = false }; worker.join() }
 
     private fun loop() {
         var decoder: MediaCodec? = null; var surface: Surface? = null; var format: CallVideoFormat? = null
+        var output: Output? = null; var lastRendered = 0L
         var pending: ByteArray? = null; var meta = LongArray(12)
         var held = -1; var heldPts = -1L; var inflight = 0; var draining = false
         var sourceEpoch = 0L; var lastOutput = -1L; var lastProgress = System.nanoTime()
@@ -28,16 +37,21 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin
             while (running) {
                 try {
                     val now = System.nanoTime()
-                    if (surface !== target || surface?.isValid == false) {
-                        close(); surface = target; pending = null; CallVideoNative.requestKey(id)
+                    while (true) {
+                        val next = outputs.poll() ?: break
+                        close(); output?.release?.invoke(); output = next; surface = next.surface
+                        pending = null; displayed.value = null; CallVideoNative.requestKey(id)
                     }
                     val valid = NativeLib.callAudioClock(id, now / 1000, clock) && clock[5] != 0L
                     if (valid && sourceEpoch != clock[1]) {
-                        if (sourceEpoch != 0L) { close(); pending = null; CallVideoNative.requestKey(id) }
+                        if (sourceEpoch != 0L) LogManager.addLog("DEBUG", "call",
+                            "video audio clock epoch $sourceEpoch -> ${clock[1]}; decoder retained generation=${format?.generation}")
                         sourceEpoch = clock[1]
                     }
                     if (held >= 0 && valid && heldPts <= clock[4]) {
-                        decoder!!.releaseOutputBuffer(held, heldPts >= clock[4] - 500_000); held = -1
+                        val render = heldPts >= clock[4] - 500_000
+                        decoder!!.releaseOutputBuffer(held, render); held = -1
+                        if (render) { displayed.value = format; lastRendered = now }
                         lastProgress = now
                     }
                     if (decoder != null && held < 0) {
@@ -92,14 +106,21 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin
                         }
                     } else if (data != null && surface == null) { pending = null; CallVideoNative.requestKey(id) }
                     if (decoder != null && held < 0 && inflight > 0 && now - lastProgress > 2_000_000_000) error("Video decoder stalled")
+                    if (lastRendered != 0L && now - lastRendered >= 5_000_000_000) {
+                        displayed.value = null; lastRendered = 0
+                        LogManager.addLog("INFO", "call", "video hidden: no rendered frames for 5s call=$id")
+                    }
                 } catch (e: Exception) {
                     LogManager.addLog("ERROR", "call", "video receive call=$id: ${e.message}")
                     val generation = if (pending != null) meta[0] else format?.generation ?: 0
                     if (generation != 0L) { CallVideoNative.decoded(id, generation, false); rejectedGeneration = generation }
-                    close(); pending = null; failure(e.message ?: "Video decoder failed")
+                    close(); pending = null; displayed.value = null; failure(e.message ?: "Video decoder failed")
                 }
                 Thread.sleep(5)
             }
-        } finally { close() }
+        } finally {
+            close(); displayed.value = null; output?.release?.invoke()
+            while (true) { val next = outputs.poll() ?: break; next.release() }
+        }
     }
 }