Render Android call camera preview directly through Camera2

tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt

@@ -3,9 +3,7 @@ package com.utun.chat.data
 import android.annotation.SuppressLint
 import android.content.Context
 import android.graphics.ImageFormat
-import android.graphics.Bitmap
+import android.graphics.SurfaceTexture
-import android.graphics.Matrix
-import android.media.Image
 import android.hardware.camera2.CameraCaptureSession
 import android.hardware.camera2.CameraCharacteristics
 import android.hardware.camera2.CameraDevice
@@ -38,9 +36,10 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
     private val manager = context.getSystemService(CameraManager::class.java)
     private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } }
     internal val display: StateFlow<CallVideoFormat?> = player.display
-    private val previewFrames = MutableStateFlow<Bitmap?>(null)
+    val preview = StreamPreview()
-    val preview: StateFlow<Bitmap?> = previewFrames
+    private var previewTarget: StreamPreviewTarget? = null
-    private var previewAt = 0L
+    private var sessionSerial = 0L
+    private val sessions = mutableMapOf<Long, Pair<CameraDevice, () -> Unit>>()
     @Volatile private var stopped = false
     private var encoders = emptyMap<Int, CallVideoCodec.Companion.Encoder>()
     private var caps = ByteArray(64)
@@ -76,6 +75,15 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
     private var excluded = emptySet<Int>()
 
     init {
+        preview.bind { target ->
+            if (!handler.post {
+                if (stopped) { target?.release(); return@post }
+                if (target === previewTarget) { target?.release(); return@post }
+                val previous = previewTarget; previewTarget = target; previous?.release()
+                val device = camera; val source = reader
+                if (device != null && source != null && !closing) createSession(device, source, cameraToken)
+            }) { target?.release(); LogManager.addLog("WARN", "call", "preview command after worker stop id=$id") }
+        }
         handler.post {
             try {
                 val decoders = CallVideoCodec.probeDecoders()
@@ -204,7 +212,6 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
         if (stopped || !wanted.enabled || opening || closing) return
         val settings = active ?: return; val f = format ?: return; val token = ++cameraToken
         val c = manager.getCameraCharacteristics(settings.camera)
-        val mirror = c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT
         realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME
         sensorOffset = null
         val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader
@@ -216,16 +223,6 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
                 if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener
                 val now = System.nanoTime()
                 lastCaptureAt = now
-                if (previewFrames.subscriptionCount.value > 0 && now - previewAt >= 200_000_000) {
-                    previewAt = now
-                    try {
-                        val preview = cameraPreview(image, f.rotation, mirror)
-                        if (previewFrames.value == null) LogManager.addLog("INFO", "call",
-                            "camera preview visible id=$id size=${preview.width}x${preview.height} rotation=${f.rotation} mirror=$mirror")
-                        previewFrames.value = preview
-                    }
-                    catch (e: Exception) { LogManager.addLog("WARN", "call", "camera preview failed: ${e.message}") }
-                }
                 val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also {
                     sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it")
                 }
@@ -265,28 +262,12 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
             override fun onOpened(device: CameraDevice) {
                 opening = false; camera = device
                 if (stopped || token != cameraToken) { device.close(); return }
-                try {
+                createSession(device, nextReader, token)
-                    device.createCaptureSession(listOf(nextReader.surface), object : CameraCaptureSession.StateCallback() {
-                        override fun onConfigured(s: CameraCaptureSession) {
-                            if (stopped || token != cameraToken) { s.close(); return }
-                            session = s
-                            try {
-                                val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty()
-                                val range = rates.filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps")
-                                val capture = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply {
-                                    addTarget(nextReader.surface); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range)
-                                }.build()
-                                s.setRepeatingRequest(capture, null, handler)
-                                report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}")
-                            } catch (e: Exception) { fail("Camera request: ${e.message}") }
-                        }
-                        override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); fail("Camera session configuration failed") }
-                    }, handler)
-                } catch (e: Exception) { fail("Camera session: ${e.message}") }
             }
             override fun onDisconnected(device: CameraDevice) { opening = false; camera = device; fail("Camera disconnected") }
             override fun onError(device: CameraDevice, error: Int) { opening = false; camera = device; fail("Camera error $error") }
             override fun onClosed(device: CameraDevice) {
+                sessions.values.filter { it.first === device }.map { it.second }.forEach { it() }
                 if (camera === device) camera = null
                 closing = false
                 if (stopped) thread.quitSafely()
@@ -294,6 +275,62 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
             }
         }, handler)
     }
+
+    /** Прямое превью использует второй output той же камеры; encoder и его PTS не меняются. */
+    private fun createSession(device: CameraDevice, source: ImageReader, token: Long, withPreview: Boolean = true) {
+        val f = format ?: return
+        val serial = ++sessionSerial
+        val output = if (withPreview) previewTarget?.retain() else null
+        var completed = false
+        fun complete() {
+            if (completed) return
+            completed = true; sessions.remove(serial); output?.release()
+            LogManager.addLog("DEBUG", "call", "camera session released id=$id session=$serial remaining=${sessions.size}")
+        }
+        fun valid() = !stopped && token == cameraToken && serial == sessionSerial
+        fun rejected(message: String) {
+            if (!valid()) { LogManager.addLog("DEBUG", "call", "stale camera session id=$id session=$serial: $message"); return }
+            if (output == null) fail(message) else {
+                LogManager.addLog("ERROR", "call", "direct preview failed id=$id: $message; retaining transmission")
+                preview.update(preview.state.value.format, failed = true)
+                createSession(device, source, token, withPreview = false)
+            }
+        }
+        sessions[serial] = device to ::complete
+        try {
+            val c = manager.getCameraCharacteristics(device.id)
+            var previewFormat: StreamPreviewFormat? = null
+            if (output != null) {
+                val sizes = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)?.getOutputSizes(SurfaceTexture::class.java).orEmpty()
+                val size = sizes.filter { it.width <= f.width && it.height <= f.height && it.width * f.height == it.height * f.width }
+                    .maxByOrNull { it.width * it.height } ?: error("No compatible preview size")
+                output.texture.setDefaultBufferSize(size.width, size.height)
+                previewFormat = StreamPreviewFormat(size.width, size.height,
+                    StreamPreviewCamera(f.rotation, c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT), realtime)
+                preview.update(previewFormat)
+            } else if (withPreview) preview.update(null)
+            val targets = listOfNotNull(source.surface, output?.surface)
+            device.createCaptureSession(targets, object : CameraCaptureSession.StateCallback() {
+                override fun onConfigured(s: CameraCaptureSession) {
+                    if (!valid()) { s.close(); return }
+                    session = s
+                    try {
+                        val range = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty()
+                            .filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps")
+                        val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply {
+                            targets.forEach(::addTarget); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range)
+                        }.build()
+                        s.setRepeatingRequest(request, null, handler)
+                        if (output != null) preview.update(previewFormat, active = true)
+                        LogManager.addLog("INFO", "call", "camera capture id=$id session=$serial range=$range preview=$previewFormat")
+                        report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}")
+                    } catch (e: Exception) { s.close(); rejected("Camera request: ${e.message}") }
+                }
+                override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); complete(); rejected("Camera session configuration failed") }
+                override fun onClosed(s: CameraCaptureSession) { if (session === s) session = null; complete() }
+            }, handler)
+        } catch (e: Exception) { complete(); rejected("Camera session: ${e.message}") }
+    }
     private fun drain() {
         val codec = encoder ?: return; val f = format ?: return
         for (p in codec.drain()) {
@@ -316,42 +353,19 @@ class CallVideoEngine(private val context: Context, private val id: Long) {
         }
     }
     private fun closeCamera() {
-        cameraToken++
+        cameraToken++; sessionSerial++
-        previewFrames.value = null; previewAt = 0
+        preview.update(null)
         try { session?.close(); session = null; reader?.close(); reader = null
             if (camera != null) { closing = true; camera!!.close() }
         } catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") }
     }
 
-    /** Миниатюра из уже захваченного YUV: второй output Camera2 и отдельная камера не нужны. */
-    private fun cameraPreview(image: Image, rotation: Int, mirror: Boolean): Bitmap {
-        val crop = image.cropRect; val width = 192; val height = (width * crop.height() / crop.width()).coerceAtLeast(1)
-        val pixels = IntArray(width * height)
-        val planes = image.planes; val buffers = planes.map { it.buffer.duplicate() }
-        fun sample(plane: Int, x: Int, y: Int): Int {
-            val p = planes[plane]; val b = buffers[plane]
-            return b.get(b.position() + y * p.rowStride + x * p.pixelStride).toInt() and 255
-        }
-        for (y in 0 until height) for (x in 0 until width) {
-            val sx = crop.left + x * crop.width() / width; val sy = crop.top + y * crop.height() / height
-            val luma = (sample(0, sx, sy) - 16).coerceAtLeast(0)
-            val u = sample(1, sx / 2, sy / 2) - 128; val v = sample(2, sx / 2, sy / 2) - 128
-            val r = ((298 * luma + 409 * v + 128) shr 8).coerceIn(0, 255)
-            val g = ((298 * luma - 100 * u - 208 * v + 128) shr 8).coerceIn(0, 255)
-            val b = ((298 * luma + 516 * u + 128) shr 8).coerceIn(0, 255)
-            pixels[y * width + x] = (255 shl 24) or (r shl 16) or (g shl 8) or b
-        }
-        val bitmap = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888)
-        val transform = Matrix().apply { postRotate(rotation.toFloat()); if (mirror) postScale(-1f, 1f) }
-        val result = Bitmap.createBitmap(bitmap, 0, 0, width, height, transform, true)
-        if (result !== bitmap) bitmap.recycle()
-        return result
-    }
     fun stop() {
         if (stopped) return
         stopped = true; player.stop(); CallVideoNative.close(id)
         handler.post {
             closeCamera(); encoder?.close(); encoder = null
+            preview.close(); previewTarget?.release(); previewTarget = null
             if (camera == null && !opening) thread.quitSafely()
             else handler.postDelayed({ thread.quitSafely() }, 3000)
             LogManager.addLog("INFO", "call", "video workers stopped id=$id")

tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt

@@ -10,16 +10,12 @@ import android.view.TextureView
 import androidx.activity.compose.rememberLauncherForActivityResult
 import androidx.activity.result.contract.ActivityResultContracts
 import androidx.compose.foundation.background
-import androidx.compose.foundation.Image
-import androidx.compose.foundation.clickable
 import androidx.compose.foundation.layout.*
 import androidx.compose.material3.*
 import androidx.compose.runtime.*
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.graphics.Color
-import androidx.compose.ui.graphics.asImageBitmap
-import androidx.compose.ui.layout.ContentScale
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.res.stringResource
 import androidx.compose.ui.unit.dp
@@ -81,27 +77,33 @@ internal enum class CallVideoSource { LOCAL, REMOTE }
 internal fun CallVideoStage(video: CallVideoEngine, expanded: CallVideoSource?, onSelect: (CallVideoSource) -> Unit,
     onTap: () -> Unit, modifier: Modifier = Modifier) {
     val remote by video.display.collectAsState()
-    val local by video.preview.collectAsState()
+    val local by video.preview.state.collectAsState()
+    val displayDegrees = StreamDisplayDegrees()
     val insets = WindowInsets.systemBarsIgnoringVisibility.union(WindowInsets.displayCutout)
     Box(modifier) {
         // Слоты не меняются местами в composition; миниатюра рисуется поверх большого кадра.
         for (source in CallVideoSource.entries) key(source) {
             val large = expanded == source
-            val available = if (source == CallVideoSource.LOCAL) local != null else remote != null
             Box(Modifier.matchParentSize().zIndex(if (large) 0f else 1f)
                 .then(if (large) Modifier else Modifier.windowInsetsPadding(insets).padding(top = 64.dp, end = 12.dp))) {
                 val frameModifier = if (large) Modifier.matchParentSize() else Modifier
                     .align(if (expanded != null) Alignment.TopEnd else Alignment.TopCenter)
-                    .offset(x = if (expanded != null || local == null || remote == null) 0.dp
+                    .offset(x = if (expanded != null || !local.active || remote == null) 0.dp
                         else if (source == CallVideoSource.LOCAL) (-60).dp else 60.dp)
                     .size(104.dp, 144.dp)
                 val description = stringResource(if (source == CallVideoSource.LOCAL) R.string.call_video_local else R.string.call_video_remote)
                 val click = { if (large) onTap() else onSelect(source) }
                 if (source == CallVideoSource.REMOTE) {
                     CallVideoPlayback(video, click, frameModifier, description)
-                } else if (available) {
+                } else BoxWithConstraints(frameModifier, contentAlignment = Alignment.Center) {
-                    Image(local!!.asImageBitmap(), contentDescription = description, contentScale = ContentScale.Fit,
+                    val aspect = local.format?.aspect(displayDegrees) ?: 1f
-                        modifier = frameModifier.clickable(onClick = click))
+                    val width = minOf(maxWidth, maxHeight * aspect)
+                    AndroidView(factory = { CameraPreviewView(it, video.preview, keepAwake = false) },
+                        modifier = Modifier.size(width, width / aspect), update = { view ->
+                            view.update(local.format, displayDegrees); view.alpha = if (local.active) 1f else 0f
+                            view.contentDescription = description
+                            view.setOnClickListener { if (local.active) click() }; view.isClickable = local.active
+                        }, onRelease = { it.close() })
                 }
             }
         }

tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt

@@ -318,7 +318,7 @@ private fun LiveStreamView(localPreview: StreamPreview, displayDegrees: Int) {
 
 /** DisplayListener замечает также 180°, когда Configuration и размеры окна не меняются. */
 @Composable
-private fun StreamDisplayDegrees(): Int {
+internal fun StreamDisplayDegrees(): Int {
     val context = LocalContext.current
     val configuration = LocalConfiguration.current
     val display = (context as? Activity)?.windowManager?.defaultDisplay
@@ -355,7 +355,7 @@ private fun CameraPreviewSurface(preview: StreamPreview, displayDegrees: Int) {
 }
 
 /** TextureView отдаёт texture владельцам Camera2; destroy не освобождает используемый камерой output. */
-private class CameraPreviewView(context: Context, private val preview: StreamPreview) : TextureView(context) {
+internal class CameraPreviewView(context: Context, private val preview: StreamPreview, keepAwake: Boolean = true) : TextureView(context) {
     private var target: StreamPreviewTarget? = null
     private var format: StreamPreviewFormat? = null
     private var displayDegrees = 0
@@ -366,7 +366,7 @@ private class CameraPreviewView(context: Context, private val preview: StreamPre
     private var observedFormat: StreamPreviewFormat? = null
 
     init {
-        keepScreenOn = true
+        keepScreenOn = keepAwake
         surfaceTextureListener = object : SurfaceTextureListener {
             override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) {
                 if (closed) return