Send Android camera frames directly to preview surfaces

tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt

@@ -2,6 +2,7 @@ package com.utun.chat.data
 
 import android.annotation.SuppressLint
 import android.content.Context
+import android.graphics.SurfaceTexture
 import android.hardware.camera2.CameraCaptureSession
 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession
 import android.hardware.camera2.CameraCharacteristics
@@ -26,7 +27,7 @@ import android.view.Surface
 import java.io.ByteArrayOutputStream
 
 /** Camera/codec принадлежат HandlerThread; микрофон и origin сохраняются при смене видео.
- * Preview читает тот же источник. Поколение отсеивает callbacks закрываемой камеры/кодека. */
+ * Превью — второй выход Camera2. Поколения видео и camera session отсеивают старые callbacks. */
 class StreamCaptureEngine(
     private val context: Context, private val id: Long, private val group: Long,
     private val initialConfig: StreamCaptureConfig, private val microphone: Boolean, private val preview: StreamPreview,
@@ -40,6 +41,10 @@ class StreamCaptureEngine(
     private var session: CameraCaptureSession? = null
     private var encoder: MediaCodec? = null
     private var input: Surface? = null
+    private var activeConfig: StreamCaptureConfig? = null
+    private var previewTarget: StreamPreviewTarget? = null
+    private var sessionGeneration = 0L
+    private var cameraSessions = 0
     private var record: AudioRecord? = null
     private var audioThread: Thread? = null
     @Volatile private var audioRunning = false
@@ -77,6 +82,19 @@ class StreamCaptureEngine(
     private var lastPts = -1L
 
     fun start() {
+        preview.bind { target ->
+            if (!handler.post {
+                if (stopping || stopped || terminalFailure) { target?.release(); return@post }
+                val previous = previewTarget; previewTarget = target; previous?.release()
+                val config = activeConfig; val device = camera
+                LogManager.addLog("DEBUG", "video", "preview output change id=$id target=${target?.let(System::identityHashCode)} " +
+                    "revision=$generation camera=${device != null} opening=$opening")
+                if (device != null && config != null && current(generation)) createSession(device, config, generation)
+            }) {
+                LogManager.addLog("WARN", "video", "preview command rejected id=$id capture handler stopped")
+                target?.release()
+            }
+        }
         handler.post {
             if (stopping || stopped) return@post
             try {
@@ -129,8 +147,9 @@ class StreamCaptureEngine(
             check(token <= 0xffffffffL) { "Video revision exhausted" }
             val cameraPose = cameraPose(mode.camera)
             val rotation = cameraPose.rotation(config.orientation.degrees)
-            preview.register(token, cameraPose)
+            activeConfig = config
-            LogManager.addLog("DEBUG", "video", "preview camera id=$id revision=$token camera=${mode.camera} " +
+            preview.update(null)
+            LogManager.addLog("DEBUG", "video", "capture camera id=$id revision=$token camera=${mode.camera} " +
                 "sensor=${cameraPose.sensor} front=${cameraPose.front} broadcastRotation=$rotation")
             csd = byteArrayOf(); partial.reset(); partialConfig.reset(); requireKey = true; lastKeyRequest = 0; waitingFirst = true
             confirming = mode.highSpeed; confirmStartNs = 0
@@ -192,60 +211,12 @@ class StreamCaptureEngine(
                     override fun onOpened(device: CameraDevice) {
                         if (openingGeneration == token) { opening = false; camera = device }
                         if (!current(token)) { device.close(); return }
-                        try {
+                        createSession(device, config, token)
-                            val callback = object : CameraCaptureSession.StateCallback() {
-                                override fun onConfigured(configured: CameraCaptureSession) {
-                                    if (!current(token)) { configured.close(); return }
-                                    session = configured
-                                    try {
-                                        val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply {
-                                            addTarget(input!!)
-                                            set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(mode.minimumFps, mode.fps))
-                                        }.build()
-                                        var sensorResults = 0L
-                                        var sensorReportCount = 0L
-                                        var sensorReportNs = System.nanoTime()
-                                        val diagnostics = object : CameraCaptureSession.CaptureCallback() {
-                                            override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) {
-                                                if (!current(token)) return
-                                                sensorResults++
-                                                val now = System.nanoTime()
-                                                if (now - sensorReportNs >= 1_000_000_000) {
-                                                    val fps = (sensorResults - sensorReportCount) * 1_000_000_000.0 / (now - sensorReportNs)
-                                                    LogManager.addLog("DEBUG", "video", "sensor id=$id revision=$token results=$sensorResults callbackFps=${"%.1f".format(fps)} " +
-                                                        "range=${result.get(CaptureResult.CONTROL_AE_TARGET_FPS_RANGE)} " +
-                                                        "durationNs=${result.get(CaptureResult.SENSOR_FRAME_DURATION)} " +
-                                                        "exposureNs=${result.get(CaptureResult.SENSOR_EXPOSURE_TIME)} timestamp=${result.get(CaptureResult.SENSOR_TIMESTAMP)}")
-                                                    sensorReportNs = now; sensorReportCount = sensorResults
-                                                }
-                                            }
-                                            override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest,
-                                                                         failed: android.hardware.camera2.CaptureFailure) {
-                                                if (current(token)) LogManager.addLog("WARN", "video", "sensor capture failed id=$id revision=$token " +
-                                                    "frame=${failed.frameNumber} reason=${failed.reason} captured=${failed.wasImageCaptured()}")
-                                            }
-                                        }
-                                        if (mode.highSpeed) {
-                                            val highSpeed = configured as CameraConstrainedHighSpeedCaptureSession
-                                            highSpeed.setRepeatingBurst(highSpeed.createHighSpeedRequestList(request), diagnostics, handler)
-                                        } else configured.setRepeatingRequest(request, diagnostics, handler)
-                                        LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " +
-                                            "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " +
-                                            "type=${mode.videoCodec.label} bitrate=${config.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs")
-                                    } catch (e: Exception) { captureError(token, "camera request: ${e.message}") }
-                                }
-                                override fun onConfigureFailed(session: CameraCaptureSession) {
-                                    session.close(); captureError(token, "camera session configuration")
-                                }
-                            }
-                            if (mode.highSpeed) device.createConstrainedHighSpeedCaptureSession(listOf(input!!), callback, handler)
-                            else device.createCaptureSession(listOf(input!!), callback, handler)
-                        } catch (e: Exception) { captureError(token, "camera session: ${e.message}") }
                     }
                     override fun onClosed(device: CameraDevice) {
                         if (camera === device) camera = null
                         if (releaseDone != null && camera == null && !opening) finishRelease(true)
-                        if (stopped && camera == null && !opening) thread.quitSafely()
+                        maybeQuit()
                     }
                     override fun onDisconnected(device: CameraDevice) {
                         if (openingGeneration == token) { opening = false; camera = device }
@@ -263,6 +234,99 @@ class StreamCaptureEngine(
 
     private fun current(token: Long) = token == generation && !stopping && !stopped && !terminalFailure && releaseDone == null
 
+    /** Меняется только набор Camera2 outputs; encoder, audio и origin не перезапускаются. */
+    private fun createSession(device: CameraDevice, config: StreamCaptureConfig, token: Long, withPreview: Boolean = true) {
+        val mode = config.mode
+        val serial = ++sessionGeneration
+        val output = if (withPreview) previewTarget?.retain() else null
+        var format: StreamPreviewFormat? = null
+        var completed = false
+        cameraSessions++
+        fun complete() {
+            if (completed) return
+            completed = true; output?.release(); cameraSessions--; maybeQuit()
+        }
+        fun valid() = current(token) && serial == sessionGeneration
+        fun rejected(detail: String) {
+            if (!valid()) { LogManager.addLog("DEBUG", "video", "stale camera session failure id=$id session=$serial: $detail"); return }
+            if (output == null) { captureError(token, detail); return }
+            LogManager.addLog("ERROR", "video", "direct preview failed id=$id session=$serial; keep encoder output: $detail")
+            preview.update(format, failed = true)
+            createSession(device, config, token, withPreview = false)
+        }
+        try {
+            if (output != null) {
+                val map = manager.getCameraCharacteristics(mode.camera).get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
+                    ?: error("Missing preview stream map")
+                val size = if (mode.highSpeed) android.util.Size(mode.width, mode.height) else
+                    map.getOutputSizes(SurfaceTexture::class.java).orEmpty().filter {
+                        it.width <= minOf(mode.width, 1280) && it.height <= minOf(mode.height, 720)
+                            && it.width.toLong() * mode.height == it.height.toLong() * mode.width
+                    }.maxByOrNull { it.width * it.height } ?: error("No compatible camera preview size")
+                output.texture.setDefaultBufferSize(size.width, size.height)
+                format = StreamPreviewFormat(size.width, size.height, cameraPose(mode.camera))
+                preview.update(format)
+            } else if (withPreview) preview.update(preview.state.value.format)
+            val targets = listOfNotNull(input, output?.surface)
+            check(targets.isNotEmpty()) { "Missing encoder surface" }
+            val callback = object : CameraCaptureSession.StateCallback() {
+                override fun onConfigured(configured: CameraCaptureSession) {
+                    if (!valid()) { configured.close(); return }
+                    session = configured
+                    try {
+                        val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply {
+                            targets.forEach(::addTarget)
+                            set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(mode.minimumFps, mode.fps))
+                        }.build()
+                        var sensorResults = 0L
+                        var sensorReportCount = 0L
+                        var sensorReportNs = System.nanoTime()
+                        val diagnostics = object : CameraCaptureSession.CaptureCallback() {
+                            override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) {
+                                if (!valid()) return
+                                sensorResults++
+                                val now = System.nanoTime()
+                                if (now - sensorReportNs < 1_000_000_000) return
+                                val fps = (sensorResults - sensorReportCount) * 1_000_000_000.0 / (now - sensorReportNs)
+                                LogManager.addLog("DEBUG", "video", "sensor id=$id revision=$token session=$serial results=$sensorResults " +
+                                    "callbackFps=${"%.1f".format(fps)} range=${result.get(CaptureResult.CONTROL_AE_TARGET_FPS_RANGE)} " +
+                                    "durationNs=${result.get(CaptureResult.SENSOR_FRAME_DURATION)} " +
+                                    "exposureNs=${result.get(CaptureResult.SENSOR_EXPOSURE_TIME)} timestamp=${result.get(CaptureResult.SENSOR_TIMESTAMP)}")
+                                sensorReportNs = now; sensorReportCount = sensorResults
+                            }
+                            override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest,
+                                                         failed: android.hardware.camera2.CaptureFailure) {
+                                if (valid()) LogManager.addLog("WARN", "video", "sensor capture failed id=$id revision=$token " +
+                                    "session=$serial frame=${failed.frameNumber} reason=${failed.reason} captured=${failed.wasImageCaptured()}")
+                            }
+                        }
+                        if (mode.highSpeed) {
+                            val highSpeed = configured as CameraConstrainedHighSpeedCaptureSession
+                            highSpeed.setRepeatingBurst(highSpeed.createHighSpeedRequestList(request), diagnostics, handler)
+                        } else configured.setRepeatingRequest(request, diagnostics, handler)
+                        if (output != null) preview.update(format, active = true)
+                        LogManager.addLog("INFO", "video", "capture started id=$id revision=$token session=$serial camera=${mode.camera} ${mode.label} " +
+                            "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} encoder=${encoder?.name} " +
+                            "preview=${output != null} previewSize=${format?.let { "${it.width}x${it.height}" }} origin=$originNs")
+                    } catch (e: Exception) { configured.close(); rejected("camera request: ${e.message}") }
+                }
+                override fun onConfigureFailed(session: CameraCaptureSession) {
+                    complete(); rejected("camera session configuration")
+                }
+                override fun onClosed(closed: CameraCaptureSession) {
+                    if (session === closed) session = null
+                    complete()
+                    LogManager.addLog("DEBUG", "video", "camera session closed id=$id session=$serial remaining=$cameraSessions")
+                }
+            }
+            LogManager.addLog("DEBUG", "video", "camera session configuring id=$id revision=$token session=$serial outputs=${targets.size}")
+            if (mode.highSpeed) device.createConstrainedHighSpeedCaptureSession(targets, callback, handler)
+            else device.createCaptureSession(targets, callback, handler)
+        } catch (e: Exception) { complete(); rejected("camera session: ${e.message}") }
+    }
+
+    private fun maybeQuit() { if (stopped && camera == null && !opening && cameraSessions == 0) thread.quitSafely() }
+
     private fun captureError(token: Long, detail: String) {
         if (!current(token)) { LogManager.addLog("DEBUG", "video", "stale capture error revision=$token: $detail"); return }
         LogManager.addLog("ERROR", "video", "capture error id=$id revision=$token: $detail")
@@ -341,6 +405,8 @@ class StreamCaptureEngine(
     private fun releaseVideo(done: (Boolean) -> Unit) {
         if (releaseDone != null) { releaseDone = done; return }
         val token = ++generation
+        sessionGeneration++
+        preview.update(preview.state.value.format)
         releaseDone = done
         try { session?.close() } catch (e: Exception) { LogManager.addLog("WARN", "video", "camera session close id=$id: ${e.message}") }
         session = null
@@ -418,6 +484,7 @@ class StreamCaptureEngine(
             stopCallbacks.add(done)
             if (stopping) return@post
             stopping = true; audioRunning = false
+            preview.close(); previewTarget?.release(); previewTarget = null
             try { record?.stop() } catch (e: Exception) { LogManager.addLog("WARN", "stream", "audio stop id=$id: ${e.message}") }
             audioThread?.join(); audioThread = null
             try { record?.release() } catch (e: Exception) { LogManager.addLog("WARN", "stream", "audio release id=$id: ${e.message}") }
@@ -426,7 +493,7 @@ class StreamCaptureEngine(
                 stopped = true
                 LogManager.addLog("INFO", "stream", "capture released id=$id frames=$frames dropped=$dropped bytes=$bytesSent")
                 stopCallbacks.toList().also { stopCallbacks.clear() }.forEach { it() }
-                if (camera == null && !opening) thread.quitSafely()
+                maybeQuit()
             }
         })) {
             LogManager.addLog("WARN", "stream", "capture handler already closed id=$id; complete stop")

tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt

@@ -1,27 +1,81 @@
 package com.utun.chat.data
 
+import android.graphics.SurfaceTexture
+import android.view.Surface
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+
 /** Положение сенсора конкретной камеры; ручной поворот относится только к трансляции. */
 data class StreamPreviewCamera(val sensor: Int, val front: Boolean) {
     fun rotation(displayDegrees: Int): Int = (sensor + (if (front) displayDegrees else -displayDegrees) + 360) % 360
 }
 
-data class StreamVideoDisplay(val rotation: Int, val top: Int?) {
+data class StreamPreviewFormat(val width: Int, val height: Int, val camera: StreamPreviewCamera) {
-    companion object {
+    fun aspect(displayDegrees: Int): Float = if (camera.rotation(displayDegrees) % 180 == 0) width.toFloat() / height else height.toFloat() / width
-        fun forFrame(broadcastRotation: Int, camera: StreamPreviewCamera?, displayDegrees: Int): StreamVideoDisplay {
+
-            if (camera == null) return StreamVideoDisplay(broadcastRotation, null)
+    /** Углы TextureView после поворота; размеры view уже учитывают поворот сенсора. */
-            val previewRotation = camera.rotation(displayDegrees)
+    fun corners(displayDegrees: Int): FloatArray = when (camera.rotation(displayDegrees)) {
-            return StreamVideoDisplay(previewRotation, (previewRotation - broadcastRotation + 360) % 360)
+        90 -> floatArrayOf(1f, 0f, 1f, 1f, 0f, 1f, 0f, 0f)
-        }
+        180 -> floatArrayOf(1f, 1f, 0f, 1f, 0f, 0f, 1f, 0f)
+        270 -> floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f)
+        else -> floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f)
+    }
+}
+
+data class StreamPreviewState(val format: StreamPreviewFormat? = null, val active: Boolean = false, val failed: Boolean = false)
+
+/** UI, ожидающая команда camera thread и каждая CameraCaptureSession держат отдельную ссылку. */
+internal class StreamPreviewLease(private val close: () -> Unit) {
+    private var references = 1
+    @Synchronized fun retain() { check(references > 0); references++ }
+    @Synchronized fun release() { check(references > 0); if (--references == 0) close() }
+}
+
+class StreamPreviewTarget internal constructor(val texture: SurfaceTexture) {
+    val surface = Surface(texture)
+    private val lease = StreamPreviewLease {
+        surface.release(); texture.release()
+        LogManager.addLog("DEBUG", "video", "direct preview surface released target=${System.identityHashCode(this)}")
     }
+    internal fun retain(): StreamPreviewTarget { lease.retain(); return this }
+    fun release() = lease.release()
 }
 
-/** Принадлежит одной публикации. Camera thread пишет immutable snapshots; viewer читает по revision своего кадра. */
+/** Прямой выход камеры. UI не владеет camera/codec; listener получает собственную ссылку на target. */
 class StreamPreview {
-    @Volatile private var cameras: List<Pair<Long, StreamPreviewCamera>> = emptyList()
+    private var target: StreamPreviewTarget? = null
+    private var owner: ((StreamPreviewTarget?) -> Unit)? = null
+    private var closed = false
+    private val mutable = MutableStateFlow(StreamPreviewState())
+    val state: StateFlow<StreamPreviewState> = mutable
 
-    internal fun register(revision: Long, camera: StreamPreviewCamera) {
+    @Synchronized fun attach(texture: SurfaceTexture): StreamPreviewTarget {
-        cameras = cameras.filterNot { it.first == revision }.takeLast(31) + (revision to camera)
+        val next = StreamPreviewTarget(texture)
+        if (closed) { LogManager.addLog("WARN", "video", "direct preview attach after capture stopped"); return next }
+        val previous = target
+        target = next.retain(); owner?.invoke(next.retain()); previous?.release()
+        LogManager.addLog("INFO", "video", "direct preview attached target=${System.identityHashCode(next)} bound=${owner != null}")
+        return next // Первую ссылку освобождает UI в onSurfaceTextureDestroyed, возвращая false.
     }
 
-    fun camera(revision: Long): StreamPreviewCamera? = cameras.firstOrNull { it.first == revision }?.second
+    @Synchronized fun detach(previous: StreamPreviewTarget) {
+        if (target !== previous) return
+        target = null; owner?.invoke(null); previous.release()
+        mutable.value = mutable.value.copy(active = false)
+        LogManager.addLog("INFO", "video", "direct preview detached target=${System.identityHashCode(previous)}")
+    }
+
+    @Synchronized internal fun bind(listener: (StreamPreviewTarget?) -> Unit) {
+        check(!closed && owner == null)
+        owner = listener; listener(target?.retain())
+    }
+
+    internal fun update(format: StreamPreviewFormat?, active: Boolean = false, failed: Boolean = false) {
+        mutable.value = StreamPreviewState(format, active, failed)
+    }
+
+    @Synchronized internal fun close() {
+        closed = true; owner = null; target?.release(); target = null
+        mutable.value = mutable.value.copy(active = false)
+    }
 }