From e8a63ec40dbd952e036318bda012df2f7a014555 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 14:58:03 +0200 Subject: [PATCH] Send Android camera frames directly to preview surfaces --- .../com/utun/chat/data/StreamCaptureEngine.kt | 175 ++++++++++++------ .../java/com/utun/chat/data/StreamPreview.kt | 78 ++++++-- 2 files changed, 187 insertions(+), 66 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt index db705c71..f8ea0c81 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt @@ -2,6 +2,7 @@ package com.utun.chat.data import android.annotation.SuppressLint import android.content.Context +import android.graphics.SurfaceTexture import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession import android.hardware.camera2.CameraCharacteristics @@ -26,7 +27,7 @@ import android.view.Surface import java.io.ByteArrayOutputStream /** Camera/codec принадлежат HandlerThread; микрофон и origin сохраняются при смене видео. - * Preview читает тот же источник. Поколение отсеивает callbacks закрываемой камеры/кодека. */ + * Превью — второй выход Camera2. Поколения видео и camera session отсеивают старые callbacks. */ class StreamCaptureEngine( private val context: Context, private val id: Long, private val group: Long, private val initialConfig: StreamCaptureConfig, private val microphone: Boolean, private val preview: StreamPreview, @@ -40,6 +41,10 @@ class StreamCaptureEngine( private var session: CameraCaptureSession? = null private var encoder: MediaCodec? = null private var input: Surface? = null + private var activeConfig: StreamCaptureConfig? = null + private var previewTarget: StreamPreviewTarget? = null + private var sessionGeneration = 0L + private var cameraSessions = 0 private var record: AudioRecord? = null private var audioThread: Thread? = null @Volatile private var audioRunning = false @@ -77,6 +82,19 @@ class StreamCaptureEngine( private var lastPts = -1L fun start() { + preview.bind { target -> + if (!handler.post { + if (stopping || stopped || terminalFailure) { target?.release(); return@post } + val previous = previewTarget; previewTarget = target; previous?.release() + val config = activeConfig; val device = camera + LogManager.addLog("DEBUG", "video", "preview output change id=$id target=${target?.let(System::identityHashCode)} " + + "revision=$generation camera=${device != null} opening=$opening") + if (device != null && config != null && current(generation)) createSession(device, config, generation) + }) { + LogManager.addLog("WARN", "video", "preview command rejected id=$id capture handler stopped") + target?.release() + } + } handler.post { if (stopping || stopped) return@post try { @@ -129,8 +147,9 @@ class StreamCaptureEngine( check(token <= 0xffffffffL) { "Video revision exhausted" } val cameraPose = cameraPose(mode.camera) val rotation = cameraPose.rotation(config.orientation.degrees) - preview.register(token, cameraPose) - LogManager.addLog("DEBUG", "video", "preview camera id=$id revision=$token camera=${mode.camera} " + + activeConfig = config + preview.update(null) + LogManager.addLog("DEBUG", "video", "capture camera id=$id revision=$token camera=${mode.camera} " + "sensor=${cameraPose.sensor} front=${cameraPose.front} broadcastRotation=$rotation") csd = byteArrayOf(); partial.reset(); partialConfig.reset(); requireKey = true; lastKeyRequest = 0; waitingFirst = true confirming = mode.highSpeed; confirmStartNs = 0 @@ -192,60 +211,12 @@ class StreamCaptureEngine( override fun onOpened(device: CameraDevice) { if (openingGeneration == token) { opening = false; camera = device } if (!current(token)) { device.close(); return } - try { - val callback = object : CameraCaptureSession.StateCallback() { - override fun onConfigured(configured: CameraCaptureSession) { - if (!current(token)) { configured.close(); return } - session = configured - try { - val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { - addTarget(input!!) - set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(mode.minimumFps, mode.fps)) - }.build() - var sensorResults = 0L - var sensorReportCount = 0L - var sensorReportNs = System.nanoTime() - val diagnostics = object : CameraCaptureSession.CaptureCallback() { - override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) { - if (!current(token)) return - sensorResults++ - val now = System.nanoTime() - if (now - sensorReportNs >= 1_000_000_000) { - val fps = (sensorResults - sensorReportCount) * 1_000_000_000.0 / (now - sensorReportNs) - LogManager.addLog("DEBUG", "video", "sensor id=$id revision=$token results=$sensorResults callbackFps=${"%.1f".format(fps)} " + - "range=${result.get(CaptureResult.CONTROL_AE_TARGET_FPS_RANGE)} " + - "durationNs=${result.get(CaptureResult.SENSOR_FRAME_DURATION)} " + - "exposureNs=${result.get(CaptureResult.SENSOR_EXPOSURE_TIME)} timestamp=${result.get(CaptureResult.SENSOR_TIMESTAMP)}") - sensorReportNs = now; sensorReportCount = sensorResults - } - } - override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, - failed: android.hardware.camera2.CaptureFailure) { - if (current(token)) LogManager.addLog("WARN", "video", "sensor capture failed id=$id revision=$token " + - "frame=${failed.frameNumber} reason=${failed.reason} captured=${failed.wasImageCaptured()}") - } - } - if (mode.highSpeed) { - val highSpeed = configured as CameraConstrainedHighSpeedCaptureSession - highSpeed.setRepeatingBurst(highSpeed.createHighSpeedRequestList(request), diagnostics, handler) - } else configured.setRepeatingRequest(request, diagnostics, handler) - LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " + - "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " + - "type=${mode.videoCodec.label} bitrate=${config.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs") - } catch (e: Exception) { captureError(token, "camera request: ${e.message}") } - } - override fun onConfigureFailed(session: CameraCaptureSession) { - session.close(); captureError(token, "camera session configuration") - } - } - if (mode.highSpeed) device.createConstrainedHighSpeedCaptureSession(listOf(input!!), callback, handler) - else device.createCaptureSession(listOf(input!!), callback, handler) - } catch (e: Exception) { captureError(token, "camera session: ${e.message}") } + createSession(device, config, token) } override fun onClosed(device: CameraDevice) { if (camera === device) camera = null if (releaseDone != null && camera == null && !opening) finishRelease(true) - if (stopped && camera == null && !opening) thread.quitSafely() + maybeQuit() } override fun onDisconnected(device: CameraDevice) { if (openingGeneration == token) { opening = false; camera = device } @@ -263,6 +234,99 @@ class StreamCaptureEngine( private fun current(token: Long) = token == generation && !stopping && !stopped && !terminalFailure && releaseDone == null + /** Меняется только набор Camera2 outputs; encoder, audio и origin не перезапускаются. */ + private fun createSession(device: CameraDevice, config: StreamCaptureConfig, token: Long, withPreview: Boolean = true) { + val mode = config.mode + val serial = ++sessionGeneration + val output = if (withPreview) previewTarget?.retain() else null + var format: StreamPreviewFormat? = null + var completed = false + cameraSessions++ + fun complete() { + if (completed) return + completed = true; output?.release(); cameraSessions--; maybeQuit() + } + fun valid() = current(token) && serial == sessionGeneration + fun rejected(detail: String) { + if (!valid()) { LogManager.addLog("DEBUG", "video", "stale camera session failure id=$id session=$serial: $detail"); return } + if (output == null) { captureError(token, detail); return } + LogManager.addLog("ERROR", "video", "direct preview failed id=$id session=$serial; keep encoder output: $detail") + preview.update(format, failed = true) + createSession(device, config, token, withPreview = false) + } + try { + if (output != null) { + val map = manager.getCameraCharacteristics(mode.camera).get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) + ?: error("Missing preview stream map") + val size = if (mode.highSpeed) android.util.Size(mode.width, mode.height) else + map.getOutputSizes(SurfaceTexture::class.java).orEmpty().filter { + it.width <= minOf(mode.width, 1280) && it.height <= minOf(mode.height, 720) + && it.width.toLong() * mode.height == it.height.toLong() * mode.width + }.maxByOrNull { it.width * it.height } ?: error("No compatible camera preview size") + output.texture.setDefaultBufferSize(size.width, size.height) + format = StreamPreviewFormat(size.width, size.height, cameraPose(mode.camera)) + preview.update(format) + } else if (withPreview) preview.update(preview.state.value.format) + val targets = listOfNotNull(input, output?.surface) + check(targets.isNotEmpty()) { "Missing encoder surface" } + val callback = object : CameraCaptureSession.StateCallback() { + override fun onConfigured(configured: CameraCaptureSession) { + if (!valid()) { configured.close(); return } + session = configured + try { + val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { + targets.forEach(::addTarget) + set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(mode.minimumFps, mode.fps)) + }.build() + var sensorResults = 0L + var sensorReportCount = 0L + var sensorReportNs = System.nanoTime() + val diagnostics = object : CameraCaptureSession.CaptureCallback() { + override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) { + if (!valid()) return + sensorResults++ + val now = System.nanoTime() + if (now - sensorReportNs < 1_000_000_000) return + val fps = (sensorResults - sensorReportCount) * 1_000_000_000.0 / (now - sensorReportNs) + LogManager.addLog("DEBUG", "video", "sensor id=$id revision=$token session=$serial results=$sensorResults " + + "callbackFps=${"%.1f".format(fps)} range=${result.get(CaptureResult.CONTROL_AE_TARGET_FPS_RANGE)} " + + "durationNs=${result.get(CaptureResult.SENSOR_FRAME_DURATION)} " + + "exposureNs=${result.get(CaptureResult.SENSOR_EXPOSURE_TIME)} timestamp=${result.get(CaptureResult.SENSOR_TIMESTAMP)}") + sensorReportNs = now; sensorReportCount = sensorResults + } + override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, + failed: android.hardware.camera2.CaptureFailure) { + if (valid()) LogManager.addLog("WARN", "video", "sensor capture failed id=$id revision=$token " + + "session=$serial frame=${failed.frameNumber} reason=${failed.reason} captured=${failed.wasImageCaptured()}") + } + } + if (mode.highSpeed) { + val highSpeed = configured as CameraConstrainedHighSpeedCaptureSession + highSpeed.setRepeatingBurst(highSpeed.createHighSpeedRequestList(request), diagnostics, handler) + } else configured.setRepeatingRequest(request, diagnostics, handler) + if (output != null) preview.update(format, active = true) + LogManager.addLog("INFO", "video", "capture started id=$id revision=$token session=$serial camera=${mode.camera} ${mode.label} " + + "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} encoder=${encoder?.name} " + + "preview=${output != null} previewSize=${format?.let { "${it.width}x${it.height}" }} origin=$originNs") + } catch (e: Exception) { configured.close(); rejected("camera request: ${e.message}") } + } + override fun onConfigureFailed(session: CameraCaptureSession) { + complete(); rejected("camera session configuration") + } + override fun onClosed(closed: CameraCaptureSession) { + if (session === closed) session = null + complete() + LogManager.addLog("DEBUG", "video", "camera session closed id=$id session=$serial remaining=$cameraSessions") + } + } + LogManager.addLog("DEBUG", "video", "camera session configuring id=$id revision=$token session=$serial outputs=${targets.size}") + if (mode.highSpeed) device.createConstrainedHighSpeedCaptureSession(targets, callback, handler) + else device.createCaptureSession(targets, callback, handler) + } catch (e: Exception) { complete(); rejected("camera session: ${e.message}") } + } + + private fun maybeQuit() { if (stopped && camera == null && !opening && cameraSessions == 0) thread.quitSafely() } + private fun captureError(token: Long, detail: String) { if (!current(token)) { LogManager.addLog("DEBUG", "video", "stale capture error revision=$token: $detail"); return } LogManager.addLog("ERROR", "video", "capture error id=$id revision=$token: $detail") @@ -341,6 +405,8 @@ class StreamCaptureEngine( private fun releaseVideo(done: (Boolean) -> Unit) { if (releaseDone != null) { releaseDone = done; return } val token = ++generation + sessionGeneration++ + preview.update(preview.state.value.format) releaseDone = done try { session?.close() } catch (e: Exception) { LogManager.addLog("WARN", "video", "camera session close id=$id: ${e.message}") } session = null @@ -418,6 +484,7 @@ class StreamCaptureEngine( stopCallbacks.add(done) if (stopping) return@post stopping = true; audioRunning = false + preview.close(); previewTarget?.release(); previewTarget = null try { record?.stop() } catch (e: Exception) { LogManager.addLog("WARN", "stream", "audio stop id=$id: ${e.message}") } audioThread?.join(); audioThread = null try { record?.release() } catch (e: Exception) { LogManager.addLog("WARN", "stream", "audio release id=$id: ${e.message}") } @@ -426,7 +493,7 @@ class StreamCaptureEngine( stopped = true LogManager.addLog("INFO", "stream", "capture released id=$id frames=$frames dropped=$dropped bytes=$bytesSent") stopCallbacks.toList().also { stopCallbacks.clear() }.forEach { it() } - if (camera == null && !opening) thread.quitSafely() + maybeQuit() } })) { LogManager.addLog("WARN", "stream", "capture handler already closed id=$id; complete stop") diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt index 9b98e28c..2328fd94 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt @@ -1,27 +1,81 @@ package com.utun.chat.data +import android.graphics.SurfaceTexture +import android.view.Surface +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow + /** Положение сенсора конкретной камеры; ручной поворот относится только к трансляции. */ data class StreamPreviewCamera(val sensor: Int, val front: Boolean) { fun rotation(displayDegrees: Int): Int = (sensor + (if (front) displayDegrees else -displayDegrees) + 360) % 360 } -data class StreamVideoDisplay(val rotation: Int, val top: Int?) { - companion object { - fun forFrame(broadcastRotation: Int, camera: StreamPreviewCamera?, displayDegrees: Int): StreamVideoDisplay { - if (camera == null) return StreamVideoDisplay(broadcastRotation, null) - val previewRotation = camera.rotation(displayDegrees) - return StreamVideoDisplay(previewRotation, (previewRotation - broadcastRotation + 360) % 360) - } +data class StreamPreviewFormat(val width: Int, val height: Int, val camera: StreamPreviewCamera) { + fun aspect(displayDegrees: Int): Float = if (camera.rotation(displayDegrees) % 180 == 0) width.toFloat() / height else height.toFloat() / width + + /** Углы TextureView после поворота; размеры view уже учитывают поворот сенсора. */ + fun corners(displayDegrees: Int): FloatArray = when (camera.rotation(displayDegrees)) { + 90 -> floatArrayOf(1f, 0f, 1f, 1f, 0f, 1f, 0f, 0f) + 180 -> floatArrayOf(1f, 1f, 0f, 1f, 0f, 0f, 1f, 0f) + 270 -> floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f) + else -> floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f) + } +} + +data class StreamPreviewState(val format: StreamPreviewFormat? = null, val active: Boolean = false, val failed: Boolean = false) + +/** UI, ожидающая команда camera thread и каждая CameraCaptureSession держат отдельную ссылку. */ +internal class StreamPreviewLease(private val close: () -> Unit) { + private var references = 1 + @Synchronized fun retain() { check(references > 0); references++ } + @Synchronized fun release() { check(references > 0); if (--references == 0) close() } +} + +class StreamPreviewTarget internal constructor(val texture: SurfaceTexture) { + val surface = Surface(texture) + private val lease = StreamPreviewLease { + surface.release(); texture.release() + LogManager.addLog("DEBUG", "video", "direct preview surface released target=${System.identityHashCode(this)}") } + internal fun retain(): StreamPreviewTarget { lease.retain(); return this } + fun release() = lease.release() } -/** Принадлежит одной публикации. Camera thread пишет immutable snapshots; viewer читает по revision своего кадра. */ +/** Прямой выход камеры. UI не владеет camera/codec; listener получает собственную ссылку на target. */ class StreamPreview { - @Volatile private var cameras: List> = emptyList() + private var target: StreamPreviewTarget? = null + private var owner: ((StreamPreviewTarget?) -> Unit)? = null + private var closed = false + private val mutable = MutableStateFlow(StreamPreviewState()) + val state: StateFlow = mutable - internal fun register(revision: Long, camera: StreamPreviewCamera) { - cameras = cameras.filterNot { it.first == revision }.takeLast(31) + (revision to camera) + @Synchronized fun attach(texture: SurfaceTexture): StreamPreviewTarget { + val next = StreamPreviewTarget(texture) + if (closed) { LogManager.addLog("WARN", "video", "direct preview attach after capture stopped"); return next } + val previous = target + target = next.retain(); owner?.invoke(next.retain()); previous?.release() + LogManager.addLog("INFO", "video", "direct preview attached target=${System.identityHashCode(next)} bound=${owner != null}") + return next // Первую ссылку освобождает UI в onSurfaceTextureDestroyed, возвращая false. } - fun camera(revision: Long): StreamPreviewCamera? = cameras.firstOrNull { it.first == revision }?.second + @Synchronized fun detach(previous: StreamPreviewTarget) { + if (target !== previous) return + target = null; owner?.invoke(null); previous.release() + mutable.value = mutable.value.copy(active = false) + LogManager.addLog("INFO", "video", "direct preview detached target=${System.identityHashCode(previous)}") + } + + @Synchronized internal fun bind(listener: (StreamPreviewTarget?) -> Unit) { + check(!closed && owner == null) + owner = listener; listener(target?.retain()) + } + + internal fun update(format: StreamPreviewFormat?, active: Boolean = false, failed: Boolean = false) { + mutable.value = StreamPreviewState(format, active, failed) + } + + @Synchronized internal fun close() { + closed = true; owner = null; target?.release(); target = null + mutable.value = mutable.value.copy(active = false) + } }