From b665191b9e6a49b38083221fcc2c8e5ae59d107a Mon Sep 17 00:00:00 2001 From: evgeny Date: Thu, 15 Oct 2026 01:59:42 +0200 Subject: [PATCH] Render Android call camera preview directly through Camera2 --- .../com/utun/chat/data/CallVideoEngine.kt | 136 ++++++++++-------- .../utun/chat/ui/screens/CallVideoPanel.kt | 22 +-- .../com/utun/chat/ui/screens/StreamScreen.kt | 6 +- 3 files changed, 90 insertions(+), 74 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt index 4aba413d..d41e5a6a 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -3,9 +3,7 @@ package com.utun.chat.data import android.annotation.SuppressLint import android.content.Context import android.graphics.ImageFormat -import android.graphics.Bitmap -import android.graphics.Matrix -import android.media.Image +import android.graphics.SurfaceTexture import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice @@ -38,9 +36,10 @@ class CallVideoEngine(private val context: Context, private val id: Long) { private val manager = context.getSystemService(CameraManager::class.java) private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } } internal val display: StateFlow = player.display - private val previewFrames = MutableStateFlow(null) - val preview: StateFlow = previewFrames - private var previewAt = 0L + val preview = StreamPreview() + private var previewTarget: StreamPreviewTarget? = null + private var sessionSerial = 0L + private val sessions = mutableMapOf Unit>>() @Volatile private var stopped = false private var encoders = emptyMap() private var caps = ByteArray(64) @@ -76,6 +75,15 @@ class CallVideoEngine(private val context: Context, private val id: Long) { private var excluded = emptySet() init { + preview.bind { target -> + if (!handler.post { + if (stopped) { target?.release(); return@post } + if (target === previewTarget) { target?.release(); return@post } + val previous = previewTarget; previewTarget = target; previous?.release() + val device = camera; val source = reader + if (device != null && source != null && !closing) createSession(device, source, cameraToken) + }) { target?.release(); LogManager.addLog("WARN", "call", "preview command after worker stop id=$id") } + } handler.post { try { val decoders = CallVideoCodec.probeDecoders() @@ -204,7 +212,6 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (stopped || !wanted.enabled || opening || closing) return val settings = active ?: return; val f = format ?: return; val token = ++cameraToken val c = manager.getCameraCharacteristics(settings.camera) - val mirror = c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME sensorOffset = null val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader @@ -216,16 +223,6 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener val now = System.nanoTime() lastCaptureAt = now - if (previewFrames.subscriptionCount.value > 0 && now - previewAt >= 200_000_000) { - previewAt = now - try { - val preview = cameraPreview(image, f.rotation, mirror) - if (previewFrames.value == null) LogManager.addLog("INFO", "call", - "camera preview visible id=$id size=${preview.width}x${preview.height} rotation=${f.rotation} mirror=$mirror") - previewFrames.value = preview - } - catch (e: Exception) { LogManager.addLog("WARN", "call", "camera preview failed: ${e.message}") } - } val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also { sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it") } @@ -265,28 +262,12 @@ class CallVideoEngine(private val context: Context, private val id: Long) { override fun onOpened(device: CameraDevice) { opening = false; camera = device if (stopped || token != cameraToken) { device.close(); return } - try { - device.createCaptureSession(listOf(nextReader.surface), object : CameraCaptureSession.StateCallback() { - override fun onConfigured(s: CameraCaptureSession) { - if (stopped || token != cameraToken) { s.close(); return } - session = s - try { - val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() - val range = rates.filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps") - val capture = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { - addTarget(nextReader.surface); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) - }.build() - s.setRepeatingRequest(capture, null, handler) - report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}") - } catch (e: Exception) { fail("Camera request: ${e.message}") } - } - override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); fail("Camera session configuration failed") } - }, handler) - } catch (e: Exception) { fail("Camera session: ${e.message}") } + createSession(device, nextReader, token) } override fun onDisconnected(device: CameraDevice) { opening = false; camera = device; fail("Camera disconnected") } override fun onError(device: CameraDevice, error: Int) { opening = false; camera = device; fail("Camera error $error") } override fun onClosed(device: CameraDevice) { + sessions.values.filter { it.first === device }.map { it.second }.forEach { it() } if (camera === device) camera = null closing = false if (stopped) thread.quitSafely() @@ -294,6 +275,62 @@ class CallVideoEngine(private val context: Context, private val id: Long) { } }, handler) } + + /** Прямое превью использует второй output той же камеры; encoder и его PTS не меняются. */ + private fun createSession(device: CameraDevice, source: ImageReader, token: Long, withPreview: Boolean = true) { + val f = format ?: return + val serial = ++sessionSerial + val output = if (withPreview) previewTarget?.retain() else null + var completed = false + fun complete() { + if (completed) return + completed = true; sessions.remove(serial); output?.release() + LogManager.addLog("DEBUG", "call", "camera session released id=$id session=$serial remaining=${sessions.size}") + } + fun valid() = !stopped && token == cameraToken && serial == sessionSerial + fun rejected(message: String) { + if (!valid()) { LogManager.addLog("DEBUG", "call", "stale camera session id=$id session=$serial: $message"); return } + if (output == null) fail(message) else { + LogManager.addLog("ERROR", "call", "direct preview failed id=$id: $message; retaining transmission") + preview.update(preview.state.value.format, failed = true) + createSession(device, source, token, withPreview = false) + } + } + sessions[serial] = device to ::complete + try { + val c = manager.getCameraCharacteristics(device.id) + var previewFormat: StreamPreviewFormat? = null + if (output != null) { + val sizes = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)?.getOutputSizes(SurfaceTexture::class.java).orEmpty() + val size = sizes.filter { it.width <= f.width && it.height <= f.height && it.width * f.height == it.height * f.width } + .maxByOrNull { it.width * it.height } ?: error("No compatible preview size") + output.texture.setDefaultBufferSize(size.width, size.height) + previewFormat = StreamPreviewFormat(size.width, size.height, + StreamPreviewCamera(f.rotation, c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT), realtime) + preview.update(previewFormat) + } else if (withPreview) preview.update(null) + val targets = listOfNotNull(source.surface, output?.surface) + device.createCaptureSession(targets, object : CameraCaptureSession.StateCallback() { + override fun onConfigured(s: CameraCaptureSession) { + if (!valid()) { s.close(); return } + session = s + try { + val range = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() + .filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps") + val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { + targets.forEach(::addTarget); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) + }.build() + s.setRepeatingRequest(request, null, handler) + if (output != null) preview.update(previewFormat, active = true) + LogManager.addLog("INFO", "call", "camera capture id=$id session=$serial range=$range preview=$previewFormat") + report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}") + } catch (e: Exception) { s.close(); rejected("Camera request: ${e.message}") } + } + override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); complete(); rejected("Camera session configuration failed") } + override fun onClosed(s: CameraCaptureSession) { if (session === s) session = null; complete() } + }, handler) + } catch (e: Exception) { complete(); rejected("Camera session: ${e.message}") } + } private fun drain() { val codec = encoder ?: return; val f = format ?: return for (p in codec.drain()) { @@ -316,42 +353,19 @@ class CallVideoEngine(private val context: Context, private val id: Long) { } } private fun closeCamera() { - cameraToken++ - previewFrames.value = null; previewAt = 0 + cameraToken++; sessionSerial++ + preview.update(null) try { session?.close(); session = null; reader?.close(); reader = null if (camera != null) { closing = true; camera!!.close() } } catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") } } - /** Миниатюра из уже захваченного YUV: второй output Camera2 и отдельная камера не нужны. */ - private fun cameraPreview(image: Image, rotation: Int, mirror: Boolean): Bitmap { - val crop = image.cropRect; val width = 192; val height = (width * crop.height() / crop.width()).coerceAtLeast(1) - val pixels = IntArray(width * height) - val planes = image.planes; val buffers = planes.map { it.buffer.duplicate() } - fun sample(plane: Int, x: Int, y: Int): Int { - val p = planes[plane]; val b = buffers[plane] - return b.get(b.position() + y * p.rowStride + x * p.pixelStride).toInt() and 255 - } - for (y in 0 until height) for (x in 0 until width) { - val sx = crop.left + x * crop.width() / width; val sy = crop.top + y * crop.height() / height - val luma = (sample(0, sx, sy) - 16).coerceAtLeast(0) - val u = sample(1, sx / 2, sy / 2) - 128; val v = sample(2, sx / 2, sy / 2) - 128 - val r = ((298 * luma + 409 * v + 128) shr 8).coerceIn(0, 255) - val g = ((298 * luma - 100 * u - 208 * v + 128) shr 8).coerceIn(0, 255) - val b = ((298 * luma + 516 * u + 128) shr 8).coerceIn(0, 255) - pixels[y * width + x] = (255 shl 24) or (r shl 16) or (g shl 8) or b - } - val bitmap = Bitmap.createBitmap(pixels, width, height, Bitmap.Config.ARGB_8888) - val transform = Matrix().apply { postRotate(rotation.toFloat()); if (mirror) postScale(-1f, 1f) } - val result = Bitmap.createBitmap(bitmap, 0, 0, width, height, transform, true) - if (result !== bitmap) bitmap.recycle() - return result - } fun stop() { if (stopped) return stopped = true; player.stop(); CallVideoNative.close(id) handler.post { closeCamera(); encoder?.close(); encoder = null + preview.close(); previewTarget?.release(); previewTarget = null if (camera == null && !opening) thread.quitSafely() else handler.postDelayed({ thread.quitSafely() }, 3000) LogManager.addLog("INFO", "call", "video workers stopped id=$id") diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt index 5c29fedc..c80d4a4f 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt @@ -10,16 +10,12 @@ import android.view.TextureView import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.background -import androidx.compose.foundation.Image -import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.* import androidx.compose.material3.* import androidx.compose.runtime.* import androidx.compose.ui.Modifier import androidx.compose.ui.Alignment import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.asImageBitmap -import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp @@ -81,27 +77,33 @@ internal enum class CallVideoSource { LOCAL, REMOTE } internal fun CallVideoStage(video: CallVideoEngine, expanded: CallVideoSource?, onSelect: (CallVideoSource) -> Unit, onTap: () -> Unit, modifier: Modifier = Modifier) { val remote by video.display.collectAsState() - val local by video.preview.collectAsState() + val local by video.preview.state.collectAsState() + val displayDegrees = StreamDisplayDegrees() val insets = WindowInsets.systemBarsIgnoringVisibility.union(WindowInsets.displayCutout) Box(modifier) { // Слоты не меняются местами в composition; миниатюра рисуется поверх большого кадра. for (source in CallVideoSource.entries) key(source) { val large = expanded == source - val available = if (source == CallVideoSource.LOCAL) local != null else remote != null Box(Modifier.matchParentSize().zIndex(if (large) 0f else 1f) .then(if (large) Modifier else Modifier.windowInsetsPadding(insets).padding(top = 64.dp, end = 12.dp))) { val frameModifier = if (large) Modifier.matchParentSize() else Modifier .align(if (expanded != null) Alignment.TopEnd else Alignment.TopCenter) - .offset(x = if (expanded != null || local == null || remote == null) 0.dp + .offset(x = if (expanded != null || !local.active || remote == null) 0.dp else if (source == CallVideoSource.LOCAL) (-60).dp else 60.dp) .size(104.dp, 144.dp) val description = stringResource(if (source == CallVideoSource.LOCAL) R.string.call_video_local else R.string.call_video_remote) val click = { if (large) onTap() else onSelect(source) } if (source == CallVideoSource.REMOTE) { CallVideoPlayback(video, click, frameModifier, description) - } else if (available) { - Image(local!!.asImageBitmap(), contentDescription = description, contentScale = ContentScale.Fit, - modifier = frameModifier.clickable(onClick = click)) + } else BoxWithConstraints(frameModifier, contentAlignment = Alignment.Center) { + val aspect = local.format?.aspect(displayDegrees) ?: 1f + val width = minOf(maxWidth, maxHeight * aspect) + AndroidView(factory = { CameraPreviewView(it, video.preview, keepAwake = false) }, + modifier = Modifier.size(width, width / aspect), update = { view -> + view.update(local.format, displayDegrees); view.alpha = if (local.active) 1f else 0f + view.contentDescription = description + view.setOnClickListener { if (local.active) click() }; view.isClickable = local.active + }, onRelease = { it.close() }) } } } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt index d7ca69c3..cc24bdcb 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt @@ -318,7 +318,7 @@ private fun LiveStreamView(localPreview: StreamPreview, displayDegrees: Int) { /** DisplayListener замечает также 180°, когда Configuration и размеры окна не меняются. */ @Composable -private fun StreamDisplayDegrees(): Int { +internal fun StreamDisplayDegrees(): Int { val context = LocalContext.current val configuration = LocalConfiguration.current val display = (context as? Activity)?.windowManager?.defaultDisplay @@ -355,7 +355,7 @@ private fun CameraPreviewSurface(preview: StreamPreview, displayDegrees: Int) { } /** TextureView отдаёт texture владельцам Camera2; destroy не освобождает используемый камерой output. */ -private class CameraPreviewView(context: Context, private val preview: StreamPreview) : TextureView(context) { +internal class CameraPreviewView(context: Context, private val preview: StreamPreview, keepAwake: Boolean = true) : TextureView(context) { private var target: StreamPreviewTarget? = null private var format: StreamPreviewFormat? = null private var displayDegrees = 0 @@ -366,7 +366,7 @@ private class CameraPreviewView(context: Context, private val preview: StreamPre private var observedFormat: StreamPreviewFormat? = null init { - keepScreenOn = true + keepScreenOn = keepAwake surfaceTextureListener = object : SurfaceTextureListener { override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) { if (closed) return