From 548e64c9634b93b77145bb9e461d90bacc4aeaab Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 14:58:03 +0200 Subject: [PATCH] Use direct camera preview instead of a local stream decoder --- .../com/utun/chat/data/StreamPlayerEngine.kt | 25 ++--- .../com/utun/chat/ui/screens/StreamScreen.kt | 97 ++++++++++++++++++- .../com/utun/chat/data/StreamPreviewTest.kt | 75 ++++++++------ 3 files changed, 144 insertions(+), 53 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt index eae13787..b4ebb537 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt @@ -21,17 +21,14 @@ import kotlin.concurrent.withLock /** Запас кодированного потока остаётся в общем RS-кольце. Worker запрашивает ближайшие кадры, * общий native controller ведёт минутную историю и темп, AudioTrack задаёт фактические часы. * Временная нехватка сохраняет decoder references; GAP/вытеснение начинают новую точку KEY. */ -class StreamPlayerEngine(private val group: Long, private val source: Long, private val epoch: Long, - private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) { +class StreamPlayerEngine(private val group: Long, private val source: Long, private val epoch: Long) { companion object { private val viewerLock = ReentrantLock() } private class UnsupportedCodec(message: String) : IllegalStateException(message) private data class Frame(override val pts: Long, val key: Boolean, val width: Int, val height: Int, - val rotation: Int, val top: Int?, val revision: Long, val codec: StreamVideoCodec, + val rotation: Int, val revision: Long, val codec: StreamVideoCodec, val config: ByteArray, val data: ByteArray) : StreamDecodePump.Frame private val mutableAspect = MutableStateFlow(16f / 9f) val aspect: StateFlow = mutableAspect - private val mutableTop = MutableStateFlow(null) - val top: StateFlow = mutableTop private val mutableStatus = MutableStateFlow(R.string.stream_connecting) val status: StateFlow = mutableStatus private val id = StreamController.nextId() @@ -121,9 +118,9 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv fun observeStatus(meta: LongArray?) { if (meta == null || meta[5] and 1 == 0L) return - ready = if (localPreview != null) meta[1] else meta[0] + ready = meta[0] if (playback != 0L) return - audioEnabled = localPreview == null && meta[5] and 2 != 0L + audioEnabled = meta[5] and 2 != 0L playback = NativeLib.streamPlaybackCreate(id, meta[3], audioEnabled) check(playback != 0L); position = meta[3]; generation = meta[4] LogManager.addLog("INFO", "stream", "playback starts id=$id PTS=$position generation=$generation fixedPrefill=0") @@ -181,7 +178,7 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv } video = createVideoPump() try { - check(NativeLib.streamOpen(id, group, source, audio = localPreview == null, epoch = epoch)) { "Viewer open refused" } + check(NativeLib.streamOpen(id, group, source, audio = true, epoch = epoch)) { "Viewer open refused" } while (running && surface.isValid) { check(NativeLib.isReady()) { "Core stopped" } observeStatus(NativeLib.streamStatus(id)) @@ -205,12 +202,6 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv if (kind in 1..2 && playback == 0L) observeStatus(NativeLib.streamStatus(id)) when (kind) { 1 -> { - val camera = localPreview?.camera(revision) - if (localPreview != null && camera == null) { - LogManager.addLog("WARN", "video", "preview camera unavailable id=$id revision=$revision") - resetTimeline(true); continue - } - val display = StreamVideoDisplay.forFrame(broadcastRotation, camera, displayDegrees) if (revision != videoRevision) { check(revision > videoRevision); resetVideo(); videoRevision = revision deadline = System.nanoTime() + 10_000_000_000 @@ -221,12 +212,12 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv val codec = StreamVideoCodec.fromId(codecId) val access = if (key) codec.unpackKey(bytes) else byteArrayOf() to bytes if (waitingKey) { - waitingKey = false; mutableTop.value = display.top - mutableAspect.value = if (display.rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width + waitingKey = false + mutableAspect.value = if (broadcastRotation % 180 == 0) width.toFloat() / height else height.toFloat() / width mutableStatus.value = R.string.stream_waiting_decoder LogManager.addLog("INFO", "video", "viewer KEY accepted id=$id revision=$revision PTS=$pts bytes=$size") } - video.offer(Frame(pts, key, width, height, display.rotation, display.top, revision, codec, access.first, access.second), + video.offer(Frame(pts, key, width, height, broadcastRotation, revision, codec, access.first, access.second), System.nanoTime() / 1000) } } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt index 5bf447ef..8db4d7ff 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt @@ -3,6 +3,8 @@ package com.utun.chat.ui.screens import android.Manifest import android.app.Activity import android.content.pm.PackageManager +import android.content.Context +import android.graphics.Matrix import android.graphics.SurfaceTexture import android.view.Surface import android.view.TextureView @@ -47,6 +49,8 @@ import com.utun.chat.data.StreamController import com.utun.chat.data.StreamPhase import com.utun.chat.data.StreamPlayerEngine import com.utun.chat.data.StreamPreview +import com.utun.chat.data.StreamPreviewFormat +import com.utun.chat.data.StreamPreviewTarget import com.utun.chat.data.StreamSource import com.utun.chat.data.StreamCaptureMode import com.utun.chat.data.StreamCaptureConfig @@ -58,7 +62,7 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -/** Выход с экрана закрывает только reader; foreground publisher останавливает явная кнопка/notification. */ +/** Выход закрывает reader либо прямой preview output; foreground publisher останавливается явно. */ @OptIn(ExperimentalMaterial3Api::class) @Composable fun StreamScreen(channelId: String, channelName: String, sources: List, selected: StreamSource?, @@ -118,7 +122,10 @@ fun StreamScreen(channelId: String, channelName: String, sources: List(null) } @@ -338,3 +348,80 @@ private fun LiveStreamSurface(group: Long, source: Long, epoch: Long, localPrevi } } } + +@Composable +private fun CameraPreviewSurface(preview: StreamPreview, displayDegrees: Int) { + val state by preview.state.collectAsState() + val aspect = state.format?.aspect(displayDegrees) ?: 9f / 16f + Column(Modifier.fillMaxSize()) { + Text(stringResource(if (state.failed) R.string.stream_failed else if (state.active) R.string.stream_live else R.string.stream_connecting)) + Box(Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) { + Box(Modifier.aspectRatio(aspect, matchHeightConstraintsFirst = true)) { + AndroidView(factory = { CameraPreviewView(it, preview) }, modifier = Modifier.matchParentSize(), + update = { it.update(state.format, displayDegrees) }, onRelease = { it.close() }) + } + } + } +} + +/** TextureView отдаёт texture владельцам Camera2; destroy не освобождает используемый камерой output. */ +private class CameraPreviewView(context: Context, private val preview: StreamPreview) : TextureView(context) { + private var target: StreamPreviewTarget? = null + private var format: StreamPreviewFormat? = null + private var displayDegrees = 0 + private var closed = false + private var frames = 0L + private var reportFrames = 0L + private var reportNs = 0L + + init { + keepScreenOn = true + surfaceTextureListener = object : SurfaceTextureListener { + override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) { + if (closed) return + target = preview.attach(texture); frames = 0; reportFrames = 0; reportNs = System.nanoTime() + transform() + } + override fun onSurfaceTextureSizeChanged(texture: SurfaceTexture, width: Int, height: Int) { transform() } + override fun onSurfaceTextureDestroyed(texture: SurfaceTexture): Boolean { + val previous = target ?: return true + preview.detach(previous); target = null; previous.release() + return false // Camera session удерживает свою ссылку до onClosed/onConfigureFailed. + } + override fun onSurfaceTextureUpdated(texture: SurfaceTexture) { + frames++ + val now = System.nanoTime() + if (frames == 1L || now - reportNs >= 1_000_000_000) { + val fps = (frames - reportFrames) * 1_000_000_000.0 / (now - reportNs).coerceAtLeast(1) + LogManager.addLog(if (frames == 1L) "INFO" else "DEBUG", "video", "direct preview frame " + + "target=${target?.let(System::identityHashCode)} frames=$frames fps=${"%.1f".format(fps)} " + + "ageUs=${(now - texture.timestamp) / 1000} timestampNs=${texture.timestamp} " + + "rotation=${format?.camera?.rotation(displayDegrees)}") + reportNs = now; reportFrames = frames + } + } + } + } + + fun update(next: StreamPreviewFormat?, display: Int) { + if (next == format && display == displayDegrees) return + format = next; displayDegrees = display; transform() + } + + private fun transform() { + val next = format ?: return + if (width == 0 || height == 0) return + val destination = next.corners(displayDegrees) + for (i in destination.indices) destination[i] *= if (i % 2 == 0) width.toFloat() else height.toFloat() + val source = floatArrayOf(0f, 0f, width.toFloat(), 0f, width.toFloat(), height.toFloat(), 0f, height.toFloat()) + val matrix = Matrix() + if (!matrix.setPolyToPoly(source, 0, destination, 0, 4)) { + LogManager.addLog("ERROR", "video", "direct preview transform failed view=${width}x$height format=$next"); return + } + setTransform(matrix) + LogManager.addLog("DEBUG", "video", "direct preview transform view=${width}x$height buffer=${next.width}x${next.height} " + + "rotation=${next.camera.rotation(displayDegrees)} front=${next.camera.front}") + } + + fun close() { closed = true; keepScreenOn = false; target?.let(preview::detach) } +} diff --git a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt index 1ebcb9be..6c7a9341 100644 --- a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt +++ b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt @@ -4,45 +4,58 @@ import org.junit.Assert.* import org.junit.Test class StreamPreviewTest { - @Test fun manualRotationMovesRearTopMarkerWithoutRotatingCameraPreview() { - val camera = StreamPreviewCamera(90, false) - val display = StreamOrientation.values().map { - StreamVideoDisplay.forFrame(it.sensorRotation(90, false), camera, 0) - } - assertEquals(listOf(90, 90, 90, 90), display.map { it.rotation }) - assertEquals(listOf(0, 90, 180, 270), display.map { it.top }) + @Test fun cameraPreviewUsesSensorAndDisplayOrientation() { + val rear = StreamPreviewFormat(1280, 720, StreamPreviewCamera(90, false)) + val front = StreamPreviewFormat(1280, 720, StreamPreviewCamera(270, true)) + assertEquals(9f / 16f, rear.aspect(0), 0f) + assertEquals(16f / 9f, rear.aspect(90), 0f) + assertEquals(9f / 16f, front.aspect(0), 0f) + assertEquals(16f / 9f, front.aspect(90), 0f) + assertArrayEquals(floatArrayOf(1f, 0f, 1f, 1f, 0f, 1f, 0f, 0f), rear.corners(0), 0f) + assertArrayEquals(floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f), front.corners(0), 0f) + assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), rear.corners(90), 0f) + assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), front.corners(90), 0f) } - @Test fun frontCameraTopMarkerUsesItsOwnSensorDirection() { - val camera = StreamPreviewCamera(270, true) - val display = StreamOrientation.values().map { - StreamVideoDisplay.forFrame(it.sensorRotation(270, true), camera, 0) + @Test fun allPreviewRotationsPreserveRectangleCorners() { + for (front in listOf(false, true)) for (sensor in listOf(0, 90, 180, 270)) for (display in listOf(0, 90, 180, 270)) { + val format = StreamPreviewFormat(640, 480, StreamPreviewCamera(sensor, front)) + val corners = format.corners(display).toList().chunked(2) + assertEquals(setOf(listOf(0f, 0f), listOf(1f, 0f), listOf(1f, 1f), listOf(0f, 1f)), corners.toSet()) + assertEquals(if (format.camera.rotation(display) % 180 == 0) 4f / 3f else 3f / 4f, format.aspect(display), 0f) } - assertEquals(listOf(270, 270, 270, 270), display.map { it.rotation }) - assertEquals(listOf(0, 270, 180, 90), display.map { it.top }) } - @Test fun previewFollowsScreenWhileBroadcastKeepsManualOrientation() { - assertEquals(StreamVideoDisplay(0, 270), StreamVideoDisplay.forFrame(90, StreamPreviewCamera(90, false), 90)) - assertEquals(StreamVideoDisplay(0, 90), StreamVideoDisplay.forFrame(270, StreamPreviewCamera(270, true), 90)) + @Test fun destroyedViewDoesNotReleaseCameraOrPendingCommandSurface() { + var released = 0 + val lease = StreamPreviewLease { released++ } + lease.retain() // Команда, ожидающая camera handler. + lease.retain() // Сессия камеры. + lease.release() // UI TextureView уничтожен. + assertEquals(0, released) + lease.release() // Отменена ожидающая команда. + assertEquals(0, released) + lease.release() // Последняя сессия закрыта. + assertEquals(1, released) } - @Test fun remoteViewerKeepsBroadcastRotationWithoutLocalMarker() { - for (rotation in listOf(0, 90, 180, 270)) - assertEquals(StreamVideoDisplay(rotation, null), StreamVideoDisplay.forFrame(rotation, null, 90)) + @Test fun sessionReplacementKeepsSurfaceUntilBothSessionsClose() { + var released = 0 + val lease = StreamPreviewLease { released++ } + lease.retain(); lease.retain() + lease.release() // Старая сессия. + lease.release() // UI исчез, новая сессия ещё работает. + assertEquals(0, released) + lease.release() + assertEquals(1, released) } - @Test fun cameraSwitchRetainsPoseForAlreadyQueuedFrames() { - val preview = StreamPreview() - val rear = StreamPreviewCamera(90, false) - val front = StreamPreviewCamera(270, true) - preview.register(1, rear); preview.register(3, front) - assertEquals(rear, preview.camera(1)) - assertEquals(front, preview.camera(3)) - assertNull(preview.camera(2)) - for (revision in 4L..40L) preview.register(revision, rear) - assertNull(preview.camera(1)) - assertEquals(rear, preview.camera(9)) - assertEquals(rear, preview.camera(40)) + @Test fun closedSurfaceCannotBeRetainedOrReleasedAgain() { + var released = 0 + val lease = StreamPreviewLease { released++ } + lease.release() + try { lease.retain(); fail("retained a closed surface") } catch (_: IllegalStateException) {} + try { lease.release(); fail("released a closed surface twice") } catch (_: IllegalStateException) {} + assertEquals(1, released) } }