Use direct camera preview instead of a local stream decoder

tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt

@@ -21,17 +21,14 @@ import kotlin.concurrent.withLock
 /** Запас кодированного потока остаётся в общем RS-кольце. Worker запрашивает ближайшие кадры,
  * общий native controller ведёт минутную историю и темп, AudioTrack задаёт фактические часы.
  * Временная нехватка сохраняет decoder references; GAP/вытеснение начинают новую точку KEY. */
-class StreamPlayerEngine(private val group: Long, private val source: Long, private val epoch: Long,
-                         private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) {
+class StreamPlayerEngine(private val group: Long, private val source: Long, private val epoch: Long) {
     companion object { private val viewerLock = ReentrantLock() }
     private class UnsupportedCodec(message: String) : IllegalStateException(message)
     private data class Frame(override val pts: Long, val key: Boolean, val width: Int, val height: Int,
-                             val rotation: Int, val top: Int?, val revision: Long, val codec: StreamVideoCodec,
+                             val rotation: Int, val revision: Long, val codec: StreamVideoCodec,
                              val config: ByteArray, val data: ByteArray) : StreamDecodePump.Frame
     private val mutableAspect = MutableStateFlow(16f / 9f)
     val aspect: StateFlow<Float> = mutableAspect
-    private val mutableTop = MutableStateFlow<Int?>(null)
-    val top: StateFlow<Int?> = mutableTop
     private val mutableStatus = MutableStateFlow(R.string.stream_connecting)
     val status: StateFlow<Int> = mutableStatus
     private val id = StreamController.nextId()
@@ -121,9 +118,9 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
 
         fun observeStatus(meta: LongArray?) {
             if (meta == null || meta[5] and 1 == 0L) return
-            ready = if (localPreview != null) meta[1] else meta[0]
+            ready = meta[0]
             if (playback != 0L) return
-            audioEnabled = localPreview == null && meta[5] and 2 != 0L
+            audioEnabled = meta[5] and 2 != 0L
             playback = NativeLib.streamPlaybackCreate(id, meta[3], audioEnabled)
             check(playback != 0L); position = meta[3]; generation = meta[4]
             LogManager.addLog("INFO", "stream", "playback starts id=$id PTS=$position generation=$generation fixedPrefill=0")
@@ -181,7 +178,7 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
         }
         video = createVideoPump()
         try {
-            check(NativeLib.streamOpen(id, group, source, audio = localPreview == null, epoch = epoch)) { "Viewer open refused" }
+            check(NativeLib.streamOpen(id, group, source, audio = true, epoch = epoch)) { "Viewer open refused" }
             while (running && surface.isValid) {
                 check(NativeLib.isReady()) { "Core stopped" }
                 observeStatus(NativeLib.streamStatus(id))
@@ -205,12 +202,6 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
                     if (kind in 1..2 && playback == 0L) observeStatus(NativeLib.streamStatus(id))
                     when (kind) {
                         1 -> {
-                            val camera = localPreview?.camera(revision)
-                            if (localPreview != null && camera == null) {
-                                LogManager.addLog("WARN", "video", "preview camera unavailable id=$id revision=$revision")
-                                resetTimeline(true); continue
-                            }
-                            val display = StreamVideoDisplay.forFrame(broadcastRotation, camera, displayDegrees)
                             if (revision != videoRevision) {
                                 check(revision > videoRevision); resetVideo(); videoRevision = revision
                                 deadline = System.nanoTime() + 10_000_000_000
@@ -221,12 +212,12 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
                                 val codec = StreamVideoCodec.fromId(codecId)
                                 val access = if (key) codec.unpackKey(bytes) else byteArrayOf() to bytes
                                 if (waitingKey) {
-                                    waitingKey = false; mutableTop.value = display.top
-                                    mutableAspect.value = if (display.rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width
+                                    waitingKey = false
+                                    mutableAspect.value = if (broadcastRotation % 180 == 0) width.toFloat() / height else height.toFloat() / width
                                     mutableStatus.value = R.string.stream_waiting_decoder
                                     LogManager.addLog("INFO", "video", "viewer KEY accepted id=$id revision=$revision PTS=$pts bytes=$size")
                                 }
-                                video.offer(Frame(pts, key, width, height, display.rotation, display.top, revision, codec, access.first, access.second),
+                                video.offer(Frame(pts, key, width, height, broadcastRotation, revision, codec, access.first, access.second),
                                     System.nanoTime() / 1000)
                             }
                         }

tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt

@@ -3,6 +3,8 @@ package com.utun.chat.ui.screens
 import android.Manifest
 import android.app.Activity
 import android.content.pm.PackageManager
+import android.content.Context
+import android.graphics.Matrix
 import android.graphics.SurfaceTexture
 import android.view.Surface
 import android.view.TextureView
@@ -47,6 +49,8 @@ import com.utun.chat.data.StreamController
 import com.utun.chat.data.StreamPhase
 import com.utun.chat.data.StreamPlayerEngine
 import com.utun.chat.data.StreamPreview
+import com.utun.chat.data.StreamPreviewFormat
+import com.utun.chat.data.StreamPreviewTarget
 import com.utun.chat.data.StreamSource
 import com.utun.chat.data.StreamCaptureMode
 import com.utun.chat.data.StreamCaptureConfig
@@ -58,7 +62,7 @@ import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 
-/** Выход с экрана закрывает только reader; foreground publisher останавливает явная кнопка/notification. */
+/** Выход закрывает reader либо прямой preview output; foreground publisher останавливается явно. */
 @OptIn(ExperimentalMaterial3Api::class)
 @Composable
 fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSource>, selected: StreamSource?,
@@ -118,7 +122,10 @@ fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSou
                     StreamTxPanel(local.id)
                 }
                 Box(Modifier.weight(1f).fillMaxWidth()) {
-                    key(group, selected.id, selected.epoch) { LiveStreamView(group, selected.id, selected.epoch, if (selected.self) local?.preview else null) }
+                    if (selected.self && local?.ongoing != true) Text(stringResource(R.string.stream_ended))
+                    else key(group, selected.id, selected.epoch) {
+                        LiveStreamView(group, selected.id, selected.epoch, if (selected.self) local?.preview else null)
+                    }
                 }
                 TextButton(onClick = { onSelect(null); onBack() }) { Text(stringResource(R.string.stream_close_view)) }
                 if (selected.self && local?.ongoing == true) Button(onClick = StreamController::stop,
@@ -303,12 +310,15 @@ private fun LiveStreamView(group: Long, source: Long, epoch: Long, localPreview:
     val displayDegrees = if (localPreview != null) remember(configuration) {
         ((context as? Activity)?.windowManager?.defaultDisplay?.rotation ?: Surface.ROTATION_0) * 90
     } else 0
-    if (visible) key(localPreview, displayDegrees) { LiveStreamSurface(group, source, epoch, localPreview, displayDegrees) }
+    if (visible) {
+        if (localPreview != null) CameraPreviewSurface(localPreview, displayDegrees)
+        else LiveStreamSurface(group, source, epoch)
+    }
 }
 
 @Composable
-private fun LiveStreamSurface(group: Long, source: Long, epoch: Long, localPreview: StreamPreview?, displayDegrees: Int) {
-    val engine = remember(group, source, epoch, localPreview) { StreamPlayerEngine(group, source, epoch, localPreview, displayDegrees) }
+private fun LiveStreamSurface(group: Long, source: Long, epoch: Long) {
+    val engine = remember(group, source, epoch) { StreamPlayerEngine(group, source, epoch) }
     val aspect by engine.aspect.collectAsState()
     val status by engine.status.collectAsState()
     var surface by remember { mutableStateOf<Surface?>(null) }
@@ -338,3 +348,80 @@ private fun LiveStreamSurface(group: Long, source: Long, epoch: Long, localPrevi
         }
     }
 }
+
+@Composable
+private fun CameraPreviewSurface(preview: StreamPreview, displayDegrees: Int) {
+    val state by preview.state.collectAsState()
+    val aspect = state.format?.aspect(displayDegrees) ?: 9f / 16f
+    Column(Modifier.fillMaxSize()) {
+        Text(stringResource(if (state.failed) R.string.stream_failed else if (state.active) R.string.stream_live else R.string.stream_connecting))
+        Box(Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) {
+            Box(Modifier.aspectRatio(aspect, matchHeightConstraintsFirst = true)) {
+                AndroidView(factory = { CameraPreviewView(it, preview) }, modifier = Modifier.matchParentSize(),
+                    update = { it.update(state.format, displayDegrees) }, onRelease = { it.close() })
+            }
+        }
+    }
+}
+
+/** TextureView отдаёт texture владельцам Camera2; destroy не освобождает используемый камерой output. */
+private class CameraPreviewView(context: Context, private val preview: StreamPreview) : TextureView(context) {
+    private var target: StreamPreviewTarget? = null
+    private var format: StreamPreviewFormat? = null
+    private var displayDegrees = 0
+    private var closed = false
+    private var frames = 0L
+    private var reportFrames = 0L
+    private var reportNs = 0L
+
+    init {
+        keepScreenOn = true
+        surfaceTextureListener = object : SurfaceTextureListener {
+            override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) {
+                if (closed) return
+                target = preview.attach(texture); frames = 0; reportFrames = 0; reportNs = System.nanoTime()
+                transform()
+            }
+            override fun onSurfaceTextureSizeChanged(texture: SurfaceTexture, width: Int, height: Int) { transform() }
+            override fun onSurfaceTextureDestroyed(texture: SurfaceTexture): Boolean {
+                val previous = target ?: return true
+                preview.detach(previous); target = null; previous.release()
+                return false // Camera session удерживает свою ссылку до onClosed/onConfigureFailed.
+            }
+            override fun onSurfaceTextureUpdated(texture: SurfaceTexture) {
+                frames++
+                val now = System.nanoTime()
+                if (frames == 1L || now - reportNs >= 1_000_000_000) {
+                    val fps = (frames - reportFrames) * 1_000_000_000.0 / (now - reportNs).coerceAtLeast(1)
+                    LogManager.addLog(if (frames == 1L) "INFO" else "DEBUG", "video", "direct preview frame " +
+                        "target=${target?.let(System::identityHashCode)} frames=$frames fps=${"%.1f".format(fps)} " +
+                        "ageUs=${(now - texture.timestamp) / 1000} timestampNs=${texture.timestamp} " +
+                        "rotation=${format?.camera?.rotation(displayDegrees)}")
+                    reportNs = now; reportFrames = frames
+                }
+            }
+        }
+    }
+
+    fun update(next: StreamPreviewFormat?, display: Int) {
+        if (next == format && display == displayDegrees) return
+        format = next; displayDegrees = display; transform()
+    }
+
+    private fun transform() {
+        val next = format ?: return
+        if (width == 0 || height == 0) return
+        val destination = next.corners(displayDegrees)
+        for (i in destination.indices) destination[i] *= if (i % 2 == 0) width.toFloat() else height.toFloat()
+        val source = floatArrayOf(0f, 0f, width.toFloat(), 0f, width.toFloat(), height.toFloat(), 0f, height.toFloat())
+        val matrix = Matrix()
+        if (!matrix.setPolyToPoly(source, 0, destination, 0, 4)) {
+            LogManager.addLog("ERROR", "video", "direct preview transform failed view=${width}x$height format=$next"); return
+        }
+        setTransform(matrix)
+        LogManager.addLog("DEBUG", "video", "direct preview transform view=${width}x$height buffer=${next.width}x${next.height} " +
+            "rotation=${next.camera.rotation(displayDegrees)} front=${next.camera.front}")
+    }
+
+    fun close() { closed = true; keepScreenOn = false; target?.let(preview::detach) }
+}

tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt

@@ -4,45 +4,58 @@ import org.junit.Assert.*
 import org.junit.Test
 
 class StreamPreviewTest {
-    @Test fun manualRotationMovesRearTopMarkerWithoutRotatingCameraPreview() {
-        val camera = StreamPreviewCamera(90, false)
-        val display = StreamOrientation.values().map {
-            StreamVideoDisplay.forFrame(it.sensorRotation(90, false), camera, 0)
+    @Test fun cameraPreviewUsesSensorAndDisplayOrientation() {
+        val rear = StreamPreviewFormat(1280, 720, StreamPreviewCamera(90, false))
+        val front = StreamPreviewFormat(1280, 720, StreamPreviewCamera(270, true))
+        assertEquals(9f / 16f, rear.aspect(0), 0f)
+        assertEquals(16f / 9f, rear.aspect(90), 0f)
+        assertEquals(9f / 16f, front.aspect(0), 0f)
+        assertEquals(16f / 9f, front.aspect(90), 0f)
+        assertArrayEquals(floatArrayOf(1f, 0f, 1f, 1f, 0f, 1f, 0f, 0f), rear.corners(0), 0f)
+        assertArrayEquals(floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f), front.corners(0), 0f)
+        assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), rear.corners(90), 0f)
+        assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), front.corners(90), 0f)
+    }
+
+    @Test fun allPreviewRotationsPreserveRectangleCorners() {
+        for (front in listOf(false, true)) for (sensor in listOf(0, 90, 180, 270)) for (display in listOf(0, 90, 180, 270)) {
+            val format = StreamPreviewFormat(640, 480, StreamPreviewCamera(sensor, front))
+            val corners = format.corners(display).toList().chunked(2)
+            assertEquals(setOf(listOf(0f, 0f), listOf(1f, 0f), listOf(1f, 1f), listOf(0f, 1f)), corners.toSet())
+            assertEquals(if (format.camera.rotation(display) % 180 == 0) 4f / 3f else 3f / 4f, format.aspect(display), 0f)
         }
-        assertEquals(listOf(90, 90, 90, 90), display.map { it.rotation })
-        assertEquals(listOf(0, 90, 180, 270), display.map { it.top })
     }
 
-    @Test fun frontCameraTopMarkerUsesItsOwnSensorDirection() {
-        val camera = StreamPreviewCamera(270, true)
-        val display = StreamOrientation.values().map {
-            StreamVideoDisplay.forFrame(it.sensorRotation(270, true), camera, 0)
-        }
-        assertEquals(listOf(270, 270, 270, 270), display.map { it.rotation })
-        assertEquals(listOf(0, 270, 180, 90), display.map { it.top })
+    @Test fun destroyedViewDoesNotReleaseCameraOrPendingCommandSurface() {
+        var released = 0
+        val lease = StreamPreviewLease { released++ }
+        lease.retain() // Команда, ожидающая camera handler.
+        lease.retain() // Сессия камеры.
+        lease.release() // UI TextureView уничтожен.
+        assertEquals(0, released)
+        lease.release() // Отменена ожидающая команда.
+        assertEquals(0, released)
+        lease.release() // Последняя сессия закрыта.
+        assertEquals(1, released)
     }
 
-    @Test fun previewFollowsScreenWhileBroadcastKeepsManualOrientation() {
-        assertEquals(StreamVideoDisplay(0, 270), StreamVideoDisplay.forFrame(90, StreamPreviewCamera(90, false), 90))
-        assertEquals(StreamVideoDisplay(0, 90), StreamVideoDisplay.forFrame(270, StreamPreviewCamera(270, true), 90))
+    @Test fun sessionReplacementKeepsSurfaceUntilBothSessionsClose() {
+        var released = 0
+        val lease = StreamPreviewLease { released++ }
+        lease.retain(); lease.retain()
+        lease.release() // Старая сессия.
+        lease.release() // UI исчез, новая сессия ещё работает.
+        assertEquals(0, released)
+        lease.release()
+        assertEquals(1, released)
     }
 
-    @Test fun remoteViewerKeepsBroadcastRotationWithoutLocalMarker() {
-        for (rotation in listOf(0, 90, 180, 270))
-            assertEquals(StreamVideoDisplay(rotation, null), StreamVideoDisplay.forFrame(rotation, null, 90))
-    }
-
-    @Test fun cameraSwitchRetainsPoseForAlreadyQueuedFrames() {
-        val preview = StreamPreview()
-        val rear = StreamPreviewCamera(90, false)
-        val front = StreamPreviewCamera(270, true)
-        preview.register(1, rear); preview.register(3, front)
-        assertEquals(rear, preview.camera(1))
-        assertEquals(front, preview.camera(3))
-        assertNull(preview.camera(2))
-        for (revision in 4L..40L) preview.register(revision, rear)
-        assertNull(preview.camera(1))
-        assertEquals(rear, preview.camera(9))
-        assertEquals(rear, preview.camera(40))
+    @Test fun closedSurfaceCannotBeRetainedOrReleasedAgain() {
+        var released = 0
+        val lease = StreamPreviewLease { released++ }
+        lease.release()
+        try { lease.retain(); fail("retained a closed surface") } catch (_: IllegalStateException) {}
+        try { lease.release(); fail("released a closed surface twice") } catch (_: IllegalStateException) {}
+        assertEquals(1, released)
     }
 }