Avoid double sensor rotation in direct Android camera preview

tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPreview.kt

@@ -13,8 +13,10 @@ data class StreamPreviewCamera(val sensor: Int, val front: Boolean) {
 data class StreamPreviewFormat(val width: Int, val height: Int, val camera: StreamPreviewCamera, val realtimeTimestamp: Boolean = false) {
     fun aspect(displayDegrees: Int): Float = if (camera.rotation(displayDegrees) % 180 == 0) width.toFloat() / height else height.toFloat() / width
 
-    /** Углы TextureView после поворота; размеры view уже учитывают поворот сенсора. */
+    /** TextureView уже учитывает SENSOR_ORIENTATION; остаётся компенсация поворота экрана. */
-    fun corners(displayDegrees: Int): FloatArray = when (camera.rotation(displayDegrees)) {
+    fun textureRotation(displayDegrees: Int): Int = (360 - displayDegrees) % 360
+
+    fun corners(displayDegrees: Int): FloatArray = when (textureRotation(displayDegrees)) {
         90 -> floatArrayOf(1f, 0f, 1f, 1f, 0f, 1f, 0f, 0f)
         180 -> floatArrayOf(1f, 1f, 0f, 1f, 0f, 0f, 1f, 0f)
         270 -> floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f)

tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/StreamScreen.kt

@@ -6,6 +6,9 @@ import android.content.pm.PackageManager
 import android.content.Context
 import android.graphics.Matrix
 import android.graphics.SurfaceTexture
+import android.hardware.display.DisplayManager
+import android.os.Handler
+import android.os.Looper
 import android.os.SystemClock
 import android.view.Surface
 import android.view.TextureView
@@ -69,6 +72,7 @@ import kotlinx.coroutines.withContext
 fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSource>, selected: StreamSource?,
                  sourcesFailed: Boolean, onSelect: (StreamSource?) -> Unit, onBack: () -> Unit) {
     val context = LocalContext.current
+    val displayDegrees = StreamDisplayDegrees()
     val group = remember(channelId) { channelId.toULongOrNull()?.toLong() ?: 0L }
     val state by StreamController.state.collectAsState()
     val local = state?.takeIf { it.group == group }
@@ -117,7 +121,7 @@ fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSou
             if (selected != null) {
                 Text(selected.displayName, style = MaterialTheme.typography.titleSmall)
                 if (selected.self && local?.ongoing == true) {
-                    CaptureOptions(cameras, local.pendingConfig ?: local.config, local.phase == StreamPhase.LIVE && local.pendingConfig == null,
+                    CaptureOptions(cameras, local.pendingConfig ?: local.config, local.phase == StreamPhase.LIVE && local.pendingConfig == null, displayDegrees,
                         StreamController::change)
                     CaptureModeStatus(local.config, local.pendingConfig, local.notice)
                     StreamTxPanel(local.id)
@@ -125,7 +129,7 @@ fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSou
                 Box(Modifier.weight(1f).fillMaxWidth()) {
                     if (selected.self && local?.ongoing != true) Text(stringResource(R.string.stream_ended))
                     else key(group, selected.id, selected.epoch) {
-                        LiveStreamView(group, selected.id, selected.epoch, if (selected.self) local?.preview else null)
+                        LiveStreamView(group, selected.id, selected.epoch, if (selected.self) local?.preview else null, displayDegrees)
                     }
                 }
                 TextButton(onClick = { onSelect(null); onBack() }) { Text(stringResource(R.string.stream_close_view)) }
@@ -133,7 +137,7 @@ fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSou
                     enabled = local.phase != StreamPhase.STOPPING) { Text(stringResource(R.string.stream_stop)) }
             } else {
                 if (local?.ongoing == true) {
-                    CaptureOptions(cameras, local.pendingConfig ?: local.config, local.phase == StreamPhase.LIVE && local.pendingConfig == null,
+                    CaptureOptions(cameras, local.pendingConfig ?: local.config, local.phase == StreamPhase.LIVE && local.pendingConfig == null, displayDegrees,
                         StreamController::change)
                     CaptureModeStatus(local.config, local.pendingConfig, local.notice)
                     StreamTxPanel(local.id)
@@ -144,7 +148,7 @@ fun StreamScreen(channelId: String, channelName: String, sources: List<StreamSou
                     }))
                     Button(onClick = StreamController::stop, enabled = local.phase != StreamPhase.STOPPING) { Text(stringResource(R.string.stream_stop)) }
                 } else {
-                    CaptureOptions(cameras, config, !loadingModes) { mode = it.mode; orientation = it.orientation; videoBitrate = it.videoBitrate }
+                    CaptureOptions(cameras, config, !loadingModes, displayDegrees) { mode = it.mode; orientation = it.orientation; videoBitrate = it.videoBitrate }
                     if (!loadingModes && cameras.isEmpty()) Text(stringResource(R.string.stream_no_modes), color = MaterialTheme.colorScheme.error)
                     Row { Checkbox(checked = audio, onCheckedChange = { audio = it }); Text(stringResource(R.string.microphone), Modifier.padding(top = 12.dp)) }
                     Button(onClick = {
@@ -188,7 +192,7 @@ private fun StreamTxPanel(id: Long) {
 }
 
 @Composable
-private fun CaptureOptions(cameras: List<StreamCamera>, config: StreamCaptureConfig?, enabled: Boolean,
+private fun CaptureOptions(cameras: List<StreamCamera>, config: StreamCaptureConfig?, enabled: Boolean, display: Int,
                            onChange: (StreamCaptureConfig) -> Unit) {
     val mode = config?.mode
     Row(Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(8.dp)) {
@@ -239,9 +243,6 @@ private fun CaptureOptions(cameras: List<StreamCamera>, config: StreamCaptureCon
             }
         }
     }
-    val context = LocalContext.current
-    val configuration = LocalConfiguration.current
-    val display = remember(configuration) { ((context as? Activity)?.windowManager?.defaultDisplay?.rotation ?: 0) * 90 }
     var expanded by remember { mutableStateOf(false) }
     Row(horizontalArrangement = Arrangement.spacedBy(6.dp), verticalAlignment = Alignment.CenterVertically) {
         Box {
@@ -295,7 +296,7 @@ private fun StreamRotationMark(top: Int) {
 }
 
 @Composable
-private fun LiveStreamView(group: Long, source: Long, epoch: Long, localPreview: StreamPreview?) {
+private fun LiveStreamView(group: Long, source: Long, epoch: Long, localPreview: StreamPreview?, displayDegrees: Int) {
     val owner = LocalLifecycleOwner.current
     var visible by remember(owner) { mutableStateOf(owner.lifecycle.currentState.isAtLeast(Lifecycle.State.STARTED)) }
     DisposableEffect(owner, localPreview) {
@@ -307,17 +308,35 @@ private fun LiveStreamView(group: Long, source: Long, epoch: Long, localPreview:
         owner.lifecycle.addObserver(observer)
         onDispose { owner.lifecycle.removeObserver(observer); localPreview?.setVisible(false) }
     }
-    val context = LocalContext.current
-    val configuration = LocalConfiguration.current
-    val displayDegrees = if (localPreview != null) remember(configuration) {
-        ((context as? Activity)?.windowManager?.defaultDisplay?.rotation ?: Surface.ROTATION_0) * 90
-    } else 0
     if (visible) {
         if (localPreview != null) CameraPreviewSurface(localPreview, displayDegrees)
         else LiveStreamSurface(group, source, epoch)
     }
 }
 
+/** DisplayListener замечает также 180°, когда Configuration и размеры окна не меняются. */
+@Composable
+private fun StreamDisplayDegrees(): Int {
+    val context = LocalContext.current
+    val configuration = LocalConfiguration.current
+    val display = (context as? Activity)?.windowManager?.defaultDisplay
+    var degrees by remember(display) { mutableStateOf((display?.rotation ?: 0) * 90) }
+    DisposableEffect(context, display, configuration) {
+        degrees = (display?.rotation ?: 0) * 90
+        val manager = context.getSystemService(DisplayManager::class.java)
+        val listener = object : DisplayManager.DisplayListener {
+            override fun onDisplayAdded(id: Int) {}
+            override fun onDisplayRemoved(id: Int) {}
+            override fun onDisplayChanged(id: Int) {
+                if (id == display?.displayId) degrees = display.rotation * 90
+            }
+        }
+        manager.registerDisplayListener(listener, Handler(Looper.getMainLooper()))
+        onDispose { manager.unregisterDisplayListener(listener) }
+    }
+    return degrees
+}
+
 @Composable
 private fun LiveStreamSurface(group: Long, source: Long, epoch: Long) {
     val engine = remember(group, source, epoch) { StreamPlayerEngine(group, source, epoch) }
@@ -356,7 +375,7 @@ private fun CameraPreviewSurface(preview: StreamPreview, displayDegrees: Int) {
     val state by preview.state.collectAsState()
     val aspect = state.format?.aspect(displayDegrees) ?: 9f / 16f
     Column(Modifier.fillMaxSize()) {
-        Text(stringResource(if (state.failed) R.string.stream_failed else if (state.active) R.string.stream_live else R.string.stream_connecting))
+        Text(stringResource(if (state.failed) R.string.stream_preview_failed else if (state.active) R.string.stream_live else R.string.stream_connecting))
         Box(Modifier.weight(1f).fillMaxWidth(), contentAlignment = Alignment.Center) {
             Box(Modifier.aspectRatio(aspect, matchHeightConstraintsFirst = true)) {
                 AndroidView(factory = { CameraPreviewView(it, preview) }, modifier = Modifier.matchParentSize(),
@@ -375,13 +394,14 @@ private class CameraPreviewView(context: Context, private val preview: StreamPre
     private var frames = 0L
     private var reportFrames = 0L
     private var reportNs = 0L
+    private var observedFormat: StreamPreviewFormat? = null
 
     init {
         keepScreenOn = true
         surfaceTextureListener = object : SurfaceTextureListener {
             override fun onSurfaceTextureAvailable(texture: SurfaceTexture, width: Int, height: Int) {
                 if (closed) return
-                target = preview.attach(texture); frames = 0; reportFrames = 0; reportNs = System.nanoTime()
+                target = preview.attach(texture); frames = 0; reportFrames = 0; reportNs = System.nanoTime(); observedFormat = null
                 transform()
             }
             override fun onSurfaceTextureSizeChanged(texture: SurfaceTexture, width: Int, height: Int) { transform() }
@@ -393,6 +413,12 @@ private class CameraPreviewView(context: Context, private val preview: StreamPre
             override fun onSurfaceTextureUpdated(texture: SurfaceTexture) {
                 val timestamp = texture.timestamp
                 if (timestamp == 0L) return // Очистка TextureView до первого кадра камеры.
+                if (format != observedFormat) {
+                    val native = FloatArray(16); texture.getTransformMatrix(native)
+                    LogManager.addLog("INFO", "video", "direct preview native transform sensor=${format?.camera?.sensor} " +
+                        "front=${format?.camera?.front} native=${listOf(native[0], native[1], native[4], native[5])}")
+                    observedFormat = format
+                }
                 frames++
                 val now = System.nanoTime()
                 if (frames == 1L || now - reportNs >= 1_000_000_000) {
@@ -401,7 +427,7 @@ private class CameraPreviewView(context: Context, private val preview: StreamPre
                     LogManager.addLog(if (frames == 1L) "INFO" else "DEBUG", "video", "direct preview frame " +
                         "target=${target?.let(System::identityHashCode)} frames=$frames fps=${"%.1f".format(fps)} " +
                         "ageUs=$age timestampNs=$timestamp realtime=${format?.realtimeTimestamp} " +
-                        "rotation=${format?.camera?.rotation(displayDegrees)}")
+                        "sensor=${format?.camera?.sensor} display=$displayDegrees textureRotation=${format?.textureRotation(displayDegrees)}")
                     reportNs = now; reportFrames = frames
                 }
             }
@@ -425,7 +451,7 @@ private class CameraPreviewView(context: Context, private val preview: StreamPre
         }
         setTransform(matrix)
         LogManager.addLog("DEBUG", "video", "direct preview transform view=${width}x$height buffer=${next.width}x${next.height} " +
-            "rotation=${next.camera.rotation(displayDegrees)} front=${next.camera.front}")
+            "sensor=${next.camera.sensor} display=$displayDegrees textureRotation=${next.textureRotation(displayDegrees)} front=${next.camera.front}")
     }
 
     fun close() { closed = true; keepScreenOn = false; target?.let(preview::detach) }

tools/chatgui-android/app/src/test/java/com/utun/chat/data/StreamPreviewTest.kt

@@ -11,10 +11,20 @@ class StreamPreviewTest {
         assertEquals(16f / 9f, rear.aspect(90), 0f)
         assertEquals(9f / 16f, front.aspect(0), 0f)
         assertEquals(16f / 9f, front.aspect(90), 0f)
-        assertArrayEquals(floatArrayOf(1f, 0f, 1f, 1f, 0f, 1f, 0f, 0f), rear.corners(0), 0f)
+        assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), rear.corners(0), 0f)
-        assertArrayEquals(floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f), front.corners(0), 0f)
+        assertArrayEquals(rear.corners(0), front.corners(0), 0f)
-        assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), rear.corners(90), 0f)
+        assertArrayEquals(floatArrayOf(0f, 1f, 0f, 0f, 1f, 0f, 1f, 1f), rear.corners(90), 0f)
-        assertArrayEquals(floatArrayOf(0f, 0f, 1f, 0f, 1f, 1f, 0f, 1f), front.corners(90), 0f)
+        assertArrayEquals(rear.corners(90), front.corners(90), 0f)
+    }
+
+    @Test fun textureViewNeverAppliesSensorRotationTwice() {
+        for (front in listOf(false, true)) for (sensor in listOf(0, 90, 180, 270)) {
+            val format = StreamPreviewFormat(1280, 720, StreamPreviewCamera(sensor, front))
+            assertEquals(0, format.textureRotation(0))
+            assertEquals(270, format.textureRotation(90))
+            assertEquals(180, format.textureRotation(180))
+            assertEquals(90, format.textureRotation(270))
+        }
     }
 
     @Test fun allPreviewRotationsPreserveRectangleCorners() {