Add Android call camera and keyframe encoder replacement worker

5 days ago · 04f4d29e4f
1 changed files with 250 additions and 0 deletions
--- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt
+++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt
@ -0,0 +1,250 @@
+package com.utun.chat.data
+
+import android.annotation.SuppressLint
+import android.content.Context
+import android.graphics.ImageFormat
+import android.hardware.camera2.CameraCaptureSession
+import android.hardware.camera2.CameraCharacteristics
+import android.hardware.camera2.CameraDevice
+import android.hardware.camera2.CameraManager
+import android.hardware.camera2.CaptureRequest
+import android.media.ImageReader
+import android.os.Handler
+import android.os.HandlerThread
+import android.os.SystemClock
+import android.util.Size
+import android.view.Surface
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+
+data class CallVideoCamera(val id: String, val name: String, val sizes: List<Size>)
+data class CallVideoSettings(val enabled: Boolean = false, val camera: String = "", val width: Int = 640,
+    val height: Int = 480, val codec: Int = 0)
+data class CallVideoState(val ready: Boolean = false, val settings: CallVideoSettings = CallVideoSettings(),
+    val cameras: List<CallVideoCamera> = emptyList(), val codecs: List<Int> = emptyList(), val bitrate: Int = 0,
+    val range: CallVideoBitrateRange = CallVideoBitrateRange(), val status: String = "")
+
+/** Сервис владеет capture и RX; экран только выбирает режим и передаёт Surface. */
+class CallVideoEngine(private val context: Context, private val id: Long) {
+    private val mutable = MutableStateFlow(CallVideoState())
+    val state: StateFlow<CallVideoState> = mutable
+    private val thread = HandlerThread("call-video-tx").apply { start() }
+    private val handler = Handler(thread.looper)
+    private val manager = context.getSystemService(CameraManager::class.java)
+    private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } }
+    @Volatile private var stopped = false
+    private var encoders = emptyMap<Int, String>()
+    private var caps = ByteArray(80)
+    private var wanted = CallVideoSettings()
+    private var active: CallVideoSettings? = null
+    private var format: CallVideoFormat? = null
+    private var encoder: CallVideoCodec? = null
+    private var camera: CameraDevice? = null
+    private var session: CameraCaptureSession? = null
+    private var reader: ImageReader? = null
+    private var opening = false
+    private var closing = false
+    private var cameraToken = 0L
+    private var generation = 0L
+    private var frame = 0L
+    private var lastPts = -1L
+    private var lastKey = -1L
+    private var keySerial = -1L
+    private var needKey = true
+    private var request = 0L
+    private var pendingRequest = 0L
+    private var pendingRate = 0
+    private var confirmingRequest = 0L
+    private var realtime = false
+    private var sensorOffset: Long? = null
+    private var openedAt = 0L
+    private var reportAt = 0L
+    private var encodedBytes = 0L
+    private var encodedFrames = 0
+
+    init {
+        handler.post {
+            try {
+                encoders = CallVideoCodec.probe()
+                if (stopped) return@post
+                val cameras = CameraLensManager.enumerate(manager).mapNotNull { lens ->
+                    val c = manager.getCameraCharacteristics(lens.cameraId)
+                    val sizes = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)?.getOutputSizes(ImageFormat.YUV_420_888)
+                        .orEmpty().filter { it == Size(640, 480) || it == Size(1280, 720) }
+                    if (sizes.isEmpty()) null else CallVideoCamera(lens.cameraId, lens.label, sizes)
+                }
+                val bytes = ByteBuffer.wrap(caps).order(ByteOrder.BIG_ENDIAN)
+                repeat(2) { for (codec in 1..2) {
+                    if (encoders.containsKey(codec)) bytes.putShort(1280).putShort(720).putShort(30).putInt(1280 * 720 * 30)
+                        .putInt(4_000_000).putInt(1_000_000).put(if (codec == 1) 31 else 93).put(0)
+                    else bytes.position(bytes.position() + 20)
+                } }
+                check(CallVideoNative.open(id, caps)) { "Video port unavailable" }
+                wanted = wanted.copy(camera = cameras.firstOrNull()?.id.orEmpty())
+                mutable.value = mutable.value.copy(ready = true, cameras = cameras, codecs = encoders.keys.toList(), settings = wanted)
+                report(if (encoders.isEmpty()) "No verified video codec" else "Video ready")
+                tick()
+            } catch (e: Exception) { fail("Video setup: ${e.message}") }
+        }
+    }
+
+    fun surface(surface: Surface?) = player.surface(surface)
+    fun select(settings: CallVideoSettings) { handler.post {
+        if (!stopped) { wanted = settings; mutable.value = mutable.value.copy(settings = settings) }
+    } }
+    fun range(value: CallVideoBitrateRange) { handler.post {
+        if (CallVideoNative.range(id, value.minimum * 1000, value.maximum * 1000)) mutable.value = mutable.value.copy(range = value)
+        else report("Bitrate range rejected")
+    } }
+    private fun report(message: String) {
+        mutable.value = mutable.value.copy(status = message)
+        LogManager.addLog("INFO", "call", "video call=$id $message")
+    }
+    private fun fail(message: String) {
+        LogManager.addLog("ERROR", "call", "video call=$id $message")
+        wanted = wanted.copy(enabled = false); mutable.value = mutable.value.copy(settings = wanted, status = message)
+        closeCamera(); encoder?.close(); encoder = null; active = null
+        if (confirmingRequest != 0L) CallVideoNative.applied(id, format?.generation ?: 0, confirmingRequest, 0)
+        confirmingRequest = 0
+    }
+    private fun tick() {
+        if (stopped) return
+        try {
+            val s = CallVideoNative.status(id) ?: return
+            if (s[6] >= 300000 && s[7] >= s[6]) mutable.value = mutable.value.copy(range = CallVideoBitrateRange((s[6] / 1000).toInt(), (s[7] / 1000).toInt()))
+            if (keySerial != s[0]) { keySerial = s[0]; needKey = true }
+            if (s[1] == format?.generation && s[2] > request) {
+                request = s[2]; pendingRequest = request; pendingRate = s[3].toInt()
+            }
+            if (!wanted.enabled) {
+                if (active != null) { closeCamera(); encoder?.close(); encoder = null; active = null; report("Video off") }
+            } else if (wanted != active && !opening && !closing && (lastKey < 0 || System.nanoTime() / 1000 - lastKey >= 1_000_000)) {
+                val peer = CallVideoNative.peer(id)
+                if (peer != null) change(peer)
+            }
+            drain()
+            if (active != null && System.nanoTime() - openedAt > 8_000_000_000 && lastPts < 0) fail("Camera produced no frames")
+        } catch (e: Exception) { fail("Video worker: ${e.message}") }
+        if (!stopped) handler.postDelayed(::tick, 10)
+    }
+    private fun change(peer: ByteArray) {
+        val cameraInfo = manager.getCameraCharacteristics(wanted.camera)
+        val rotation = cameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
+        val desired = CallVideoFormat(++generation, wanted.width, wanted.height, rotation = rotation, codec = wanted.codec)
+        val selected = CallVideoNative.choose(caps, peer, desired.array()) ?: error("No common codec for selected resolution")
+        // MediaCodec получает фиксированный 3.1; ряд backend не умеет понижать маркировку level для меньшего размера.
+        val next = CallVideoFormat.from(selected).let { it.copy(level = if (it.codec == 1) 31 else 93) }
+        val candidate = CallVideoCodec(next, 1_000_000, checkNotNull(encoders[next.codec]))
+        closeCamera(); encoder?.close(); encoder = candidate; format = next; active = wanted
+        pendingRequest = 0; confirmingRequest = 0; request = 0; lastKey = -1; needKey = true; lastPts = -1
+        check(CallVideoNative.format(id, next.array(), candidate.bitrate)) { "Video format rejected" }
+        mutable.value = mutable.value.copy(bitrate = candidate.bitrate)
+        if (!closing) openCamera()
+    }
+    @SuppressLint("MissingPermission")
+    private fun openCamera() {
+        if (stopped || !wanted.enabled || opening || closing) return
+        val settings = active ?: return; val f = format ?: return; val token = ++cameraToken
+        val c = manager.getCameraCharacteristics(settings.camera)
+        realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME
+        sensorOffset = null
+        val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader
+        nextReader.setOnImageAvailableListener({ source ->
+            val image = source.acquireLatestImage() ?: return@setOnImageAvailableListener
+            try {
+                if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener
+                val now = System.nanoTime()
+                val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also {
+                    sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it")
+                }
+                val pts = (image.timestamp + offset) / 1000
+                if (pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) return@setOnImageAvailableListener
+                val key = needKey || lastKey < 0 || pts - lastKey >= 1_000_000
+                if (key && pendingRequest != 0L) {
+                    val requestId = pendingRequest; pendingRequest = 0
+                    try {
+                        val candidate = CallVideoCodec(f, pendingRate, checkNotNull(encoders[f.codec]))
+                        encoder?.close(); encoder = candidate; confirmingRequest = requestId; needKey = true
+                        report("Applying ${pendingRate / 1000} kbit/s on IDR")
+                    } catch (e: Exception) {
+                        CallVideoNative.applied(id, f.generation, requestId, 0); throw e
+                    }
+                }
+                if (encoder?.submit(image, pts, key) == true) { lastPts = pts; if (key) lastKey = pts }
+                drain()
+            } catch (e: Exception) { fail("Camera encode: ${e.message}") }
+            finally { image.close() }
+        }, handler)
+        opening = true; openedAt = System.nanoTime()
+        manager.openCamera(settings.camera, object : CameraDevice.StateCallback() {
+            override fun onOpened(device: CameraDevice) {
+                opening = false; camera = device
+                if (stopped || token != cameraToken) { device.close(); return }
+                try {
+                    device.createCaptureSession(listOf(nextReader.surface), object : CameraCaptureSession.StateCallback() {
+                        override fun onConfigured(s: CameraCaptureSession) {
+                            if (stopped || token != cameraToken) { s.close(); return }
+                            session = s
+                            try {
+                                val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty()
+                                val range = rates.filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps")
+                                val capture = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply {
+                                    addTarget(nextReader.surface); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range)
+                                }.build()
+                                s.setRepeatingRequest(capture, null, handler)
+                                report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}")
+                            } catch (e: Exception) { fail("Camera request: ${e.message}") }
+                        }
+                        override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); fail("Camera session configuration failed") }
+                    }, handler)
+                } catch (e: Exception) { fail("Camera session: ${e.message}") }
+            }
+            override fun onDisconnected(device: CameraDevice) { opening = false; camera = device; fail("Camera disconnected") }
+            override fun onError(device: CameraDevice, error: Int) { opening = false; camera = device; fail("Camera error $error") }
+            override fun onClosed(device: CameraDevice) {
+                if (camera === device) camera = null
+                closing = false
+                if (stopped) thread.quitSafely()
+                else if (active != null && wanted.enabled) try { openCamera() } catch (e: Exception) { fail("Camera reopen: ${e.message}") }
+            }
+        }, handler)
+    }
+    private fun drain() {
+        val codec = encoder ?: return; val f = format ?: return
+        for (p in codec.drain()) {
+            encodedBytes += p.bytes.size; encodedFrames++
+            if (confirmingRequest != 0L && p.key) {
+                CallVideoNative.applied(id, f.generation, confirmingRequest, codec.bitrate); confirmingRequest = 0
+                mutable.value = mutable.value.copy(bitrate = codec.bitrate)
+                LogManager.addLog("INFO", "call", "video bitrate applied id=$id generation=${f.generation} bitrate=${codec.bitrate} pts=${p.pts}")
+            }
+            if (needKey && !p.key) continue
+            val metadata = f.array() + longArrayOf(++frame, p.pts, if (p.key) 1 else 0)
+            needKey = CallVideoNative.send(id, metadata, p.bytes) != 1
+        }
+        val now = System.nanoTime()
+        if (now - reportAt > 1_000_000_000) {
+            if (reportAt != 0L) LogManager.addLog("DEBUG", "call", "video output id=$id fps=$encodedFrames " +
+                "bps=${encodedBytes * 8_000_000_000 / (now - reportAt)} target=${codec.bitrate} needKey=$needKey")
+            reportAt = now; encodedFrames = 0; encodedBytes = 0
+        }
+    }
+    private fun closeCamera() {
+        cameraToken++
+        try { session?.close(); session = null; reader?.close(); reader = null
+            if (camera != null) { closing = true; camera!!.close() }
+        } catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") }
+    }
+    fun stop() {
+        if (stopped) return
+        stopped = true; player.stop(); CallVideoNative.close(id)
+        handler.post {
+            closeCamera(); encoder?.close(); encoder = null
+            if (camera == null && !opening) thread.quitSafely()
+            else handler.postDelayed({ thread.quitSafely() }, 3000)
+            LogManager.addLog("INFO", "call", "video workers stopped id=$id")
+        }
+    }
+}