diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt new file mode 100644 index 00000000..4f63f370 --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -0,0 +1,250 @@ +package com.utun.chat.data + +import android.annotation.SuppressLint +import android.content.Context +import android.graphics.ImageFormat +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.hardware.camera2.CaptureRequest +import android.media.ImageReader +import android.os.Handler +import android.os.HandlerThread +import android.os.SystemClock +import android.util.Size +import android.view.Surface +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import java.nio.ByteBuffer +import java.nio.ByteOrder + +data class CallVideoCamera(val id: String, val name: String, val sizes: List) +data class CallVideoSettings(val enabled: Boolean = false, val camera: String = "", val width: Int = 640, + val height: Int = 480, val codec: Int = 0) +data class CallVideoState(val ready: Boolean = false, val settings: CallVideoSettings = CallVideoSettings(), + val cameras: List = emptyList(), val codecs: List = emptyList(), val bitrate: Int = 0, + val range: CallVideoBitrateRange = CallVideoBitrateRange(), val status: String = "") + +/** Сервис владеет capture и RX; экран только выбирает режим и передаёт Surface. */ +class CallVideoEngine(private val context: Context, private val id: Long) { + private val mutable = MutableStateFlow(CallVideoState()) + val state: StateFlow = mutable + private val thread = HandlerThread("call-video-tx").apply { start() } + private val handler = Handler(thread.looper) + private val manager = context.getSystemService(CameraManager::class.java) + private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } } + @Volatile private var stopped = false + private var encoders = emptyMap() + private var caps = ByteArray(80) + private var wanted = CallVideoSettings() + private var active: CallVideoSettings? = null + private var format: CallVideoFormat? = null + private var encoder: CallVideoCodec? = null + private var camera: CameraDevice? = null + private var session: CameraCaptureSession? = null + private var reader: ImageReader? = null + private var opening = false + private var closing = false + private var cameraToken = 0L + private var generation = 0L + private var frame = 0L + private var lastPts = -1L + private var lastKey = -1L + private var keySerial = -1L + private var needKey = true + private var request = 0L + private var pendingRequest = 0L + private var pendingRate = 0 + private var confirmingRequest = 0L + private var realtime = false + private var sensorOffset: Long? = null + private var openedAt = 0L + private var reportAt = 0L + private var encodedBytes = 0L + private var encodedFrames = 0 + + init { + handler.post { + try { + encoders = CallVideoCodec.probe() + if (stopped) return@post + val cameras = CameraLensManager.enumerate(manager).mapNotNull { lens -> + val c = manager.getCameraCharacteristics(lens.cameraId) + val sizes = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)?.getOutputSizes(ImageFormat.YUV_420_888) + .orEmpty().filter { it == Size(640, 480) || it == Size(1280, 720) } + if (sizes.isEmpty()) null else CallVideoCamera(lens.cameraId, lens.label, sizes) + } + val bytes = ByteBuffer.wrap(caps).order(ByteOrder.BIG_ENDIAN) + repeat(2) { for (codec in 1..2) { + if (encoders.containsKey(codec)) bytes.putShort(1280).putShort(720).putShort(30).putInt(1280 * 720 * 30) + .putInt(4_000_000).putInt(1_000_000).put(if (codec == 1) 31 else 93).put(0) + else bytes.position(bytes.position() + 20) + } } + check(CallVideoNative.open(id, caps)) { "Video port unavailable" } + wanted = wanted.copy(camera = cameras.firstOrNull()?.id.orEmpty()) + mutable.value = mutable.value.copy(ready = true, cameras = cameras, codecs = encoders.keys.toList(), settings = wanted) + report(if (encoders.isEmpty()) "No verified video codec" else "Video ready") + tick() + } catch (e: Exception) { fail("Video setup: ${e.message}") } + } + } + + fun surface(surface: Surface?) = player.surface(surface) + fun select(settings: CallVideoSettings) { handler.post { + if (!stopped) { wanted = settings; mutable.value = mutable.value.copy(settings = settings) } + } } + fun range(value: CallVideoBitrateRange) { handler.post { + if (CallVideoNative.range(id, value.minimum * 1000, value.maximum * 1000)) mutable.value = mutable.value.copy(range = value) + else report("Bitrate range rejected") + } } + private fun report(message: String) { + mutable.value = mutable.value.copy(status = message) + LogManager.addLog("INFO", "call", "video call=$id $message") + } + private fun fail(message: String) { + LogManager.addLog("ERROR", "call", "video call=$id $message") + wanted = wanted.copy(enabled = false); mutable.value = mutable.value.copy(settings = wanted, status = message) + closeCamera(); encoder?.close(); encoder = null; active = null + if (confirmingRequest != 0L) CallVideoNative.applied(id, format?.generation ?: 0, confirmingRequest, 0) + confirmingRequest = 0 + } + private fun tick() { + if (stopped) return + try { + val s = CallVideoNative.status(id) ?: return + if (s[6] >= 300000 && s[7] >= s[6]) mutable.value = mutable.value.copy(range = CallVideoBitrateRange((s[6] / 1000).toInt(), (s[7] / 1000).toInt())) + if (keySerial != s[0]) { keySerial = s[0]; needKey = true } + if (s[1] == format?.generation && s[2] > request) { + request = s[2]; pendingRequest = request; pendingRate = s[3].toInt() + } + if (!wanted.enabled) { + if (active != null) { closeCamera(); encoder?.close(); encoder = null; active = null; report("Video off") } + } else if (wanted != active && !opening && !closing && (lastKey < 0 || System.nanoTime() / 1000 - lastKey >= 1_000_000)) { + val peer = CallVideoNative.peer(id) + if (peer != null) change(peer) + } + drain() + if (active != null && System.nanoTime() - openedAt > 8_000_000_000 && lastPts < 0) fail("Camera produced no frames") + } catch (e: Exception) { fail("Video worker: ${e.message}") } + if (!stopped) handler.postDelayed(::tick, 10) + } + private fun change(peer: ByteArray) { + val cameraInfo = manager.getCameraCharacteristics(wanted.camera) + val rotation = cameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 + val desired = CallVideoFormat(++generation, wanted.width, wanted.height, rotation = rotation, codec = wanted.codec) + val selected = CallVideoNative.choose(caps, peer, desired.array()) ?: error("No common codec for selected resolution") + // MediaCodec получает фиксированный 3.1; ряд backend не умеет понижать маркировку level для меньшего размера. + val next = CallVideoFormat.from(selected).let { it.copy(level = if (it.codec == 1) 31 else 93) } + val candidate = CallVideoCodec(next, 1_000_000, checkNotNull(encoders[next.codec])) + closeCamera(); encoder?.close(); encoder = candidate; format = next; active = wanted + pendingRequest = 0; confirmingRequest = 0; request = 0; lastKey = -1; needKey = true; lastPts = -1 + check(CallVideoNative.format(id, next.array(), candidate.bitrate)) { "Video format rejected" } + mutable.value = mutable.value.copy(bitrate = candidate.bitrate) + if (!closing) openCamera() + } + @SuppressLint("MissingPermission") + private fun openCamera() { + if (stopped || !wanted.enabled || opening || closing) return + val settings = active ?: return; val f = format ?: return; val token = ++cameraToken + val c = manager.getCameraCharacteristics(settings.camera) + realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME + sensorOffset = null + val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader + nextReader.setOnImageAvailableListener({ source -> + val image = source.acquireLatestImage() ?: return@setOnImageAvailableListener + try { + if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener + val now = System.nanoTime() + val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also { + sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it") + } + val pts = (image.timestamp + offset) / 1000 + if (pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) return@setOnImageAvailableListener + val key = needKey || lastKey < 0 || pts - lastKey >= 1_000_000 + if (key && pendingRequest != 0L) { + val requestId = pendingRequest; pendingRequest = 0 + try { + val candidate = CallVideoCodec(f, pendingRate, checkNotNull(encoders[f.codec])) + encoder?.close(); encoder = candidate; confirmingRequest = requestId; needKey = true + report("Applying ${pendingRate / 1000} kbit/s on IDR") + } catch (e: Exception) { + CallVideoNative.applied(id, f.generation, requestId, 0); throw e + } + } + if (encoder?.submit(image, pts, key) == true) { lastPts = pts; if (key) lastKey = pts } + drain() + } catch (e: Exception) { fail("Camera encode: ${e.message}") } + finally { image.close() } + }, handler) + opening = true; openedAt = System.nanoTime() + manager.openCamera(settings.camera, object : CameraDevice.StateCallback() { + override fun onOpened(device: CameraDevice) { + opening = false; camera = device + if (stopped || token != cameraToken) { device.close(); return } + try { + device.createCaptureSession(listOf(nextReader.surface), object : CameraCaptureSession.StateCallback() { + override fun onConfigured(s: CameraCaptureSession) { + if (stopped || token != cameraToken) { s.close(); return } + session = s + try { + val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() + val range = rates.filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps") + val capture = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { + addTarget(nextReader.surface); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) + }.build() + s.setRepeatingRequest(capture, null, handler) + report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}") + } catch (e: Exception) { fail("Camera request: ${e.message}") } + } + override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); fail("Camera session configuration failed") } + }, handler) + } catch (e: Exception) { fail("Camera session: ${e.message}") } + } + override fun onDisconnected(device: CameraDevice) { opening = false; camera = device; fail("Camera disconnected") } + override fun onError(device: CameraDevice, error: Int) { opening = false; camera = device; fail("Camera error $error") } + override fun onClosed(device: CameraDevice) { + if (camera === device) camera = null + closing = false + if (stopped) thread.quitSafely() + else if (active != null && wanted.enabled) try { openCamera() } catch (e: Exception) { fail("Camera reopen: ${e.message}") } + } + }, handler) + } + private fun drain() { + val codec = encoder ?: return; val f = format ?: return + for (p in codec.drain()) { + encodedBytes += p.bytes.size; encodedFrames++ + if (confirmingRequest != 0L && p.key) { + CallVideoNative.applied(id, f.generation, confirmingRequest, codec.bitrate); confirmingRequest = 0 + mutable.value = mutable.value.copy(bitrate = codec.bitrate) + LogManager.addLog("INFO", "call", "video bitrate applied id=$id generation=${f.generation} bitrate=${codec.bitrate} pts=${p.pts}") + } + if (needKey && !p.key) continue + val metadata = f.array() + longArrayOf(++frame, p.pts, if (p.key) 1 else 0) + needKey = CallVideoNative.send(id, metadata, p.bytes) != 1 + } + val now = System.nanoTime() + if (now - reportAt > 1_000_000_000) { + if (reportAt != 0L) LogManager.addLog("DEBUG", "call", "video output id=$id fps=$encodedFrames " + + "bps=${encodedBytes * 8_000_000_000 / (now - reportAt)} target=${codec.bitrate} needKey=$needKey") + reportAt = now; encodedFrames = 0; encodedBytes = 0 + } + } + private fun closeCamera() { + cameraToken++ + try { session?.close(); session = null; reader?.close(); reader = null + if (camera != null) { closing = true; camera!!.close() } + } catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") } + } + fun stop() { + if (stopped) return + stopped = true; player.stop(); CallVideoNative.close(id) + handler.post { + closeCamera(); encoder?.close(); encoder = null + if (camera == null && !opening) thread.quitSafely() + else handler.postDelayed({ thread.quitSafely() }, 3000) + LogManager.addLog("INFO", "call", "video workers stopped id=$id") + } + } +}