From 2f1de49e87f22c32701f87575caa0f5a655b0af1 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 23:13:01 +0200 Subject: [PATCH] Negotiate actual Android modes and replace single encoder on IDR --- .../com/utun/chat/data/CallVideoEngine.kt | 75 +++++++++++++------ .../com/utun/chat/data/CallVideoPlayer.kt | 6 +- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt index 4f63f370..2067752a 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -35,8 +35,8 @@ class CallVideoEngine(private val context: Context, private val id: Long) { private val manager = context.getSystemService(CameraManager::class.java) private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } } @Volatile private var stopped = false - private var encoders = emptyMap() - private var caps = ByteArray(80) + private var encoders = emptyMap() + private var caps = ByteArray(64) private var wanted = CallVideoSettings() private var active: CallVideoSettings? = null private var format: CallVideoFormat? = null @@ -67,6 +67,8 @@ class CallVideoEngine(private val context: Context, private val id: Long) { init { handler.post { try { + val decoders = CallVideoCodec.probeDecoders() + player.decoders(decoders) encoders = CallVideoCodec.probe() if (stopped) return@post val cameras = CameraLensManager.enumerate(manager).mapNotNull { lens -> @@ -76,15 +78,17 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (sizes.isEmpty()) null else CallVideoCamera(lens.cameraId, lens.label, sizes) } val bytes = ByteBuffer.wrap(caps).order(ByteOrder.BIG_ENDIAN) - repeat(2) { for (codec in 1..2) { - if (encoders.containsKey(codec)) bytes.putShort(1280).putShort(720).putShort(30).putInt(1280 * 720 * 30) - .putInt(4_000_000).putInt(1_000_000).put(if (codec == 1) 31 else 93).put(0) - else bytes.position(bytes.position() + 20) + repeat(2) { direction -> for (codec in 1..2) { + val profile = if (direction == 0) encoders[codec]?.profile else decoders[codec]?.profiles + val level = if (direction == 0) encoders[codec]?.level else if (codec == 1) 31 else 93 + if (profile != null && level != null) bytes.putShort(1280).putShort(720).putShort(30).putInt(1280 * 720 * 30) + .putInt(4_000_000).put(level.toByte()).put(profile.toByte()) + else bytes.position(bytes.position() + 16) } } check(CallVideoNative.open(id, caps)) { "Video port unavailable" } wanted = wanted.copy(camera = cameras.firstOrNull()?.id.orEmpty()) mutable.value = mutable.value.copy(ready = true, cameras = cameras, codecs = encoders.keys.toList(), settings = wanted) - report(if (encoders.isEmpty()) "No verified video codec" else "Video ready") + report(if (encoders.isEmpty()) "Receive-only video" else "Video ready") tick() } catch (e: Exception) { fail("Video setup: ${e.message}") } } @@ -133,15 +137,35 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val cameraInfo = manager.getCameraCharacteristics(wanted.camera) val rotation = cameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 val desired = CallVideoFormat(++generation, wanted.width, wanted.height, rotation = rotation, codec = wanted.codec) - val selected = CallVideoNative.choose(caps, peer, desired.array()) ?: error("No common codec for selected resolution") - // MediaCodec получает фиксированный 3.1; ряд backend не умеет понижать маркировку level для меньшего размера. - val next = CallVideoFormat.from(selected).let { it.copy(level = if (it.codec == 1) 31 else 93) } - val candidate = CallVideoCodec(next, 1_000_000, checkNotNull(encoders[next.codec])) - closeCamera(); encoder?.close(); encoder = candidate; format = next; active = wanted - pendingRequest = 0; confirmingRequest = 0; request = 0; lastKey = -1; needKey = true; lastPts = -1 - check(CallVideoNative.format(id, next.array(), candidate.bitrate)) { "Video format rejected" } - mutable.value = mutable.value.copy(bitrate = candidate.bitrate) - if (!closing) openCamera() + val previous = encoder; val previousFormat = format; val previousSettings = active + val previousName = previous?.name; val previousRate = previous?.bitrate + previous?.close(); encoder = null + for (codec in if (wanted.codec == 0) listOf(2, 1) else listOf(wanted.codec)) { + val implementation = encoders[codec] ?: continue + val selected = CallVideoNative.choose(caps, peer, desired.copy(codec = codec).array()) ?: continue + var candidate: CallVideoCodec? = null + try { + candidate = CallVideoCodec(CallVideoFormat.from(selected), 1_000_000, implementation.name) + candidate.prepare() + val next = candidate.format + check(CallVideoNative.supported(caps, peer, next.array())) { "Actual encoder format exceeds peer capabilities: $next" } + check(CallVideoNative.format(id, next.array(), candidate.bitrate)) { "Video format rejected" } + closeCamera(); encoder = candidate; format = next; active = wanted + pendingRequest = 0; confirmingRequest = 0; request = 0; lastKey = -1; needKey = true + mutable.value = mutable.value.copy(bitrate = candidate.bitrate) + if (!closing) openCamera() + return + } catch (e: Exception) { + candidate?.close() + LogManager.addLog("WARN", "call", "video mode rejected codec=$codec name=${implementation.name}: ${e.message}") + } + } + if (previousFormat != null && previousSettings != null && previousName != null && previousRate != null) { + encoder = CallVideoCodec(previousFormat, previousRate, previousName).also { it.prepare(); check(it.format == previousFormat) } + wanted = previousSettings; needKey = true + mutable.value = mutable.value.copy(settings = wanted) + report("Selected mode unavailable; previous video restored") + } else fail("No compatible encoder for selected video mode") } @SuppressLint("MissingPermission") private fun openCamera() { @@ -152,7 +176,9 @@ class CallVideoEngine(private val context: Context, private val id: Long) { sensorOffset = null val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader nextReader.setOnImageAvailableListener({ source -> - val image = source.acquireLatestImage() ?: return@setOnImageAvailableListener + val image = try { source.acquireLatestImage() } catch (e: IllegalStateException) { + LogManager.addLog("WARN", "call", "closed camera reader token=$token/$cameraToken: ${e.message}"); null + } ?: return@setOnImageAvailableListener try { if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener val now = System.nanoTime() @@ -164,15 +190,21 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val key = needKey || lastKey < 0 || pts - lastKey >= 1_000_000 if (key && pendingRequest != 0L) { val requestId = pendingRequest; pendingRequest = 0 + val old = checkNotNull(encoder); val oldRate = old.bitrate; val name = old.name + old.close(); encoder = null try { - val candidate = CallVideoCodec(f, pendingRate, checkNotNull(encoders[f.codec])) - encoder?.close(); encoder = candidate; confirmingRequest = requestId; needKey = true + encoder = CallVideoCodec(f, pendingRate, name) + encoder!!.prepare(); check(encoder!!.format == f) { "Bitrate change altered encoder format" } + confirmingRequest = requestId; needKey = true report("Applying ${pendingRate / 1000} kbit/s on IDR") } catch (e: Exception) { - CallVideoNative.applied(id, f.generation, requestId, 0); throw e + LogManager.addLog("WARN", "call", "video bitrate replacement failed: ${e.message}; restoring $oldRate") + encoder?.close(); encoder = CallVideoCodec(f, oldRate, name) + encoder!!.prepare(); check(encoder!!.format == f) + confirmingRequest = requestId; needKey = true } } - if (encoder?.submit(image, pts, key) == true) { lastPts = pts; if (key) lastKey = pts } + if (encoder?.submit(image, pts, key) == true) lastPts = pts drain() } catch (e: Exception) { fail("Camera encode: ${e.message}") } finally { image.close() } @@ -215,6 +247,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val codec = encoder ?: return; val f = format ?: return for (p in codec.drain()) { encodedBytes += p.bytes.size; encodedFrames++ + if (p.key) lastKey = p.pts if (confirmingRequest != 0L && p.key) { CallVideoNative.applied(id, f.generation, confirmingRequest, codec.bitrate); confirmingRequest = 0 mutable.value = mutable.value.copy(bitrate = codec.bitrate) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt index daedb8f3..64252282 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoPlayer.kt @@ -7,8 +7,10 @@ import android.view.Surface internal class CallVideoPlayer(private val id: Long, private val failure: (String) -> Unit) { @Volatile private var running = true @Volatile private var target: Surface? = null + @Volatile private var verified = emptyMap() private val worker = Thread(::loop, "call-video-rx").apply { start() } fun surface(surface: Surface?) { target = surface } + fun decoders(value: Map) { verified = value } fun stop() { running = false; worker.join() } private fun loop() { @@ -65,7 +67,9 @@ internal class CallVideoPlayer(private val id: Long, private val failure: (Strin if (!key) { pending = null; CallVideoNative.requestKey(id); continue } CallVideoCodec.validateKey(data, next) val type = StreamVideoCodec.fromId(next.codec); val config = type.unpackKey(data).first - decoder = MediaCodec.createDecoderByType(type.mime) + val implementation = checkNotNull(verified[next.codec]) { "No verified decoder" } + check(implementation.profiles and next.profile != 0) { "Unsupported decoder profile" } + decoder = MediaCodec.createByCodecName(implementation.name) decoder!!.configure(type.decoderFormat(next.width, next.height, next.rotation, config), surface, null, 0) decoder!!.start(); format = next; lastProgress = now LogManager.addLog("INFO", "call", "video decoder started call=$id format=$next")