diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt index 98681689..6ce394ad 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt @@ -81,7 +81,7 @@ class StreamCaptureEngine( val mode = initialConfig.mode val rotation = cameraPose(mode.camera).rotation(initialConfig.orientation.degrees) check(NativeLib.streamOpen(id, group, width = mode.width, height = mode.height, - rotation = rotation, audio = microphone)) { "Publisher open rejected" } + rotation = rotation, audio = microphone, videoCodec = mode.videoCodec.id)) { "Publisher open rejected" } originNs = System.nanoTime() if (microphone) startAudio() configure(initialConfig) @@ -95,6 +95,17 @@ class StreamCaptureEngine( LogManager.addLog("WARN", "video", "capture change rejected id=$id busy=$waitingFirst stopping=$stopping"); return@post } if (config == lastGood) { ready(config, false); return@post } + val previous = lastGood + if (previous != null && config.copy(videoBitrate = previous.videoBitrate) == previous) { + try { + encoder!!.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, config.bitrate) }) + lastGood = config; ready(config, false) + LogManager.addLog("INFO", "video", "bitrate changed id=$id revision=$generation ${previous.bitrate}->${config.bitrate}") + return@post + } catch (e: Exception) { + LogManager.addLog("WARN", "video", "live bitrate refused id=$id target=${config.bitrate}; reconfigure: ${e.message}") + } + } waitingFirst = true; reverting = false; switchStartedNs = System.nanoTime() LogManager.addLog("INFO", "video", "switch capture id=$id camera=${config.mode.camera} ${config.mode.label} orientation=${config.orientation.degrees}") releaseVideo { closed -> if (closed) configure(config) else failed("camera close timed out during switch") } @@ -123,11 +134,12 @@ class StreamCaptureEngine( confirming = mode.highSpeed; confirmStartNs = 0 reportNs = System.nanoTime(); reportFrames = encodedFrames; reportBytes = bytesSent if (switchStartedNs == 0L) switchStartedNs = reportNs - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, mode.width, mode.height).apply { + val format = MediaFormat.createVideoFormat(mode.videoCodec.mime, mode.width, mode.height).apply { setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) - setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline) - setInteger(MediaFormat.KEY_BIT_RATE, mode.bitrate) + mode.videoCodec.profile?.let { setInteger(MediaFormat.KEY_PROFILE, it) } + setInteger(MediaFormat.KEY_BIT_RATE, config.bitrate) setInteger(MediaFormat.KEY_FRAME_RATE, mode.fps); setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + if (Build.VERSION.SDK_INT >= 29) setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) } encoder = MediaCodec.createByCodecName(mode.codec) encoder!!.also { codec -> @@ -138,9 +150,9 @@ class StreamCaptureEngine( if (!current(token)) return try { csd = listOf("csd-0", "csd-1").mapNotNull { name -> format.getByteBuffer(name)?.duplicate()?.let { b -> - ByteArray(b.remaining()).also { b.get(it) }.let(H264AnnexB::normalize) + ByteArray(b.remaining()).also { b.get(it) }.let(mode.videoCodec::normalize) } }.fold(byteArrayOf()) { a, b -> a + b } - check(H264AnnexB.nals(csd).map { it.first }.toSet().containsAll(setOf(7, 8))) { "AVC format missing SPS/PPS" } + check(csd.size <= StreamVideoCodec.MAX_CONFIG) { "Codec configuration too large" } LogManager.addLog("INFO", "video", "encoder format id=$id revision=$token codec=${codec.name} csd=${csd.size} format=$format") } catch (e: Exception) { captureError(token, "encoder format: ${e.message}") } } @@ -153,8 +165,8 @@ class StreamCaptureEngine( buffer.position(info.offset); buffer.limit(info.offset + info.size) val bytes = ByteArray(info.size).also { buffer.get(it) } if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { - val codecConfig = H264AnnexB.normalize(bytes) - if (H264AnnexB.nals(codecConfig).map { it.first }.toSet().containsAll(setOf(7, 8))) csd = codecConfig + val codecConfig = mode.videoCodec.normalize(bytes) + check(codecConfig.size <= StreamVideoCodec.MAX_CONFIG); csd = codecConfig } else encoded(bytes, info, config, rotation, token) } if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) problem = "unexpected encoder EOS" @@ -214,7 +226,7 @@ class StreamCaptureEngine( } else configured.setRepeatingRequest(request, diagnostics, handler) LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " + "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " + - "bitrate=${mode.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs") + "type=${mode.videoCodec.label} bitrate=${config.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs") } catch (e: Exception) { captureError(token, "camera request: ${e.message}") } } override fun onConfigureFailed(session: CameraCaptureSession) { @@ -263,15 +275,16 @@ class StreamCaptureEngine( check(partial.size() + bytes.size <= H264AnnexB.MAX_FRAME) { "AVC frame exceeds limit" } partial.write(bytes); partialKey = partialKey || info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0 if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return - val data = H264AnnexB.normalize(partial.toByteArray()); partial.reset() - val key = partialKey && H264AnnexB.nals(data).any { it.first == 5 } + val data = mode.videoCodec.normalize(partial.toByteArray()); partial.reset() + val key = partialKey val pts = partialPts - originNs / 1000 check(pts >= 0 && pts > lastPts) { "AVC PTS outside timeline pts=$pts last=$lastPts" } lastPts = pts; encodedFrames++ if (requireKey && !key) { dropped++; requestKey() } else { - val output = if (key) H264AnnexB.keyFrame(data, csd) else data - when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone)) { + val output = if (key) mode.videoCodec.keyFrame(data, csd) else data + if (key) mode.videoCodec.unpackKey(output) + when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone, mode.videoCodec.id)) { 1 -> { requireKey = false; frames++; bytesSent += output.size if (waitingFirst) { diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt index fb44a721..7b3a602a 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt @@ -23,7 +23,8 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) { companion object { private val viewerLock = ReentrantLock() } private data class Frame(val pts: Long, val key: Boolean, val width: Int, val height: Int, - val rotation: Int, val top: Int?, val revision: Long, val data: ByteArray) + val rotation: Int, val top: Int?, val revision: Long, val codec: StreamVideoCodec, + val config: ByteArray, val data: ByteArray) private val mutableAspect = MutableStateFlow(16f / 9f) val aspect: StateFlow = mutableAspect private val mutableTop = MutableStateFlow(null) @@ -148,12 +149,14 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv repeat(4) { val raw = if (ended) null else NativeLib.streamPull(id, position, videoAhead, audioAhead) if (raw == null) return@repeat - check(raw.size >= 24) + check(raw.size >= 28) val b = ByteBuffer.wrap(raw).order(ByteOrder.BIG_ENDIAN) val kind = b.get().toInt(); val flags = b.get().toInt() and 255 val width = b.short.toInt() and 65535; val height = b.short.toInt() and 65535 val broadcastRotation = b.short.toInt() and 65535; val pts = b.long; val size = b.int val revision = b.int.toLong() and 0xffffffffL + val codecId = b.get().toInt() and 255 + check(b.get() == 0.toByte() && b.get() == 0.toByte() && b.get() == 0.toByte()) check(size == b.remaining()) if (kind in 1..2 && playback == 0L) observeStatus(NativeLib.streamStatus(id)) when (kind) { @@ -171,13 +174,14 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv val key = flags and 1 != 0 if (!waitingKey || key) { val bytes = ByteArray(size).also { b.get(it) } + val codec = StreamVideoCodec.fromId(codecId) + val access = if (key) codec.unpackKey(bytes) else byteArrayOf() to bytes if (waitingKey) { - check(H264AnnexB.nals(bytes).map { it.first }.toSet().containsAll(setOf(5, 7, 8))) waitingKey = false; mutableTop.value = display.top mutableAspect.value = if (display.rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width } check(videos.size < 4) { "Video preparation queue exceeded" } - videos.addLast(Frame(pts, key, width, height, display.rotation, display.top, revision, bytes)) + videos.addLast(Frame(pts, key, width, height, display.rotation, display.top, revision, codec, access.first, access.second)) } } 2 -> if (audioEnabled && playback != 0L) { @@ -224,13 +228,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv val frame = videos.first if (decoder == null) { check(frame.key) - val nals = H264AnnexB.nals(frame.data) - val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, frame.width, frame.height).apply { - setByteBuffer("csd-0", ByteBuffer.wrap(nals.first { it.first == 7 }.second)) - setByteBuffer("csd-1", ByteBuffer.wrap(nals.first { it.first == 8 }.second)) - setInteger(MediaFormat.KEY_ROTATION, frame.rotation); setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, H264AnnexB.MAX_FRAME) - } - decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC) + val format = frame.codec.decoderFormat(frame.width, frame.height, frame.rotation, frame.config) + val name = android.media.MediaCodecList(android.media.MediaCodecList.REGULAR_CODECS).findDecoderForFormat(format) + ?: error("Unsupported decoder ${frame.codec.label} ${frame.width}x${frame.height}") + decoder = MediaCodec.createByCodecName(name) + LogManager.addLog("INFO", "video", "viewer codec id=$id type=${frame.codec.label} decoder=$name config=${frame.config.size}") decoder!!.configure(format, surface, null, 0); decoder!!.start() } val index = decoder!!.dequeueInputBuffer(0)