jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Capture and play codec specific access units with live bitrate changes

master
evgeny 6 days ago
parent
d209049f71
commit
1ac7a40432
2 changed files with 39 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 39
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt
  2. 24
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt

39
tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt
Unescape View File

@ -81,7 +81,7 @@ class StreamCaptureEngine(
val mode = initialConfig.mode val mode = initialConfig.mode
val rotation = cameraPose(mode.camera).rotation(initialConfig.orientation.degrees) val rotation = cameraPose(mode.camera).rotation(initialConfig.orientation.degrees)
check(NativeLib.streamOpen(id, group, width = mode.width, height = mode.height, check(NativeLib.streamOpen(id, group, width = mode.width, height = mode.height,
rotation = rotation, audio = microphone)) { "Publisher open rejected" } rotation = rotation, audio = microphone, videoCodec = mode.videoCodec.id)) { "Publisher open rejected" }
originNs = System.nanoTime() originNs = System.nanoTime()
if (microphone) startAudio() if (microphone) startAudio()
configure(initialConfig) configure(initialConfig)
@ -95,6 +95,17 @@ class StreamCaptureEngine(
LogManager.addLog("WARN", "video", "capture change rejected id=$id busy=$waitingFirst stopping=$stopping"); return@post LogManager.addLog("WARN", "video", "capture change rejected id=$id busy=$waitingFirst stopping=$stopping"); return@post
} }
if (config == lastGood) { ready(config, false); return@post } if (config == lastGood) { ready(config, false); return@post }
val previous = lastGood
if (previous != null && config.copy(videoBitrate = previous.videoBitrate) == previous) {
try {
encoder!!.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, config.bitrate) })
lastGood = config; ready(config, false)
LogManager.addLog("INFO", "video", "bitrate changed id=$id revision=$generation ${previous.bitrate}->${config.bitrate}")
return@post
} catch (e: Exception) {
LogManager.addLog("WARN", "video", "live bitrate refused id=$id target=${config.bitrate}; reconfigure: ${e.message}")
}
}
waitingFirst = true; reverting = false; switchStartedNs = System.nanoTime() waitingFirst = true; reverting = false; switchStartedNs = System.nanoTime()
LogManager.addLog("INFO", "video", "switch capture id=$id camera=${config.mode.camera} ${config.mode.label} orientation=${config.orientation.degrees}") LogManager.addLog("INFO", "video", "switch capture id=$id camera=${config.mode.camera} ${config.mode.label} orientation=${config.orientation.degrees}")
releaseVideo { closed -> if (closed) configure(config) else failed("camera close timed out during switch") } releaseVideo { closed -> if (closed) configure(config) else failed("camera close timed out during switch") }
@ -123,11 +134,12 @@ class StreamCaptureEngine(
confirming = mode.highSpeed; confirmStartNs = 0 confirming = mode.highSpeed; confirmStartNs = 0
reportNs = System.nanoTime(); reportFrames = encodedFrames; reportBytes = bytesSent reportNs = System.nanoTime(); reportFrames = encodedFrames; reportBytes = bytesSent
if (switchStartedNs == 0L) switchStartedNs = reportNs if (switchStartedNs == 0L) switchStartedNs = reportNs
val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, mode.width, mode.height).apply { val format = MediaFormat.createVideoFormat(mode.videoCodec.mime, mode.width, mode.height).apply {
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline) mode.videoCodec.profile?.let { setInteger(MediaFormat.KEY_PROFILE, it) }
setInteger(MediaFormat.KEY_BIT_RATE, mode.bitrate) setInteger(MediaFormat.KEY_BIT_RATE, config.bitrate)
setInteger(MediaFormat.KEY_FRAME_RATE, mode.fps); setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) setInteger(MediaFormat.KEY_FRAME_RATE, mode.fps); setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
if (Build.VERSION.SDK_INT >= 29) setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0)
} }
encoder = MediaCodec.createByCodecName(mode.codec) encoder = MediaCodec.createByCodecName(mode.codec)
encoder!!.also { codec -> encoder!!.also { codec ->
@ -138,9 +150,9 @@ class StreamCaptureEngine(
if (!current(token)) return if (!current(token)) return
try { try {
csd = listOf("csd-0", "csd-1").mapNotNull { name -> format.getByteBuffer(name)?.duplicate()?.let { b -> csd = listOf("csd-0", "csd-1").mapNotNull { name -> format.getByteBuffer(name)?.duplicate()?.let { b ->
ByteArray(b.remaining()).also { b.get(it) }.let(H264AnnexB::normalize) ByteArray(b.remaining()).also { b.get(it) }.let(mode.videoCodec::normalize)
} }.fold(byteArrayOf()) { a, b -> a + b } } }.fold(byteArrayOf()) { a, b -> a + b }
check(H264AnnexB.nals(csd).map { it.first }.toSet().containsAll(setOf(7, 8))) { "AVC format missing SPS/PPS" } check(csd.size <= StreamVideoCodec.MAX_CONFIG) { "Codec configuration too large" }
LogManager.addLog("INFO", "video", "encoder format id=$id revision=$token codec=${codec.name} csd=${csd.size} format=$format") LogManager.addLog("INFO", "video", "encoder format id=$id revision=$token codec=${codec.name} csd=${csd.size} format=$format")
} catch (e: Exception) { captureError(token, "encoder format: ${e.message}") } } catch (e: Exception) { captureError(token, "encoder format: ${e.message}") }
} }
@ -153,8 +165,8 @@ class StreamCaptureEngine(
buffer.position(info.offset); buffer.limit(info.offset + info.size) buffer.position(info.offset); buffer.limit(info.offset + info.size)
val bytes = ByteArray(info.size).also { buffer.get(it) } val bytes = ByteArray(info.size).also { buffer.get(it) }
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
val codecConfig = H264AnnexB.normalize(bytes) val codecConfig = mode.videoCodec.normalize(bytes)
if (H264AnnexB.nals(codecConfig).map { it.first }.toSet().containsAll(setOf(7, 8))) csd = codecConfig check(codecConfig.size <= StreamVideoCodec.MAX_CONFIG); csd = codecConfig
} else encoded(bytes, info, config, rotation, token) } else encoded(bytes, info, config, rotation, token)
} }
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) problem = "unexpected encoder EOS" if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) problem = "unexpected encoder EOS"
@ -214,7 +226,7 @@ class StreamCaptureEngine(
} else configured.setRepeatingRequest(request, diagnostics, handler) } else configured.setRepeatingRequest(request, diagnostics, handler)
LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " + LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " +
"range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " + "range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " +
"bitrate=${mode.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs") "type=${mode.videoCodec.label} bitrate=${config.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs")
} catch (e: Exception) { captureError(token, "camera request: ${e.message}") } } catch (e: Exception) { captureError(token, "camera request: ${e.message}") }
} }
override fun onConfigureFailed(session: CameraCaptureSession) { override fun onConfigureFailed(session: CameraCaptureSession) {
@ -263,15 +275,16 @@ class StreamCaptureEngine(
check(partial.size() + bytes.size <= H264AnnexB.MAX_FRAME) { "AVC frame exceeds limit" } check(partial.size() + bytes.size <= H264AnnexB.MAX_FRAME) { "AVC frame exceeds limit" }
partial.write(bytes); partialKey = partialKey || info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0 partial.write(bytes); partialKey = partialKey || info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return
val data = H264AnnexB.normalize(partial.toByteArray()); partial.reset() val data = mode.videoCodec.normalize(partial.toByteArray()); partial.reset()
val key = partialKey && H264AnnexB.nals(data).any { it.first == 5 } val key = partialKey
val pts = partialPts - originNs / 1000 val pts = partialPts - originNs / 1000
check(pts >= 0 && pts > lastPts) { "AVC PTS outside timeline pts=$pts last=$lastPts" } check(pts >= 0 && pts > lastPts) { "AVC PTS outside timeline pts=$pts last=$lastPts" }
lastPts = pts; encodedFrames++ lastPts = pts; encodedFrames++
if (requireKey && !key) { dropped++; requestKey() } if (requireKey && !key) { dropped++; requestKey() }
else { else {
val output = if (key) H264AnnexB.keyFrame(data, csd) else data val output = if (key) mode.videoCodec.keyFrame(data, csd) else data
when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone)) { if (key) mode.videoCodec.unpackKey(output)
when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone, mode.videoCodec.id)) {
1 -> { 1 -> {
requireKey = false; frames++; bytesSent += output.size requireKey = false; frames++; bytesSent += output.size
if (waitingFirst) { if (waitingFirst) {

24
tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt
Unescape View File

@ -23,7 +23,8 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) { private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) {
companion object { private val viewerLock = ReentrantLock() } companion object { private val viewerLock = ReentrantLock() }
private data class Frame(val pts: Long, val key: Boolean, val width: Int, val height: Int, private data class Frame(val pts: Long, val key: Boolean, val width: Int, val height: Int,
val rotation: Int, val top: Int?, val revision: Long, val data: ByteArray) val rotation: Int, val top: Int?, val revision: Long, val codec: StreamVideoCodec,
val config: ByteArray, val data: ByteArray)
private val mutableAspect = MutableStateFlow(16f / 9f) private val mutableAspect = MutableStateFlow(16f / 9f)
val aspect: StateFlow<Float> = mutableAspect val aspect: StateFlow<Float> = mutableAspect
private val mutableTop = MutableStateFlow<Int?>(null) private val mutableTop = MutableStateFlow<Int?>(null)
@ -148,12 +149,14 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
repeat(4) { repeat(4) {
val raw = if (ended) null else NativeLib.streamPull(id, position, videoAhead, audioAhead) val raw = if (ended) null else NativeLib.streamPull(id, position, videoAhead, audioAhead)
if (raw == null) return@repeat if (raw == null) return@repeat
check(raw.size >= 24) check(raw.size >= 28)
val b = ByteBuffer.wrap(raw).order(ByteOrder.BIG_ENDIAN) val b = ByteBuffer.wrap(raw).order(ByteOrder.BIG_ENDIAN)
val kind = b.get().toInt(); val flags = b.get().toInt() and 255 val kind = b.get().toInt(); val flags = b.get().toInt() and 255
val width = b.short.toInt() and 65535; val height = b.short.toInt() and 65535 val width = b.short.toInt() and 65535; val height = b.short.toInt() and 65535
val broadcastRotation = b.short.toInt() and 65535; val pts = b.long; val size = b.int val broadcastRotation = b.short.toInt() and 65535; val pts = b.long; val size = b.int
val revision = b.int.toLong() and 0xffffffffL val revision = b.int.toLong() and 0xffffffffL
val codecId = b.get().toInt() and 255
check(b.get() == 0.toByte() && b.get() == 0.toByte() && b.get() == 0.toByte())
check(size == b.remaining()) check(size == b.remaining())
if (kind in 1..2 && playback == 0L) observeStatus(NativeLib.streamStatus(id)) if (kind in 1..2 && playback == 0L) observeStatus(NativeLib.streamStatus(id))
when (kind) { when (kind) {
@ -171,13 +174,14 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
val key = flags and 1 != 0 val key = flags and 1 != 0
if (!waitingKey || key) { if (!waitingKey || key) {
val bytes = ByteArray(size).also { b.get(it) } val bytes = ByteArray(size).also { b.get(it) }
val codec = StreamVideoCodec.fromId(codecId)
val access = if (key) codec.unpackKey(bytes) else byteArrayOf() to bytes
if (waitingKey) { if (waitingKey) {
check(H264AnnexB.nals(bytes).map { it.first }.toSet().containsAll(setOf(5, 7, 8)))
waitingKey = false; mutableTop.value = display.top waitingKey = false; mutableTop.value = display.top
mutableAspect.value = if (display.rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width mutableAspect.value = if (display.rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width
} }
check(videos.size < 4) { "Video preparation queue exceeded" } check(videos.size < 4) { "Video preparation queue exceeded" }
videos.addLast(Frame(pts, key, width, height, display.rotation, display.top, revision, bytes)) videos.addLast(Frame(pts, key, width, height, display.rotation, display.top, revision, codec, access.first, access.second))
} }
} }
2 -> if (audioEnabled && playback != 0L) { 2 -> if (audioEnabled && playback != 0L) {
@ -224,13 +228,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
val frame = videos.first val frame = videos.first
if (decoder == null) { if (decoder == null) {
check(frame.key) check(frame.key)
val nals = H264AnnexB.nals(frame.data) val format = frame.codec.decoderFormat(frame.width, frame.height, frame.rotation, frame.config)
val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, frame.width, frame.height).apply { val name = android.media.MediaCodecList(android.media.MediaCodecList.REGULAR_CODECS).findDecoderForFormat(format)
setByteBuffer("csd-0", ByteBuffer.wrap(nals.first { it.first == 7 }.second)) ?: error("Unsupported decoder ${frame.codec.label} ${frame.width}x${frame.height}")
setByteBuffer("csd-1", ByteBuffer.wrap(nals.first { it.first == 8 }.second)) decoder = MediaCodec.createByCodecName(name)
setInteger(MediaFormat.KEY_ROTATION, frame.rotation); setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, H264AnnexB.MAX_FRAME) LogManager.addLog("INFO", "video", "viewer codec id=$id type=${frame.codec.label} decoder=$name config=${frame.config.size}")
}
decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
decoder!!.configure(format, surface, null, 0); decoder!!.start() decoder!!.configure(format, surface, null, 0); decoder!!.start()
} }
val index = decoder!!.dequeueInputBuffer(0) val index = decoder!!.dequeueInputBuffer(0)

Write Preview
Loading…
Cancel
Save