Capture and play codec specific access units with live bitrate changes
This commit is contained in:
evgeny committed
1 parent
d209049f71
commit
1ac7a40432
2 files changed
+39
-24
No files matched your search
@@ -81,7 +81,7 @@ class StreamCaptureEngine(
|
||||
val mode = initialConfig.mode
|
||||
val rotation = cameraPose(mode.camera).rotation(initialConfig.orientation.degrees)
|
||||
check(NativeLib.streamOpen(id, group, width = mode.width, height = mode.height,
|
||||
rotation = rotation, audio = microphone)) { "Publisher open rejected" }
|
||||
rotation = rotation, audio = microphone, videoCodec = mode.videoCodec.id)) { "Publisher open rejected" }
|
||||
originNs = System.nanoTime()
|
||||
if (microphone) startAudio()
|
||||
configure(initialConfig)
|
||||
@@ -95,6 +95,17 @@ class StreamCaptureEngine(
|
||||
LogManager.addLog("WARN", "video", "capture change rejected id=$id busy=$waitingFirst stopping=$stopping"); return@post
|
||||
}
|
||||
if (config == lastGood) { ready(config, false); return@post }
|
||||
val previous = lastGood
|
||||
if (previous != null && config.copy(videoBitrate = previous.videoBitrate) == previous) {
|
||||
try {
|
||||
encoder!!.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, config.bitrate) })
|
||||
lastGood = config; ready(config, false)
|
||||
LogManager.addLog("INFO", "video", "bitrate changed id=$id revision=$generation ${previous.bitrate}->${config.bitrate}")
|
||||
return@post
|
||||
} catch (e: Exception) {
|
||||
LogManager.addLog("WARN", "video", "live bitrate refused id=$id target=${config.bitrate}; reconfigure: ${e.message}")
|
||||
}
|
||||
}
|
||||
waitingFirst = true; reverting = false; switchStartedNs = System.nanoTime()
|
||||
LogManager.addLog("INFO", "video", "switch capture id=$id camera=${config.mode.camera} ${config.mode.label} orientation=${config.orientation.degrees}")
|
||||
releaseVideo { closed -> if (closed) configure(config) else failed("camera close timed out during switch") }
|
||||
@@ -123,11 +134,12 @@ class StreamCaptureEngine(
|
||||
confirming = mode.highSpeed; confirmStartNs = 0
|
||||
reportNs = System.nanoTime(); reportFrames = encodedFrames; reportBytes = bytesSent
|
||||
if (switchStartedNs == 0L) switchStartedNs = reportNs
|
||||
val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, mode.width, mode.height).apply {
|
||||
val format = MediaFormat.createVideoFormat(mode.videoCodec.mime, mode.width, mode.height).apply {
|
||||
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
|
||||
setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline)
|
||||
setInteger(MediaFormat.KEY_BIT_RATE, mode.bitrate)
|
||||
mode.videoCodec.profile?.let { setInteger(MediaFormat.KEY_PROFILE, it) }
|
||||
setInteger(MediaFormat.KEY_BIT_RATE, config.bitrate)
|
||||
setInteger(MediaFormat.KEY_FRAME_RATE, mode.fps); setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
|
||||
if (Build.VERSION.SDK_INT >= 29) setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0)
|
||||
}
|
||||
encoder = MediaCodec.createByCodecName(mode.codec)
|
||||
encoder!!.also { codec ->
|
||||
@@ -138,9 +150,9 @@ class StreamCaptureEngine(
|
||||
if (!current(token)) return
|
||||
try {
|
||||
csd = listOf("csd-0", "csd-1").mapNotNull { name -> format.getByteBuffer(name)?.duplicate()?.let { b ->
|
||||
ByteArray(b.remaining()).also { b.get(it) }.let(H264AnnexB::normalize)
|
||||
ByteArray(b.remaining()).also { b.get(it) }.let(mode.videoCodec::normalize)
|
||||
} }.fold(byteArrayOf()) { a, b -> a + b }
|
||||
check(H264AnnexB.nals(csd).map { it.first }.toSet().containsAll(setOf(7, 8))) { "AVC format missing SPS/PPS" }
|
||||
check(csd.size <= StreamVideoCodec.MAX_CONFIG) { "Codec configuration too large" }
|
||||
LogManager.addLog("INFO", "video", "encoder format id=$id revision=$token codec=${codec.name} csd=${csd.size} format=$format")
|
||||
} catch (e: Exception) { captureError(token, "encoder format: ${e.message}") }
|
||||
}
|
||||
@@ -153,8 +165,8 @@ class StreamCaptureEngine(
|
||||
buffer.position(info.offset); buffer.limit(info.offset + info.size)
|
||||
val bytes = ByteArray(info.size).also { buffer.get(it) }
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
|
||||
val codecConfig = H264AnnexB.normalize(bytes)
|
||||
if (H264AnnexB.nals(codecConfig).map { it.first }.toSet().containsAll(setOf(7, 8))) csd = codecConfig
|
||||
val codecConfig = mode.videoCodec.normalize(bytes)
|
||||
check(codecConfig.size <= StreamVideoCodec.MAX_CONFIG); csd = codecConfig
|
||||
} else encoded(bytes, info, config, rotation, token)
|
||||
}
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) problem = "unexpected encoder EOS"
|
||||
@@ -214,7 +226,7 @@ class StreamCaptureEngine(
|
||||
} else configured.setRepeatingRequest(request, diagnostics, handler)
|
||||
LogManager.addLog("INFO", "video", "capture started id=$id revision=$token camera=${mode.camera} ${mode.label} " +
|
||||
"range=${mode.minimumFps}..${mode.fps} highSpeed=${mode.highSpeed} codec=${mode.codec} " +
|
||||
"bitrate=${mode.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs")
|
||||
"type=${mode.videoCodec.label} bitrate=${config.bitrate} rotation=$rotation orientation=${config.orientation.degrees} audio=$microphone origin=$originNs")
|
||||
} catch (e: Exception) { captureError(token, "camera request: ${e.message}") }
|
||||
}
|
||||
override fun onConfigureFailed(session: CameraCaptureSession) {
|
||||
@@ -263,15 +275,16 @@ class StreamCaptureEngine(
|
||||
check(partial.size() + bytes.size <= H264AnnexB.MAX_FRAME) { "AVC frame exceeds limit" }
|
||||
partial.write(bytes); partialKey = partialKey || info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return
|
||||
val data = H264AnnexB.normalize(partial.toByteArray()); partial.reset()
|
||||
val key = partialKey && H264AnnexB.nals(data).any { it.first == 5 }
|
||||
val data = mode.videoCodec.normalize(partial.toByteArray()); partial.reset()
|
||||
val key = partialKey
|
||||
val pts = partialPts - originNs / 1000
|
||||
check(pts >= 0 && pts > lastPts) { "AVC PTS outside timeline pts=$pts last=$lastPts" }
|
||||
lastPts = pts; encodedFrames++
|
||||
if (requireKey && !key) { dropped++; requestKey() }
|
||||
else {
|
||||
val output = if (key) H264AnnexB.keyFrame(data, csd) else data
|
||||
when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone)) {
|
||||
val output = if (key) mode.videoCodec.keyFrame(data, csd) else data
|
||||
if (key) mode.videoCodec.unpackKey(output)
|
||||
when (NativeLib.streamFeed(id, true, pts, key, output, revision, mode.width, mode.height, rotation, microphone, mode.videoCodec.id)) {
|
||||
1 -> {
|
||||
requireKey = false; frames++; bytesSent += output.size
|
||||
if (waitingFirst) {
|
||||
|
||||
@@ -23,7 +23,8 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
private val localPreview: StreamPreview? = null, private val displayDegrees: Int = 0) {
|
||||
companion object { private val viewerLock = ReentrantLock() }
|
||||
private data class Frame(val pts: Long, val key: Boolean, val width: Int, val height: Int,
|
||||
val rotation: Int, val top: Int?, val revision: Long, val data: ByteArray)
|
||||
val rotation: Int, val top: Int?, val revision: Long, val codec: StreamVideoCodec,
|
||||
val config: ByteArray, val data: ByteArray)
|
||||
private val mutableAspect = MutableStateFlow(16f / 9f)
|
||||
val aspect: StateFlow<Float> = mutableAspect
|
||||
private val mutableTop = MutableStateFlow<Int?>(null)
|
||||
@@ -148,12 +149,14 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
repeat(4) {
|
||||
val raw = if (ended) null else NativeLib.streamPull(id, position, videoAhead, audioAhead)
|
||||
if (raw == null) return@repeat
|
||||
check(raw.size >= 24)
|
||||
check(raw.size >= 28)
|
||||
val b = ByteBuffer.wrap(raw).order(ByteOrder.BIG_ENDIAN)
|
||||
val kind = b.get().toInt(); val flags = b.get().toInt() and 255
|
||||
val width = b.short.toInt() and 65535; val height = b.short.toInt() and 65535
|
||||
val broadcastRotation = b.short.toInt() and 65535; val pts = b.long; val size = b.int
|
||||
val revision = b.int.toLong() and 0xffffffffL
|
||||
val codecId = b.get().toInt() and 255
|
||||
check(b.get() == 0.toByte() && b.get() == 0.toByte() && b.get() == 0.toByte())
|
||||
check(size == b.remaining())
|
||||
if (kind in 1..2 && playback == 0L) observeStatus(NativeLib.streamStatus(id))
|
||||
when (kind) {
|
||||
@@ -171,13 +174,14 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
val key = flags and 1 != 0
|
||||
if (!waitingKey || key) {
|
||||
val bytes = ByteArray(size).also { b.get(it) }
|
||||
val codec = StreamVideoCodec.fromId(codecId)
|
||||
val access = if (key) codec.unpackKey(bytes) else byteArrayOf() to bytes
|
||||
if (waitingKey) {
|
||||
check(H264AnnexB.nals(bytes).map { it.first }.toSet().containsAll(setOf(5, 7, 8)))
|
||||
waitingKey = false; mutableTop.value = display.top
|
||||
mutableAspect.value = if (display.rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width
|
||||
}
|
||||
check(videos.size < 4) { "Video preparation queue exceeded" }
|
||||
videos.addLast(Frame(pts, key, width, height, display.rotation, display.top, revision, bytes))
|
||||
videos.addLast(Frame(pts, key, width, height, display.rotation, display.top, revision, codec, access.first, access.second))
|
||||
}
|
||||
}
|
||||
2 -> if (audioEnabled && playback != 0L) {
|
||||
@@ -224,13 +228,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
val frame = videos.first
|
||||
if (decoder == null) {
|
||||
check(frame.key)
|
||||
val nals = H264AnnexB.nals(frame.data)
|
||||
val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, frame.width, frame.height).apply {
|
||||
setByteBuffer("csd-0", ByteBuffer.wrap(nals.first { it.first == 7 }.second))
|
||||
setByteBuffer("csd-1", ByteBuffer.wrap(nals.first { it.first == 8 }.second))
|
||||
setInteger(MediaFormat.KEY_ROTATION, frame.rotation); setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, H264AnnexB.MAX_FRAME)
|
||||
}
|
||||
decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
|
||||
val format = frame.codec.decoderFormat(frame.width, frame.height, frame.rotation, frame.config)
|
||||
val name = android.media.MediaCodecList(android.media.MediaCodecList.REGULAR_CODECS).findDecoderForFormat(format)
|
||||
?: error("Unsupported decoder ${frame.codec.label} ${frame.width}x${frame.height}")
|
||||
decoder = MediaCodec.createByCodecName(name)
|
||||
LogManager.addLog("INFO", "video", "viewer codec id=$id type=${frame.codec.label} decoder=$name config=${frame.config.size}")
|
||||
decoder!!.configure(format, surface, null, 0); decoder!!.start()
|
||||
}
|
||||
val index = decoder!!.dequeueInputBuffer(0)
|
||||
|
||||
Reference in new issue
Block a user