jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Probe Android call encoders through real YUV encode and decode

master
evgeny 5 days ago
parent
c986221cb6
commit
5dbc5bcf59
1 changed files with 207 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 207
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt

207
tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCodec.kt
Unescape View File

@ -0,0 +1,207 @@
package com.utun.chat.data
import android.media.Image
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaCodecList
import android.media.MediaFormat
import android.os.Build
import android.os.Bundle
import java.nio.ByteBuffer
/** Codec принадлежит одному worker. YUV input отделён от камеры: замена на IDR не меняет capture PTS. */
internal class CallVideoCodec(val format: CallVideoFormat, val bitrate: Int, name: String) : AutoCloseable {
data class Packet(val pts: Long, val key: Boolean, val bytes: ByteArray)
private val type = StreamVideoCodec.fromId(format.codec)
private val codec = MediaCodec.createByCodecName(name)
private var config = byteArrayOf()
private val partial = java.io.ByteArrayOutputStream()
private var partialPts = -1L
private var partialFlags = 0
init {
try {
val media = MediaFormat.createVideoFormat(type.mime, format.width, format.height).apply {
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
setInteger(MediaFormat.KEY_PROFILE, if (format.codec == 1) MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline
else MediaCodecInfo.CodecProfileLevel.HEVCProfileMain)
setInteger(MediaFormat.KEY_LEVEL, platformLevel(format.codec, format.level))
setInteger(MediaFormat.KEY_BIT_RATE, bitrate); setInteger(MediaFormat.KEY_FRAME_RATE, format.fps)
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR)
if (Build.VERSION.SDK_INT >= 29) setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0)
setInteger(MediaFormat.KEY_LATENCY, 0)
}
codec.configure(media, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); codec.start()
LogManager.addLog("INFO", "call", "video encoder open name=$name format=$format bitrate=$bitrate actual=${codec.inputFormat}")
} catch (e: Exception) { codec.release(); throw e }
}
fun submit(image: Image?, pts: Long, key: Boolean): Boolean {
val index = codec.dequeueInputBuffer(0)
if (index < 0) return false
if (key) codec.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0) })
val input = checkNotNull(codec.getInputImage(index)) { "Encoder lacks flexible YUV input" }
check(input.width >= format.width && input.height >= format.height)
for (plane in 0..2) {
val w = format.width shr (if (plane == 0) 0 else 1)
val h = format.height shr (if (plane == 0) 0 else 1)
val dst = input.planes[plane]; val src = image?.planes?.get(plane)
val row = ByteArray(w)
for (y in 0 until h) {
if (src == null) row.fill(if (plane == 0) 64 else 128.toByte())
else {
val buffer = src.buffer.duplicate(); val start = buffer.position() + y * src.rowStride
if (src.pixelStride == 1) { buffer.position(start); buffer.get(row) }
else for (x in 0 until w) row[x] = buffer.get(start + x * src.pixelStride)
}
val buffer = dst.buffer.duplicate(); val start = buffer.position() + y * dst.rowStride
if (dst.pixelStride == 1) { buffer.position(start); buffer.put(row) }
else for (x in 0 until w) buffer.put(start + x * dst.pixelStride, row[x])
}
}
input.close(); codec.queueInputBuffer(index, 0, format.width * format.height * 3 / 2, pts, 0); return true
}
fun drain(): List<Packet> {
val packets = mutableListOf<Packet>(); val info = MediaCodec.BufferInfo()
repeat(16) {
val index = codec.dequeueOutputBuffer(info, 0)
if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
val f = codec.outputFormat
config = listOf("csd-0", "csd-1").mapNotNull { f.getByteBuffer(it)?.duplicate()?.let { b ->
ByteArray(b.remaining()).also(b::get).let(type::normalize)
} }.fold(byteArrayOf()) { a, b -> a + b }
check(config.size <= StreamVideoCodec.MAX_CONFIG)
LogManager.addLog("INFO", "call", "video encoder output format=$f config=${config.size}")
return@repeat
}
if (index < 0) return packets
try {
if (info.size == 0) return@repeat
val b = checkNotNull(codec.getOutputBuffer(index)).duplicate()
b.position(info.offset); b.limit(info.offset + info.size)
val bytes = ByteArray(info.size).also(b::get)
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
config = type.mergeConfig(config, bytes); return@repeat
}
if (partial.size() == 0) { partialPts = info.presentationTimeUs; partialFlags = 0 }
check(partialPts == info.presentationTimeUs && partial.size() + bytes.size <= H264AnnexB.MAX_FRAME)
partial.write(bytes); partialFlags = partialFlags or info.flags
if (info.flags and MediaCodec.BUFFER_FLAG_PARTIAL_FRAME != 0) return@repeat
val data = type.normalize(partial.toByteArray()); partial.reset()
val key = partialFlags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
val output = if (key) type.keyFrame(data, config) else data
if (key) validateKey(output, format)
packets += Packet(info.presentationTimeUs, key, output)
} finally { codec.releaseOutputBuffer(index, false) }
}
return packets
}
override fun close() {
try { codec.stop() } catch (e: Exception) { LogManager.addLog("WARN", "call", "encoder stop: ${e.message}") }
codec.release()
}
companion object {
fun platformLevel(codec: Int, level: Int): Int = if (codec == 1) when (level) {
30 -> MediaCodecInfo.CodecProfileLevel.AVCLevel3
31 -> MediaCodecInfo.CodecProfileLevel.AVCLevel31
else -> error("Unsupported AVC level $level")
} else when (level) {
90 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel3
93 -> MediaCodecInfo.CodecProfileLevel.HEVCMainTierLevel31
else -> error("Unsupported HEVC level $level")
}
/** Реальная проба input → KEY → decoder; не объявляем codec только по списку MediaCodecInfo. */
fun probe(): Map<Int, String> {
val result = mutableMapOf<Int, String>()
for (type in listOf(StreamVideoCodec.H265, StreamVideoCodec.H264)) {
for (entry in MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder }) {
if (entry.supportedTypes.none { it == type.mime }) continue
try {
val f = CallVideoFormat(1, 1280, 720, codec = type.id, level = if (type.id == 1) 31 else 93)
val caps = entry.getCapabilitiesForType(type.mime)
if (!caps.videoCapabilities.areSizeAndRateSupported(1280, 720, 30.0)
|| !caps.encoderCapabilities.isBitrateModeSupported(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR)) continue
CallVideoCodec(f, 1_000_000, entry.name).use { encoder ->
val packets = mutableListOf<Packet>(); var input = 0
val deadline = System.nanoTime() + 2_000_000_000
while (packets.size < 4 && System.nanoTime() < deadline) {
if (input < 8 && encoder.submit(null, 1_000_000L + input * 33333, input == 0)) input++
packets += encoder.drain(); Thread.sleep(2)
}
check(packets.firstOrNull()?.key == true) { "No initial KEY" }
verifyDecode(f, packets)
}
result[type.id] = entry.name
LogManager.addLog("INFO", "call", "video codec probe passed type=${type.label} encoder=${entry.name}"); break
} catch (e: Exception) { LogManager.addLog("WARN", "call", "video codec probe rejected ${entry.name}: ${e.message}") }
}
}
return result
}
private fun verifyDecode(f: CallVideoFormat, packets: List<Packet>) {
val type = StreamVideoCodec.fromId(f.codec)
val config = type.unpackKey(packets.first().bytes).first
val decoder = MediaCodec.createDecoderByType(type.mime)
try {
decoder.configure(type.decoderFormat(f.width, f.height, 0, config), null, null, 0); decoder.start()
var sent = 0; var decoded = 0; val info = MediaCodec.BufferInfo(); val deadline = System.nanoTime() + 2_000_000_000
while (decoded < packets.size && System.nanoTime() < deadline) {
if (sent < packets.size) {
val i = decoder.dequeueInputBuffer(0)
if (i >= 0) {
val p = packets[sent++]; val data = if (p.key) type.unpackKey(p.bytes).second else p.bytes
decoder.getInputBuffer(i)!!.put(data); decoder.queueInputBuffer(i, 0, data.size, p.pts, 0)
}
}
val out = decoder.dequeueOutputBuffer(info, 1000)
if (out >= 0) {
check(decoded < packets.size && info.presentationTimeUs == packets[decoded].pts) { "Decoder reordered PTS" }
val actual = decoder.outputFormat
check(actual.getInteger(MediaFormat.KEY_WIDTH) >= f.width && actual.getInteger(MediaFormat.KEY_HEIGHT) >= f.height)
decoded++; decoder.releaseOutputBuffer(out, false)
}
}
check(decoded == packets.size) { "Decoder did not return all probe frames $decoded/${packets.size}" }
decoder.stop()
} finally { decoder.release() }
}
/** Проверяем профиль и level в SPS, а не принятый configure option. */
fun validateKey(bytes: ByteArray, f: CallVideoFormat) {
val type = StreamVideoCodec.fromId(f.codec); val (config, data) = type.unpackKey(bytes)
val units = H264AnnexB.nals(config + data).map { (_, n) -> n.copyOfRange(if (n[2] == 1.toByte()) 3 else 4, n.size) }
val sps = units.first { if (f.codec == 1) it[0].toInt() and 31 == 7 else (it[0].toInt() ushr 1) and 63 == 33 }
val raw = java.io.ByteArrayOutputStream(); var zeros = 0
for (b in sps.drop(if (f.codec == 1) 1 else 2)) {
if (zeros >= 2 && b == 3.toByte()) { zeros = 0; continue }
raw.write(b.toInt()); zeros = if (b == 0.toByte()) zeros + 1 else 0
}
val bits = Bits(raw.toByteArray())
if (f.codec == 1) {
check(bits.read(8) == 66 && bits.read(8) and 64 != 0) { "AVC is not Constrained Baseline" }
check(bits.read(8) <= f.level); bits.ue(); bits.ue()
when (bits.ue()) { 0 -> bits.ue(); 2 -> Unit; else -> error("Unsupported AVC POC") }
check(bits.ue() <= 1) { "AVC uses more than one reference" }
} else {
bits.read(4); check(bits.read(3) == 0) { "HEVC temporal layers" }; bits.read(1)
bits.read(2); check(bits.read(1) == 0 && bits.read(5) == 1) { "HEVC is not Main/Main tier" }
repeat(10) { bits.read(8) }; check(bits.read(8) <= f.level)
bits.ue(); check(bits.ue() == 1); bits.ue(); bits.ue()
if (bits.read(1) != 0) repeat(4) { bits.ue() }
check(bits.ue() == 0 && bits.ue() == 0); bits.ue(); bits.read(1)
val dpb = bits.ue(); check(bits.ue() == 0) { "HEVC reordered frames" }
check(dpb <= 1) { "HEVC requires more than two decoded pictures" }
}
}
}
private class Bits(val bytes: ByteArray) {
var at = 0
fun read(n: Int): Int { require(n in 0..24 && at + n <= bytes.size * 8); var v = 0
repeat(n) { v = (v shl 1) or ((bytes[at / 8].toInt() ushr (7 - at % 8)) and 1); at++ }; return v }
fun ue(): Int { var n = 0; while (read(1) == 0) { n++; require(n <= 20) }; return (1 shl n) - 1 + read(n) }
}
}
Write Preview
Loading…
Cancel
Save