diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt index a81b3d7c..0b2284ca 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCameraModes.kt @@ -10,7 +10,9 @@ import android.os.Build /** Режим является пересечением возможностей камеры и конкретного Surface AVC encoder. */ data class StreamCaptureMode(val camera: String, val width: Int, val height: Int, val fps: Int, - val minimumFps: Int, val highSpeed: Boolean, val codec: String, val bitrate: Int) { + val minimumFps: Int, val highSpeed: Boolean, val codec: String, val bitrate: Int, + val videoCodec: StreamVideoCodec = StreamVideoCodec.H264, + val minimumBitrate: Int = 1, val maximumBitrate: Int = Int.MAX_VALUE) { val label get() = "${width}×${height} · $fps fps" } data class StreamCamera(val id: String, val lens: LensOption, val modes: List) @@ -18,9 +20,8 @@ data class StreamCamera(val id: String, val lens: LensOption, val modes: List { val manager = context.getSystemService(CameraManager::class.java) - val codecs = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { codec -> - codec.isEncoder && codec.supportedTypes.any { it.equals(MediaFormat.MIMETYPE_VIDEO_AVC, true) } - } + val codecs = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.filter { it.isEncoder } + .sortedBy { if (Build.VERSION.SDK_INT >= 29 && it.isHardwareAccelerated) 0 else 1 } return CameraLensManager.enumerate(manager).mapNotNull { lens -> try { val c = manager.getCameraCharacteristics(lens.cameraId) @@ -32,23 +33,25 @@ object StreamCameraModes { .maxByOrNull { it.width * it.height }?.let { listOf(it) }.orEmpty() } val modes = mutableListOf() fun add(width: Int, height: Int, fps: Int, lower: Int, highSpeed: Boolean) { + for (type in StreamVideoCodec.values()) { for (codec in codecs) { try { + if (codec.supportedTypes.none { it.equals(type.mime, true) }) continue if (highSpeed && Build.VERSION.SDK_INT >= 29 && !codec.isHardwareAccelerated) continue - val caps = codec.getCapabilitiesForType(MediaFormat.MIMETYPE_VIDEO_AVC) + val caps = codec.getCapabilitiesForType(type.mime) val video = caps.videoCapabilities if (!caps.colorFormats.contains(MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) || - caps.profileLevels.none { it.profile == MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline } || + (type.profile != null && caps.profileLevels.none { it.profile == type.profile }) || !video.areSizeAndRateSupported(width, height, fps.toDouble())) continue val bitrate = (width.toLong() * height * fps / 14).coerceIn(512_000L, 16_000_000L).toInt() modes.add(StreamCaptureMode(lens.cameraId, width, height, fps, lower, highSpeed, codec.name, - video.bitrateRange.clamp(bitrate))) + video.bitrateRange.clamp(bitrate), type, video.bitrateRange.lower, video.bitrateRange.upper)) LogManager.addLog("INFO", "video", "camera mode camera=${lens.cameraId} ${width}x$height fps=$lower..$fps " + "highSpeed=$highSpeed codec=${codec.name} bitrate=${modes.last().bitrate} achievable=${video.getAchievableFrameRatesFor(width, height)}") - return + break } catch (e: Exception) { LogManager.addLog("WARN", "video", "codec probe ${codec.name}: ${e.message}") } } - LogManager.addLog("DEBUG", "video", "camera mode rejected: no AVC encoder camera=${lens.cameraId} ${width}x$height fps=$fps") + } } val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() val normalFps = listOf(30, 60).filter { fps -> rates.any { it.upper == fps } } @@ -70,13 +73,15 @@ object StreamCameraModes { } } if (modes.isEmpty()) { LogManager.addLog("WARN", "video", "camera has no compatible live modes id=${lens.cameraId}"); null } - else StreamCamera(lens.cameraId, lens, modes.distinct().sortedWith(compareBy({ it.fps }, { it.width * it.height }))) + else StreamCamera(lens.cameraId, lens, modes.distinct().sortedWith(compareBy({ it.videoCodec.id }, { it.fps }, { it.width * it.height }))) } catch (e: Exception) { LogManager.addLog("ERROR", "video", "camera mode enumeration id=${lens.cameraId}: ${e.message}"); null } }.sortedBy { when (it.lens.type) { LensType.MAIN -> 0; LensType.SELFIE -> 1; else -> 2 } } } /** Смена камеры сохраняет точный режим, затем разрешение с меньшей FPS, затем штатный режим. */ fun choose(modes: List, previous: StreamCaptureMode? = null): StreamCaptureMode? { + val preferred = modes.filter { it.videoCodec == (previous?.videoCodec ?: StreamVideoCodec.H264) } + if (preferred.isNotEmpty() && preferred.size != modes.size) return choose(preferred, previous) if (previous != null) { modes.firstOrNull { it.width == previous.width && it.height == previous.height && it.fps == previous.fps }?.let { return it } modes.filter { it.width == previous.width && it.height == previous.height && it.fps <= previous.fps } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt index e3230b92..4670b256 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureConfig.kt @@ -8,4 +8,10 @@ enum class StreamOrientation(val degrees: Int) { } /** Режим камеры и ориентация подтверждаются и восстанавливаются одним значением. */ -data class StreamCaptureConfig(val mode: StreamCaptureMode, val orientation: StreamOrientation = StreamOrientation.UPRIGHT) +data class StreamCaptureConfig(val mode: StreamCaptureMode, val orientation: StreamOrientation = StreamOrientation.UPRIGHT, + val videoBitrate: Int? = null) { + val bitrate get() = videoBitrate ?: mode.bitrate + init { require(bitrate in mode.minimumBitrate..mode.maximumBitrate) } + fun withMode(next: StreamCaptureMode) = copy(mode = next, + videoBitrate = videoBitrate?.coerceIn(next.minimumBitrate, next.maximumBitrate)) +} diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamVideoCodec.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamVideoCodec.kt new file mode 100644 index 00000000..0d26b5f2 --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamVideoCodec.kt @@ -0,0 +1,68 @@ +package com.utun.chat.data + +import android.media.MediaCodecInfo +import android.media.MediaFormat +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** IDs uAV2; MIME определяется форматом, encoder name остаётся свойством режима устройства. */ +enum class StreamVideoCodec(val id: Int, val label: String, val mime: String, val profile: Int?, private val annexB: Boolean) { + H264(1, "H.264", "video/avc", MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline, true), + H265(2, "H.265", "video/hevc", MediaCodecInfo.CodecProfileLevel.HEVCProfileMain, true), + VP9(3, "VP9", "video/x-vnd.on2.vp9", MediaCodecInfo.CodecProfileLevel.VP9Profile0, false), + AV1(4, "AV1", "video/av01", MediaCodecInfo.CodecProfileLevel.AV1ProfileMain8, false), + H266(5, "H.266", "video/vvc", null, true); + + companion object { + const val MAX_CONFIG = 65536 + fun fromId(id: Int): StreamVideoCodec = values().firstOrNull { it.id == id } ?: error("Unsupported video codec id=$id") + } + fun normalize(bytes: ByteArray): ByteArray { + require(bytes.isNotEmpty() && bytes.size <= H264AnnexB.MAX_FRAME) + return if (annexB) H264AnnexB.normalize(bytes) else bytes.copyOf() + } + private fun nalTypes(bytes: ByteArray): Set = H264AnnexB.nals(bytes).map { (_, nal) -> + val at = if (nal[2] == 1.toByte()) 3 else 4 + when (this) { + H264 -> nal[at].toInt() and 31 + H265 -> { require(nal.size > at + 1); (nal[at].toInt() ushr 1) and 63 } + H266 -> { require(nal.size > at + 1); (nal[at + 1].toInt() ushr 3) and 31 } + else -> error("NAL format expected") + } + }.toSet() + + fun keyFrame(frame: ByteArray, config: ByteArray): ByteArray { + require(config.size <= MAX_CONFIG) + val data = normalize(frame) + if (this == H264) require(5 in nalTypes(data)) { "AVC KEY missing IDR" } + if (this == H265) require(nalTypes(data).any { it in 16..21 }) { "HEVC KEY missing random access picture" } + if (this == H266) require(nalTypes(data).any { it in 7..9 }) { "VVC KEY missing random access picture" } + require(data.size + config.size + 4 <= H264AnnexB.MAX_FRAME) + return ByteBuffer.allocate(4 + config.size + data.size).order(ByteOrder.BIG_ENDIAN) + .putInt(config.size).put(config).put(data).array() + } + + /** Конфигурация KEY отделена от access unit и может быть пустой для самодостаточного потока. */ + fun unpackKey(bytes: ByteArray): Pair { + require(bytes.size in 5..H264AnnexB.MAX_FRAME) + val b = ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN) + val count = b.int + require(count in 0..MAX_CONFIG && count < b.remaining()) + val config = ByteArray(count).also { b.get(it) }; val frame = ByteArray(b.remaining()).also { b.get(it) } + if (this == H264) require(nalTypes(config + frame).containsAll(setOf(5, 7, 8))) { "AVC KEY missing configuration" } + if (this == H265) require(nalTypes(config + frame).containsAll(setOf(32, 33, 34))) { "HEVC KEY missing configuration" } + return config to frame + } + + fun decoderFormat(width: Int, height: Int, rotation: Int, config: ByteArray): MediaFormat = + MediaFormat.createVideoFormat(mime, width, height).apply { + setInteger(MediaFormat.KEY_ROTATION, rotation); setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, H264AnnexB.MAX_FRAME) + if (config.isNotEmpty()) { + if (this@StreamVideoCodec == H264) { + val nals = H264AnnexB.nals(config) + setByteBuffer("csd-0", ByteBuffer.wrap(nals.filter { it.first == 7 }.fold(byteArrayOf()) { a, n -> a + n.second })) + setByteBuffer("csd-1", ByteBuffer.wrap(nals.filter { it.first == 8 }.fold(byteArrayOf()) { a, n -> a + n.second })) + } else setByteBuffer("csd-0", ByteBuffer.wrap(config)) + } + } +}