diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCadence.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCadence.kt new file mode 100644 index 00000000..ec359994 --- /dev/null +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoCadence.kt @@ -0,0 +1,15 @@ +package com.utun.chat.data + +/** Отбор raw кадров по фазе, как в desktop: допуск дрожания PTS, без burst после паузы. */ +internal class CallVideoCadence(private val fps: Int) { + private var lastPts = -1L + private var next = -1L + fun accept(pts: Long): Boolean { + if (fps !in 1..60 || pts < 0 || pts <= lastPts || pts > (Long.MAX_VALUE - 1_250_000) / fps) return false + lastPts = pts + val scaled = pts * fps + if (next >= 0 && scaled < next - 250_000) return false + next = if (next < 0 || scaled - next > 250_000) scaled + 1_000_000 else next + 1_000_000 + return true + } +} diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt index 9f13c875..67f3d2fe 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/CallVideoEngine.kt @@ -22,18 +22,20 @@ import kotlinx.coroutines.flow.StateFlow import java.nio.ByteBuffer import java.nio.ByteOrder -data class CallVideoCamera(val id: String, val name: String, val sizes: List, val sensor: Int, val front: Boolean) +data class CallVideoMode(val width: Int, val height: Int, val fps: Int) +data class CallVideoCamera(val id: String, val name: String, val modes: List, val sensor: Int, val front: Boolean) data class CallVideoSettings(val enabled: Boolean = false, val camera: String = "", val width: Int = 640, - val height: Int = 480, val codec: Int = 0) + val height: Int = 480, val codec: Int = 0, val fps: Int = 30) data class CallVideoState(val ready: Boolean = false, val settings: CallVideoSettings = CallVideoSettings(), val cameras: List = emptyList(), val codecs: List = emptyList(), val bitrate: Int = 0, val range: CallVideoBitrateRange = CallVideoBitrateRange(), val status: String = "", val transmitting: Boolean = false, - val codecSizes: Map> = emptyMap(), val notice: Int? = null, + val codecModes: Map> = emptyMap(), val notice: Int? = null, val activeCodec: Int = 0) { // Фактически отправленный codec; 0 до первого принятого TX кадра. - fun sizes(camera: String = settings.camera, codec: Int = settings.codec): List { - val allowed = if (codec == 0) codecSizes.values.flatten() else codecSizes[codec].orEmpty() - return cameras.firstOrNull { it.id == camera }?.sizes.orEmpty().filter { it in allowed } + fun modes(camera: String = settings.camera, codec: Int = settings.codec): List { + val allowed = if (codec == 0) codecModes.values.flatten() else codecModes[codec].orEmpty() + return cameras.firstOrNull { it.id == camera }?.modes.orEmpty().filter { it in allowed } } + fun sizes(camera: String = settings.camera, codec: Int = settings.codec) = modes(camera, codec).map { Size(it.width, it.height) }.distinct() } /** Сервис владеет capture и RX; экран только выбирает режим и передаёт Surface. */ @@ -143,14 +145,17 @@ class CallVideoEngine(private val context: Context, private val id: Long) { return@mapNotNull null } val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() - val sizes = map.getOutputSizes(ImageFormat.YUV_420_888).orEmpty().filter { - (it == Size(640, 480) || it == Size(1280, 720) || it == Size(1920, 1080)) && - rates.any { range -> range.upper == 30 } && - map.getOutputMinFrameDuration(ImageFormat.YUV_420_888, it) <= 33_366_700L - } - val supported = sizes.filter { size -> encoders.values.any { size in it.sizes } } + val modes = map.getOutputSizes(ImageFormat.YUV_420_888).orEmpty() + .filter { it == Size(640, 480) || it == Size(1280, 720) || it == Size(1920, 1080) }.flatMap { size -> + listOf(30, 50, 60).filter { fps -> rates.any { it.upper >= fps } && + map.getOutputMinFrameDuration(ImageFormat.YUV_420_888, size) <= 1_000_000_000L / fps + 1000 + }.map { CallVideoMode(size.width, size.height, it) } + } + val supported = modes.filter { mode -> encoders.values.any { encoder -> encoder.modes.any { + it.size == Size(mode.width, mode.height) && it.fps == mode.fps + } } } LogManager.addLog(if (supported.isEmpty()) "WARN" else "INFO", "call", - "video camera id=${lens.cameraId} 30fpsYuv=$sizes verified=$supported") + "video camera id=${lens.cameraId} aeRanges=${rates.toList()} yuv=$modes verified=$supported") if (supported.isEmpty()) null else CallVideoCamera(lens.cameraId, lens.label, supported, c.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0, c.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT) @@ -161,7 +166,9 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val level = if (direction == 0) encoders[codec]?.level else decoders[codec]?.level if (profile != null && level != null) { val size = if (direction == 0) encoders.getValue(codec).sizes.maxBy { it.width * it.height } else decoders.getValue(codec).size - bytes.putShort(size.width.toShort()).putShort(size.height.toShort()).putShort(30).putInt(size.width * size.height * 30) + val fps = if (direction == 0) encoders.getValue(codec).fps else decoders.getValue(codec).fps + val pps = if (direction == 0) encoders.getValue(codec).pixelsPerSecond else decoders.getValue(codec).pixelsPerSecond + bytes.putShort(size.width.toShort()).putShort(size.height.toShort()).putShort(fps.toShort()).putInt(pps) .putInt(4_000_000).put(level.toByte()).put(profile.toByte()) } else bytes.position(bytes.position() + 16) @@ -169,11 +176,12 @@ class CallVideoEngine(private val context: Context, private val id: Long) { caps = checkNotNull(CallVideoNative.rotatableCaps(caps)) { "Invalid verified video capabilities" } check(CallVideoNative.open(id, caps)) { "Video port unavailable" } val firstCamera = cameras.firstOrNull() - val firstSize = firstCamera?.sizes?.firstOrNull { it.width == wanted.width && it.height == wanted.height } - ?: firstCamera?.sizes?.minByOrNull { it.width * it.height } - wanted = wanted.copy(camera = firstCamera?.id.orEmpty(), width = firstSize?.width ?: 640, height = firstSize?.height ?: 480) + val firstMode = firstCamera?.modes?.firstOrNull { it.width == wanted.width && it.height == wanted.height && it.fps == wanted.fps } + ?: firstCamera?.modes?.minWithOrNull(compareBy { it.width * it.height }.thenBy { it.fps }) + wanted = wanted.copy(camera = firstCamera?.id.orEmpty(), width = firstMode?.width ?: 640, + height = firstMode?.height ?: 480, fps = firstMode?.fps ?: 30) mutable.value = mutable.value.copy(ready = true, cameras = cameras, codecs = encoders.keys.toList(), settings = wanted, - codecSizes = encoders.mapValues { it.value.sizes }) + codecModes = encoders.mapValues { entry -> entry.value.modes.map { CallVideoMode(it.size.width, it.size.height, it.fps) } }) report(if (encoders.isEmpty()) "Receive-only video" else "Video ready") tick() } catch (e: Exception) { fail("Video setup: ${e.message}") } @@ -189,8 +197,8 @@ class CallVideoEngine(private val context: Context, private val id: Long) { if (stopped) { LogManager.addLog("WARN", "call", "video selection after stop id=$id settings=$settings"); return@post } val selected = if (settings.camera.isEmpty()) wanted.copy(enabled = settings.enabled) else settings val current = mutable.value - if (selected.enabled && (!current.ready || current.sizes(selected.camera, selected.codec).none { - it.width == selected.width && it.height == selected.height + if (selected.enabled && (!current.ready || current.modes(selected.camera, selected.codec).none { + it.width == selected.width && it.height == selected.height && it.fps == selected.fps })) { LogManager.addLog("WARN", "call", "video selection unavailable id=$id ready=${current.ready} " + "requested=$settings resolved=$selected status=${current.status}") @@ -270,8 +278,8 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val height = if (turn % 180 == 0) settings.height else settings.width for (codec in if (settings.codec == 0) listOf(2, 1) else listOf(settings.codec)) { if (codec in excluded) continue - val implementation = encoders[codec]?.modes?.firstOrNull { it.size == Size(settings.width, settings.height) } ?: continue - val desired = CallVideoFormat(++generation, width, height, codec = codec, profile = implementation.profile) + val implementation = encoders[codec]?.modes?.firstOrNull { it.size == Size(settings.width, settings.height) && it.fps == settings.fps } ?: continue + val desired = CallVideoFormat(++generation, width, height, fps = settings.fps, codec = codec, profile = implementation.profile) val selected = CallVideoNative.choose(caps, peer, desired.array()) ?: continue var candidate: CallVideoCodec? = null try { @@ -309,7 +317,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { check(CallVideoNative.format(id, candidate.format.array(), candidate.bitrate)) { "Video format rejected" } } catch (e: Exception) { candidate.close(); throw e } val sourceChanged = previousSettings == null || settings.camera != previousSettings.camera - || settings.width != previousSettings.width || settings.height != previousSettings.height + || settings.width != previousSettings.width || settings.height != previousSettings.height || settings.fps != previousSettings.fps if (sourceChanged) closeCamera() encoder = candidate; format = candidate.format; active = settings; wanted = settings pendingRequest = 0; confirmingRequest = 0; request = 0; lastKeyRequest = -1; needKey = true @@ -321,7 +329,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { "cameraRestart=$sourceChanged preview=${preview.state.value.format}") if (restored) LogManager.addLog("WARN", "call", "video mode unavailable id=$id requested=${targets.first()} restored=$settings") report(if (restored) "Selected mode unavailable; previous settings restored" else - "${if (candidate.format.codec == 2) "H.265" else "H.264"} · ${candidate.format.width}×${candidate.format.height}") + "${if (candidate.format.codec == 2) "H.265" else "H.264"} · ${candidate.format.width}×${candidate.format.height} · ${candidate.format.fps} fps") if (camera == null && !closing && !opening) openCamera() return } @@ -334,6 +342,7 @@ class CallVideoEngine(private val context: Context, private val id: Long) { val c = manager.getCameraCharacteristics(settings.camera) realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME sensorOffset = null + val cadence = CallVideoCadence(settings.fps) val nextReader = ImageReader.newInstance(settings.width, settings.height, ImageFormat.YUV_420_888, 3); reader = nextReader nextReader.setOnImageAvailableListener({ source -> val image = try { source.acquireLatestImage() } catch (e: IllegalStateException) { @@ -347,8 +356,8 @@ class CallVideoEngine(private val context: Context, private val id: Long) { sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it") } val pts = (image.timestamp + offset) / 1000 - // Camera2 уже ограничена 30 fps. Порог 33333 мкс отбрасывал каждый второй кадр при дрожании часов. - if (pts <= lastPts) return@setOnImageAvailableListener + // Отбираем raw до копирования/encode; частота передачи не зависит от долга приёмника. + if (pts <= lastPts || !cadence.accept(pts)) return@setOnImageAvailableListener drain() // До восьми принятых входов/готовых выходов: backpressure действует до кодирования. if (switching || !canEncode || pendingFrames.isNotEmpty() || (encoder?.pendingFrames ?: 0) >= 8) @@ -464,13 +473,14 @@ class CallVideoEngine(private val context: Context, private val id: Long) { session = s try { val range = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty() - .filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps") + .filter { it.upper >= settings.fps }.minWithOrNull(compareBy> { it.upper }.thenByDescending { it.lower }) + ?: error("Camera lacks ${settings.fps} fps") val request = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply { targets.forEach(::addTarget); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range) }.build() s.setRepeatingRequest(request, null, handler) if (output != null) preview.update(previewFormat, active = true) - LogManager.addLog("INFO", "call", "camera capture id=$id session=$serial range=$range preview=$previewFormat") + LogManager.addLog("INFO", "call", "camera capture id=$id session=$serial range=$range outputFps=${settings.fps} preview=$previewFormat") } catch (e: Exception) { s.close(); rejected("Camera request: ${e.message}") } } override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); complete(); rejected("Camera session configuration failed") } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt index 49db91b2..4af748b9 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/CallVideoPanel.kt @@ -65,10 +65,11 @@ internal fun CallVideoToolbar(video: CallVideoEngine, onInteraction: (Boolean) - state.cameras.filter { state.sizes(it.id).isNotEmpty() }.forEach { option -> CaptureOptionItem(option.name, option.id == state.settings.camera, onClick = { menu = null - val sizes = state.sizes(option.id) - val size = sizes.firstOrNull { it.width == state.settings.width && it.height == state.settings.height } ?: sizes.first() - LogManager.addLog("INFO", "call", "video toolbar camera=${option.id} size=$size") - CallController.setVideo(state.settings.copy(camera = option.id, width = size.width, height = size.height)) + val modes = state.modes(option.id) + val mode = modes.firstOrNull { it.width == state.settings.width && it.height == state.settings.height && it.fps == state.settings.fps } + ?: modes.firstOrNull { it.fps == state.settings.fps } ?: modes.first() + LogManager.addLog("INFO", "call", "video toolbar camera=${option.id} mode=$mode") + CallController.setVideo(state.settings.copy(camera = option.id, width = mode.width, height = mode.height, fps = mode.fps)) }) } } @@ -84,12 +85,13 @@ internal fun CallVideoToolbar(video: CallVideoEngine, onInteraction: (Boolean) - DropdownMenu(menu == CallVideoOption.CODEC, onDismissRequest = { menu = null }) { for (codec in listOf(0) + state.codecs.sortedDescending()) { val label = when (codec) { 0 -> stringResource(R.string.call_video_auto); 2 -> "H.265"; else -> "H.264" } - val sizes = state.sizes(codec = codec) - if (sizes.isNotEmpty()) CaptureOptionItem(label, codec == state.settings.codec, onClick = { + val modes = state.modes(codec = codec) + if (modes.isNotEmpty()) CaptureOptionItem(label, codec == state.settings.codec, onClick = { menu = null - val size = sizes.firstOrNull { it.width == state.settings.width && it.height == state.settings.height } ?: sizes.first() - LogManager.addLog("INFO", "call", "video toolbar codec=$codec size=$size") - CallController.setVideo(state.settings.copy(codec = codec, width = size.width, height = size.height)) + val mode = modes.firstOrNull { it.width == state.settings.width && it.height == state.settings.height && it.fps == state.settings.fps } + ?: modes.firstOrNull { it.fps == state.settings.fps } ?: modes.first() + LogManager.addLog("INFO", "call", "video toolbar codec=$codec mode=$mode") + CallController.setVideo(state.settings.copy(codec = codec, width = mode.width, height = mode.height, fps = mode.fps)) }) } } @@ -111,16 +113,16 @@ internal fun CallVideoToolbar(video: CallVideoEngine, onInteraction: (Boolean) - } } Box { - CaptureControlButton("$resolutionLabel: ${state.settings.width}×${state.settings.height}", + CaptureControlButton("$resolutionLabel: ${state.settings.width}×${state.settings.height} · ${state.settings.fps} fps", enabled = state.ready && state.sizes().isNotEmpty(), onClick = { menu = CallVideoOption.RESOLUTION }) { - CaptureValueIcon(CaptureIconShape.SCREEN, "${minOf(state.settings.width, state.settings.height)}p", detail = "30fps") + CaptureValueIcon(CaptureIconShape.SCREEN, "${minOf(state.settings.width, state.settings.height)}p", detail = "${state.settings.fps} fps") } DropdownMenu(menu == CallVideoOption.RESOLUTION, onDismissRequest = { menu = null }) { - state.sizes().forEach { size -> CaptureOptionItem("${size.width}×${size.height}", - size.width == state.settings.width && size.height == state.settings.height, onClick = { + state.modes().forEach { mode -> CaptureOptionItem("${mode.width}×${mode.height} · ${mode.fps} fps", + mode.width == state.settings.width && mode.height == state.settings.height && mode.fps == state.settings.fps, onClick = { menu = null - LogManager.addLog("INFO", "call", "video toolbar resolution=$size") - CallController.setVideo(state.settings.copy(width = size.width, height = size.height)) + LogManager.addLog("INFO", "call", "video toolbar mode=$mode") + CallController.setVideo(state.settings.copy(width = mode.width, height = mode.height, fps = mode.fps)) }) } } }