Add Android call camera and keyframe encoder replacement worker
This commit is contained in:
evgeny committed
1 parent
edf7e4d2c5
commit
04f4d29e4f
1 file changed
+250
@@ -0,0 +1,250 @@
|
|||||||
|
package com.utun.chat.data
|
||||||
|
|
||||||
|
import android.annotation.SuppressLint
|
||||||
|
import android.content.Context
|
||||||
|
import android.graphics.ImageFormat
|
||||||
|
import android.hardware.camera2.CameraCaptureSession
|
||||||
|
import android.hardware.camera2.CameraCharacteristics
|
||||||
|
import android.hardware.camera2.CameraDevice
|
||||||
|
import android.hardware.camera2.CameraManager
|
||||||
|
import android.hardware.camera2.CaptureRequest
|
||||||
|
import android.media.ImageReader
|
||||||
|
import android.os.Handler
|
||||||
|
import android.os.HandlerThread
|
||||||
|
import android.os.SystemClock
|
||||||
|
import android.util.Size
|
||||||
|
import android.view.Surface
|
||||||
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
|
import java.nio.ByteBuffer
|
||||||
|
import java.nio.ByteOrder
|
||||||
|
|
||||||
|
data class CallVideoCamera(val id: String, val name: String, val sizes: List<Size>)
|
||||||
|
data class CallVideoSettings(val enabled: Boolean = false, val camera: String = "", val width: Int = 640,
|
||||||
|
val height: Int = 480, val codec: Int = 0)
|
||||||
|
data class CallVideoState(val ready: Boolean = false, val settings: CallVideoSettings = CallVideoSettings(),
|
||||||
|
val cameras: List<CallVideoCamera> = emptyList(), val codecs: List<Int> = emptyList(), val bitrate: Int = 0,
|
||||||
|
val range: CallVideoBitrateRange = CallVideoBitrateRange(), val status: String = "")
|
||||||
|
|
||||||
|
/** Сервис владеет capture и RX; экран только выбирает режим и передаёт Surface. */
|
||||||
|
class CallVideoEngine(private val context: Context, private val id: Long) {
|
||||||
|
private val mutable = MutableStateFlow(CallVideoState())
|
||||||
|
val state: StateFlow<CallVideoState> = mutable
|
||||||
|
private val thread = HandlerThread("call-video-tx").apply { start() }
|
||||||
|
private val handler = Handler(thread.looper)
|
||||||
|
private val manager = context.getSystemService(CameraManager::class.java)
|
||||||
|
private val player = CallVideoPlayer(id) { message -> handler.post { report(message) } }
|
||||||
|
@Volatile private var stopped = false
|
||||||
|
private var encoders = emptyMap<Int, String>()
|
||||||
|
private var caps = ByteArray(80)
|
||||||
|
private var wanted = CallVideoSettings()
|
||||||
|
private var active: CallVideoSettings? = null
|
||||||
|
private var format: CallVideoFormat? = null
|
||||||
|
private var encoder: CallVideoCodec? = null
|
||||||
|
private var camera: CameraDevice? = null
|
||||||
|
private var session: CameraCaptureSession? = null
|
||||||
|
private var reader: ImageReader? = null
|
||||||
|
private var opening = false
|
||||||
|
private var closing = false
|
||||||
|
private var cameraToken = 0L
|
||||||
|
private var generation = 0L
|
||||||
|
private var frame = 0L
|
||||||
|
private var lastPts = -1L
|
||||||
|
private var lastKey = -1L
|
||||||
|
private var keySerial = -1L
|
||||||
|
private var needKey = true
|
||||||
|
private var request = 0L
|
||||||
|
private var pendingRequest = 0L
|
||||||
|
private var pendingRate = 0
|
||||||
|
private var confirmingRequest = 0L
|
||||||
|
private var realtime = false
|
||||||
|
private var sensorOffset: Long? = null
|
||||||
|
private var openedAt = 0L
|
||||||
|
private var reportAt = 0L
|
||||||
|
private var encodedBytes = 0L
|
||||||
|
private var encodedFrames = 0
|
||||||
|
|
||||||
|
init {
|
||||||
|
handler.post {
|
||||||
|
try {
|
||||||
|
encoders = CallVideoCodec.probe()
|
||||||
|
if (stopped) return@post
|
||||||
|
val cameras = CameraLensManager.enumerate(manager).mapNotNull { lens ->
|
||||||
|
val c = manager.getCameraCharacteristics(lens.cameraId)
|
||||||
|
val sizes = c.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)?.getOutputSizes(ImageFormat.YUV_420_888)
|
||||||
|
.orEmpty().filter { it == Size(640, 480) || it == Size(1280, 720) }
|
||||||
|
if (sizes.isEmpty()) null else CallVideoCamera(lens.cameraId, lens.label, sizes)
|
||||||
|
}
|
||||||
|
val bytes = ByteBuffer.wrap(caps).order(ByteOrder.BIG_ENDIAN)
|
||||||
|
repeat(2) { for (codec in 1..2) {
|
||||||
|
if (encoders.containsKey(codec)) bytes.putShort(1280).putShort(720).putShort(30).putInt(1280 * 720 * 30)
|
||||||
|
.putInt(4_000_000).putInt(1_000_000).put(if (codec == 1) 31 else 93).put(0)
|
||||||
|
else bytes.position(bytes.position() + 20)
|
||||||
|
} }
|
||||||
|
check(CallVideoNative.open(id, caps)) { "Video port unavailable" }
|
||||||
|
wanted = wanted.copy(camera = cameras.firstOrNull()?.id.orEmpty())
|
||||||
|
mutable.value = mutable.value.copy(ready = true, cameras = cameras, codecs = encoders.keys.toList(), settings = wanted)
|
||||||
|
report(if (encoders.isEmpty()) "No verified video codec" else "Video ready")
|
||||||
|
tick()
|
||||||
|
} catch (e: Exception) { fail("Video setup: ${e.message}") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun surface(surface: Surface?) = player.surface(surface)
|
||||||
|
fun select(settings: CallVideoSettings) { handler.post {
|
||||||
|
if (!stopped) { wanted = settings; mutable.value = mutable.value.copy(settings = settings) }
|
||||||
|
} }
|
||||||
|
fun range(value: CallVideoBitrateRange) { handler.post {
|
||||||
|
if (CallVideoNative.range(id, value.minimum * 1000, value.maximum * 1000)) mutable.value = mutable.value.copy(range = value)
|
||||||
|
else report("Bitrate range rejected")
|
||||||
|
} }
|
||||||
|
private fun report(message: String) {
|
||||||
|
mutable.value = mutable.value.copy(status = message)
|
||||||
|
LogManager.addLog("INFO", "call", "video call=$id $message")
|
||||||
|
}
|
||||||
|
private fun fail(message: String) {
|
||||||
|
LogManager.addLog("ERROR", "call", "video call=$id $message")
|
||||||
|
wanted = wanted.copy(enabled = false); mutable.value = mutable.value.copy(settings = wanted, status = message)
|
||||||
|
closeCamera(); encoder?.close(); encoder = null; active = null
|
||||||
|
if (confirmingRequest != 0L) CallVideoNative.applied(id, format?.generation ?: 0, confirmingRequest, 0)
|
||||||
|
confirmingRequest = 0
|
||||||
|
}
|
||||||
|
private fun tick() {
|
||||||
|
if (stopped) return
|
||||||
|
try {
|
||||||
|
val s = CallVideoNative.status(id) ?: return
|
||||||
|
if (s[6] >= 300000 && s[7] >= s[6]) mutable.value = mutable.value.copy(range = CallVideoBitrateRange((s[6] / 1000).toInt(), (s[7] / 1000).toInt()))
|
||||||
|
if (keySerial != s[0]) { keySerial = s[0]; needKey = true }
|
||||||
|
if (s[1] == format?.generation && s[2] > request) {
|
||||||
|
request = s[2]; pendingRequest = request; pendingRate = s[3].toInt()
|
||||||
|
}
|
||||||
|
if (!wanted.enabled) {
|
||||||
|
if (active != null) { closeCamera(); encoder?.close(); encoder = null; active = null; report("Video off") }
|
||||||
|
} else if (wanted != active && !opening && !closing && (lastKey < 0 || System.nanoTime() / 1000 - lastKey >= 1_000_000)) {
|
||||||
|
val peer = CallVideoNative.peer(id)
|
||||||
|
if (peer != null) change(peer)
|
||||||
|
}
|
||||||
|
drain()
|
||||||
|
if (active != null && System.nanoTime() - openedAt > 8_000_000_000 && lastPts < 0) fail("Camera produced no frames")
|
||||||
|
} catch (e: Exception) { fail("Video worker: ${e.message}") }
|
||||||
|
if (!stopped) handler.postDelayed(::tick, 10)
|
||||||
|
}
|
||||||
|
private fun change(peer: ByteArray) {
|
||||||
|
val cameraInfo = manager.getCameraCharacteristics(wanted.camera)
|
||||||
|
val rotation = cameraInfo.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
|
||||||
|
val desired = CallVideoFormat(++generation, wanted.width, wanted.height, rotation = rotation, codec = wanted.codec)
|
||||||
|
val selected = CallVideoNative.choose(caps, peer, desired.array()) ?: error("No common codec for selected resolution")
|
||||||
|
// MediaCodec получает фиксированный 3.1; ряд backend не умеет понижать маркировку level для меньшего размера.
|
||||||
|
val next = CallVideoFormat.from(selected).let { it.copy(level = if (it.codec == 1) 31 else 93) }
|
||||||
|
val candidate = CallVideoCodec(next, 1_000_000, checkNotNull(encoders[next.codec]))
|
||||||
|
closeCamera(); encoder?.close(); encoder = candidate; format = next; active = wanted
|
||||||
|
pendingRequest = 0; confirmingRequest = 0; request = 0; lastKey = -1; needKey = true; lastPts = -1
|
||||||
|
check(CallVideoNative.format(id, next.array(), candidate.bitrate)) { "Video format rejected" }
|
||||||
|
mutable.value = mutable.value.copy(bitrate = candidate.bitrate)
|
||||||
|
if (!closing) openCamera()
|
||||||
|
}
|
||||||
|
@SuppressLint("MissingPermission")
|
||||||
|
private fun openCamera() {
|
||||||
|
if (stopped || !wanted.enabled || opening || closing) return
|
||||||
|
val settings = active ?: return; val f = format ?: return; val token = ++cameraToken
|
||||||
|
val c = manager.getCameraCharacteristics(settings.camera)
|
||||||
|
realtime = c.get(CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE) == CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME
|
||||||
|
sensorOffset = null
|
||||||
|
val nextReader = ImageReader.newInstance(f.width, f.height, ImageFormat.YUV_420_888, 3); reader = nextReader
|
||||||
|
nextReader.setOnImageAvailableListener({ source ->
|
||||||
|
val image = source.acquireLatestImage() ?: return@setOnImageAvailableListener
|
||||||
|
try {
|
||||||
|
if (stopped || token != cameraToken || !wanted.enabled) return@setOnImageAvailableListener
|
||||||
|
val now = System.nanoTime()
|
||||||
|
val offset = if (realtime) now - SystemClock.elapsedRealtimeNanos() else sensorOffset ?: (now - image.timestamp).also {
|
||||||
|
sensorOffset = it; LogManager.addLog("WARN", "call", "camera timestamp calibrated from arrival id=$id offset=$it")
|
||||||
|
}
|
||||||
|
val pts = (image.timestamp + offset) / 1000
|
||||||
|
if (pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) return@setOnImageAvailableListener
|
||||||
|
val key = needKey || lastKey < 0 || pts - lastKey >= 1_000_000
|
||||||
|
if (key && pendingRequest != 0L) {
|
||||||
|
val requestId = pendingRequest; pendingRequest = 0
|
||||||
|
try {
|
||||||
|
val candidate = CallVideoCodec(f, pendingRate, checkNotNull(encoders[f.codec]))
|
||||||
|
encoder?.close(); encoder = candidate; confirmingRequest = requestId; needKey = true
|
||||||
|
report("Applying ${pendingRate / 1000} kbit/s on IDR")
|
||||||
|
} catch (e: Exception) {
|
||||||
|
CallVideoNative.applied(id, f.generation, requestId, 0); throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (encoder?.submit(image, pts, key) == true) { lastPts = pts; if (key) lastKey = pts }
|
||||||
|
drain()
|
||||||
|
} catch (e: Exception) { fail("Camera encode: ${e.message}") }
|
||||||
|
finally { image.close() }
|
||||||
|
}, handler)
|
||||||
|
opening = true; openedAt = System.nanoTime()
|
||||||
|
manager.openCamera(settings.camera, object : CameraDevice.StateCallback() {
|
||||||
|
override fun onOpened(device: CameraDevice) {
|
||||||
|
opening = false; camera = device
|
||||||
|
if (stopped || token != cameraToken) { device.close(); return }
|
||||||
|
try {
|
||||||
|
device.createCaptureSession(listOf(nextReader.surface), object : CameraCaptureSession.StateCallback() {
|
||||||
|
override fun onConfigured(s: CameraCaptureSession) {
|
||||||
|
if (stopped || token != cameraToken) { s.close(); return }
|
||||||
|
session = s
|
||||||
|
try {
|
||||||
|
val rates = c.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES).orEmpty()
|
||||||
|
val range = rates.filter { it.upper == 30 }.maxByOrNull { it.lower } ?: error("Camera lacks 30 fps")
|
||||||
|
val capture = device.createCaptureRequest(CameraDevice.TEMPLATE_RECORD).apply {
|
||||||
|
addTarget(nextReader.surface); set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, range)
|
||||||
|
}.build()
|
||||||
|
s.setRepeatingRequest(capture, null, handler)
|
||||||
|
report("${if (f.codec == 2) "H.265" else "H.264"} · ${f.width}×${f.height}")
|
||||||
|
} catch (e: Exception) { fail("Camera request: ${e.message}") }
|
||||||
|
}
|
||||||
|
override fun onConfigureFailed(s: CameraCaptureSession) { s.close(); fail("Camera session configuration failed") }
|
||||||
|
}, handler)
|
||||||
|
} catch (e: Exception) { fail("Camera session: ${e.message}") }
|
||||||
|
}
|
||||||
|
override fun onDisconnected(device: CameraDevice) { opening = false; camera = device; fail("Camera disconnected") }
|
||||||
|
override fun onError(device: CameraDevice, error: Int) { opening = false; camera = device; fail("Camera error $error") }
|
||||||
|
override fun onClosed(device: CameraDevice) {
|
||||||
|
if (camera === device) camera = null
|
||||||
|
closing = false
|
||||||
|
if (stopped) thread.quitSafely()
|
||||||
|
else if (active != null && wanted.enabled) try { openCamera() } catch (e: Exception) { fail("Camera reopen: ${e.message}") }
|
||||||
|
}
|
||||||
|
}, handler)
|
||||||
|
}
|
||||||
|
private fun drain() {
|
||||||
|
val codec = encoder ?: return; val f = format ?: return
|
||||||
|
for (p in codec.drain()) {
|
||||||
|
encodedBytes += p.bytes.size; encodedFrames++
|
||||||
|
if (confirmingRequest != 0L && p.key) {
|
||||||
|
CallVideoNative.applied(id, f.generation, confirmingRequest, codec.bitrate); confirmingRequest = 0
|
||||||
|
mutable.value = mutable.value.copy(bitrate = codec.bitrate)
|
||||||
|
LogManager.addLog("INFO", "call", "video bitrate applied id=$id generation=${f.generation} bitrate=${codec.bitrate} pts=${p.pts}")
|
||||||
|
}
|
||||||
|
if (needKey && !p.key) continue
|
||||||
|
val metadata = f.array() + longArrayOf(++frame, p.pts, if (p.key) 1 else 0)
|
||||||
|
needKey = CallVideoNative.send(id, metadata, p.bytes) != 1
|
||||||
|
}
|
||||||
|
val now = System.nanoTime()
|
||||||
|
if (now - reportAt > 1_000_000_000) {
|
||||||
|
if (reportAt != 0L) LogManager.addLog("DEBUG", "call", "video output id=$id fps=$encodedFrames " +
|
||||||
|
"bps=${encodedBytes * 8_000_000_000 / (now - reportAt)} target=${codec.bitrate} needKey=$needKey")
|
||||||
|
reportAt = now; encodedFrames = 0; encodedBytes = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private fun closeCamera() {
|
||||||
|
cameraToken++
|
||||||
|
try { session?.close(); session = null; reader?.close(); reader = null
|
||||||
|
if (camera != null) { closing = true; camera!!.close() }
|
||||||
|
} catch (e: Exception) { LogManager.addLog("WARN", "call", "camera close: ${e.message}") }
|
||||||
|
}
|
||||||
|
fun stop() {
|
||||||
|
if (stopped) return
|
||||||
|
stopped = true; player.stop(); CallVideoNative.close(id)
|
||||||
|
handler.post {
|
||||||
|
closeCamera(); encoder?.close(); encoder = null
|
||||||
|
if (camera == null && !opening) thread.quitSafely()
|
||||||
|
else handler.postDelayed({ thread.quitSafely() }, 3000)
|
||||||
|
LogManager.addLog("INFO", "call", "video workers stopped id=$id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user