jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Capture encode and play stereo audio in Android live streams

master
evgeny 6 days ago
parent
22b62699ac
commit
e06e9b0e22
3 changed files with 42 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt
  2. 13
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt
  3. 49
      tools/chatgui-android/jni_bridge/android_stream_jni.c

11
tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt
Unescape View File

@ -345,11 +345,12 @@ class StreamCaptureEngine(
}
@SuppressLint("MissingPermission")
private fun startAudio() {
val min = AudioRecord.getMinBufferSize(48000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
val min = AudioRecord.getMinBufferSize(48000, AudioFormat.CHANNEL_IN_STEREO, AudioFormat.ENCODING_PCM_16BIT)
check(min > 0)
val rec = AudioRecord(MediaRecorder.AudioSource.CAMCORDER, 48000, AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT, (min * 2).coerceAtLeast(960 * 8))
val rec = AudioRecord(MediaRecorder.AudioSource.CAMCORDER, 48000, AudioFormat.CHANNEL_IN_STEREO,
AudioFormat.ENCODING_PCM_16BIT, (min * 2).coerceAtLeast(1920 * 8))
record = rec; check(rec.state == AudioRecord.STATE_INITIALIZED)
check(rec.channelCount == 2) { "CAMCORDER stereo capture unavailable: channels=${rec.channelCount}" }
rec.startRecording(); check(rec.recordingState == AudioRecord.RECORDSTATE_RECORDING) { "CAMCORDER capture did not start" }
audioRunning = true
LogManager.addLog("INFO", "stream", "audio capture started id=$id source=CAMCORDER(${rec.audioSource}) " +
@ -358,7 +359,7 @@ class StreamCaptureEngine(
var opus = 0L
try {
opus = NativeLib.streamOpusCreate(true); check(opus != 0L)
val pcm = ShortArray(960); val stamp = AudioTimestamp()
val pcm = ShortArray(1920); val stamp = AudioTimestamp()
var total = 0L; var waitingStamp = true; var last = -1L
while (audioRunning) {
var filled = 0
@ -368,7 +369,7 @@ class StreamCaptureEngine(
if (n > 0) filled += n
}
if (!audioRunning || filled != pcm.size) break
val first = total; total += filled
val first = total; total += filled / 2 // AudioTimestamp считает кадры на канал, а read — interleaved сэмплы.
if (rec.getTimestamp(stamp, AudioTimestamp.TIMEBASE_MONOTONIC) != AudioRecord.SUCCESS) {
if (waitingStamp) LogManager.addLog("DEBUG", "stream", "audio timestamp pending id=$id samples=$total")
continue

13
tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt
Unescape View File

@ -117,7 +117,10 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
while (running && offset < pcm.size) {
val count = t.write(pcm, offset, pcm.size - offset, AudioTrack.WRITE_NON_BLOCKING)
check(count >= 0) { "AudioTrack write=$count" }
if (count > 0) { offset += count; written += count }
if (count > 0) {
check(count % 2 == 0) { "AudioTrack returned an incomplete stereo frame: samples=$count" }
offset += count; written += count / 2
}
else {
check(System.nanoTime() < deadline) { "AudioTrack stalled for 100ms" }
Thread.sleep(2)
@ -207,13 +210,13 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
opus = NativeLib.streamOpusCreate(false); check(opus != 0L)
track = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE).build()).setAudioFormat(AudioFormat.Builder()
.setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_MONO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build())
.setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT)
.coerceAtLeast(9600)).setTransferMode(AudioTrack.MODE_STREAM).build()
.setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_STEREO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build())
.setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT)
.coerceAtLeast(19200)).setTransferMode(AudioTrack.MODE_STREAM).build()
check(track!!.state == AudioTrack.STATE_INITIALIZED)
audioBase = audios.first.pts
track!!.play()
LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase")
LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase rate=48000 channels=${track!!.channelCount}")
}
val t = track
if (t != null && audios.isNotEmpty()) {

49
tools/chatgui-android/jni_bridge/android_stream_jni.c
Unescape View File

@ -11,6 +11,8 @@
#include "../libutun_lite/stream_bridge.h"
#include "../../../src/stream/stream_media.h"
enum { STREAM_AUDIO_CHANNELS = 2, STREAM_AUDIO_FRAMES = 960, STREAM_AUDIO_SAMPLES = 1920 };
struct android_stream_opus {
opus_codec_encoder_t *encoder;
opus_codec_decoder_t *decoder;
@ -41,7 +43,7 @@ JNIEXPORT jint JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamFeed(
jbyte *bytes = (*env)->GetByteArrayElements(env, data, NULL);
if (!bytes) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream feed array access failed"); return -1; }
struct stream_media_format format = {(uint16_t)width, (uint16_t)height, (uint16_t)rotation,
STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS : 0, (uint32_t)revision};
STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS_STEREO : 0, (uint32_t)revision};
int result = android_stream_feed((uint64_t)id, video, (uint64_t)pts, key, (const uint8_t *)bytes, (size_t)length, video ? &format : NULL);
(*env)->ReleaseByteArrayElements(env, data, bytes, JNI_ABORT);
return result;
@ -67,16 +69,17 @@ JNIEXPORT jlong JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusCreate
struct android_stream_opus *codec = u_calloc(1, sizeof(*codec));
if (!codec) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus allocation failed"); return 0; }
if (encode) {
codec->encoder = opus_codec_encoder_create(48000, 1);
codec->encoder = opus_codec_encoder_create(48000, STREAM_AUDIO_CHANNELS);
if (!codec->encoder || opus_codec_encoder_application_set(codec->encoder, OPUS_CODEC_APP_AUDIO) < 0
|| opus_codec_encoder_bitrate_set(codec->encoder, 64000) < 0) goto fail;
|| opus_codec_encoder_bitrate_set(codec->encoder, 128000) < 0) goto fail;
codec->compressor = audio_compressor_create();
if (!codec->compressor) goto fail;
} else {
codec->decoder = opus_codec_decoder_create(48000, 1);
codec->decoder = opus_codec_decoder_create(48000, STREAM_AUDIO_CHANNELS);
if (!codec->decoder) goto fail;
}
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "JNI stream Opus created codec=%p encode=%d rate=48000 channels=1 frame=960", (void *)codec, encode);
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "JNI stream Opus created codec=%p encode=%d rate=48000 channels=2 frame=960 bitrate=128000",
(void *)codec, encode);
return (jlong)(intptr_t)codec;
fail:
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus initialization failed encode=%d", encode);
@ -88,7 +91,7 @@ JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusE
JNIEnv *env, jobject obj, jlong handle, jshortArray samples, jboolean enabled, jint max_gain_db, jint rise_rate) {
(void)obj;
struct android_stream_opus *codec = (struct android_stream_opus *)(intptr_t)handle;
if (!codec || !codec->encoder || !codec->compressor || !samples || (*env)->GetArrayLength(env, samples) != 960) {
if (!codec || !codec->encoder || !codec->compressor || !samples || (*env)->GetArrayLength(env, samples) != STREAM_AUDIO_SAMPLES) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus encode invalid handle/frame"); return NULL;
}
if (max_gain_db < 1 || max_gain_db > 60 || rise_rate < 1 || rise_rate > 100) {
@ -96,7 +99,7 @@ JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusE
}
if (codec->compressor_enabled != !!enabled || codec->max_gain_db != max_gain_db || codec->rise_rate != rise_rate) {
audio_compressor_config_t config = {0};
config.sample_rate = 48000; config.channels = 1; config.block_duration_ms = 20;
config.sample_rate = 48000; config.channels = STREAM_AUDIO_CHANNELS; config.block_duration_ms = 20;
config.lookback_ms = 200; config.lookahead_ms = 0; config.target_level = 0.25f;
config.max_gain_db = max_gain_db; config.rise_rate_per_sec = rise_rate; config.release_rate_per_sec = rise_rate * 5.0f;
if (audio_compressor_configure(codec->compressor, &config) != 0) {
@ -108,23 +111,27 @@ JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusE
"JNI stream compressor applied codec=%p enabled=%d max_gain=%ddB rise=%d release=%d target=0.25 history=200ms delay=0ms",
(void *)codec, !!enabled, max_gain_db, rise_rate, rise_rate * 5);
}
int16_t pcm[960]; uint8_t packet[1275];
(*env)->GetShortArrayRegion(env, samples, 0, 960, pcm);
int16_t pcm[STREAM_AUDIO_SAMPLES]; uint8_t packet[1275];
(*env)->GetShortArrayRegion(env, samples, 0, STREAM_AUDIO_SAMPLES, pcm);
if ((*env)->ExceptionCheck(env)) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream PCM read failed"); return NULL; }
int report = codec->frames % 50 == 0, input_peak = 0, output_peak = 0;
if (report) for (unsigned i = 0; i < 960; i++) if (abs((int)pcm[i]) > input_peak) input_peak = abs((int)pcm[i]);
if (audio_compressor_process_frame(codec->compressor, pcm, pcm, 960) != 0) {
int report = codec->frames % 50 == 0, input_peak = 0, output_peak = 0, stereo_difference = 0;
if (report) {
for (unsigned i = 0; i < STREAM_AUDIO_SAMPLES; i++) if (abs((int)pcm[i]) > input_peak) input_peak = abs((int)pcm[i]);
for (unsigned i = 0; i < STREAM_AUDIO_FRAMES; i++) stereo_difference += abs((int)pcm[2 * i] - pcm[2 * i + 1]);
}
if (audio_compressor_process_frame(codec->compressor, pcm, pcm, STREAM_AUDIO_SAMPLES) != 0) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream compressor processing failed codec=%p", (void *)codec); return NULL;
}
int length = opus_codec_encode(codec->encoder, pcm, 960, packet, sizeof(packet));
int length = opus_codec_encode(codec->encoder, pcm, STREAM_AUDIO_FRAMES, packet, sizeof(packet));
if (length < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus encode failed result=%d", length); return NULL; }
codec->frames++;
if (report) {
for (unsigned i = 0; i < 960; i++) if (abs((int)pcm[i]) > output_peak) output_peak = abs((int)pcm[i]);
for (unsigned i = 0; i < STREAM_AUDIO_SAMPLES; i++) if (abs((int)pcm[i]) > output_peak) output_peak = abs((int)pcm[i]);
uint32_t clips = audio_compressor_clip_count(codec->compressor);
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "JNI stream audio codec=%p frames=%llu enabled=%d gain=%.1fdB peaks=%d->%d clips=%u packet=%d",
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM,
"JNI stream audio codec=%p frames=%llu channels=2 enabled=%d gain=%.1fdB peaks=%d->%d lrDifference=%d clips=%u packet=%d",
(void *)codec, (unsigned long long)codec->frames, codec->compressor_enabled,
audio_compressor_gain_db(codec->compressor), input_peak, output_peak, clips, length);
audio_compressor_gain_db(codec->compressor), input_peak, output_peak, stereo_difference / STREAM_AUDIO_FRAMES, clips, length);
if (clips != codec->reported_clips) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "JNI stream audio clipped codec=%p samples=%u gain=%.1fdB",
(void *)codec, clips - codec->reported_clips, audio_compressor_gain_db(codec->compressor));
codec->reported_clips = clips;
@ -142,13 +149,13 @@ JNIEXPORT jshortArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpus
if (!codec || !codec->decoder || !packet || (*env)->GetArrayLength(env, packet) > 1275) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus decode invalid handle/packet"); return NULL;
}
jsize length = (*env)->GetArrayLength(env, packet); uint8_t bytes[1275]; int16_t pcm[960];
jsize length = (*env)->GetArrayLength(env, packet); uint8_t bytes[1275]; int16_t pcm[STREAM_AUDIO_SAMPLES];
(*env)->GetByteArrayRegion(env, packet, 0, length, (jbyte *)bytes);
if ((*env)->ExceptionCheck(env)) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus packet read failed"); return NULL; }
int count = opus_codec_decode(codec->decoder, length ? bytes : NULL, length, pcm, 960);
if (count != 960) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus decode failed count=%d", count); return NULL; }
jshortArray output = (*env)->NewShortArray(env, count);
if (output) (*env)->SetShortArrayRegion(env, output, 0, count, pcm);
int count = opus_codec_decode(codec->decoder, length ? bytes : NULL, length, pcm, STREAM_AUDIO_FRAMES);
if (count != STREAM_AUDIO_FRAMES) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus decode failed count=%d", count); return NULL; }
jshortArray output = (*env)->NewShortArray(env, STREAM_AUDIO_SAMPLES);
if (output) (*env)->SetShortArrayRegion(env, output, 0, STREAM_AUDIO_SAMPLES, pcm);
else DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus PCM allocation failed");
return output;
}

Write Preview
Loading…
Cancel
Save