Browse Source

voice: waveform recording + display, playback wiring

- voice_recorder: waveform_levels[100] accumulated during feed, normalized to 100 values in stop()
  stored as 'wf=...;dur=...;' in chat_msg_submit data
- bridge: filePath in get_messages_json from local_attrs.fp, g_db_path moved global
- Message model: waveform: List<Float>, voiceDurationMs: Int, filePath: String
- VoiceBubble: Canvas waveform (100 bars, log scale), play button, duration
- ChatViewModel.playVoiceMessage → AudioRecorderManager.playVoiceFile
topo_upd
evgeny 2 months ago
parent
commit
94a537ebdd
  1. 30
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/ChatRepository.kt
  2. 78
      tools/chatgui-android/app/src/main/java/com/utun/chat/ui/components/MessageBubble.kt
  3. 4
      tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/ChatScreen.kt
  4. 5
      tools/chatgui-android/app/src/main/java/com/utun/chat/viewmodel/ChatViewModel.kt
  5. 31
      tools/chatgui-android/jni_bridge/android_jni_bridge.c
  6. 49
      tools/chatgui-android/libutun_lite/voice_recorder.c

30
tools/chatgui-android/app/src/main/java/com/utun/chat/data/ChatRepository.kt

@ -7,7 +7,9 @@ import org.json.JSONObject
data class Channel(val id: String, val name: String, val lastMsgAt: Long = 0, val lastMsg: String = "", data class Channel(val id: String, val name: String, val lastMsgAt: Long = 0, val lastMsg: String = "",
val peersOnline: Int = 0) val peersOnline: Int = 0)
data class Message(val id: Long, val author: String, val text: String, val ts: Long, data class Message(val id: Long, val author: String, val text: String, val ts: Long,
val isOutgoing: Boolean = false, val contentType: String = "text/plain") val isOutgoing: Boolean = false, val contentType: String = "text/plain",
val filePath: String = "", val waveform: List<Float> = emptyList(),
val voiceDurationMs: Int = 0)
class ChatRepository { class ChatRepository {
private val myNodeId: Long = NativeLib.getMyNodeId() private val myNodeId: Long = NativeLib.getMyNodeId()
@ -47,13 +49,35 @@ class ChatRepository {
val arr = JSONArray(NativeLib.getMessages(channelId, limit)) val arr = JSONArray(NativeLib.getMessages(channelId, limit))
for (i in 0 until arr.length()) { for (i in 0 until arr.length()) {
val obj = arr.getJSONObject(i) val obj = arr.getJSONObject(i)
val text = obj.optString("text", "")
val contentType = obj.optString("contentType", "text/plain")
val filePath = obj.optString("filePath", "")
/* Parse waveform from voice message text (format: "wf=0.1,0.2,...;dur=1234;") */
var waveform = emptyList<Float>()
var voiceDurationMs = 0
if (contentType == "audio/opus" && text.startsWith("wf=")) {
try {
val wfEnd = text.indexOf(";dur=")
if (wfEnd > 0) {
val wfPart = text.substring(3, wfEnd)
waveform = wfPart.split(",").map { it.toFloat() }
val durPart = text.substring(wfEnd + 5).takeWhile { it != ';' }
voiceDurationMs = durPart.toIntOrNull() ?: 0
}
} catch (_: Exception) {}
}
list.add(Message( list.add(Message(
id = obj.getLong("id"), id = obj.getLong("id"),
author = obj.getString("author"), author = obj.getString("author"),
text = obj.getString("text"), text = text,
ts = obj.getLong("ts"), ts = obj.getLong("ts"),
isOutgoing = obj.optBoolean("isOutgoing", false), isOutgoing = obj.optBoolean("isOutgoing", false),
contentType = obj.optString("contentType", "text/plain") contentType = contentType,
filePath = filePath,
waveform = waveform,
voiceDurationMs = voiceDurationMs
)) ))
} }
} catch (e: Exception) { Log.w("utun-gui", "getMessages error", e) } } catch (e: Exception) { Log.w("utun-gui", "getMessages error", e) }

78
tools/chatgui-android/app/src/main/java/com/utun/chat/ui/components/MessageBubble.kt

@ -4,6 +4,7 @@ import android.Manifest
import android.content.pm.PackageManager import android.content.pm.PackageManager
import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background import androidx.compose.foundation.background
import androidx.compose.foundation.clickable import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.* import androidx.compose.foundation.layout.*
@ -14,6 +15,8 @@ import androidx.compose.runtime.*
import androidx.compose.ui.Alignment import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip import androidx.compose.ui.draw.clip
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.Color
import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.platform.LocalConfiguration
@ -22,13 +25,14 @@ import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp import androidx.compose.ui.unit.sp
import kotlin.math.log10
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import com.utun.chat.data.Message import com.utun.chat.data.Message
import com.utun.chat.data.NativeLib import com.utun.chat.data.NativeLib
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
@Composable @Composable
fun MessageBubble(message: Message) { fun MessageBubble(message: Message, onPlayVoice: (String) -> Unit = {}) {
val bgColor = if (message.isOutgoing) Color(0xFF4A90D9) else Color(0xFFE8E8E8) val bgColor = if (message.isOutgoing) Color(0xFF4A90D9) else Color(0xFFE8E8E8)
val textColor = if (message.isOutgoing) Color.White else Color.Black val textColor = if (message.isOutgoing) Color.White else Color.Black
val alignment = if (message.isOutgoing) Arrangement.End else Arrangement.Start val alignment = if (message.isOutgoing) Arrangement.End else Arrangement.Start
@ -48,21 +52,15 @@ fun MessageBubble(message: Message) {
.background(bgColor) .background(bgColor)
.padding(12.dp) .padding(12.dp)
) { ) {
Text(message.author, fontSize = 13.sp, fontWeight = FontWeight.Bold, color = textColor.copy(alpha = 0.7f)) if (!isVoice) Text(message.author, fontSize = 13.sp, fontWeight = FontWeight.Bold, color = textColor.copy(alpha = 0.7f))
Spacer(Modifier.height(2.dp)) Spacer(Modifier.height(2.dp))
when { when {
isVoice -> { isVoice -> VoiceBubble(message, textColor, onPlayVoice)
Row(verticalAlignment = Alignment.CenterVertically) { isMedia -> Text("\uD83D\uDCC4 ${message.text}", fontSize = 15.sp, color = textColor)
Text("\u25B6", fontSize = 20.sp, color = textColor)
Spacer(Modifier.width(8.dp))
Text("Voice message", fontSize = 15.sp, color = textColor)
}
}
isMedia -> {
Text("\uD83D\uDCC4 ${message.text}", fontSize = 15.sp, color = textColor)
}
else -> { else -> {
Text(message.author, fontSize = 13.sp, fontWeight = FontWeight.Bold, color = textColor.copy(alpha = 0.7f))
Spacer(Modifier.height(2.dp))
Text(message.text, fontSize = 17.sp, color = textColor) Text(message.text, fontSize = 17.sp, color = textColor)
} }
} }
@ -78,6 +76,62 @@ fun MessageBubble(message: Message) {
} }
} }
@Composable
private fun VoiceBubble(message: Message, textColor: Color, onPlay: (String) -> Unit) {
val durationSec = message.voiceDurationMs / 1000f
val wf = message.waveform
val wfCount = wf.size
Row(verticalAlignment = Alignment.CenterVertically) {
/* Play button */
Box(
modifier = Modifier.size(32.dp).background(Color(0xFF3390EC), CircleShape)
.clickable { if (message.filePath.isNotEmpty()) onPlay(message.filePath) },
contentAlignment = Alignment.Center
) {
Text("\u25B6", fontSize = 14.sp, color = Color.White)
}
Spacer(Modifier.width(8.dp))
/* Waveform */
if (wfCount >= 2) {
Box(modifier = Modifier.weight(1f).height(32.dp)) {
Canvas(modifier = Modifier.fillMaxSize()) {
val barW = size.width / wfCount
val ph = size.height.toFloat()
for (i in 0 until wfCount) {
val v = wf[i]
val db = if (v > 0.00001f) 20f * kotlin.math.log10(v.toDouble()).toFloat() else -100f
val logV = ((db + 60f) / 60f).coerceIn(0f, 1f)
val barH = (2f + logV * (ph - 2f)).coerceAtMost(ph)
val x = i.toFloat() * barW
val y = ph - barH
drawRect(
color = Color(0xFF3390EC).copy(alpha = 0.5f),
topLeft = Offset(x, y),
size = Size(barW * 0.8f, barH)
)
}
}
}
Spacer(Modifier.width(4.dp))
}
/* Duration */
Text(
formatDurationMs(message.voiceDurationMs),
fontSize = 13.sp,
color = textColor.copy(alpha = 0.7f),
modifier = Modifier.padding(start = 4.dp)
)
}
}
private fun formatDurationMs(ms: Int): String {
val secs = ms / 1000
return "%d:%02d".format(secs / 60, secs % 60)
}
private val LOCK_SWIPE_THRESHOLD_DP = 80f private val LOCK_SWIPE_THRESHOLD_DP = 80f
@Composable @Composable

4
tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/ChatScreen.kt

@ -63,7 +63,9 @@ fun ChatScreen(
state = listState state = listState
) { ) {
items(messages.size) { idx -> items(messages.size) { idx ->
MessageBubble(messages[idx]) MessageBubble(messages[idx], onPlayVoice = { path ->
viewModel.playVoiceMessage(path)
})
} }
item { Spacer(Modifier.height(8.dp)) } item { Spacer(Modifier.height(8.dp)) }
} }

5
tools/chatgui-android/app/src/main/java/com/utun/chat/viewmodel/ChatViewModel.kt

@ -264,6 +264,11 @@ class ChatViewModel : ViewModel() {
NativeLib.voiceSetCompressorConfig(maxGainDb, lookbackMs, lookaheadMs, riseRate, targetLevelDb) NativeLib.voiceSetCompressorConfig(maxGainDb, lookbackMs, lookaheadMs, riseRate, targetLevelDb)
} }
fun playVoiceMessage(filePath: String) {
LogManager.addLog("INFO", "VM", "play voice: $filePath")
audioRecorder.playVoiceFile(filePath)
}
fun clearState() { fun clearState() {
repo?.close() repo?.close()
repo = null repo = null

31
tools/chatgui-android/jni_bridge/android_jni_bridge.c

@ -52,6 +52,8 @@ static int blevel_to_android(int blevel) {
static utun_log_fn g_on_log = NULL; static utun_log_fn g_on_log = NULL;
static utun_event_fn g_on_event = NULL; static utun_event_fn g_on_event = NULL;
static char g_db_path[512];
static void bridge_log(int level, const char* fmt, ...) { static void bridge_log(int level, const char* fmt, ...) {
va_list ap; va_list ap;
va_start(ap, fmt); va_start(ap, fmt);
@ -356,7 +358,7 @@ char* utun_bridge_get_messages_json(const char* channel_id, int limit) {
msg_table_name(channel_id, tbl_msg, sizeof(tbl_msg)); msg_table_name(channel_id, tbl_msg, sizeof(tbl_msg));
char sql[256]; char sql[256];
snprintf(sql, sizeof(sql), "SELECT id, node_id, data, timestamp FROM \"%s\" ORDER BY timestamp ASC LIMIT %d", tbl_msg, limit); snprintf(sql, sizeof(sql), "SELECT id, node_id, data, timestamp, local_attrs FROM \"%s\" ORDER BY timestamp ASC LIMIT %d", tbl_msg, limit);
sqlite3_stmt* st = NULL; sqlite3_stmt* st = NULL;
if (sqlite3_prepare_v2(db, sql, -1, &st, NULL) != SQLITE_OK) return strdup("[]"); if (sqlite3_prepare_v2(db, sql, -1, &st, NULL) != SQLITE_OK) return strdup("[]");
@ -373,6 +375,7 @@ char* utun_bridge_get_messages_json(const char* channel_id, int limit) {
int64_t ts = sqlite3_column_int64(st, 3); int64_t ts = sqlite3_column_int64(st, 3);
const uint8_t* dptr = sqlite3_column_blob(st, 2); const uint8_t* dptr = sqlite3_column_blob(st, 2);
int dlen = sqlite3_column_bytes(st, 2); int dlen = sqlite3_column_bytes(st, 2);
const char* lattr = (const char*)sqlite3_column_text(st, 4);
char txt[256] = "", ct[64] = "text/plain"; char txt[256] = "", ct[64] = "text/plain";
if (dptr && dlen > 0) { if (dptr && dlen > 0) {
@ -392,12 +395,29 @@ char* utun_bridge_get_messages_json(const char* channel_id, int limit) {
if (!author_name[0]) if (!author_name[0])
snprintf(author_name, sizeof(author_name), "0x%016llx", (unsigned long long)node_id); snprintf(author_name, sizeof(author_name), "0x%016llx", (unsigned long long)node_id);
size_t needed = snprintf(NULL, 0, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"}", /* build filePath for voice/attachment messages */
sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct); char file_path[1024] = "";
if (lattr) {
const char* fps = strstr(lattr, "\"fp\":\"");
if (fps) { fps += 6;
char* fpe = strchr((char*)fps, '"');
if (fpe) { size_t fpl = (size_t)(fpe - fps);
snprintf(file_path, sizeof(file_path), "%s/media/%s/%.*s",
g_db_path, channel_id, (int)fpl, fps);
}
}
}
char* esc_fp = file_path[0] ? json_escape_alloc(file_path) : NULL;
size_t needed = snprintf(NULL, 0, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"%s%s%s}",
sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct,
esc_fp ? ",\"filePath\":\"" : "", esc_fp ? esc_fp : "", esc_fp ? "\"" : "");
while (pos + needed + 2 > cap) { cap *= 2; char* tmp = u_realloc(json, cap); if (!tmp) break; json = tmp; } while (pos + needed + 2 > cap) { cap *= 2; char* tmp = u_realloc(json, cap); if (!tmp) break; json = tmp; }
if (pos + needed + 2 <= cap) if (pos + needed + 2 <= cap)
pos += (size_t)snprintf(json + pos, cap - pos, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"}", pos += (size_t)snprintf(json + pos, cap - pos, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"%s%s%s}",
sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct); sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct,
esc_fp ? ",\"filePath\":\"" : "", esc_fp ? esc_fp : "", esc_fp ? "\"" : "");
if (esc_fp) u_free(esc_fp);
u_free(esc_ct); u_free(esc_ct);
u_free(esc_txt); u_free(esc_txt);
} }
@ -584,7 +604,6 @@ static jobject g_config_callback = NULL;
static jobject g_log_callback = NULL; static jobject g_log_callback = NULL;
static jobject g_event_callback = NULL; static jobject g_event_callback = NULL;
static jmethodID g_event_mid = NULL; static jmethodID g_event_mid = NULL;
static char g_db_path[512];
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) { JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) {
(void)reserved; (void)reserved;

49
tools/chatgui-android/libutun_lite/voice_recorder.c

@ -47,6 +47,9 @@ struct voice_recorder {
int preset; int preset;
float peak_level; float peak_level;
float waveform_levels[100];
int waveform_count;
int64_t next_wf_time_ms;
int64_t start_time_ms; int64_t start_time_ms;
}; };
@ -54,6 +57,11 @@ struct voice_recorder {
static struct voice_recorder* g_rec = NULL; static struct voice_recorder* g_rec = NULL;
static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
static int64_t now_ms(void) {
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL;
}
int voice_recorder_init(const char* db_path) { int voice_recorder_init(const char* db_path) {
pthread_mutex_lock(&g_init_mtx); pthread_mutex_lock(&g_init_mtx);
if (g_rec) { pthread_mutex_unlock(&g_init_mtx); return 0; } if (g_rec) { pthread_mutex_unlock(&g_init_mtx); return 0; }
@ -128,6 +136,8 @@ int voice_recorder_start(const char* channel_id, int sample_rate, int channels)
g_rec->pcm_count = 0; g_rec->pcm_count = 0;
g_rec->active = 1; g_rec->active = 1;
g_rec->waveform_count = 0;
g_rec->next_wf_time_ms = 0;
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
g_rec->start_time_ms = ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL; g_rec->start_time_ms = ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL;
@ -158,6 +168,14 @@ int voice_recorder_feed(const int16_t* samples, int count) {
sum_sq += v * v; float av = fabsf(v); if (av > peak) peak = av; sum_sq += v * v; float av = fabsf(v); if (av > peak) peak = av;
} }
g_rec->peak_level = (sqrtf(sum_sq / (float)count) * 2.0f + peak) / 2.0f; g_rec->peak_level = (sqrtf(sum_sq / (float)count) * 2.0f + peak) / 2.0f;
/* accumulate waveform level every ~100ms */
{ int64_t now = now_ms();
if (g_rec->waveform_count < 100 && now >= g_rec->next_wf_time_ms) {
g_rec->waveform_levels[g_rec->waveform_count++] = g_rec->peak_level;
g_rec->next_wf_time_ms = now + 100;
}
}
} }
if (g_rec->compressor && g_rec->compressor_enabled) { if (g_rec->compressor && g_rec->compressor_enabled) {
@ -189,11 +207,6 @@ static uint32_t media_calc_block_size(uint64_t file_size) {
return (uint32_t)target; return (uint32_t)target;
} }
static int64_t now_ms(void) {
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL;
}
static void voice_cleanup_locked(struct voice_recorder* rec) { static void voice_cleanup_locked(struct voice_recorder* rec) {
u_free(rec->pcm_buffer); rec->pcm_buffer = NULL; u_free(rec->pcm_buffer); rec->pcm_buffer = NULL;
rec->pcm_count = 0; rec->pcm_cap = 0; rec->pcm_count = 0; rec->pcm_cap = 0;
@ -383,8 +396,27 @@ int voice_recorder_stop(int* out_duration_ms) {
fclose(src); fclose(src);
unlink(temp_path); unlink(temp_path);
/* build and post chat_msg_submit */ /* build and post chat_msg_submit with waveform data */
struct chat_msg_submit* req = u_calloc(1, sizeof(struct chat_msg_submit) + 1); char wf_buf[1024];
int wf_off = 0;
{ int wfN = g_rec->waveform_count > 0 ? g_rec->waveform_count : 1;
int step100 = wfN > 100 ? wfN : 100;
for (int i = 0; i < 100; i++) {
int s = i * wfN / 100, e = (i + 1) * wfN / 100;
if (s >= wfN) s = wfN - 1; if (e <= s) e = s + 1; if (e > wfN) e = wfN;
float sum = 0; int cnt = 0;
for (int j = s; j < e; j++) { sum += g_rec->waveform_levels[j]; cnt++; }
float v = cnt > 0 ? sum / cnt : 0;
if (i > 0) wf_off += snprintf(wf_buf + wf_off, sizeof(wf_buf) - wf_off, ",");
wf_off += snprintf(wf_buf + wf_off, sizeof(wf_buf) - wf_off, "%.3f", v);
}
}
char wf_data[1280];
int wf_data_len = snprintf(wf_data, sizeof(wf_data), "wf=%s;dur=%lld;", wf_buf, (long long)elapsed_ms);
size_t total_data_len = (size_t)wf_data_len;
struct chat_msg_submit* req = u_calloc(1, sizeof(struct chat_msg_submit) + total_data_len + 1);
if (!req) { voice_cleanup_locked(g_rec); pthread_mutex_unlock(&g_rec->mtx); pthread_mutex_unlock(&g_init_mtx); return -1; } if (!req) { voice_cleanup_locked(g_rec); pthread_mutex_unlock(&g_rec->mtx); pthread_mutex_unlock(&g_init_mtx); return -1; }
snprintf(req->channel_id, sizeof(req->channel_id), "%s", g_rec->channel_id); snprintf(req->channel_id, sizeof(req->channel_id), "%s", g_rec->channel_id);
@ -395,7 +427,8 @@ int voice_recorder_stop(int* out_duration_ms) {
snprintf(req->media_ext, sizeof(req->media_ext), "opus"); snprintf(req->media_ext, sizeof(req->media_ext), "opus");
req->media_num_blocks = (uint32_t)num_blocks; req->media_num_blocks = (uint32_t)num_blocks;
req->data = (uint8_t*)(req + 1); req->data = (uint8_t*)(req + 1);
req->data_len = 0; req->data_len = (uint32_t)total_data_len;
memcpy(req->data, wf_data, total_data_len);
req->timestamp = 0; req->timestamp = 0;
DEBUG_INFO(DEBUG_CATEGORY_GENERAL, "voice_recorder_stop: posting media ch=%s dt=%s stem=%s blocks=%d", DEBUG_INFO(DEBUG_CATEGORY_GENERAL, "voice_recorder_stop: posting media ch=%s dt=%s stem=%s blocks=%d",

Loading…
Cancel
Save