Files
utun2/src/chat/chat_whisper.c
T

531 lines
19 KiB
C

/*
* chat_whisper.c — Whisper speech-to-text интеграция для голосовых сообщений
*
* Поток данных:
* on_msg_inserted (content_type == "voice")
* → md_auto_download (скачиваем .wav/.opus)
* → chat_whisper_transcribe_async
* → [worker thread]: PCM decode → resample 16kHz → whisper_full → текст
* → [uasync callback]: chat_core_submit_message(ct="voice_transcription")
*/
#include "chat_whisper.h"
#include "chat_setting.h"
#include "../media_async/media_async.h"
#include "../../lib/u_async.h"
#include "../../lib/mem.h"
#include "../../lib/platform_compat.h"
#include "../../lib/debug_config.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <math.h>
#ifdef HAVE_WHISPER
#include <whisper.h>
#endif
#define CW_ID "chat_whisper"
/* ─── состояние ─── */
#ifdef HAVE_WHISPER
static struct whisper_context *g_whisper_ctx = NULL;
#endif
static int g_initialized = 0;
/* очередь транскрипций: whisper не thread-safe, обрабатываем по одной */
struct wh_job {
struct UTUN_INSTANCE* inst;
char audio_path[1024];
char channel_id[64];
uint64_t reply_to_ts;
uint64_t reply_to_node;
chat_whisper_done_fn done_cb;
void* done_arg;
struct media_async* ma;
struct UASYNC* ua;
};
static struct wh_job* g_pending_job = NULL;
static int g_processing = 0;
/* ─── wav reader ─── */
struct wav_pcm {
float* samples; /* float32 PCM моно */
int n_samples; /* количество семплов */
int sample_rate; /* исходный sample rate */
};
static int read_u16_le(FILE* f, uint16_t* v) { return fread(v, 2, 1, f) == 1; }
static int read_u32_le(FILE* f, uint32_t* v) { return fread(v, 4, 1, f) == 1; }
static struct wav_pcm* wav_read(const char* path) {
FILE* f = fopen(path, "rb");
if (!f) { DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: cannot open %s", CW_ID, path); return NULL; }
char riff[4], wave[4];
if (fread(riff, 1, 4, f) != 4 || memcmp(riff, "RIFF", 4) != 0) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: no RIFF header", CW_ID); return NULL; }
uint32_t file_size; read_u32_le(f, &file_size);
if (fread(wave, 1, 4, f) != 4 || memcmp(wave, "WAVE", 4) != 0) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: no WAVE", CW_ID); return NULL; }
uint16_t audio_format = 0, channels = 0;
uint32_t sample_rate = 0, byte_rate = 0, data_size = 0;
uint16_t block_align = 0, bits_per_sample = 0;
for (;;) {
char chunk_id[4]; size_t rd = fread(chunk_id, 1, 4, f);
if (rd < 4) break;
uint32_t chunk_size; read_u32_le(f, &chunk_size);
if (memcmp(chunk_id, "fmt ", 4) == 0) {
read_u16_le(f, &audio_format); read_u16_le(f, &channels);
read_u32_le(f, &sample_rate); read_u32_le(f, &byte_rate);
read_u16_le(f, &block_align); read_u16_le(f, &bits_per_sample);
if (chunk_size > 16) fseek(f, chunk_size - 16, SEEK_CUR);
} else if (memcmp(chunk_id, "data", 4) == 0) {
data_size = chunk_size;
break;
} else {
fseek(f, chunk_size, SEEK_CUR);
}
}
if (audio_format != 1) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: not PCM (fmt=%u)", CW_ID, audio_format); return NULL; }
if (channels < 1 || channels > 2) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: bad channels %u", CW_ID, channels); return NULL; }
if (bits_per_sample != 16) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: not 16-bit PCM", CW_ID); return NULL; }
if (sample_rate < 8000 || sample_rate > 48000) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: bad sample_rate %u", CW_ID, sample_rate); return NULL; }
int frame_size = (int)(bits_per_sample / 8) * channels;
int total_frames = (int)(data_size / frame_size);
struct wav_pcm* pcm = u_calloc(1, sizeof(*pcm));
if (!pcm) { fclose(f); return NULL; }
pcm->samples = u_malloc((size_t)total_frames * sizeof(float));
if (!pcm->samples) { fclose(f); u_free(pcm); return NULL; }
pcm->sample_rate = (int)sample_rate;
pcm->n_samples = 0;
for (int i = 0; i < total_frames; i++) {
int16_t raw[2] = {0, 0};
if (fread(raw, frame_size < 4 ? 2 : frame_size, 1, f) != 1) break;
float s = (float)raw[0];
if (channels == 2) s = (s + (float)raw[1]) * 0.5f;
pcm->samples[pcm->n_samples++] = s / 32768.0f;
}
fclose(f);
DEBUG_DEBUG(DEBUG_CATEGORY_CHAT, "%s: wav_read %s: %d samples %dHz mono", CW_ID, path, pcm->n_samples, pcm->sample_rate);
return pcm;
}
static void wav_free(struct wav_pcm* pcm) {
if (!pcm) return;
u_free(pcm->samples);
u_free(pcm);
}
/* ─── opus frame reader (простой header: "OPUS" + sample_rate(4LE) + channels(4LE) + frames...) ─── */
#include "opus_codec.h"
struct opus_pcm {
float* samples;
int n_samples;
int sample_rate;
};
static struct opus_pcm* opus_read(const char* path) {
FILE* f = fopen(path, "rb");
if (!f) { DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: cannot open %s", CW_ID, path); return NULL; }
char magic[4];
if (fread(magic, 1, 4, f) != 4 || memcmp(magic, "OPUS", 4) != 0) { fclose(f); return NULL; }
uint32_t sr, ch;
if (fread(&sr, 4, 1, f) != 1 || fread(&ch, 4, 1, f) != 1) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: bad opus header", CW_ID); return NULL; }
opus_codec_decoder_t* dec = opus_codec_decoder_create((int)sr, (int)ch);
if (!dec) { fclose(f); DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: opus decoder create failed", CW_ID); return NULL; }
int frame_samples = (int)sr * 20 / 1000;
size_t alloc_samples = 0;
float* buffer = NULL;
int total_samples = 0;
for (;;) {
uint8_t len_buf[2];
if (fread(len_buf, 1, 2, f) != 2) break;
uint16_t pkt_len = (uint16_t)((len_buf[0] << 8) | len_buf[1]);
if (pkt_len == 0 || pkt_len > 16384) break;
uint8_t* pkt = u_malloc(pkt_len);
if (!pkt) break;
if (fread(pkt, 1, pkt_len, f) != pkt_len) { u_free(pkt); break; }
if (total_samples + frame_samples > (int)alloc_samples) {
alloc_samples = (total_samples + frame_samples + 48000) & ~1023;
float* nb = u_realloc(buffer, alloc_samples * sizeof(float));
if (!nb) { u_free(pkt); break; }
buffer = nb;
}
int16_t pcm_buf[5760]; /* max frame at 48kHz stereo */
int decoded = opus_codec_decode(dec, pkt, (int)pkt_len, pcm_buf, frame_samples);
u_free(pkt);
if (decoded <= 0) continue;
for (int k = 0; k < decoded; k++)
buffer[total_samples + k] = (float)pcm_buf[k] / 32768.0f;
total_samples += decoded;
}
fclose(f);
opus_codec_decoder_destroy(dec);
struct opus_pcm* pcm = u_calloc(1, sizeof(*pcm));
if (!pcm) { u_free(buffer); return NULL; }
pcm->samples = buffer;
pcm->n_samples = total_samples;
pcm->sample_rate = (int)sr;
DEBUG_DEBUG(DEBUG_CATEGORY_CHAT, "%s: opus_read %s: %d samples %dHz", CW_ID, path, total_samples, (int)sr);
return pcm;
}
static void opus_free(struct opus_pcm* pcm) {
if (!pcm) return;
u_free(pcm->samples);
u_free(pcm);
}
/* ─── ресемпл в 16kHz float32 моно (линейная интерполяция) ─── */
static float* resample_16k(const float* src, int n_src, int src_rate, int* n_dst_out) {
if (src_rate == 16000) {
float* dst = u_malloc((size_t)n_src * sizeof(float));
if (dst) { memcpy(dst, src, (size_t)n_src * sizeof(float)); *n_dst_out = n_src; }
return dst;
}
double ratio = 16000.0 / (double)src_rate;
int n_dst = (int)((double)n_src * ratio) + 16;
float* dst = u_malloc((size_t)n_dst * sizeof(float));
if (!dst) { *n_dst_out = 0; return NULL; }
for (int i = 0; i < n_dst; i++) {
double pos = (double)i / ratio;
int idx = (int)pos;
double frac = pos - (double)idx;
float v0 = (idx < n_src) ? src[idx] : 0.0f;
float v1 = (idx + 1 < n_src) ? src[idx + 1] : v0;
dst[i] = v0 + (float)((v1 - v0) * frac);
}
*n_dst_out = n_dst;
return dst;
}
/* ─── worker thread ─── */
struct wh_work_ctx;
static void chat_whisper_transcribe_async_impl(
struct UTUN_INSTANCE* inst,
struct media_async* ma, struct UASYNC* ua,
const char* audio_path, const char* channel_id,
uint64_t reply_to_ts, uint64_t reply_to_node,
chat_whisper_done_fn done_cb, void* done_arg);
static void chat_whisper_transcribe_async(
struct UTUN_INSTANCE* inst,
struct media_async* ma, struct UASYNC* ua,
const char* audio_path, const char* channel_id,
uint64_t reply_to_ts, uint64_t reply_to_node,
chat_whisper_done_fn done_cb, void* done_arg);
struct wh_work_ctx {
struct wh_job job;
char* text; /* результат транскрипции (u_malloc) */
int err;
struct UASYNC* ua;
};
static void wh_work_fn(void* raw) {
struct wh_work_ctx* w = (struct wh_work_ctx*)raw;
#ifndef HAVE_WHISPER
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: whisper not compiled in", CW_ID);
w->err = -1;
return;
#else
if (!g_whisper_ctx) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: whisper not initialized", CW_ID);
w->err = -1;
return;
}
/* 1. Load PCM from file */
float* pcm_samples = NULL;
int pcm_n = 0;
int pcm_rate = 0;
/* try wav first, then opus */
struct wav_pcm* wpcm = wav_read(w->job.audio_path);
if (wpcm) {
pcm_samples = wpcm->samples; pcm_n = wpcm->n_samples; pcm_rate = wpcm->sample_rate;
wpcm->samples = NULL; wav_free(wpcm);
} else {
struct opus_pcm* opcm = opus_read(w->job.audio_path);
if (opcm) {
pcm_samples = opcm->samples; pcm_n = opcm->n_samples; pcm_rate = opcm->sample_rate;
opcm->samples = NULL; opus_free(opcm);
}
}
if (!pcm_samples || pcm_n <= 0) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: cannot decode %s", CW_ID, w->job.audio_path);
w->err = -1;
return;
}
/* 2. Resample to 16kHz */
int n_16k = 0;
float* samples_16k = resample_16k(pcm_samples, pcm_n, pcm_rate, &n_16k);
u_free(pcm_samples);
if (!samples_16k || n_16k <= 0) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: resample failed", CW_ID);
w->err = -1;
return;
}
DEBUG_DEBUG(DEBUG_CATEGORY_CHAT, "%s: %d samples @ 16kHz, running whisper_full", CW_ID, n_16k);
/* 3. Whisper */
struct whisper_full_params wparams = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
wparams.language = "ru";
wparams.n_threads = 4;
wparams.no_timestamps = 1;
wparams.single_segment = 1;
int ret = whisper_full(g_whisper_ctx, wparams, samples_16k, n_16k);
u_free(samples_16k);
if (ret != 0) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: whisper_full failed ret=%d", CW_ID, ret);
w->err = -1;
return;
}
/* 4. Collect text */
int n_seg = whisper_full_n_segments(g_whisper_ctx);
if (n_seg <= 0) {
DEBUG_WARN(DEBUG_CATEGORY_CHAT, "%s: whisper returned no segments", CW_ID);
w->text = u_strdup("");
} else {
size_t total = 0;
for (int i = 0; i < n_seg; i++) {
const char* seg = whisper_full_get_segment_text(g_whisper_ctx, i);
if (seg) total += strlen(seg);
}
w->text = u_malloc(total + 1);
if (w->text) {
w->text[0] = '\0';
for (int i = 0; i < n_seg; i++) {
const char* seg = whisper_full_get_segment_text(g_whisper_ctx, i);
if (seg) strcat(w->text, seg);
}
DEBUG_INFO(DEBUG_CATEGORY_CHAT, "%s: transcribed [%s]", CW_ID, w->text);
}
}
w->err = 0;
#endif /* HAVE_WHISPER */
}
static void wh_done_fn(void* raw, int err) {
struct wh_work_ctx* w = (struct wh_work_ctx*)raw;
if (err) w->err = err;
g_processing = 0;
if (w->job.done_cb) {
w->job.done_cb(w->job.done_arg, w->job.channel_id,
w->text, w->err,
w->job.reply_to_ts, w->job.reply_to_node);
}
u_free(w->text);
u_free(w);
/* запустить следующую из очереди */
if (g_pending_job) {
struct wh_job* nj = g_pending_job;
g_pending_job = NULL;
chat_whisper_transcribe_async_impl(nj->inst, nj->ma, nj->ua, nj->audio_path, nj->channel_id,
nj->reply_to_ts, nj->reply_to_node,
nj->done_cb, nj->done_arg);
u_free(nj);
}
}
/* ─── авто-поиск модели ─── */
static int find_model_path(struct UTUN_INSTANCE* inst, char* out, size_t out_sz) {
const char* setting = chat_setting_get(inst, "whisper_model_path");
if (setting && setting[0] != '\0') {
/* если путь относительный, не проверяем доступ — вернём как есть */
if (access(setting, R_OK) == 0) { snprintf(out, out_sz, "%s", setting); return 0; }
/* если задан явно но не доступен — ошибка */
DEBUG_WARN(DEBUG_CATEGORY_CHAT, "%s: model_path from setting not accessible: %s", CW_ID, setting);
}
const char* search_paths[] = {
"ggml-base.bin",
"/usr/share/utun/models/ggml-base.bin",
NULL
};
const char* home = getenv("HOME");
char home_path[512];
if (home) { snprintf(home_path, sizeof(home_path), "%s/.local/share/utun/models/ggml-base.bin", home); }
for (int i = 0; search_paths[i]; i++) {
if (access(search_paths[i], R_OK) == 0) { snprintf(out, out_sz, "%s", search_paths[i]); return 0; }
}
if (home && access(home_path, R_OK) == 0) { snprintf(out, out_sz, "%s", home_path); return 0; }
return -1;
}
/* ─── публичный API ─── */
int chat_whisper_init(struct UTUN_INSTANCE* inst) {
if (g_initialized) return 0;
g_initialized = 1;
#ifndef HAVE_WHISPER
DEBUG_WARN(DEBUG_CATEGORY_CHAT, "%s: compiled without whisper support", CW_ID);
return -1;
#else
if (!chat_setting_get_int(inst, "whisper_enabled", 0)) {
DEBUG_INFO(DEBUG_CATEGORY_CHAT, "%s: whisper disabled in settings", CW_ID);
return -1;
}
char model_path[512];
if (find_model_path(inst, model_path, sizeof(model_path)) != 0) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: model not found (set whisper_model_path)", CW_ID);
return -1;
}
DEBUG_INFO(DEBUG_CATEGORY_CHAT, "%s: loading model %s..." , CW_ID, model_path);
struct whisper_context_params cparams = whisper_context_default_params();
cparams.use_gpu = 0;
g_whisper_ctx = whisper_init_from_file_with_params(model_path, cparams);
if (!g_whisper_ctx) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: whisper_init_from_file failed for %s", CW_ID, model_path);
return -1;
}
DEBUG_INFO(DEBUG_CATEGORY_CHAT, "%s: model loaded successfully", CW_ID);
g_chat_whisper_trigger = chat_whisper_transcribe_async;
return 0;
#endif /* HAVE_WHISPER */
}
int chat_whisper_available(struct UTUN_INSTANCE* inst) {
#ifdef HAVE_WHISPER
return g_whisper_ctx != NULL && chat_setting_get_int(inst, "whisper_enabled", 0);
#else
return 0;
#endif
}
void chat_whisper_destroy(void) {
#ifdef HAVE_WHISPER
if (g_whisper_ctx) {
whisper_free(g_whisper_ctx);
g_whisper_ctx = NULL;
DEBUG_INFO(DEBUG_CATEGORY_CHAT, "%s: model unloaded", CW_ID);
}
#endif
/* отменяем ожидающие задачи */
if (g_pending_job) { u_free(g_pending_job); g_pending_job = NULL; }
g_initialized = 0;
g_processing = 0;
}
void chat_whisper_transcribe_async(
struct UTUN_INSTANCE* inst,
struct media_async* ma, struct UASYNC* ua,
const char* audio_path, const char* channel_id,
uint64_t reply_to_ts, uint64_t reply_to_node,
chat_whisper_done_fn done_cb, void* done_arg)
{
chat_whisper_transcribe_async_impl(inst, ma, ua, audio_path, channel_id,
reply_to_ts, reply_to_node,
done_cb, done_arg);
}
static void chat_whisper_transcribe_async_impl(
struct UTUN_INSTANCE* inst,
struct media_async* ma, struct UASYNC* ua,
const char* audio_path, const char* channel_id,
uint64_t reply_to_ts, uint64_t reply_to_node,
chat_whisper_done_fn done_cb, void* done_arg)
{
if (!audio_path || !channel_id || !done_cb || !ma || !ua) {
DEBUG_ERROR(DEBUG_CATEGORY_CHAT, "%s: invalid args", CW_ID);
if (done_cb) done_cb(done_arg, channel_id ? channel_id : "", NULL, -1, reply_to_ts, reply_to_node);
return;
}
if (!chat_whisper_available(inst)) {
int rc = chat_whisper_init(inst);
if (rc != 0) {
DEBUG_WARN(DEBUG_CATEGORY_CHAT, "%s: whisper not available, skip transcription", CW_ID);
done_cb(done_arg, channel_id, NULL, -1, reply_to_ts, reply_to_node);
return;
}
}
if (g_processing) {
/* поставить в очередь */
if (g_pending_job) { u_free(g_pending_job); } /* заменяем */
g_pending_job = u_malloc(sizeof(*g_pending_job));
if (!g_pending_job) { done_cb(done_arg, channel_id, NULL, -1, reply_to_ts, reply_to_node); return; }
g_pending_job->inst = inst;
snprintf(g_pending_job->audio_path, sizeof(g_pending_job->audio_path), "%s", audio_path);
snprintf(g_pending_job->channel_id, sizeof(g_pending_job->channel_id), "%s", channel_id);
g_pending_job->reply_to_ts = reply_to_ts;
g_pending_job->reply_to_node = reply_to_node;
g_pending_job->done_cb = done_cb;
g_pending_job->done_arg = done_arg;
g_pending_job->ma = ma;
g_pending_job->ua = ua;
DEBUG_DEBUG(DEBUG_CATEGORY_CHAT, "%s: queued transcription for ch=%s", CW_ID, channel_id);
return;
}
g_processing = 1;
struct wh_work_ctx* w = u_calloc(1, sizeof(*w));
if (!w) { g_processing = 0; done_cb(done_arg, channel_id, NULL, -1, reply_to_ts, reply_to_node); return; }
snprintf(w->job.audio_path, sizeof(w->job.audio_path), "%s", audio_path);
snprintf(w->job.channel_id, sizeof(w->job.channel_id), "%s", channel_id);
w->job.reply_to_ts = reply_to_ts;
w->job.reply_to_node = reply_to_node;
w->job.done_cb = done_cb;
w->job.done_arg = done_arg;
w->job.ma = ma;
w->job.ua = ua;
w->ua = ua;
w->err = 0;
DEBUG_INFO(DEBUG_CATEGORY_CHAT, "%s: starting transcription ch=%s file=%s", CW_ID, channel_id, audio_path);
media_async_submit(ma, ua, wh_work_fn, w, wh_done_fn, w);
}