@ -30,6 +30,8 @@ chat_whisper_trigger_fn g_chat_whisper_trigger = NULL;
void chat_core_submit_media_message ( struct chat_msg_submit * req ) ;
static void wh_transcribe_done_cb ( void * arg , const char * ch_id , const char * text , int err ,
uint64_t reply_ts , uint64_t reply_node ) ;
static void chat_media_cache_cleanup_orphans ( void ) ;
static void chat_media_cache_enforce ( void ) ;
/* ─── отправка сообщения (GUI → uasync) ─── */
@ -574,6 +576,9 @@ static void md_download_done_cb(void* arg, int err) {
ctx - > ts , ctx - > author_node_id ,
wh_transcribe_done_cb , NULL ) ;
}
/* после докачки — применить лимит медиакеша */
chat_media_cache_enforce ( ) ;
} else {
DEBUG_ERROR ( DEBUG_CATEGORY_GENERAL , " %s: media download failed ch=%s err=%d " , CC_ID , ctx - > channel_id , err ) ;
/* пометить сообщение ошибкой, чтобы UI показал «повторить» */
@ -750,25 +755,25 @@ static int md_start_download(struct UTUN_INSTANCE* inst,
return 0 ;
}
static void md_auto_download ( struct UTUN_INSTANCE * inst ,
static int md_auto_download ( struct UTUN_INSTANCE * inst ,
const char * data_str , size_t data_len ,
const char * ch_id , const char * base_filename ,
uint64_t ts , const uint8_t * author_sig ,
uint64_t author , int64_t msg_id ,
const char * content_type ) {
if ( ! chat_setting_get_int ( " storage_autoload " , 1 ) ) return ;
if ( ! chat_setting_get_int ( " storage_autoload " , 1 ) ) return 0 ;
uint64_t max_size = g_cc . inst - > config - > global . chatserver_storage_unit_size ;
const char * p = data_str ;
while ( * p & & * p ! = ' | ' ) p + + ;
if ( ! * p ) return ; p + + ;
if ( ! * p ) return 0 ; p + + ;
long long fsize = strtoll ( p , ( char * * ) & p , 10 ) ;
if ( max_size > 0 & & fsize > ( long long ) max_size ) {
DEBUG_INFO ( DEBUG_CATEGORY_CHAT_SYNC , " %s: media too large (%lld > %llu bytes), skip auto-download " ,
CC_ID , fsize , ( unsigned long long ) max_size ) ;
return ;
DEBUG_WARN ( DEBUG_CATEGORY_GENERAL , " %s: media too large (%lld > %llu bytes), skip auto-download ch=%s file=%s " ,
CC_ID , fsize , ( unsigned long long ) max_size , ch_id , base_filename ) ;
return 0 ;
}
md_start_download ( inst , data_str , data_len , ch_id , base_filename , ts , author_sig , author , msg_id , content_type ) ;
return md_start_download ( inst , data_str , data_len , ch_id , base_filename , ts , author_sig , author , msg_id , content_type ) = = 0 ? 1 : 0 ;
}
/* ─── db_sync callback ─── */
@ -1041,7 +1046,7 @@ void chat_media_autodownload_backfill(void) {
if ( last_slash ) snprintf ( media_base , sizeof ( media_base ) , " %.*s " , ( int ) ( last_slash - g_cc . db_path ) , g_cc . db_path ) ;
else snprintf ( media_base , sizeof ( media_base ) , " %s " , g_cc . db_path ) ;
int scanned = 0 , present = 0 , started = 0 ;
int scanned = 0 , present = 0 , started = 0 , skipped = 0 ;
sqlite3_stmt * cs = NULL ;
if ( sqlite3_prepare_v2 ( g_cc . db , " SELECT channel_id FROM channels " , - 1 , & cs , NULL ) ! = SQLITE_OK ) return ;
@ -1106,8 +1111,10 @@ void chat_media_autodownload_backfill(void) {
uint8_t author_sig [ 64 ] ; memcpy ( author_sig , sig_blob , 64 ) ;
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: autodownload missing ch=%s id=%lld file=%s size=%lld " ,
CC_ID , ch_id , ( long long ) msg_id , final_name , ( long long ) result . file_size ) ;
md_auto_download ( g_cc . inst , body , dlen , ch_id , base_filename , ts , author_sig , author , msg_id , content_type ) ;
if ( md_auto_download ( g_cc . inst , body , dlen , ch_id , base_filename , ts , author_sig , author , msg_id , content_type ) )
started + + ;
else
skipped + + ;
u_free ( body ) ;
media_index_result_free ( & result ) ;
}
@ -1115,8 +1122,245 @@ void chat_media_autodownload_backfill(void) {
}
sqlite3_finalize ( cs ) ;
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: autodownload backfill done scanned=%d present=%d started=%d days=%d " ,
CC_ID , scanned , present , started , days ) ;
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: autodownload backfill done scanned=%d present=%d started=%d skipped=%d days=%d " ,
CC_ID , scanned , present , started , skipped , days ) ;
}
/* ─── Кеш медиафайлов: очистка сирот + лимит storage_total_size ───
* В о з р а с т ф а й л а = timestamp и с х о д н о г о с о о б щ е н и я ( msg_ < ch > . timestamp ) .
* С и р о т ы ( ф а й л ы б е з с о о б щ е н и я , в к л ю ч а я . chunk_ * о с т а т к и ) у д а л я ю т с я н а с т а р т е .
* Л и м и т : п р и п р е в ы ш е н и и storage_total_size у д а л я ю т с я с а м ы е с т а р ы е ф а й л ы , н о т о л ь к о
* с т а р ш е storage_backfill_days ( и х backfill у ж е н е д о к а ч а е т н а с л е д у ю щ е м с т а р т е ) . */
struct media_msg_file {
char rel_path [ 1536 ] ; // media/<ch_id>/<final_name>
char full_path [ 1536 ] ; // <media_base>/media/<ch_id>/<final_name>
uint64_t ts_ms ;
} ;
typedef void ( * media_msg_file_cb ) ( const struct media_msg_file * f , void * arg ) ;
static void chat_media_base ( char * out , size_t out_sz ) {
const char * ls = strrchr ( g_cc . db_path , ' / ' ) ;
if ( ls ) snprintf ( out , out_sz , " %.*s " , ( int ) ( ls - g_cc . db_path ) , g_cc . db_path ) ;
else snprintf ( out , out_sz , " %s " , g_cc . db_path ) ;
}
/* Итератор по всем медиасообщениям всех каналов (без фильтра по node_id/времени). */
static void chat_media_foreach_media_message ( const char * media_base , media_msg_file_cb cb , void * arg ) {
if ( ! g_cc . db | | ! cb ) return ;
sqlite3_stmt * cs = NULL ;
if ( sqlite3_prepare_v2 ( g_cc . db , " SELECT channel_id FROM channels " , - 1 , & cs , NULL ) ! = SQLITE_OK ) return ;
while ( sqlite3_step ( cs ) = = SQLITE_ROW ) {
const char * ch_id = ( const char * ) sqlite3_column_text ( cs , 0 ) ;
if ( ! ch_id | | ! ch_id [ 0 ] ) continue ;
char tbl [ 80 ] ; msg_table_name ( ch_id , tbl , sizeof ( tbl ) ) ;
char sql [ 256 ] ;
snprintf ( sql , sizeof ( sql ) , " SELECT timestamp, data FROM \" %s \" " , tbl ) ;
sqlite3_stmt * ms = NULL ;
if ( sqlite3_prepare_v2 ( g_cc . db , sql , - 1 , & ms , NULL ) ! = SQLITE_OK ) continue ;
while ( sqlite3_step ( ms ) = = SQLITE_ROW ) {
uint64_t ts = ( uint64_t ) sqlite3_column_int64 ( ms , 0 ) ;
const char * jdata = ( const char * ) sqlite3_column_text ( ms , 1 ) ;
if ( ! jdata ) continue ;
const char * ds = strstr ( jdata , " \" d \" : \" " ) ;
if ( ! ds ) continue ;
const char * d_start = ds + 5 ;
const char * d_end = strchr ( d_start , ' " ' ) ;
if ( ! d_end ) continue ;
size_t dlen = ( size_t ) ( d_end - d_start ) ;
if ( dlen = = 0 | | dlen > 65536 ) continue ;
char * body = u_malloc ( dlen + 1 ) ;
if ( ! body ) { DEBUG_ERROR ( DEBUG_CATEGORY_GENERAL , " %s: cache scan malloc failed ch=%s " , CC_ID , ch_id ) ; continue ; }
memcpy ( body , d_start , dlen ) ; body [ dlen ] = ' \0 ' ;
struct media_index_result result ;
if ( chat_msg_parse_media_body ( body , & result ) ! = 0 ) { u_free ( body ) ; continue ; }
char content_type [ 32 ] = { 0 } ;
const char * ct = strstr ( jdata , " \" ct \" : \" " ) ;
if ( ct ) {
const char * cv = ct + 6 ;
int i = 0 ;
while ( cv [ i ] & & cv [ i ] ! = ' " ' & & i < ( int ) sizeof ( content_type ) - 1 ) content_type [ i + + ] = cv [ i ] ;
content_type [ i ] = ' \0 ' ;
}
char base_filename [ 256 ] ;
extract_base_filename ( body , base_filename , sizeof ( base_filename ) ) ;
char final_name [ 256 ] ;
media_final_name ( & result , content_type , base_filename , final_name , sizeof ( final_name ) ) ;
struct media_msg_file f ;
snprintf ( f . rel_path , sizeof ( f . rel_path ) , " media/%s/%s " , ch_id , final_name ) ;
snprintf ( f . full_path , sizeof ( f . full_path ) , " %s/media/%s/%s " , media_base , ch_id , final_name ) ;
f . ts_ms = ts ;
cb ( & f , arg ) ;
u_free ( body ) ;
media_index_result_free ( & result ) ;
}
sqlite3_finalize ( ms ) ;
}
sqlite3_finalize ( cs ) ;
}
struct cache_file { char full_path [ 1536 ] ; uint64_t ts_ms ; int64_t size ; } ;
struct cache_rel_list { char * * items ; int count , cap ; } ;
struct cache_file_list { struct cache_file * files ; int count , cap ; } ;
static int cache_str_cmp ( const void * a , const void * b ) {
return strcmp ( * ( const char * const * ) a , * ( const char * const * ) b ) ;
}
static int cache_file_cmp_ts ( const void * a , const void * b ) {
const struct cache_file * x = ( const struct cache_file * ) a ;
const struct cache_file * y = ( const struct cache_file * ) b ;
if ( x - > ts_ms < y - > ts_ms ) return - 1 ;
if ( x - > ts_ms > y - > ts_ms ) return 1 ;
return 0 ;
}
static int cache_rel_add ( struct cache_rel_list * l , const char * rel ) {
if ( l - > count = = l - > cap ) {
int nc = l - > cap ? l - > cap * 2 : 64 ;
char * * ni = u_realloc ( l - > items , ( size_t ) nc * sizeof ( char * ) ) ;
if ( ! ni ) { DEBUG_ERROR ( DEBUG_CATEGORY_GENERAL , " %s: cache rel list realloc failed (%d) " , CC_ID , nc ) ; return - 1 ; }
l - > items = ni ; l - > cap = nc ;
}
l - > items [ l - > count ] = u_strdup ( rel ) ;
if ( ! l - > items [ l - > count ] ) { DEBUG_ERROR ( DEBUG_CATEGORY_GENERAL , " %s: cache rel strdup failed " , CC_ID ) ; return - 1 ; }
l - > count + + ;
return 0 ;
}
static int cache_file_add ( struct cache_file_list * l , const char * full_path , uint64_t ts_ms , int64_t size ) {
if ( l - > count = = l - > cap ) {
int nc = l - > cap ? l - > cap * 2 : 64 ;
struct cache_file * ni = u_realloc ( l - > files , ( size_t ) nc * sizeof ( struct cache_file ) ) ;
if ( ! ni ) { DEBUG_ERROR ( DEBUG_CATEGORY_GENERAL , " %s: cache file list realloc failed (%d) " , CC_ID , nc ) ; return - 1 ; }
l - > files = ni ; l - > cap = nc ;
}
snprintf ( l - > files [ l - > count ] . full_path , sizeof ( l - > files [ l - > count ] . full_path ) , " %s " , full_path ) ;
l - > files [ l - > count ] . ts_ms = ts_ms ;
l - > files [ l - > count ] . size = size ;
l - > count + + ;
return 0 ;
}
static void cache_rel_collect_cb ( const struct media_msg_file * f , void * arg ) {
cache_rel_add ( ( struct cache_rel_list * ) arg , f - > rel_path ) ;
}
static void cache_file_collect_cb ( const struct media_msg_file * f , void * arg ) {
int fsz = ma_file_size ( f - > full_path ) ;
if ( fsz > 0 ) cache_file_add ( ( struct cache_file_list * ) arg , f - > full_path , f - > ts_ms , ( int64_t ) fsz ) ;
}
struct orphan_walk_ctx {
const char * media_base ;
size_t mb_len ;
char * * rels ; int rel_count ;
int removed ; uint64_t removed_bytes ;
} ;
static void orphan_walk_cb ( void * arg , const char * full_path , int64_t size ) {
struct orphan_walk_ctx * ow = ( struct orphan_walk_ctx * ) arg ;
const char * rel = full_path ;
if ( strncmp ( full_path , ow - > media_base , ow - > mb_len ) = = 0 & & full_path [ ow - > mb_len ] = = ' / ' )
rel = full_path + ow - > mb_len + 1 ;
else return ;
char * found = bsearch ( & rel , ow - > rels , ( size_t ) ow - > rel_count , sizeof ( char * ) , cache_str_cmp ) ;
if ( found ) return ;
if ( remove ( full_path ) = = 0 ) {
ow - > removed + + ;
ow - > removed_bytes + = ( uint64_t ) ( size > 0 ? size : 0 ) ;
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: cache orphan removed: %s (%lld bytes) " ,
CC_ID , rel , ( long long ) size ) ;
} else {
DEBUG_WARN ( DEBUG_CATEGORY_GENERAL , " %s: cache orphan remove failed: %s " , CC_ID , full_path ) ;
}
}
static void chat_media_cache_cleanup_orphans ( void ) {
if ( ! g_cc . initialized | | ! g_cc . db | | ! g_cc . inst ) return ;
char media_base [ 512 ] ; chat_media_base ( media_base , sizeof ( media_base ) ) ;
char media_dir [ 600 ] ; snprintf ( media_dir , sizeof ( media_dir ) , " %s/media " , media_base ) ;
struct cache_rel_list rels = { 0 } ;
chat_media_foreach_media_message ( media_base , cache_rel_collect_cb , & rels ) ;
if ( rels . count > 1 ) qsort ( rels . items , ( size_t ) rels . count , sizeof ( char * ) , cache_str_cmp ) ;
struct orphan_walk_ctx ow ;
ow . media_base = media_base ; ow . mb_len = strlen ( media_base ) ;
ow . rels = rels . items ; ow . rel_count = rels . count ;
ow . removed = 0 ; ow . removed_bytes = 0 ;
ma_dir_walk ( media_dir , orphan_walk_cb , & ow ) ;
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: cache orphan cleanup done removed=%d bytes=%llu " ,
CC_ID , ow . removed , ( unsigned long long ) ow . removed_bytes ) ;
for ( int i = 0 ; i < rels . count ; i + + ) u_free ( rels . items [ i ] ) ;
u_free ( rels . items ) ;
}
static void chat_media_cache_enforce ( void ) {
if ( ! g_cc . initialized | | ! g_cc . db | | ! g_cc . inst ) return ;
uint64_t limit = g_cc . inst - > config - > global . chatserver_storage_total_size ;
if ( limit = = 0 ) return ;
char media_base [ 512 ] ; chat_media_base ( media_base , sizeof ( media_base ) ) ;
struct cache_file_list files = { 0 } ;
chat_media_foreach_media_message ( media_base , cache_file_collect_cb , & files ) ;
uint64_t total = 0 ;
for ( int i = 0 ; i < files . count ; i + + ) total + = ( uint64_t ) ( files . files [ i ] . size > 0 ? files . files [ i ] . size : 0 ) ;
if ( total < = limit ) {
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: cache limit ok total=%llu limit=%llu files=%d " ,
CC_ID , ( unsigned long long ) total , ( unsigned long long ) limit , files . count ) ;
u_free ( files . files ) ;
return ;
}
if ( files . count > 1 ) qsort ( files . files , ( size_t ) files . count , sizeof ( struct cache_file ) , cache_file_cmp_ts ) ;
int days = chat_setting_get_int ( " storage_backfill_days " , 7 ) ;
uint64_t now_ms = ( uint64_t ) ntp_time_get_seconds ( g_cc . inst ) * 1000ULL ;
uint64_t cutoff_ms = now_ms - ( uint64_t ) days * 86400ULL * 1000ULL ;
int deleted = 0 ; uint64_t deleted_bytes = 0 ;
for ( int i = 0 ; i < files . count & & total > limit ; i + + ) {
if ( files . files [ i ] . ts_ms > = cutoff_ms ) continue ;
if ( remove ( files . files [ i ] . full_path ) = = 0 ) {
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: cache evict: %s ts=%llu size=%lld " ,
CC_ID , files . files [ i ] . full_path ,
( unsigned long long ) files . files [ i ] . ts_ms , ( long long ) files . files [ i ] . size ) ;
total - = ( uint64_t ) ( files . files [ i ] . size > 0 ? files . files [ i ] . size : 0 ) ;
deleted_bytes + = ( uint64_t ) ( files . files [ i ] . size > 0 ? files . files [ i ] . size : 0 ) ;
deleted + + ;
} else {
DEBUG_WARN ( DEBUG_CATEGORY_GENERAL , " %s: cache evict failed: %s " , CC_ID , files . files [ i ] . full_path ) ;
}
}
if ( total > limit )
DEBUG_WARN ( DEBUG_CATEGORY_GENERAL , " %s: cache still over limit total=%llu limit=%llu (older-than-%d-days files exhausted) " ,
CC_ID , ( unsigned long long ) total , ( unsigned long long ) limit , days ) ;
else
DEBUG_INFO ( DEBUG_CATEGORY_GENERAL , " %s: cache limit enforced total=%llu limit=%llu deleted=%d bytes=%llu " ,
CC_ID , ( unsigned long long ) total , ( unsigned long long ) limit , deleted , ( unsigned long long ) deleted_bytes ) ;
u_free ( files . files ) ;
}
/* ─── Триггер: после первого SYNC_DONE (+ дебаунс) — анонс локальных блоков + докачка ─── */
@ -1132,6 +1376,9 @@ static void media_startup_backfill_timer_cb(void* arg) {
/* 1) переанонс локальных блоков суперузлам (block_availability), 2) докачка недостающего */
media_delivery_announce_local_blocks ( inst ) ;
chat_media_autodownload_backfill ( ) ;
/* 3) сверка кеша: удалить сироты и применить лимит storage_total_size */
chat_media_cache_cleanup_orphans ( ) ;
chat_media_cache_enforce ( ) ;
}
static void media_startup_backfill_done_cb ( struct DB_SYNC_INSTANCE * si , uint64_t peer_node_id , void * arg ) {