From d1034cb4ea5537768e4f13c8c4a46b8290076ec5 Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 6 Oct 2026 11:25:18 +0200 Subject: [PATCH] Maintain file and directory statistics incrementally per user --- src/member_files/storage.c | 85 +++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/src/member_files/storage.c b/src/member_files/storage.c index 1180fe03..d6ac9653 100644 --- a/src/member_files/storage.c +++ b/src/member_files/storage.c @@ -15,7 +15,7 @@ #define STORE_USERS 256 struct store_file { char path[MF_PATH_MAX + 80]; int fd; uint64_t touched; }; -struct store_user { struct store_user* next; uint64_t owner, used, reserved; int valid; }; +struct store_user { struct store_user* next; uint64_t owner, used, reserved, files, directories; int valid; }; struct store_upload { uint64_t group, peer, owner, size, previous, reservation, touched; struct store_user* user; @@ -58,18 +58,27 @@ static void store_users_free(struct mf_storage* s) { s->user_count = 0; } -/* Один worker изменяет размер и резерв вместе. Повторы квитанций сюда не попадают. */ -static int store_user_update(struct store_user* u, uint64_t previous, uint64_t size, uint64_t release, const char* operation) { - if (!u->valid || previous > u->used || release > u->reserved || size > UINT64_MAX - (u->used - previous)) { +/* Один worker корректирует счётчики вместе. file_delta/dir_delta равны -1, 0 или +1; обхода дерева здесь нет. */ +static int store_user_update(struct store_user* u, uint64_t previous, uint64_t size, uint64_t release, + int file_delta, int dir_delta, const char* operation) { + if (!u->valid || previous > u->used || release > u->reserved || size > UINT64_MAX - (u->used - previous) || + (file_delta < 0 && !u->files) || (file_delta > 0 && u->files == UINT64_MAX) || + (dir_delta < 0 && !u->directories) || (dir_delta > 0 && u->directories == UINT64_MAX)) { DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, - "member_files: quota accounting invalid owner=%llu operation=%s used=%llu previous=%llu size=%llu reserved=%llu release=%llu", + "member_files: quota accounting invalid owner=%llu operation=%s used=%llu previous=%llu size=%llu reserved=%llu release=%llu" + " files=%llu file_delta=%d directories=%llu dir_delta=%d", (unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)previous, - (unsigned long long)size, (unsigned long long)u->reserved, (unsigned long long)release); + (unsigned long long)size, (unsigned long long)u->reserved, (unsigned long long)release, + (unsigned long long)u->files, file_delta, (unsigned long long)u->directories, dir_delta); u->valid = 0; return MF_IO_ERROR; } u->used = u->used - previous + size; u->reserved -= release; - DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota updated owner=%llu operation=%s used=%llu reserved=%llu", - (unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)u->reserved); + if (file_delta > 0) u->files++; else if (file_delta < 0) u->files--; + if (dir_delta > 0) u->directories++; else if (dir_delta < 0) u->directories--; + DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, + "member_files: quota updated owner=%llu operation=%s used=%llu reserved=%llu files=%llu directories=%llu", + (unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)u->reserved, + (unsigned long long)u->files, (unsigned long long)u->directories); return MF_OK; } @@ -125,7 +134,7 @@ struct scan_context { int cleanup; }; -/* Размеры UID суммируются по всем группам. Каталоги и .uploads не занимают пользовательскую квоту. */ +/* Стартовый обход: содержимое UID по всем группам; технические корни и .uploads не входят в счётчики. */ static int store_scan_entry(void* arg, const char* name, const struct mf_disk_info* info) { struct scan_context* c = arg; char child[MF_PATH_MAX + 80]; @@ -147,11 +156,18 @@ static int store_scan_entry(void* arg, const char* name, const struct mf_disk_in if (c->depth == 1 && !(user = store_user_get(c->s, id, 1))) return -MF_IO_ERROR; } if (info->type == MF_FILE) { - if (info->size > UINT64_MAX - user->used) { + if (info->size > UINT64_MAX - user->used || user->files == UINT64_MAX) { DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan size overflow owner=%llu", (unsigned long long)user->owner); return -MF_IO_ERROR; } - user->used += info->size; return 0; + user->used += info->size; user->files++; return 0; + } + if (c->depth >= 2) { + if (user->directories == UINT64_MAX) { + DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan directory count overflow owner=%llu", (unsigned long long)user->owner); + return -MF_IO_ERROR; + } + user->directories++; } if (c->depth >= 64) return -MF_INVALID_PATH; struct scan_context next = { .s = c->s, .user = user, .depth = c->depth + 1 }; @@ -167,7 +183,7 @@ int mf_storage_initialize(struct mf_storage* s) { DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "member_files: quota scan started root=%s user_quota=%llu", s->path, (unsigned long long)s->quota); int rc = mf_disk_root_open(&s->root, s->path); if (rc) goto failed; - rc = mf_disk_mkdir(&s->root, ".uploads"); + rc = mf_disk_mkdir(&s->root, ".uploads", NULL); if (rc == MF_EXISTS) rc = MF_OK; struct scan_context c = { .s = s, .cleanup = 1 }; snprintf(c.path, sizeof(c.path), ".uploads"); @@ -177,8 +193,9 @@ int mf_storage_initialize(struct mf_storage* s) { if (rc) { mf_disk_root_close(&s->root); goto failed; } s->ready = 1; s->init_status = MF_OK; for (unsigned i = 0; i < STORE_USERS; i++) for (struct store_user* u = s->users[i]; u; u = u->next) { - DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota scan owner=%llu used=%llu limit=%llu", - (unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota); + DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota scan owner=%llu used=%llu limit=%llu files=%llu directories=%llu", + (unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota, + (unsigned long long)u->files, (unsigned long long)u->directories); if (u->used > s->quota) DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "member_files: user exceeds quota owner=%llu used=%llu limit=%llu", (unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota); } @@ -218,7 +235,7 @@ static int store_upload_discard(struct mf_storage* s, struct store_upload* u) { struct mf_disk_info after; if (mf_disk_stat(&s->root, u->temp, &after) != MF_NOT_FOUND) return rc; } - int updated = store_user_update(u->user, 0, 0, u->reservation, "abort"); u->active = 0; + int updated = store_user_update(u->user, 0, 0, u->reservation, 0, 0, "abort"); u->active = 0; return rc && rc != MF_NOT_FOUND ? rc : updated; } @@ -256,11 +273,11 @@ void mf_storage_destroy(struct mf_storage* s) { static int store_base(struct mf_storage* s, uint64_t group, uint64_t owner) { char path[64]; struct mf_disk_info info; snprintf(path, sizeof(path), "%llu", (unsigned long long)group); - int rc = mf_disk_mkdir(&s->root, path); + int rc = mf_disk_mkdir(&s->root, path, NULL); if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info); if (rc) return rc; snprintf(path, sizeof(path), "%llu/%llu", (unsigned long long)group, (unsigned long long)owner); - rc = mf_disk_mkdir(&s->root, path); + rc = mf_disk_mkdir(&s->root, path, NULL); if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info); return rc; } @@ -376,10 +393,13 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co return MF_IO_ERROR; } mf_put64(r->data, user ? user->used : 0); mf_put64(r->data + 8, s->quota); - mf_put64(r->data + 16, user ? user->reserved : 0); r->data_len = MF_QUOTA_BYTES; - DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota response owner=%llu used=%llu reserved=%llu limit=%llu", + mf_put64(r->data + 16, user ? user->reserved : 0); + mf_put64(r->data + 24, user ? user->files : 0); mf_put64(r->data + 32, user ? user->directories : 0); + r->data_len = MF_QUOTA_BYTES; + DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota response owner=%llu used=%llu reserved=%llu limit=%llu files=%llu directories=%llu", (unsigned long long)q->owner, (unsigned long long)(user ? user->used : 0), - (unsigned long long)(user ? user->reserved : 0), (unsigned long long)s->quota); + (unsigned long long)(user ? user->reserved : 0), (unsigned long long)s->quota, + (unsigned long long)(user ? user->files : 0), (unsigned long long)(user ? user->directories : 0)); return MF_OK; } if (q->command == MF_BEGIN_WRITE) return store_begin(s, group, peer, path, q); @@ -407,6 +427,7 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co return MF_IO_ERROR; } uint64_t previous = info.size; + int new_file = info.type != MF_FILE; // Нулевой размер существующего файла не означает создание нового. store_invalidate(s); rc = mf_disk_rename(&s->root, u->temp, path); /* Даже при ошибке fsync rename уже мог произойти; сверяем существование temp. */ @@ -420,7 +441,7 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co } return rc ? rc : temp_status ? temp_status : MF_IO_ERROR; } - int updated = store_user_update(u->user, previous, u->size, u->reservation, "commit"); u->active = 0; + int updated = store_user_update(u->user, previous, u->size, u->reservation, new_file, 0, "commit"); u->active = 0; if (rc) return rc; if (updated) return updated; rc = mf_disk_stat(&s->root, path, &info); @@ -434,8 +455,21 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co if (q->command == MF_MKDIR) { for (unsigned i = 0; i < STORE_UPLOADS; i++) if (s->uploads[i].active && !strcmp(s->uploads[i].path, path)) return MF_BUSY; + struct store_user* user = store_user_get(s, q->owner, 1); + if (!user) return MF_IO_ERROR; + if (!user->valid) { + DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: mkdir quota account invalid owner=%llu", (unsigned long long)q->owner); + return MF_IO_ERROR; + } rc = store_base(s, group, q->owner); - if (!rc) { store_invalidate(s); rc = mf_disk_mkdir(&s->root, path); } + if (!rc) { + int created; + store_invalidate(s); rc = mf_disk_mkdir(&s->root, path, &created); + if (created) { + int updated = store_user_update(user, 0, 0, 0, 0, 1, "mkdir"); + if (!rc) rc = updated; + } + } return rc; } rc = mf_disk_stat(&s->root, path, &info); @@ -477,8 +511,8 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co } store_invalidate(s); if (q->command == MF_REMOVE) { - struct store_user* user = info.type == MF_FILE ? store_user_get(s, q->owner, 1) : NULL; - if (info.type == MF_FILE && !user) return MF_IO_ERROR; + struct store_user* user = store_user_get(s, q->owner, 1); + if (!user) return MF_IO_ERROR; rc = mf_disk_remove(&s->root, path, info.type == MF_DIRECTORY); int removed = rc == MF_OK; if (rc) { @@ -490,7 +524,8 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co } } if (removed && user) { - int updated = store_user_update(user, info.size, 0, 0, "remove"); + int updated = store_user_update(user, info.size, 0, 0, info.type == MF_FILE ? -1 : 0, + info.type == MF_DIRECTORY ? -1 : 0, "remove"); if (!rc) rc = updated; } return rc;