jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Maintain file and directory statistics incrementally per user

Browse Source
This commit is contained in:
evgeny committed 2026-10-06 11:25:18 +02:00
1 parent c8236373b1
commit d1034cb4ea
1 file changed
+60 -25
Download Patch File Download Diff File Expand all files Collapse all files
src/member_files/storage.c
+60 -25
View File
@@ -15,7 +15,7 @@
#define STORE_USERS 256
struct store_file { char path[MF_PATH_MAX + 80]; int fd; uint64_t touched; };
struct store_user { struct store_user* next; uint64_t owner, used, reserved; int valid; };
struct store_user { struct store_user* next; uint64_t owner, used, reserved, files, directories; int valid; };
struct store_upload {
uint64_t group, peer, owner, size, previous, reservation, touched;
struct store_user* user;
@@ -58,18 +58,27 @@ static void store_users_free(struct mf_storage* s) {
s->user_count = 0;
}
/* Один worker изменяет размер и резерв вместе. Повторы квитанций сюда не попадают. */
static int store_user_update(struct store_user* u, uint64_t previous, uint64_t size, uint64_t release, const char* operation) {
if (!u->valid || previous > u->used || release > u->reserved || size > UINT64_MAX - (u->used - previous)) {
/* Один worker корректирует счётчики вместе. file_delta/dir_delta равны -1, 0 или +1; обхода дерева здесь нет. */
static int store_user_update(struct store_user* u, uint64_t previous, uint64_t size, uint64_t release,
int file_delta, int dir_delta, const char* operation) {
if (!u->valid || previous > u->used || release > u->reserved || size > UINT64_MAX - (u->used - previous) ||
(file_delta < 0 && !u->files) || (file_delta > 0 && u->files == UINT64_MAX) ||
(dir_delta < 0 && !u->directories) || (dir_delta > 0 && u->directories == UINT64_MAX)) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA,
"member_files: quota accounting invalid owner=%llu operation=%s used=%llu previous=%llu size=%llu reserved=%llu release=%llu",
"member_files: quota accounting invalid owner=%llu operation=%s used=%llu previous=%llu size=%llu reserved=%llu release=%llu"
" files=%llu file_delta=%d directories=%llu dir_delta=%d",
(unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)previous,
(unsigned long long)size, (unsigned long long)u->reserved, (unsigned long long)release);
(unsigned long long)size, (unsigned long long)u->reserved, (unsigned long long)release,
(unsigned long long)u->files, file_delta, (unsigned long long)u->directories, dir_delta);
u->valid = 0; return MF_IO_ERROR;
}
u->used = u->used - previous + size; u->reserved -= release;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota updated owner=%llu operation=%s used=%llu reserved=%llu",
(unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)u->reserved);
if (file_delta > 0) u->files++; else if (file_delta < 0) u->files--;
if (dir_delta > 0) u->directories++; else if (dir_delta < 0) u->directories--;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG,
"member_files: quota updated owner=%llu operation=%s used=%llu reserved=%llu files=%llu directories=%llu",
(unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)u->reserved,
(unsigned long long)u->files, (unsigned long long)u->directories);
return MF_OK;
}
@@ -125,7 +134,7 @@ struct scan_context {
int cleanup;
};
/* Размеры UID суммируются по всем группам. Каталоги и .uploads не занимают пользовательскую квоту. */
/* Стартовый обход: содержимое UID по всем группам; технические корни и .uploads не входят в счётчики. */
static int store_scan_entry(void* arg, const char* name, const struct mf_disk_info* info) {
struct scan_context* c = arg;
char child[MF_PATH_MAX + 80];
@@ -147,11 +156,18 @@ static int store_scan_entry(void* arg, const char* name, const struct mf_disk_in
if (c->depth == 1 && !(user = store_user_get(c->s, id, 1))) return -MF_IO_ERROR;
}
if (info->type == MF_FILE) {
if (info->size > UINT64_MAX - user->used) {
if (info->size > UINT64_MAX - user->used || user->files == UINT64_MAX) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan size overflow owner=%llu", (unsigned long long)user->owner);
return -MF_IO_ERROR;
}
user->used += info->size; return 0;
user->used += info->size; user->files++; return 0;
}
if (c->depth >= 2) {
if (user->directories == UINT64_MAX) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan directory count overflow owner=%llu", (unsigned long long)user->owner);
return -MF_IO_ERROR;
}
user->directories++;
}
if (c->depth >= 64) return -MF_INVALID_PATH;
struct scan_context next = { .s = c->s, .user = user, .depth = c->depth + 1 };
@@ -167,7 +183,7 @@ int mf_storage_initialize(struct mf_storage* s) {
DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "member_files: quota scan started root=%s user_quota=%llu", s->path, (unsigned long long)s->quota);
int rc = mf_disk_root_open(&s->root, s->path);
if (rc) goto failed;
rc = mf_disk_mkdir(&s->root, ".uploads");
rc = mf_disk_mkdir(&s->root, ".uploads", NULL);
if (rc == MF_EXISTS) rc = MF_OK;
struct scan_context c = { .s = s, .cleanup = 1 };
snprintf(c.path, sizeof(c.path), ".uploads");
@@ -177,8 +193,9 @@ int mf_storage_initialize(struct mf_storage* s) {
if (rc) { mf_disk_root_close(&s->root); goto failed; }
s->ready = 1; s->init_status = MF_OK;
for (unsigned i = 0; i < STORE_USERS; i++) for (struct store_user* u = s->users[i]; u; u = u->next) {
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota scan owner=%llu used=%llu limit=%llu",
(unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota);
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota scan owner=%llu used=%llu limit=%llu files=%llu directories=%llu",
(unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota,
(unsigned long long)u->files, (unsigned long long)u->directories);
if (u->used > s->quota) DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "member_files: user exceeds quota owner=%llu used=%llu limit=%llu",
(unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota);
}
@@ -218,7 +235,7 @@ static int store_upload_discard(struct mf_storage* s, struct store_upload* u) {
struct mf_disk_info after;
if (mf_disk_stat(&s->root, u->temp, &after) != MF_NOT_FOUND) return rc;
}
int updated = store_user_update(u->user, 0, 0, u->reservation, "abort"); u->active = 0;
int updated = store_user_update(u->user, 0, 0, u->reservation, 0, 0, "abort"); u->active = 0;
return rc && rc != MF_NOT_FOUND ? rc : updated;
}
@@ -256,11 +273,11 @@ void mf_storage_destroy(struct mf_storage* s) {
static int store_base(struct mf_storage* s, uint64_t group, uint64_t owner) {
char path[64]; struct mf_disk_info info;
snprintf(path, sizeof(path), "%llu", (unsigned long long)group);
int rc = mf_disk_mkdir(&s->root, path);
int rc = mf_disk_mkdir(&s->root, path, NULL);
if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info);
if (rc) return rc;
snprintf(path, sizeof(path), "%llu/%llu", (unsigned long long)group, (unsigned long long)owner);
rc = mf_disk_mkdir(&s->root, path);
rc = mf_disk_mkdir(&s->root, path, NULL);
if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info);
return rc;
}
@@ -376,10 +393,13 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
return MF_IO_ERROR;
}
mf_put64(r->data, user ? user->used : 0); mf_put64(r->data + 8, s->quota);
mf_put64(r->data + 16, user ? user->reserved : 0); r->data_len = MF_QUOTA_BYTES;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota response owner=%llu used=%llu reserved=%llu limit=%llu",
mf_put64(r->data + 16, user ? user->reserved : 0);
mf_put64(r->data + 24, user ? user->files : 0); mf_put64(r->data + 32, user ? user->directories : 0);
r->data_len = MF_QUOTA_BYTES;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota response owner=%llu used=%llu reserved=%llu limit=%llu files=%llu directories=%llu",
(unsigned long long)q->owner, (unsigned long long)(user ? user->used : 0),
(unsigned long long)(user ? user->reserved : 0), (unsigned long long)s->quota);
(unsigned long long)(user ? user->reserved : 0), (unsigned long long)s->quota,
(unsigned long long)(user ? user->files : 0), (unsigned long long)(user ? user->directories : 0));
return MF_OK;
}
if (q->command == MF_BEGIN_WRITE) return store_begin(s, group, peer, path, q);
@@ -407,6 +427,7 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
return MF_IO_ERROR;
}
uint64_t previous = info.size;
int new_file = info.type != MF_FILE; // Нулевой размер существующего файла не означает создание нового.
store_invalidate(s);
rc = mf_disk_rename(&s->root, u->temp, path);
/* Даже при ошибке fsync rename уже мог произойти; сверяем существование temp. */
@@ -420,7 +441,7 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
}
return rc ? rc : temp_status ? temp_status : MF_IO_ERROR;
}
int updated = store_user_update(u->user, previous, u->size, u->reservation, "commit"); u->active = 0;
int updated = store_user_update(u->user, previous, u->size, u->reservation, new_file, 0, "commit"); u->active = 0;
if (rc) return rc;
if (updated) return updated;
rc = mf_disk_stat(&s->root, path, &info);
@@ -434,8 +455,21 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
if (q->command == MF_MKDIR) {
for (unsigned i = 0; i < STORE_UPLOADS; i++)
if (s->uploads[i].active && !strcmp(s->uploads[i].path, path)) return MF_BUSY;
struct store_user* user = store_user_get(s, q->owner, 1);
if (!user) return MF_IO_ERROR;
if (!user->valid) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: mkdir quota account invalid owner=%llu", (unsigned long long)q->owner);
return MF_IO_ERROR;
}
rc = store_base(s, group, q->owner);
if (!rc) { store_invalidate(s); rc = mf_disk_mkdir(&s->root, path); }
if (!rc) {
int created;
store_invalidate(s); rc = mf_disk_mkdir(&s->root, path, &created);
if (created) {
int updated = store_user_update(user, 0, 0, 0, 0, 1, "mkdir");
if (!rc) rc = updated;
}
}
return rc;
}
rc = mf_disk_stat(&s->root, path, &info);
@@ -477,8 +511,8 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
}
store_invalidate(s);
if (q->command == MF_REMOVE) {
struct store_user* user = info.type == MF_FILE ? store_user_get(s, q->owner, 1) : NULL;
if (info.type == MF_FILE && !user) return MF_IO_ERROR;
struct store_user* user = store_user_get(s, q->owner, 1);
if (!user) return MF_IO_ERROR;
rc = mf_disk_remove(&s->root, path, info.type == MF_DIRECTORY);
int removed = rc == MF_OK;
if (rc) {
@@ -490,7 +524,8 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
}
}
if (removed && user) {
int updated = store_user_update(user, info.size, 0, 0, "remove");
int updated = store_user_update(user, info.size, 0, 0, info.type == MF_FILE ? -1 : 0,
info.type == MF_DIRECTORY ? -1 : 0, "remove");
if (!rc) rc = updated;
}
return rc;