jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Maintain file and directory statistics incrementally per user

master
evgeny 6 days ago
parent
c8236373b1
commit
d1034cb4ea
1 changed files with 60 additions and 25 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 85
      src/member_files/storage.c

85
src/member_files/storage.c
Unescape View File

@ -15,7 +15,7 @@
#define STORE_USERS 256 #define STORE_USERS 256
struct store_file { char path[MF_PATH_MAX + 80]; int fd; uint64_t touched; }; struct store_file { char path[MF_PATH_MAX + 80]; int fd; uint64_t touched; };
struct store_user { struct store_user* next; uint64_t owner, used, reserved; int valid; }; struct store_user { struct store_user* next; uint64_t owner, used, reserved, files, directories; int valid; };
struct store_upload { struct store_upload {
uint64_t group, peer, owner, size, previous, reservation, touched; uint64_t group, peer, owner, size, previous, reservation, touched;
struct store_user* user; struct store_user* user;
@ -58,18 +58,27 @@ static void store_users_free(struct mf_storage* s) {
s->user_count = 0; s->user_count = 0;
} }
/* Один worker изменяет размер и резерв вместе. Повторы квитанций сюда не попадают. */ /* Один worker корректирует счётчики вместе. file_delta/dir_delta равны -1, 0 или +1; обхода дерева здесь нет. */
static int store_user_update(struct store_user* u, uint64_t previous, uint64_t size, uint64_t release, const char* operation) { static int store_user_update(struct store_user* u, uint64_t previous, uint64_t size, uint64_t release,
if (!u->valid || previous > u->used || release > u->reserved || size > UINT64_MAX - (u->used - previous)) { int file_delta, int dir_delta, const char* operation) {
if (!u->valid || previous > u->used || release > u->reserved || size > UINT64_MAX - (u->used - previous) ||
(file_delta < 0 && !u->files) || (file_delta > 0 && u->files == UINT64_MAX) ||
(dir_delta < 0 && !u->directories) || (dir_delta > 0 && u->directories == UINT64_MAX)) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, DEBUG_ERROR(DEBUG_CATEGORY_MEDIA,
"member_files: quota accounting invalid owner=%llu operation=%s used=%llu previous=%llu size=%llu reserved=%llu release=%llu", "member_files: quota accounting invalid owner=%llu operation=%s used=%llu previous=%llu size=%llu reserved=%llu release=%llu"
" files=%llu file_delta=%d directories=%llu dir_delta=%d",
(unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)previous, (unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)previous,
(unsigned long long)size, (unsigned long long)u->reserved, (unsigned long long)release); (unsigned long long)size, (unsigned long long)u->reserved, (unsigned long long)release,
(unsigned long long)u->files, file_delta, (unsigned long long)u->directories, dir_delta);
u->valid = 0; return MF_IO_ERROR; u->valid = 0; return MF_IO_ERROR;
} }
u->used = u->used - previous + size; u->reserved -= release; u->used = u->used - previous + size; u->reserved -= release;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota updated owner=%llu operation=%s used=%llu reserved=%llu", if (file_delta > 0) u->files++; else if (file_delta < 0) u->files--;
(unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)u->reserved); if (dir_delta > 0) u->directories++; else if (dir_delta < 0) u->directories--;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG,
"member_files: quota updated owner=%llu operation=%s used=%llu reserved=%llu files=%llu directories=%llu",
(unsigned long long)u->owner, operation, (unsigned long long)u->used, (unsigned long long)u->reserved,
(unsigned long long)u->files, (unsigned long long)u->directories);
return MF_OK; return MF_OK;
} }
@ -125,7 +134,7 @@ struct scan_context {
int cleanup; int cleanup;
}; };
/* Размеры UID суммируются по всем группам. Каталоги и .uploads не занимают пользовательскую квоту. */ /* Стартовый обход: содержимое UID по всем группам; технические корни и .uploads не входят в счётчики. */
static int store_scan_entry(void* arg, const char* name, const struct mf_disk_info* info) { static int store_scan_entry(void* arg, const char* name, const struct mf_disk_info* info) {
struct scan_context* c = arg; struct scan_context* c = arg;
char child[MF_PATH_MAX + 80]; char child[MF_PATH_MAX + 80];
@ -147,11 +156,18 @@ static int store_scan_entry(void* arg, const char* name, const struct mf_disk_in
if (c->depth == 1 && !(user = store_user_get(c->s, id, 1))) return -MF_IO_ERROR; if (c->depth == 1 && !(user = store_user_get(c->s, id, 1))) return -MF_IO_ERROR;
} }
if (info->type == MF_FILE) { if (info->type == MF_FILE) {
if (info->size > UINT64_MAX - user->used) { if (info->size > UINT64_MAX - user->used || user->files == UINT64_MAX) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan size overflow owner=%llu", (unsigned long long)user->owner); DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan size overflow owner=%llu", (unsigned long long)user->owner);
return -MF_IO_ERROR; return -MF_IO_ERROR;
} }
user->used += info->size; return 0; user->used += info->size; user->files++; return 0;
}
if (c->depth >= 2) {
if (user->directories == UINT64_MAX) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: scan directory count overflow owner=%llu", (unsigned long long)user->owner);
return -MF_IO_ERROR;
}
user->directories++;
} }
if (c->depth >= 64) return -MF_INVALID_PATH; if (c->depth >= 64) return -MF_INVALID_PATH;
struct scan_context next = { .s = c->s, .user = user, .depth = c->depth + 1 }; struct scan_context next = { .s = c->s, .user = user, .depth = c->depth + 1 };
@ -167,7 +183,7 @@ int mf_storage_initialize(struct mf_storage* s) {
DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "member_files: quota scan started root=%s user_quota=%llu", s->path, (unsigned long long)s->quota); DEBUG_INFO(DEBUG_CATEGORY_MEDIA, "member_files: quota scan started root=%s user_quota=%llu", s->path, (unsigned long long)s->quota);
int rc = mf_disk_root_open(&s->root, s->path); int rc = mf_disk_root_open(&s->root, s->path);
if (rc) goto failed; if (rc) goto failed;
rc = mf_disk_mkdir(&s->root, ".uploads"); rc = mf_disk_mkdir(&s->root, ".uploads", NULL);
if (rc == MF_EXISTS) rc = MF_OK; if (rc == MF_EXISTS) rc = MF_OK;
struct scan_context c = { .s = s, .cleanup = 1 }; struct scan_context c = { .s = s, .cleanup = 1 };
snprintf(c.path, sizeof(c.path), ".uploads"); snprintf(c.path, sizeof(c.path), ".uploads");
@ -177,8 +193,9 @@ int mf_storage_initialize(struct mf_storage* s) {
if (rc) { mf_disk_root_close(&s->root); goto failed; } if (rc) { mf_disk_root_close(&s->root); goto failed; }
s->ready = 1; s->init_status = MF_OK; s->ready = 1; s->init_status = MF_OK;
for (unsigned i = 0; i < STORE_USERS; i++) for (struct store_user* u = s->users[i]; u; u = u->next) { for (unsigned i = 0; i < STORE_USERS; i++) for (struct store_user* u = s->users[i]; u; u = u->next) {
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota scan owner=%llu used=%llu limit=%llu", DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota scan owner=%llu used=%llu limit=%llu files=%llu directories=%llu",
(unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota); (unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota,
(unsigned long long)u->files, (unsigned long long)u->directories);
if (u->used > s->quota) DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "member_files: user exceeds quota owner=%llu used=%llu limit=%llu", if (u->used > s->quota) DEBUG_WARN(DEBUG_CATEGORY_MEDIA, "member_files: user exceeds quota owner=%llu used=%llu limit=%llu",
(unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota); (unsigned long long)u->owner, (unsigned long long)u->used, (unsigned long long)s->quota);
} }
@ -218,7 +235,7 @@ static int store_upload_discard(struct mf_storage* s, struct store_upload* u) {
struct mf_disk_info after; struct mf_disk_info after;
if (mf_disk_stat(&s->root, u->temp, &after) != MF_NOT_FOUND) return rc; if (mf_disk_stat(&s->root, u->temp, &after) != MF_NOT_FOUND) return rc;
} }
int updated = store_user_update(u->user, 0, 0, u->reservation, "abort"); u->active = 0; int updated = store_user_update(u->user, 0, 0, u->reservation, 0, 0, "abort"); u->active = 0;
return rc && rc != MF_NOT_FOUND ? rc : updated; return rc && rc != MF_NOT_FOUND ? rc : updated;
} }
@ -256,11 +273,11 @@ void mf_storage_destroy(struct mf_storage* s) {
static int store_base(struct mf_storage* s, uint64_t group, uint64_t owner) { static int store_base(struct mf_storage* s, uint64_t group, uint64_t owner) {
char path[64]; struct mf_disk_info info; char path[64]; struct mf_disk_info info;
snprintf(path, sizeof(path), "%llu", (unsigned long long)group); snprintf(path, sizeof(path), "%llu", (unsigned long long)group);
int rc = mf_disk_mkdir(&s->root, path); int rc = mf_disk_mkdir(&s->root, path, NULL);
if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info); if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info);
if (rc) return rc; if (rc) return rc;
snprintf(path, sizeof(path), "%llu/%llu", (unsigned long long)group, (unsigned long long)owner); snprintf(path, sizeof(path), "%llu/%llu", (unsigned long long)group, (unsigned long long)owner);
rc = mf_disk_mkdir(&s->root, path); rc = mf_disk_mkdir(&s->root, path, NULL);
if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info); if (rc == MF_EXISTS) rc = mf_disk_stat(&s->root, path, &info);
return rc; return rc;
} }
@ -376,10 +393,13 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
return MF_IO_ERROR; return MF_IO_ERROR;
} }
mf_put64(r->data, user ? user->used : 0); mf_put64(r->data + 8, s->quota); mf_put64(r->data, user ? user->used : 0); mf_put64(r->data + 8, s->quota);
mf_put64(r->data + 16, user ? user->reserved : 0); r->data_len = MF_QUOTA_BYTES; mf_put64(r->data + 16, user ? user->reserved : 0);
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota response owner=%llu used=%llu reserved=%llu limit=%llu", mf_put64(r->data + 24, user ? user->files : 0); mf_put64(r->data + 32, user ? user->directories : 0);
r->data_len = MF_QUOTA_BYTES;
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "member_files: quota response owner=%llu used=%llu reserved=%llu limit=%llu files=%llu directories=%llu",
(unsigned long long)q->owner, (unsigned long long)(user ? user->used : 0), (unsigned long long)q->owner, (unsigned long long)(user ? user->used : 0),
(unsigned long long)(user ? user->reserved : 0), (unsigned long long)s->quota); (unsigned long long)(user ? user->reserved : 0), (unsigned long long)s->quota,
(unsigned long long)(user ? user->files : 0), (unsigned long long)(user ? user->directories : 0));
return MF_OK; return MF_OK;
} }
if (q->command == MF_BEGIN_WRITE) return store_begin(s, group, peer, path, q); if (q->command == MF_BEGIN_WRITE) return store_begin(s, group, peer, path, q);
@ -407,6 +427,7 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
return MF_IO_ERROR; return MF_IO_ERROR;
} }
uint64_t previous = info.size; uint64_t previous = info.size;
int new_file = info.type != MF_FILE; // Нулевой размер существующего файла не означает создание нового.
store_invalidate(s); store_invalidate(s);
rc = mf_disk_rename(&s->root, u->temp, path); rc = mf_disk_rename(&s->root, u->temp, path);
/* Даже при ошибке fsync rename уже мог произойти; сверяем существование temp. */ /* Даже при ошибке fsync rename уже мог произойти; сверяем существование temp. */
@ -420,7 +441,7 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
} }
return rc ? rc : temp_status ? temp_status : MF_IO_ERROR; return rc ? rc : temp_status ? temp_status : MF_IO_ERROR;
} }
int updated = store_user_update(u->user, previous, u->size, u->reservation, "commit"); u->active = 0; int updated = store_user_update(u->user, previous, u->size, u->reservation, new_file, 0, "commit"); u->active = 0;
if (rc) return rc; if (rc) return rc;
if (updated) return updated; if (updated) return updated;
rc = mf_disk_stat(&s->root, path, &info); rc = mf_disk_stat(&s->root, path, &info);
@ -434,8 +455,21 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
if (q->command == MF_MKDIR) { if (q->command == MF_MKDIR) {
for (unsigned i = 0; i < STORE_UPLOADS; i++) for (unsigned i = 0; i < STORE_UPLOADS; i++)
if (s->uploads[i].active && !strcmp(s->uploads[i].path, path)) return MF_BUSY; if (s->uploads[i].active && !strcmp(s->uploads[i].path, path)) return MF_BUSY;
struct store_user* user = store_user_get(s, q->owner, 1);
if (!user) return MF_IO_ERROR;
if (!user->valid) {
DEBUG_ERROR(DEBUG_CATEGORY_MEDIA, "member_files: mkdir quota account invalid owner=%llu", (unsigned long long)q->owner);
return MF_IO_ERROR;
}
rc = store_base(s, group, q->owner); rc = store_base(s, group, q->owner);
if (!rc) { store_invalidate(s); rc = mf_disk_mkdir(&s->root, path); } if (!rc) {
int created;
store_invalidate(s); rc = mf_disk_mkdir(&s->root, path, &created);
if (created) {
int updated = store_user_update(user, 0, 0, 0, 0, 1, "mkdir");
if (!rc) rc = updated;
}
}
return rc; return rc;
} }
rc = mf_disk_stat(&s->root, path, &info); rc = mf_disk_stat(&s->root, path, &info);
@ -477,8 +511,8 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
} }
store_invalidate(s); store_invalidate(s);
if (q->command == MF_REMOVE) { if (q->command == MF_REMOVE) {
struct store_user* user = info.type == MF_FILE ? store_user_get(s, q->owner, 1) : NULL; struct store_user* user = store_user_get(s, q->owner, 1);
if (info.type == MF_FILE && !user) return MF_IO_ERROR; if (!user) return MF_IO_ERROR;
rc = mf_disk_remove(&s->root, path, info.type == MF_DIRECTORY); rc = mf_disk_remove(&s->root, path, info.type == MF_DIRECTORY);
int removed = rc == MF_OK; int removed = rc == MF_OK;
if (rc) { if (rc) {
@ -490,7 +524,8 @@ static int store_command(struct mf_storage* s, uint64_t group, uint64_t peer, co
} }
} }
if (removed && user) { if (removed && user) {
int updated = store_user_update(user, info.size, 0, 0, "remove"); int updated = store_user_update(user, info.size, 0, 0, info.type == MF_FILE ? -1 : 0,
info.type == MF_DIRECTORY ? -1 : 0, "remove");
if (!rc) rc = updated; if (!rc) rc = updated;
} }
return rc; return rc;

Write Preview
Loading…
Cancel
Save