diff --git a/.gitignore b/.gitignore index 6de205ad..f06fcdef 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,9 @@ tests/*.log tests/*.trs tests/logs/ +# Mass update logs +logs/ + # Test binaries (files without extension in tests/) tests/test_* tests/bench_* diff --git a/tools/mass_update.py b/tools/mass_update.py index d7e29f0f..6970455c 100755 --- a/tools/mass_update.py +++ b/tools/mass_update.py @@ -19,7 +19,6 @@ SSH keys: tools/keys/ (in script directory) import argparse import json -import os import subprocess import sys from concurrent.futures import ThreadPoolExecutor, as_completed @@ -67,8 +66,8 @@ def load_servers(config_path, script_dir): return servers -def update_server(server, command): - """Run update on a single server via SSH.""" +def update_server(server, command, logs_dir): + """Run update on a single server via SSH. Returns result dict with logs_dir for log writing.""" host = server["host"] port = server.get("port", 22) workdir = server.get("workdir", "~/utun") @@ -94,26 +93,50 @@ def update_server(server, command): text=True, timeout=SSH_TIMEOUT ) - return { + return write_log({ "host": host, "returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr - } + }, logs_dir) except subprocess.TimeoutExpired: - return { + return write_log({ "host": host, "returncode": -1, "stdout": "", "stderr": f"Timeout after {SSH_TIMEOUT}s" - } + }, logs_dir) except Exception as e: - return { + return write_log({ "host": host, "returncode": -1, "stdout": "", "stderr": str(e) - } + }, logs_dir) + + +def write_log(result, logs_dir): + """Write session log to file. Returns result dict.""" + host = result["host"] + rc = result["returncode"] + log_path = logs_dir / f"{host}.log" + try: + with open(log_path, 'w') as f: + f.write(f"Host: {host} | Exit code: {rc}\n") + f.write("=" * 60 + "\n") + if result["stdout"]: + f.write(result["stdout"]) + if not result["stdout"].endswith("\n"): + f.write("\n") + if result["stderr"]: + f.write("STDERR:\n") + f.write(result["stderr"]) + if not result["stderr"].endswith("\n"): + f.write("\n") + result["_log_file"] = str(log_path) + except Exception as e: + result["_log_file"] = f"ERROR writing log: {e}" + return result def main(): @@ -167,9 +190,13 @@ def main(): print("ERROR: No servers found in config", file=sys.stderr) sys.exit(1) + logs_dir = Path("logs") + logs_dir.mkdir(parents=True, exist_ok=True) + print(f"Updating {len(servers)} servers in '{args.command}' mode...") print(f"Parallel jobs: {args.jobs}") print(f"Config: {config_path}") + print(f"Logs: {logs_dir.resolve()}") print() success = 0 @@ -177,7 +204,7 @@ def main(): with ThreadPoolExecutor(max_workers=args.jobs) as executor: futures = { - executor.submit(update_server, s, args.command): s + executor.submit(update_server, s, args.command, logs_dir): s for s in servers } @@ -186,19 +213,17 @@ def main(): host = result["host"] rc = result["returncode"] - print(f"{'='*60}") - print(f"Host: {host} | Exit code: {rc}") - print(f"{'='*60}") - - if result["stdout"]: - print(result["stdout"]) - if result["stderr"]: - print(result["stderr"], file=sys.stderr, end="") - if rc == 0: success += 1 else: failed += 1 + print(f"{'='*60}") + print(f"Host: {host} | Exit code: {rc} | Log: {result.get('_log_file', 'N/A')}") + print(f"{'='*60}") + if result["stdout"]: + print(result["stdout"]) + if result["stderr"]: + print(result["stderr"], file=sys.stderr, end="") print(f"{'='*60}") print(f"Done: {success} succeeded, {failed} failed")