You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
208 lines
12 KiB
208 lines
12 KiB
"""Сетевые операции допускаются только в namespace, созданных этим запуском.""" |
|
|
|
import json |
|
import os |
|
from pathlib import Path |
|
import select |
|
import subprocess |
|
import time |
|
|
|
from topology import COUNTRIES, NATS, NODES, WANS, wan |
|
|
|
|
|
def nat_rules(gateway): |
|
clients = [node for node in NODES if node.nat == gateway.name] |
|
destinations = sorted({n.ip for n in NODES if not n.nat} | {g.ip for g in NATS}) |
|
lines = ["table ip lab {", |
|
" set active { type inet_service; flags dynamic,timeout; timeout 60s; }", |
|
" set contacts { type ipv4_addr . inet_service . inet_service; flags dynamic,timeout; timeout 60s; }", |
|
" chain refresh { type filter hook postrouting priority mangle; policy accept;"] |
|
for node in clients: |
|
match = f' oifname "wan" ip saddr {node.ip} udp sport {node.udp_port}' |
|
if gateway.mode == "cone": |
|
lines.append(f"{match} update @active {{ {node.mapped_port} timeout 60s }}") |
|
elif gateway.mode == "restricted": |
|
lines.append(f"{match} update @contacts {{ ip daddr . udp dport . {node.mapped_port} timeout 60s }}") |
|
lines += [" }", " chain prerouting { type nat hook prerouting priority dstnat; policy accept;"] |
|
for node in clients: |
|
match = f' iifname "wan" udp dport {node.mapped_port}' |
|
if gateway.mode == "cone": |
|
lines.append(f"{match} udp dport @active dnat to {node.ip}:{node.udp_port}") |
|
elif gateway.mode == "restricted": |
|
lines.append(f"{match} ip saddr . udp sport . udp dport @contacts dnat to {node.ip}:{node.udp_port}") |
|
lines += [" }", " chain postrouting { type nat hook postrouting priority srcnat; policy accept;"] |
|
for node in clients: |
|
match = f' oifname "wan" ip saddr {node.ip} udp sport {node.udp_port}' |
|
if gateway.mode == "strict": |
|
for index, destination in enumerate(destinations): |
|
port = 40000 + node.number * 100 + index |
|
lines.append(f"{match} ip daddr {destination} snat to {gateway.ip}:{port}") |
|
else: |
|
lines.append(f"{match} snat to {gateway.ip}:{node.mapped_port}") |
|
lines += [' oifname "wan" masquerade', " }", |
|
" chain forward { type filter hook forward priority filter; policy drop;", |
|
" ct state established,related accept", ' iifname "lan" oifname "wan" accept', |
|
" ct status dnat accept", " counter drop", " }", "}"] |
|
return "\n".join(lines) + "\n" |
|
|
|
|
|
class Network: |
|
def __init__(self, directory, seed, host_namespace): |
|
self.directory = Path(directory) |
|
self.seed = seed |
|
self.host_namespace = host_namespace |
|
self.controller = os.readlink("/proc/self/ns/net") |
|
if self.controller == host_namespace or not host_namespace: |
|
raise RuntimeError("Refusing network changes: controller must be in a new network namespace") |
|
self.namespaces = {} |
|
self.children = [] |
|
self.serial = 0 |
|
self.log = (self.directory / "network.jsonl").open("a", buffering=1) |
|
|
|
def command(self, name, arguments, *, input=None, check=True): |
|
prefix = self.prefix(name) |
|
started = time.monotonic() |
|
result = subprocess.run(prefix + list(arguments), input=input, text=True, capture_output=True, timeout=20) |
|
inode = self.namespaces[name][1] if name else self.controller |
|
self.log.write(json.dumps({"time": started, "namespace": name, "inode": inode, "argv": list(arguments), |
|
"rc": result.returncode, "stdout": result.stdout, "stderr": result.stderr}) + "\n") |
|
if check and result.returncode: |
|
raise RuntimeError(f"{name}: {' '.join(arguments)}: {result.stderr.strip() or result.stdout.strip()}") |
|
return result |
|
|
|
def prefix(self, name): |
|
if os.readlink("/proc/self/ns/net") != self.controller: |
|
raise RuntimeError("Controller changed network namespace") |
|
if name is None: |
|
return [] |
|
child, inode = self.namespaces[name] |
|
path = f"/proc/{child.pid}/ns/net" |
|
if child.poll() is not None or os.readlink(path) != inode or inode == self.host_namespace: |
|
raise RuntimeError(f"Namespace {name} is no longer owned by this run") |
|
return ["nsenter", f"--net={path}", "--"] |
|
|
|
def spawn(self, name, arguments, **kwargs): |
|
child = subprocess.Popen(self.prefix(name) + list(arguments), **kwargs) |
|
self.children.append(child) |
|
return child |
|
|
|
def create_namespace(self, name): |
|
keeper = "import os,sys; print(os.getpid(),flush=True); sys.stdin.buffer.read()" |
|
child = subprocess.Popen(["unshare", "--net", "--", "python3", "-u", "-c", keeper], |
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) |
|
self.children.append(child) |
|
if not select.select([child.stdout], [], [], 5)[0] or child.stdout.readline().strip() != str(child.pid): |
|
raise RuntimeError(f"Cannot create namespace {name}") |
|
inode = os.readlink(f"/proc/{child.pid}/ns/net") |
|
if inode in {self.controller, self.host_namespace}: |
|
raise RuntimeError(f"Namespace isolation failed for {name}") |
|
self.namespaces[name] = (child, inode) |
|
self.command(name, ["ip", "link", "set", "lo", "up"]) |
|
self.command(name, ["sysctl", "-qw", "net.ipv6.conf.all.disable_ipv6=1"]) |
|
self.command(name, ["sysctl", "-qw", "net.ipv4.conf.all.rp_filter=0"]) |
|
self.command(name, ["sysctl", "-qw", "net.ipv4.conf.default.rp_filter=0"]) |
|
|
|
def link(self, left, left_interface, right, right_interface): |
|
self.serial += 1 |
|
a, b = f"v{self.serial}a", f"v{self.serial}b" |
|
self.command(None, ["ip", "link", "add", a, "type", "veth", "peer", "name", b]) |
|
for temporary, name, interface in ((a, left, left_interface), (b, right, right_interface)): |
|
self.command(None, ["ip", "link", "set", temporary, "netns", str(self.namespaces[name][0].pid)]) |
|
self.command(name, ["ip", "link", "set", temporary, "name", interface]) |
|
self.command(name, ["ethtool", "-K", interface, "gro", "off", "gso", "off", "tso", "off"]) |
|
self.command(name, ["ip", "link", "set", interface, "up"]) |
|
|
|
def bridge(self, name, address): |
|
self.command(name, ["ip", "link", "add", "lan", "type", "bridge"]) |
|
self.command(name, ["ip", "addr", "add", address + "/24", "dev", "lan"]) |
|
self.command(name, ["ip", "link", "set", "lan", "up"]) |
|
self.command(name, ["sysctl", "-qw", "net.ipv4.ip_forward=1"]) |
|
|
|
def attach(self, node, gateway, port): |
|
self.link(node, "eth0" if node.startswith("n") else "wan", gateway, port) |
|
self.command(gateway, ["ip", "link", "set", port, "master", "lan"]) |
|
|
|
def netem(self, node, source_country, *, rtt=None, loss=None): |
|
index = next(i + 1 for i, c in enumerate(COUNTRIES) if c.name == source_country) |
|
same = source_country == node.country |
|
link = None if same else wan(source_country, node.country) |
|
delay = (2 if same else link.rtt_ms) / 2 if rtt is None else rtt / 2 |
|
jitter = 0.2 if same else link.jitter_ms |
|
loss = (0 if same else link.loss_percent) if loss is None else loss |
|
self.command(node.name, ["tc", "qdisc", "replace", "dev", "ifb0", "parent", f"1:{index}", |
|
"handle", f"{index + 10}:", "netem", "limit", "4096", "delay", f"{delay}ms", |
|
f"{jitter}ms", "distribution", "normal", "loss", "random", f"{loss}%", |
|
"seed", str(self.seed + node.number * 10 + index)]) |
|
|
|
def receiver(self, node): |
|
self.command(node.name, ["ip", "link", "add", "ifb0", "type", "ifb"]) |
|
self.command(node.name, ["ip", "link", "set", "ifb0", "up"]) |
|
self.command(node.name, ["tc", "qdisc", "add", "dev", "eth0", "clsact"]) |
|
self.command(node.name, ["tc", "filter", "add", "dev", "eth0", "ingress", "matchall", "action", |
|
"mirred", "egress", "redirect", "dev", "ifb0"]) |
|
own = next(i + 1 for i, c in enumerate(COUNTRIES) if c.name == node.country) |
|
self.command(node.name, ["tc", "qdisc", "add", "dev", "ifb0", "root", "handle", "1:", "htb", "default", str(own)]) |
|
for index, country in enumerate(COUNTRIES, 1): |
|
self.command(node.name, ["tc", "class", "add", "dev", "ifb0", "parent", "1:", "classid", f"1:{index}", |
|
"htb", "rate", "1gbit", "ceil", "1gbit"]) |
|
self.netem(node, country.name) |
|
self.command(node.name, ["tc", "filter", "add", "dev", "ifb0", "parent", "1:", "protocol", "ip", |
|
"prio", str(index), "flower", "src_ip", country.subnet, "classid", f"1:{index}"]) |
|
|
|
def bandwidth(self, link, mbps): |
|
for country in (link.a, link.b): |
|
self.command("r" + country, ["tc", "qdisc", "replace", "dev", "w_" + link.a + link.b, |
|
"root", "tbf", "rate", f"{mbps}mbit", "burst", "64kb", "latency", "100ms"]) |
|
|
|
def build(self): |
|
for name in [n.name for n in NODES] + [c.router for c in COUNTRIES] + [g.name for g in NATS]: |
|
self.create_namespace(name) |
|
for country in COUNTRIES: |
|
self.bridge(country.router, country.gateway) |
|
for gateway in NATS: |
|
self.bridge(gateway.name, "10.64.1.1") |
|
self.attach(gateway.name, "r" + gateway.country, "p" + gateway.name) |
|
self.command(gateway.name, ["ip", "addr", "add", gateway.ip + "/24", "dev", "wan"]) |
|
country = next(c for c in COUNTRIES if c.name == gateway.country) |
|
self.command(gateway.name, ["ip", "route", "add", "default", "via", country.gateway]) |
|
rules = nat_rules(gateway) |
|
(self.directory / f"{gateway.name}.nft").write_text(rules) |
|
self.command(gateway.name, ["nft", "-f", "-"], input=rules) |
|
for node in NODES: |
|
self.attach(node.name, node.nat or "r" + node.country, "p" + node.name) |
|
self.command(node.name, ["ip", "addr", "add", node.ip + "/24", "dev", "eth0"]) |
|
self.command(node.name, ["ip", "route", "add", "default", "via", node.gateway]) |
|
self.receiver(node) |
|
for link in WANS: |
|
interface = "w_" + link.a + link.b |
|
self.link("r" + link.a, interface, "r" + link.b, interface) |
|
for country, address, peer, destination in ((link.a, link.ip_a, link.ip_b, link.b), |
|
(link.b, link.ip_b, link.ip_a, link.a)): |
|
subnet = next(c.subnet for c in COUNTRIES if c.name == destination) |
|
self.command("r" + country, ["ip", "addr", "add", address + "/30", "dev", interface]) |
|
self.command("r" + country, ["ip", "route", "add", subnet, "via", peer]) |
|
self.bandwidth(link, link.mbps) |
|
state = {name: {"pid": child.pid, "namespace": inode} for name, (child, inode) in self.namespaces.items()} |
|
(self.directory / "namespaces.json").write_text(json.dumps(state, indent=2) + "\n") |
|
|
|
def snapshot(self, label): |
|
snapshot = {} |
|
for name in self.namespaces: |
|
snapshot[name] = {"links": json.loads(self.command(name, ["ip", "-j", "link"]).stdout), |
|
"routes": json.loads(self.command(name, ["ip", "-j", "route"]).stdout), |
|
"qdisc": json.loads(self.command(name, ["tc", "-j", "-s", "qdisc"]).stdout)} |
|
if name.startswith("g_"): |
|
snapshot[name]["nft"] = json.loads(self.command(name, ["nft", "-j", "list", "ruleset"]).stdout) |
|
(self.directory / f"network-{label}.json").write_text(json.dumps(snapshot, indent=2) + "\n") |
|
|
|
def close(self): |
|
for child in reversed(self.children): |
|
if child.poll() is None: |
|
child.terminate() |
|
for child in reversed(self.children): |
|
try: |
|
child.wait(timeout=3) |
|
except subprocess.TimeoutExpired: |
|
child.kill() |
|
child.wait(timeout=3) |
|
self.log.close()
|
|
|