diff --git a/containers/server-n0/devenv.nix b/containers/server-n0/devenv.nix deleted file mode 100644 index 400238f..0000000 --- a/containers/server-n0/devenv.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - name, - hostInfo, - dsl, - ... -}: -let - inherit (dsl) mkContainer mkService; - inherit (hostInfo.paths) data secrets; -in -mkContainer { - inherit name; - net.access = [ { to = "@internet"; } ]; - svc.autoStart = true; - resources.fs = { - "/config" = { - hostPath = "${data}/web/devenv/config"; - isReadOnly = false; - }; - "/workspace" = { - hostPath = "${data}/web/devenv/workspace"; - isReadOnly = false; - }; - "/run/secrets/code-server-env" = { - hostPath = "${secrets}/code-server-env"; - isReadOnly = true; - }; - }; - users = [ - { - name = "coder"; - uid = 1000; - home = "/config"; - } - ]; - service = - { pkgs, lib, ... }: - mkService { - description = "code-server"; - execStart = "${lib.getExe' pkgs.code-server "code-server"} --bind-addr 0.0.0.0:8080 --user-data-dir /config/data --extensions-dir /config/extensions --auth password /workspace"; - networkOnline = true; - serviceConfig = { - EnvironmentFile = "/run/secrets/code-server-env"; - User = "coder"; - ReadWritePaths = [ - "/config" - "/workspace" - ]; - }; - }; - net.expose.tcp = [ 8080 ]; - resources.mem = "4G"; - resources.cpu = 2; -} diff --git a/containers/server-n0/nginx.nix b/containers/server-n0/nginx.nix index 68a1b8f..1c8a266 100644 --- a/containers/server-n0/nginx.nix +++ b/containers/server-n0/nginx.nix @@ -158,7 +158,9 @@ let discoveredVhosts = map ( { container, data }: - { inherit (data) domain; } + { + inherit (data) domain; + } // lib.optionalAttrs (data ? port) { upstream = up container data.port; } // lib.optionalAttrs (data ? websockets) { inherit (data) websockets; } // lib.optionalAttrs (data ? redirect) { inherit (data) redirect; } @@ -181,8 +183,7 @@ let ]; upstreamContainers = map (e: e.container) (lib.filter ({ data, ... }: data ? port) discovered); -in -let + setup = mkNginxConfig { email = "floweydev@gmail.com"; resolver = hostAddr; @@ -201,6 +202,7 @@ let hostPath = "${data}/core/nginx/acme"; isReadOnly = false; }; + miscProperties.acmeDomains = map (v: v.domain) (discoveredVhosts ++ extraVhosts); inherit (setup) containerConfig; }; inherit (c) hostAddr; diff --git a/dsl.nix b/dsl.nix index 91bf6ed..95ce857 100644 --- a/dsl.nix +++ b/dsl.nix @@ -1223,6 +1223,43 @@ rec { ) (lib.attrValues autoStartTestContainers) ); + exposedTcp = lib.mapAttrs (_: c: c.ports or [ ]) autoStartTestContainers; + + hostAccessTcp = lib.filterAttrs (_: v: v != [ ]) ( + lib.mapAttrs ( + _: c: lib.concatMap (a: if a.to == "@host" then a.tcp or [ ] else [ ]) c.access + ) autoStartTestContainers + ); + + pyForwardTcp = lib.concatStringsSep ", " ( + lib.concatMap ( + c: + map ( + p: + let + np = normalizePort p; + in + "(${toString np.host}, \"${c.name}\", ${toString np.container})" + ) c.hostPorts + ) (lib.attrValues autoStartTestContainers) + ); + + nginxBackendPorts = lib.mapAttrs (_: c: c.miscProperties.nginx.port) ( + lib.filterAttrs ( + _: c: c.miscProperties ? nginx && c.miscProperties.nginx ? port + ) autoStartTestContainers + ); + + nginxDomainMap = lib.listToAttrs ( + lib.concatMap ( + c: + lib.optional (c.miscProperties ? nginx && c.miscProperties.nginx ? domain) { + name = c.miscProperties.nginx.domain; + value = c.name; + } + ) (lib.attrValues autoStartTestContainers) + ); + bin = lib.getExe'; testTools = [ @@ -1330,6 +1367,12 @@ rec { # (src, to, tcp ports, udp ports); `to` may be "@host" or "@internet" EDGES = [${pyEdges}] + EXPOSED_TCP = ${pyDict pyInts exposedTcp} + HOST_ACCESS_TCP = ${pyDict pyInts hostAccessTcp} + FORWARD_TCP = [${pyForwardTcp}] + NGINX_BACKENDS = ${pyDict toString nginxBackendPorts} + NGINX_DOMAINS = ${pyDict pyStr nginxDomainMap} + NSENTER = "${bin pkgs.util-linux "nsenter"}" CURL = "${bin pkgs.curl "curl"}" IP = "${bin pkgs.iproute2 "ip"}" diff --git a/flake.nix b/flake.nix index 95b7e43..ca6beee 100644 --- a/flake.nix +++ b/flake.nix @@ -65,7 +65,12 @@ specialArgsFor = hostName: { hostInfo = hostInfoFor hostName; - inherit inputs customPkgs flakeRev deployLabel; + inherit + inputs + customPkgs + flakeRev + deployLabel + ; }; hostDefFor = diff --git a/tests/server-n0/integration.nix b/tests/server-n0/integration.nix index e985211..377891e 100644 --- a/tests/server-n0/integration.nix +++ b/tests/server-n0/integration.nix @@ -14,13 +14,11 @@ let it = dsl.mkIntegrationTest { inherit host testContainers pkgs; }; - nginxAcmeDomains = [ - "devenv.floweytf.com" - "maven.floweytf.com" - "git.floweytf.com" - "utils.floweytf.com" - "mail.floweytf.com" - ]; + nginxAcmeDomains = + if testContainers ? nginx && testContainers.nginx.miscProperties ? acmeDomains then + testContainers.nginx.miscProperties.acmeDomains + else + [ ]; nginxAcmeDir = domain: "${serverDataDir}/core/nginx/acme/${domain}"; @@ -67,9 +65,44 @@ let ''; sshpiperTestUser = "admin"; - hostExporterPort = 9100; - hostCadvisorPort = 9200; hostSshdPort = 5952; + + # derive data directories from container bind mounts + allDataDirs = lib.unique ( + lib.concatMap (c: lib.filter (p: lib.hasPrefix serverDataDir p) c.bindMountHostPaths) ( + lib.attrValues testContainers + ) + ); + + mkdirAllDataDirs = lib.concatMapStrings (p: ''mkdir -p "${p}"'' + "\n") allDataDirs; + + # writable data dirs need ownership set to 1000:1000 (the standard + # container user uid). services that run as a different user (loki, + # prometheus, grafana) fix ownership via StateDirectory on startup. + writableDataDirs = lib.unique ( + lib.concatMap ( + c: + if c.kind == "nspawn" then + lib.filter (p: lib.hasPrefix serverDataDir p) ( + map (bm: bm.hostPath) (lib.attrValues (lib.filterAttrs (_: bm: !bm.isReadOnly) c.bindMounts)) + ) + else + [ ] + ) (lib.attrValues testContainers) + ); + + chownWritableDataDirs = lib.concatMapStrings ( + p: ''chown -R 1000:1000 "${p}"'' + "\n" + ) writableDataDirs; + + # secret paths needed by test containers + secretPaths = lib.unique ( + lib.concatMap (c: lib.filter (p: lib.hasPrefix serverSecretsDir p) c.bindMountHostPaths) ( + lib.attrValues testContainers + ) + ); + + mkdirSecretParents = lib.concatMapStrings (p: ''mkdir -p "$(dirname '${p}')"'' + "\n") secretPaths; in { name = "integration-${host.hostName}"; @@ -90,63 +123,33 @@ in it.serverModule ]; - containers.nginx.config = { - systemd.services = lib.genAttrs (map (d: "acme-order-renew-${d}") nginxAcmeDomains) (_: { - enable = lib.mkForce false; - }); - }; - system.activationScripts.test-secrets = # bash '' - mkdir -p /nix/var/nix/db /nix/var/nix/daemon-socket - - mkdir -p /var/log/journal + mkdir -p /nix/var/nix/db /nix/var/nix/daemon-socket + mkdir -p /var/log/journal + mkdir -p "${serverSecretsDir}" + ${mkdirAllDataDirs} ${mkdirAcmeDirs} - mkdir -p "${serverSecretsDir}" \ - "${serverDataDir}/core/sshpiper" \ - "${serverDataDir}/core/mail" \ - "${serverDataDir}/overseer/loki/data" \ - "${serverDataDir}/overseer/prometheus/data" \ - "${serverDataDir}/overseer/grafana" \ - "${serverDataDir}/web/git" \ - "${serverDataDir}/web/git-ci-runner/nix-rw-store" \ - "${serverDataDir}/web/git-ci-runner/docker" \ - "${serverDataDir}/web/git-ci-runner/runner-state" \ - "${serverSecretsDir}/ci-runner" \ - "${serverDataDir}/web/reposilite" \ - "${serverDataDir}/web/devenv/config" \ - "${serverDataDir}/web/devenv/workspace" \ - "${serverDataDir}/ssh/generic" \ - "${serverDataDir}/ssh/nyaa" \ - "${serverDataDir}/mc/botm" \ - "${serverDataDir}/mc/rogule-dev" \ - "${serverDataDir}/mc/rogule-build" \ - "${serverDataDir}/mc/starwalker" \ - "${serverDataDir}/mc/tl" + ${mkdirSecretParents} + ${chownWritableDataDirs} - chown -R 1000:1000 \ - "${serverDataDir}/web/git" \ - "${serverDataDir}/web/reposilite" \ - "${serverDataDir}/web/devenv/config" \ - "${serverDataDir}/web/devenv/workspace" \ - "${serverDataDir}/core/mail" \ - "${serverDataDir}/ssh/generic" \ - "${serverDataDir}/mc/botm" \ - "${serverDataDir}/mc/rogule-dev" \ - "${serverDataDir}/mc/rogule-build" \ - "${serverDataDir}/mc/starwalker" \ - "${serverDataDir}/mc/tl" + for keytype in ed25519 rsa ecdsa; do + keyfile="${serverSecretsDir}/ssh_host_''${keytype}_key" + if [[ ! -f "$keyfile" ]]; then + ${lib.getExe' pkgs.openssh "ssh-keygen"} -q -t "$keytype" -f "$keyfile" -N "" + fi + done - for keytype in ed25519 rsa ecdsa; do - keyfile="${serverSecretsDir}/ssh_host_''${keytype}_key" - if [[ ! -f "$keyfile" ]]; then - ${lib.getExe' pkgs.openssh "ssh-keygen"} -q -t "$keytype" -f "$keyfile" -N "" - fi - done - - # stalwart rejects weak/dictionary passwords + ${installFakeCerts} + '' + + lib.optionalString (testContainers ? sshpiper) '' + mkdir -p "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}" + install -m 644 ${testSshKey}/key.pub \ + "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}/authorized_keys" + '' + + lib.optionalString (testContainers ? mail) '' for f in mail-admin-password mail-account-password; do target="${serverSecretsDir}/$f" if [[ ! -f "$target" ]]; then @@ -154,22 +157,14 @@ in chmod 600 "$target" fi done - - if [[ ! -f "${serverSecretsDir}/ci-runner/token" ]]; then - echo "TOKEN=test-registration-token" > "${serverSecretsDir}/ci-runner/token" - chmod 600 "${serverSecretsDir}/ci-runner/token" - fi - - if [[ ! -f "${serverSecretsDir}/code-server-env" ]]; then - echo "test-secret" > "${serverSecretsDir}/code-server-env" - chmod 600 "${serverSecretsDir}/code-server-env" - fi - - ${installFakeCerts} - mkdir -p "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}" - install -m 644 ${testSshKey}/key.pub \ - "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}/authorized_keys" ''; + } + // lib.optionalAttrs (testContainers ? nginx && nginxAcmeDomains != [ ]) { + containers.nginx.config = { + systemd.services = lib.genAttrs (map (d: "acme-order-renew-${d}") nginxAcmeDomains) (_: { + enable = lib.mkForce false; + }); + }; }; }; @@ -182,8 +177,6 @@ in TEST_SSH_KEY_STORE = "${testSshKey}/key" SSHPIPER_TEST_USER = "${sshpiperTestUser}" - HOST_EXPORTER_PORT = ${toString hostExporterPort} - HOST_CADVISOR_PORT = ${toString hostCadvisorPort} HOST_SSHD_PORT = ${toString hostSshdPort} '' + builtins.readFile ./integration.py; diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 328dafe..c8235cf 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -29,12 +29,15 @@ if TYPE_CHECKING: KIND: dict[str, str] = {} DEVICES: dict[str, list[str]] = {} EDGES: list[tuple[str, str, list[int], list[int]]] = [] + EXPOSED_TCP: dict[str, list[int]] = {} + HOST_ACCESS_TCP: dict[str, list[int]] = {} + FORWARD_TCP: list[tuple[int, str, int]] = [] + NGINX_BACKENDS: dict[str, int] = {} + NGINX_DOMAINS: dict[str, str] = {} SSH_KEY: str = "" SSH_OPTS: str = "" TEST_SSH_KEY_STORE: str = "" SSHPIPER_TEST_USER: str = "" - HOST_EXPORTER_PORT: int = 0 - HOST_CADVISOR_PORT: int = 0 HOST_SSHD_PORT: int = 0 SW = "/run/current-system/sw/bin" @@ -42,6 +45,12 @@ SH = f"{SW}/sh" PEER_EDGES = {(src, to) for src, to, _, _ in EDGES if to not in ("@host", "@internet")} +# pre-compute the set of all host-access ports (union across containers) +ALL_HOST_ACCESS_PORTS = {p for ps in HOST_ACCESS_TCP.values() for p in ps} + +# published host ports +FWD_HOST_PORTS = {hp for hp, _, _ in FORWARD_TCP} + def failed_units(node_arg=""): out = server.succeed(f"systemctl {node_arg} list-units --state=failed --no-legend --plain || true") @@ -144,49 +153,38 @@ with subtest("(isolation): internet egress follows the access list"): server.fail(net_cmd(name, f"{IP} route show default | grep -q .")) with subtest("(functional): container DNS still resolves"): + src = enterable[0] + dst = next(c for c in included if c != src) resolved = server.succeed( - net_cmd("git", f"{DIG} +short +time=3 @{HOST_ADDR['git']} nginx.containers") + net_cmd(src, f"{DIG} +short +time=3 @{HOST_ADDR[src]} {dst}.containers") ).strip() - assert resolved == ADDR["nginx"], f"nginx.containers resolved to {resolved!r}, expected {ADDR['nginx']}" + assert resolved == ADDR[dst], f"{dst}.containers resolved to {resolved!r}, expected {ADDR[dst]}" with subtest("(functional): web services respond"): - for name, port in [("devenv", 8080), ("maven", 8080), ("git", 3000)]: - server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2")) - -# TODO: re-enable mail -""" -with subtest("(functional): mail accepts connections"): - smtp_probe = net_cmd("mail", f"{SH} -c 'exec 3<>/dev/tcp/{ADDR['mail']}/25 && head -1 <&3'") - server.wait_until_succeeds(f"{smtp_probe} | grep -q '^220'", timeout=60) - - imap_probe = net_cmd( - "mail", - f"{TIMEOUT} 3 {OPENSSL} s_client -connect {ADDR['mail']}:993 -quiet -verify_quiet 2>/dev/null", - ) - - def imap_ready(_): - greeting = server.execute(f"echo | {imap_probe} | head -1")[1] - return greeting.strip().startswith("* OK") - - retry(imap_ready, timeout_seconds=60) -""" + for name, port in NGINX_BACKENDS.items(): + if name in enterable: + server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2")) with subtest("(functional): nginx proxies to backend containers"): - server.succeed( - net_cmd( - "nginx", - f"{CURL} -sfk --max-time 5 --resolve git.floweytf.com:443:{ADDR['nginx']} " - "https://git.floweytf.com/ >&2", - ) - ) + if "nginx" in enterable: + for domain, backend in NGINX_DOMAINS.items(): + if backend in included: + server.succeed( + net_cmd( + "nginx", + f"{CURL} -sfk --max-time 5 --resolve {domain}:443:{ADDR['nginx']} " + f"https://{domain}/ >&2", + ) + ) with subtest("(functional): sshpiper authenticates/routes properly"): - whoami = server.succeed(f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} whoami").strip() - assert whoami == "runner", f"expected to land as runner, got {whoami}" - hostname = server.succeed( - f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} hostname" - ).strip() - assert hostname == "ssh-admin", f"expected to land on ssh-admin, got {hostname}" + if "sshpiper" in included: + whoami = server.succeed(f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} whoami").strip() + assert whoami == "runner", f"expected to land as runner, got {whoami}" + hostname = server.succeed( + f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} hostname" + ).strip() + assert hostname == "ssh-admin", f"expected to land on ssh-admin, got {hostname}" with subtest("(isolation): every declared container edge connects"): for src, to, tcp_ports, _ in EDGES: @@ -196,89 +194,136 @@ with subtest("(isolation): every declared container edge connects"): server.succeed(ct_connect(src, ADDR[to], port)) with subtest("(isolation): undeclared container edges are refused"): - for src, dst, port in [ - ("devenv", "git", 3000), - ("git", "devenv", 8080), - ("maven", "git", 3000), - # ("nginx", "mail", 25), - ("ssh-botm", "ssh-admin", 22), - ("overseer", "git", 3000), - ]: - assert (src, dst) not in PEER_EDGES, f"{src} -> {dst} is a declared edge, not a negative case" - server.fail(ct_connect(src, ADDR[dst], port, timeout=2)) + tested = 0 + for src in enterable: + for dst in enterable: + if src == dst or (src, dst) in PEER_EDGES: + continue + ports = EXPOSED_TCP.get(dst, []) + if not ports: + continue + server.fail(ct_connect(src, ADDR[dst], ports[0], timeout=2)) + tested += 1 + assert tested > 0, "no negative edge cases to test" with subtest("(isolation): edges are one-way"): - assert ("nginx", "git") in PEER_EDGES - assert ("git", "nginx") not in PEER_EDGES - server.succeed(ct_connect("nginx", ADDR["git"], 3000)) - server.fail(ct_connect("git", ADDR["nginx"], 443, timeout=2)) + tested = 0 + for src, dst, tcp_ports, _ in EDGES: + if dst in ("@host", "@internet") or src not in enterable or dst not in enterable: + continue + if (dst, src) in PEER_EDGES or not tcp_ports: + continue + server.succeed(ct_connect(src, ADDR[dst], tcp_ports[0])) + tested += 1 + rev_ports = EXPOSED_TCP.get(src, []) + if rev_ports: + server.fail(ct_connect(dst, ADDR[src], rev_ports[0], timeout=2)) + assert tested > 0, "no one-way edge cases to test" with subtest("(isolation): host access allowlist"): - server.succeed(ct_connect("overseer", HOST_ADDR["overseer"], HOST_EXPORTER_PORT)) - for port in (HOST_EXPORTER_PORT, HOST_CADVISOR_PORT): - server.fail(ct_connect("git", HOST_ADDR["git"], port, timeout=2)) - server.fail(ct_connect("git", HOST_ADDR["overseer"], port, timeout=2)) + for name, ports in HOST_ACCESS_TCP.items(): + if name in enterable: + for port in ports: + server.succeed(ct_connect(name, HOST_ADDR[name], port)) + no_access = [c for c in enterable if c not in HOST_ACCESS_TCP] + if no_access and ALL_HOST_ACCESS_PORTS: + ct = no_access[0] + for port in ALL_HOST_ACCESS_PORTS: + server.fail(ct_connect(ct, HOST_ADDR[ct], port, timeout=2)) + # reaching the port through a different container's host addr is also blocked + for other in HOST_ACCESS_TCP: + if other in included: + server.fail(ct_connect(ct, HOST_ADDR[other], port, timeout=2)) with subtest("(isolation): host services outside the access list"): internet.succeed(connect(server_ip, HOST_SSHD_PORT)) - for name in ("git", "overseer"): - server.fail(ct_connect(name, HOST_ADDR[name], HOST_SSHD_PORT, timeout=2)) - server.fail(ct_connect(name, server_ip, HOST_SSHD_PORT, timeout=2)) + for name in enterable: + if HOST_SSHD_PORT not in HOST_ACCESS_TCP.get(name, []): + server.fail(ct_connect(name, HOST_ADDR[name], HOST_SSHD_PORT, timeout=2)) + server.fail(ct_connect(name, server_ip, HOST_SSHD_PORT, timeout=2)) with subtest("(isolation): source address spoofing is dropped"): + src = enterable[0] + other = enterable[1] + before = spoof_counter() - server.succeed(net_cmd("git", f"{PING} -c 1 -W 2 {HOST_ADDR['git']} >&2")) + server.succeed(net_cmd(src, f"{PING} -c 1 -W 2 {HOST_ADDR[src]} >&2")) assert spoof_counter() == before, "an unspoofed packet hit the source guard" - iface = IFACE["git"] - server.succeed(net_cmd("git", f"{IP} addr add {ADDR['nginx']}/32 dev {iface}")) + iface = IFACE[src] + server.succeed(net_cmd(src, f"{IP} addr add {ADDR[other]}/32 dev {iface}")) try: server.execute( - net_cmd("git", f"{PING} -c 2 -W 2 -I {ADDR['nginx']} {HOST_ADDR['git']}") + net_cmd(src, f"{PING} -c 2 -W 2 -I {ADDR[other]} {HOST_ADDR[src]}") ) assert spoof_counter() > before, "a forged source address was not dropped" finally: - server.succeed(net_cmd("git", f"{IP} addr del {ADDR['nginx']}/32 dev {iface}")) + server.succeed(net_cmd(src, f"{IP} addr del {ADDR[other]}/32 dev {iface}")) with subtest("(isolation): renaming the interface changes nothing"): - iface, host_addr = IFACE["nginx"], HOST_ADDR["nginx"] + # find a container with a declared peer edge + rename_src = rename_dst = rename_port = None + for src, dst, tcp_ports, _ in EDGES: + if dst in ("@host", "@internet") or src not in enterable or dst not in included: + continue + if tcp_ports: + rename_src, rename_dst, rename_port = src, dst, tcp_ports[0] + break + + assert rename_src is not None and rename_dst is not None and rename_port is not None, \ + "no peer edge to test interface rename" + + iface, host_addr = IFACE[rename_src], HOST_ADDR[rename_src] def rename(frm, to): - server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} down")) - server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} name {to}")) - server.succeed(net_cmd("nginx", f"{IP} link set dev {to} up")) + server.succeed(net_cmd(rename_src, f"{IP} link set dev {frm} down")) + server.succeed(net_cmd(rename_src, f"{IP} link set dev {frm} name {to}")) + server.succeed(net_cmd(rename_src, f"{IP} link set dev {to} up")) for route in (host_addr, f"10.0.0.0/8 via {host_addr}", f"default via {host_addr}"): - server.succeed(net_cmd("nginx", f"{IP} route replace {route} dev {to}")) + server.succeed(net_cmd(rename_src, f"{IP} route replace {route} dev {to}")) rename(iface, "evil0") try: - server.succeed(ct_connect("nginx", ADDR["git"], 3000)) - # server.fail(ct_connect("nginx", ADDR["mail"], 25, timeout=2)) + server.succeed(ct_connect(rename_src, ADDR[rename_dst], rename_port)) finally: rename("evil0", iface) with subtest("(isolation): the veth cannot be moved out of the namespace"): - server.fail(net_cmd("git", f"{IP} link set dev {IFACE['git']} netns 1")) + ct = enterable[0] + server.fail(net_cmd(ct, f"{IP} link set dev {IFACE[ct]} netns 1")) with subtest("(isolation): route bypass"): - iface = IFACE["hopper"] + bypass_ct = next((c for c in enterable if c in no_gateway), None) + if bypass_ct is None: + bypass_ct = next((c for c in enterable if c not in allow_internet), None) + assert bypass_ct is not None, "no container without internet to test route bypass" + + iface = IFACE[bypass_ct] server.succeed( - net_cmd("hopper", f"{IP} route replace default via {HOST_ADDR['hopper']} dev {iface} onlink") + net_cmd(bypass_ct, f"{IP} route replace default via {HOST_ADDR[bypass_ct]} dev {iface} onlink") ) - server.fail(net_cmd("hopper", f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) - server.fail(ct_connect("hopper", ADDR["git"], 3000, timeout=2)) + server.fail(net_cmd(bypass_ct, f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) + unreachable = next( + (c for c in included if c != bypass_ct and (bypass_ct, c) not in PEER_EDGES and EXPOSED_TCP.get(c, [])), + None, + ) + if unreachable: + server.fail(ct_connect(bypass_ct, ADDR[unreachable], EXPOSED_TCP[unreachable][0], timeout=2)) with subtest("(isolation): no IPv6 link-local addresses"): - for name in ["git", "overseer"]: + for name in enterable: assert not has_ipv6_link_local(name), f"{name} still has an IPv6 link-local address" with subtest("(isolation): published ports grant no reach between containers"): - for port in (80, 443, 25565, 22, 25): - server.fail(ct_connect("overseer", HOST_ADDR["overseer"], port, timeout=2)) - server.fail(ct_connect("git", server_ip, port, timeout=2)) + if FWD_HOST_PORTS: + ct = next((c for c in enterable if c not in HOST_ACCESS_TCP), enterable[0]) + for port in FWD_HOST_PORTS: + server.fail(ct_connect(ct, HOST_ADDR[ct], port, timeout=2)) + server.fail(ct_connect(ct, server_ip, next(iter(FWD_HOST_PORTS)), timeout=2)) with subtest("(isolation): privilege escalation / container escape"): - leader = container_pid("git") + ct = enterable[0] + leader = container_pid(ct) host_uid = server.succeed(f"awk '/^Uid:/{{print $2}}' /proc/{leader}/status").strip() assert host_uid != "0", "container's PID 1 is not idmapped away from host uid 0" @@ -287,28 +332,16 @@ with subtest("(isolation): privilege escalation / container escape"): container_pidns = server.succeed(f"readlink /proc/{leader}/ns/pid").strip() assert host_pidns != container_pidns, "container shares the host's PID namespace" - server.succeed( - f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " - f"{SH} -c 'echo x >> /var/lib/gitea/canary' >&2" - ) - - sshpiper_pid = container_pid("sshpiper") - server.fail( - f"{NSENTER} --mount=/proc/{sshpiper_pid}/ns/mnt --root=/proc/{sshpiper_pid}/root -- " - f"{SH} -c 'echo x >> /etc/ssh/ssh_host_ed25519_key' >&2" - ) - server.fail( - f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " - f"{SH} -c 'touch /nix/store/canary' >&2" - ) + server.fail(ct_cmd(ct, "touch /nix/store/canary")) server.fail( f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " "/run/current-system/sw/bin/test -d /persist >&2" ) with subtest("(functional): external clients reach published ports"): - for port in (80, 25565, 22): - internet.succeed(ext_connect(port)) + for hp, _, _ in FORWARD_TCP: + internet.succeed(ext_connect(hp)) + # a non-published port should be unreachable internet.fail(ext_connect(8080)) with subtest("(functional): no failing-reason restarts"): @@ -317,21 +350,27 @@ with subtest("(functional): no failing-reason restarts"): assert ( failed_units(f"-M {name}") == [] ), f"{name} has failed units: {failed_units(f'-M {name}')}" - restarts = server.succeed(f"systemctl show container@{name}.service -p NRestarts --value").strip() - assert restarts == "0", f"container@{name}.service restarted {restarts} times" + unit = "container" if KIND[name] == "nspawn" else "microvm" + restarts = server.succeed(f"systemctl show {unit}@{name}.service -p NRestarts --value").strip() + assert restarts == "0", f"{unit}@{name}.service restarted {restarts} times" with subtest("(functional): port forwards survive an nftables reload"): - server.succeed("systemctl reload nftables.service") - elements = server.succeed("nft list map ip container-portforward tcp-fwd") - assert re.search(r"\b80\s*:", elements), f"nftables reload emptied the map: {elements!r}" - internet.wait_until_succeeds(ext_connect(80), timeout=30) + if FORWARD_TCP: + hp, ct_name, _ = FORWARD_TCP[0] + server.succeed("systemctl reload nftables.service") + elements = server.succeed("nft list map ip container-portforward tcp-fwd") + assert re.search(rf"\b{hp}\s*:", elements), f"nftables reload emptied the map: {elements!r}" + internet.wait_until_succeeds(ext_connect(hp), timeout=30) with subtest("(functional): published ports follow the container's lifetime"): - server.succeed("systemctl stop container@nginx") - internet.fail(ext_connect(80)) - server.succeed("systemctl start container@nginx") - server.wait_for_unit("container@nginx.service") - internet.wait_until_succeeds(ext_connect(80), timeout=30) + if FORWARD_TCP: + hp, ct_name, _ = FORWARD_TCP[0] + unit = "container" if KIND[ct_name] == "nspawn" else "microvm" + server.succeed(f"systemctl stop {unit}@{ct_name}") + internet.fail(ext_connect(hp)) + server.succeed(f"systemctl start {unit}@{ct_name}") + server.wait_for_unit(f"{unit}@{ct_name}.service") + internet.wait_until_succeeds(ext_connect(hp), timeout=30) with subtest("(functional): declared devices are usable in-container"): for name, devices in DEVICES.items():