From 9877005d3507527e31ddf5d879378486ee2208e2 Mon Sep 17 00:00:00 2001 From: "gitea-actions[bot]" Date: Thu, 17 Sep 2026 04:17:59 +0000 Subject: [PATCH 1/5] flake.lock: update inputs --- flake.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 6e14954..0c21124 100644 --- a/flake.lock +++ b/flake.lock @@ -26,11 +26,11 @@ "spectrum": "spectrum" }, "locked": { - "lastModified": 1786300091, - "narHash": "sha256-4UFJOVGpaYtHW5yasSv80MaJxTBYdk2zyf3jhKtt0wA=", + "lastModified": 1789506833, + "narHash": "sha256-4L8SOCF84de1irB72Ka9VHcu9wrAJ9Amuey4lU9oh4Y=", "owner": "microvm-nix", "repo": "microvm.nix", - "rev": "71beea0076cd46dafcee97a5a2e7d00cbba5bd2f", + "rev": "187b0a390ee054028106a674e7b01b1cb940cbba", "type": "github" }, "original": { @@ -63,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1786384358, - "narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=", + "lastModified": 1789546076, + "narHash": "sha256-zVxLZiSnmaaPLwnhj7pwmqe3axBg/C6nG5JZsJMh2g4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a", + "rev": "b1b875982b17dabde9b4a37f3e229e74913e6db3", "type": "github" }, "original": { @@ -87,11 +87,11 @@ "spectrum": { "flake": false, "locked": { - "lastModified": 1785761586, - "narHash": "sha256-MWOMVqJJERwjQGgRIv8d6rlEBqbLM3VZWxHkNKIIZNw=", + "lastModified": 1786032750, + "narHash": "sha256-+Ba/kIn5BjvV7PN8bWOo7RoqgTY/Yv2M7qeMf3wS2Ag=", "ref": "refs/heads/main", - "rev": "a7762d6f54b40560dd5255ce902e6e6a5d980fe9", - "revCount": 1416, + "rev": "77d2389874bb092b8dbf05ecc5b1030372e80e98", + "revCount": 1417, "type": "git", "url": "https://spectrum-os.org/git/spectrum" }, -- 2.55.0 From 363a02f8650cb0eca3a8691f84cd3ac140e1a6a4 Mon Sep 17 00:00:00 2001 From: flowey Date: Thu, 17 Sep 2026 00:18:25 -0500 Subject: [PATCH 2/5] fix flake update --- containers/server-n0/git.nix | 3 ++- dsl.nix | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/containers/server-n0/git.nix b/containers/server-n0/git.nix index b7ac5ce..050812f 100644 --- a/containers/server-n0/git.nix +++ b/containers/server-n0/git.nix @@ -176,8 +176,9 @@ in PermitEmptyPasswords = "yes"; }; }; + security.pam.services.sshd.allowNullPassword = true; users.users.root.initialHashedPassword = ""; - services.journald.extraConfig = "ForwardToConsole=yes"; + services.journald.settings.Journal.ForwardToConsole = "yes"; virtualisation.containerd.enable = true; virtualisation.containerd.settings.plugins."io.containerd.snapshotter.v1.overlayfs".mount_options = diff --git a/dsl.nix b/dsl.nix index 2af94fe..91bf6ed 100644 --- a/dsl.nix +++ b/dsl.nix @@ -1269,7 +1269,7 @@ rec { # containers don't forward journal to console by default containers = lib.genAttrs nspawnNames (_: { timeoutStartSec = "10min"; - config.services.journald.extraConfig = "ForwardToConsole=yes"; + config.services.journald.settings.Journal.ForwardToConsole = "yes"; }); # a test VM has neither the memory to hand out nor the cores to divide. -- 2.55.0 From 1b796ff9dd5ae01d0318591cad8b5b6750e00f56 Mon Sep 17 00:00:00 2001 From: "gitea-actions[bot]" Date: Tue, 22 Sep 2026 04:19:08 +0000 Subject: [PATCH 3/5] flake.lock: update inputs --- flake.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 98d5d3a..9d2a79b 100644 --- a/flake.lock +++ b/flake.lock @@ -38,11 +38,11 @@ "spectrum": "spectrum" }, "locked": { - "lastModified": 1786300091, - "narHash": "sha256-4UFJOVGpaYtHW5yasSv80MaJxTBYdk2zyf3jhKtt0wA=", + "lastModified": 1789506833, + "narHash": "sha256-4L8SOCF84de1irB72Ka9VHcu9wrAJ9Amuey4lU9oh4Y=", "owner": "microvm-nix", "repo": "microvm.nix", - "rev": "71beea0076cd46dafcee97a5a2e7d00cbba5bd2f", + "rev": "187b0a390ee054028106a674e7b01b1cb940cbba", "type": "github" }, "original": { @@ -75,11 +75,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1786384358, - "narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=", + "lastModified": 1789921291, + "narHash": "sha256-Ft/BRnIqw1MywFoXydKobjjWmDFgDdYtSpJliE8+yUw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a", + "rev": "44a91898084f46797b5fac650c7e8c9ac38c43d4", "type": "github" }, "original": { @@ -100,11 +100,11 @@ "spectrum": { "flake": false, "locked": { - "lastModified": 1785761586, - "narHash": "sha256-MWOMVqJJERwjQGgRIv8d6rlEBqbLM3VZWxHkNKIIZNw=", + "lastModified": 1786032750, + "narHash": "sha256-+Ba/kIn5BjvV7PN8bWOo7RoqgTY/Yv2M7qeMf3wS2Ag=", "ref": "refs/heads/main", - "rev": "a7762d6f54b40560dd5255ce902e6e6a5d980fe9", - "revCount": 1416, + "rev": "77d2389874bb092b8dbf05ecc5b1030372e80e98", + "revCount": 1417, "type": "git", "url": "https://spectrum-os.org/git/spectrum" }, -- 2.55.0 From 2a343542cd1f1b2ef6c744abc24d6d04acce83d4 Mon Sep 17 00:00:00 2001 From: flowey Date: Tue, 22 Sep 2026 11:11:33 -0500 Subject: [PATCH 4/5] meow --- containers/server-n0/git.nix | 4 ++ containers/server-n0/mail.nix | 3 ++ containers/server-n0/maven.nix | 4 ++ containers/server-n0/nginx.nix | 75 +++++++++++++++------------------- 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/containers/server-n0/git.nix b/containers/server-n0/git.nix index 050812f..adbca87 100644 --- a/containers/server-n0/git.nix +++ b/containers/server-n0/git.nix @@ -53,6 +53,10 @@ in systemd.services.gitea.serviceConfig.FailureAction = "exit-force"; }; net.expose.tcp = [ 3000 ]; + miscProperties.nginx = { + domain = "git.floweytf.com"; + port = 3000; + }; }; ci-runner = diff --git a/containers/server-n0/mail.nix b/containers/server-n0/mail.nix index 0581593..8fad71d 100644 --- a/containers/server-n0/mail.nix +++ b/containers/server-n0/mail.nix @@ -150,6 +150,9 @@ mkContainer { net.access = [ { to = "@internet"; } ]; svc.autoStart = true; + miscProperties.nginx = { + domain = "mail.floweytf.com"; + }; net.forward.tcp = [ 25 993 diff --git a/containers/server-n0/maven.nix b/containers/server-n0/maven.nix index bfa00aa..a0f1563 100644 --- a/containers/server-n0/maven.nix +++ b/containers/server-n0/maven.nix @@ -38,4 +38,8 @@ mkContainer { environment.JAVA_OPTS = "-Xmx128M"; }; net.expose.tcp = [ 8080 ]; + miscProperties.nginx = { + domain = "maven.floweytf.com"; + port = 8080; + }; } diff --git a/containers/server-n0/nginx.nix b/containers/server-n0/nginx.nix index 3c7b71d..68a1b8f 100644 --- a/containers/server-n0/nginx.nix +++ b/containers/server-n0/nginx.nix @@ -7,7 +7,7 @@ ... }: let - inherit (dsl) mkContainer; + inherit (dsl) mkContainer collectContainers; inherit (hostInfo.paths) data; mkNginxConfig = @@ -40,9 +40,6 @@ let defaultLoc = lib.optionalAttrs (upstream != null) { "/" = { proxyWebsockets = websockets; - # avoid startup error when containers aren't registered yet - # proxyPass is left unset for that reason, so recommendedProxySettings - # won't auto-include its headers here - set them manually to match extraConfig = '' set $backend "${upstream.container.name}:${toString upstream.port}"; proxy_pass http://$backend; @@ -157,49 +154,45 @@ let up = container: port: { inherit container port; }; - nginxSetup = - hostAddr: - mkNginxConfig { - email = "floweydev@gmail.com"; - resolver = hostAddr; - vhosts = [ - { - domain = "devenv.floweytf.com"; - upstream = up containers.devenv 8080; - } - { - domain = "maven.floweytf.com"; - upstream = up containers.maven 8080; - } - { - domain = "git.floweytf.com"; - upstream = up containers.git 3000; - } - { - domain = "utils.floweytf.com"; - redirect = false; - listenHttp = true; - extraLocations = { - "/static/" = { - alias = "/usr/share/nginx/static/"; - extraConfig = "autoindex on;"; - }; - }; - } - { - # no upstream, vhost only exists for certs - domain = "mail.floweytf.com"; - } - ]; - }; + discovered = collectContainers "nginx" containers; + + discoveredVhosts = map ( + { container, data }: + { inherit (data) domain; } + // lib.optionalAttrs (data ? port) { upstream = up container data.port; } + // lib.optionalAttrs (data ? websockets) { inherit (data) websockets; } + // lib.optionalAttrs (data ? redirect) { inherit (data) redirect; } + // lib.optionalAttrs (data ? listenHttp) { inherit (data) listenHttp; } + // lib.optionalAttrs (data ? extraLocations) { inherit (data) extraLocations; } + ) discovered; + + extraVhosts = [ + { + domain = "utils.floweytf.com"; + redirect = false; + listenHttp = true; + extraLocations = { + "/static/" = { + alias = "/usr/share/nginx/static/"; + extraConfig = "autoindex on;"; + }; + }; + } + ]; + + upstreamContainers = map (e: e.container) (lib.filter ({ data, ... }: data ? port) discovered); in let - setup = nginxSetup hostAddr; + setup = mkNginxConfig { + email = "floweydev@gmail.com"; + resolver = hostAddr; + vhosts = discoveredVhosts ++ extraVhosts; + }; c = mkContainer { inherit name; net.access = setup.access ++ [ { to = "@internet"; } ]; svc.autoStart = true; - svc.depends = lib.attrVals [ "devenv" "maven" "git" ] containers; + svc.depends = upstreamContainers; net.forward.tcp = [ 80 443 -- 2.55.0 From bea649649196627d91d909fad8879a4837725abd Mon Sep 17 00:00:00 2001 From: flowey Date: Tue, 22 Sep 2026 15:53:42 -0500 Subject: [PATCH 5/5] meow --- containers/server-n0/devenv.nix | 54 ------- containers/server-n0/nginx.nix | 8 +- dsl.nix | 43 ++++++ flake.nix | 7 +- tests/server-n0/integration.nix | 145 +++++++++--------- tests/server-n0/integration.py | 253 ++++++++++++++++++-------------- 6 files changed, 269 insertions(+), 241 deletions(-) delete mode 100644 containers/server-n0/devenv.nix diff --git a/containers/server-n0/devenv.nix b/containers/server-n0/devenv.nix deleted file mode 100644 index 400238f..0000000 --- a/containers/server-n0/devenv.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - name, - hostInfo, - dsl, - ... -}: -let - inherit (dsl) mkContainer mkService; - inherit (hostInfo.paths) data secrets; -in -mkContainer { - inherit name; - net.access = [ { to = "@internet"; } ]; - svc.autoStart = true; - resources.fs = { - "/config" = { - hostPath = "${data}/web/devenv/config"; - isReadOnly = false; - }; - "/workspace" = { - hostPath = "${data}/web/devenv/workspace"; - isReadOnly = false; - }; - "/run/secrets/code-server-env" = { - hostPath = "${secrets}/code-server-env"; - isReadOnly = true; - }; - }; - users = [ - { - name = "coder"; - uid = 1000; - home = "/config"; - } - ]; - service = - { pkgs, lib, ... }: - mkService { - description = "code-server"; - execStart = "${lib.getExe' pkgs.code-server "code-server"} --bind-addr 0.0.0.0:8080 --user-data-dir /config/data --extensions-dir /config/extensions --auth password /workspace"; - networkOnline = true; - serviceConfig = { - EnvironmentFile = "/run/secrets/code-server-env"; - User = "coder"; - ReadWritePaths = [ - "/config" - "/workspace" - ]; - }; - }; - net.expose.tcp = [ 8080 ]; - resources.mem = "4G"; - resources.cpu = 2; -} diff --git a/containers/server-n0/nginx.nix b/containers/server-n0/nginx.nix index 68a1b8f..1c8a266 100644 --- a/containers/server-n0/nginx.nix +++ b/containers/server-n0/nginx.nix @@ -158,7 +158,9 @@ let discoveredVhosts = map ( { container, data }: - { inherit (data) domain; } + { + inherit (data) domain; + } // lib.optionalAttrs (data ? port) { upstream = up container data.port; } // lib.optionalAttrs (data ? websockets) { inherit (data) websockets; } // lib.optionalAttrs (data ? redirect) { inherit (data) redirect; } @@ -181,8 +183,7 @@ let ]; upstreamContainers = map (e: e.container) (lib.filter ({ data, ... }: data ? port) discovered); -in -let + setup = mkNginxConfig { email = "floweydev@gmail.com"; resolver = hostAddr; @@ -201,6 +202,7 @@ let hostPath = "${data}/core/nginx/acme"; isReadOnly = false; }; + miscProperties.acmeDomains = map (v: v.domain) (discoveredVhosts ++ extraVhosts); inherit (setup) containerConfig; }; inherit (c) hostAddr; diff --git a/dsl.nix b/dsl.nix index 91bf6ed..95ce857 100644 --- a/dsl.nix +++ b/dsl.nix @@ -1223,6 +1223,43 @@ rec { ) (lib.attrValues autoStartTestContainers) ); + exposedTcp = lib.mapAttrs (_: c: c.ports or [ ]) autoStartTestContainers; + + hostAccessTcp = lib.filterAttrs (_: v: v != [ ]) ( + lib.mapAttrs ( + _: c: lib.concatMap (a: if a.to == "@host" then a.tcp or [ ] else [ ]) c.access + ) autoStartTestContainers + ); + + pyForwardTcp = lib.concatStringsSep ", " ( + lib.concatMap ( + c: + map ( + p: + let + np = normalizePort p; + in + "(${toString np.host}, \"${c.name}\", ${toString np.container})" + ) c.hostPorts + ) (lib.attrValues autoStartTestContainers) + ); + + nginxBackendPorts = lib.mapAttrs (_: c: c.miscProperties.nginx.port) ( + lib.filterAttrs ( + _: c: c.miscProperties ? nginx && c.miscProperties.nginx ? port + ) autoStartTestContainers + ); + + nginxDomainMap = lib.listToAttrs ( + lib.concatMap ( + c: + lib.optional (c.miscProperties ? nginx && c.miscProperties.nginx ? domain) { + name = c.miscProperties.nginx.domain; + value = c.name; + } + ) (lib.attrValues autoStartTestContainers) + ); + bin = lib.getExe'; testTools = [ @@ -1330,6 +1367,12 @@ rec { # (src, to, tcp ports, udp ports); `to` may be "@host" or "@internet" EDGES = [${pyEdges}] + EXPOSED_TCP = ${pyDict pyInts exposedTcp} + HOST_ACCESS_TCP = ${pyDict pyInts hostAccessTcp} + FORWARD_TCP = [${pyForwardTcp}] + NGINX_BACKENDS = ${pyDict toString nginxBackendPorts} + NGINX_DOMAINS = ${pyDict pyStr nginxDomainMap} + NSENTER = "${bin pkgs.util-linux "nsenter"}" CURL = "${bin pkgs.curl "curl"}" IP = "${bin pkgs.iproute2 "ip"}" diff --git a/flake.nix b/flake.nix index 95b7e43..ca6beee 100644 --- a/flake.nix +++ b/flake.nix @@ -65,7 +65,12 @@ specialArgsFor = hostName: { hostInfo = hostInfoFor hostName; - inherit inputs customPkgs flakeRev deployLabel; + inherit + inputs + customPkgs + flakeRev + deployLabel + ; }; hostDefFor = diff --git a/tests/server-n0/integration.nix b/tests/server-n0/integration.nix index e985211..377891e 100644 --- a/tests/server-n0/integration.nix +++ b/tests/server-n0/integration.nix @@ -14,13 +14,11 @@ let it = dsl.mkIntegrationTest { inherit host testContainers pkgs; }; - nginxAcmeDomains = [ - "devenv.floweytf.com" - "maven.floweytf.com" - "git.floweytf.com" - "utils.floweytf.com" - "mail.floweytf.com" - ]; + nginxAcmeDomains = + if testContainers ? nginx && testContainers.nginx.miscProperties ? acmeDomains then + testContainers.nginx.miscProperties.acmeDomains + else + [ ]; nginxAcmeDir = domain: "${serverDataDir}/core/nginx/acme/${domain}"; @@ -67,9 +65,44 @@ let ''; sshpiperTestUser = "admin"; - hostExporterPort = 9100; - hostCadvisorPort = 9200; hostSshdPort = 5952; + + # derive data directories from container bind mounts + allDataDirs = lib.unique ( + lib.concatMap (c: lib.filter (p: lib.hasPrefix serverDataDir p) c.bindMountHostPaths) ( + lib.attrValues testContainers + ) + ); + + mkdirAllDataDirs = lib.concatMapStrings (p: ''mkdir -p "${p}"'' + "\n") allDataDirs; + + # writable data dirs need ownership set to 1000:1000 (the standard + # container user uid). services that run as a different user (loki, + # prometheus, grafana) fix ownership via StateDirectory on startup. + writableDataDirs = lib.unique ( + lib.concatMap ( + c: + if c.kind == "nspawn" then + lib.filter (p: lib.hasPrefix serverDataDir p) ( + map (bm: bm.hostPath) (lib.attrValues (lib.filterAttrs (_: bm: !bm.isReadOnly) c.bindMounts)) + ) + else + [ ] + ) (lib.attrValues testContainers) + ); + + chownWritableDataDirs = lib.concatMapStrings ( + p: ''chown -R 1000:1000 "${p}"'' + "\n" + ) writableDataDirs; + + # secret paths needed by test containers + secretPaths = lib.unique ( + lib.concatMap (c: lib.filter (p: lib.hasPrefix serverSecretsDir p) c.bindMountHostPaths) ( + lib.attrValues testContainers + ) + ); + + mkdirSecretParents = lib.concatMapStrings (p: ''mkdir -p "$(dirname '${p}')"'' + "\n") secretPaths; in { name = "integration-${host.hostName}"; @@ -90,63 +123,33 @@ in it.serverModule ]; - containers.nginx.config = { - systemd.services = lib.genAttrs (map (d: "acme-order-renew-${d}") nginxAcmeDomains) (_: { - enable = lib.mkForce false; - }); - }; - system.activationScripts.test-secrets = # bash '' - mkdir -p /nix/var/nix/db /nix/var/nix/daemon-socket - - mkdir -p /var/log/journal + mkdir -p /nix/var/nix/db /nix/var/nix/daemon-socket + mkdir -p /var/log/journal + mkdir -p "${serverSecretsDir}" + ${mkdirAllDataDirs} ${mkdirAcmeDirs} - mkdir -p "${serverSecretsDir}" \ - "${serverDataDir}/core/sshpiper" \ - "${serverDataDir}/core/mail" \ - "${serverDataDir}/overseer/loki/data" \ - "${serverDataDir}/overseer/prometheus/data" \ - "${serverDataDir}/overseer/grafana" \ - "${serverDataDir}/web/git" \ - "${serverDataDir}/web/git-ci-runner/nix-rw-store" \ - "${serverDataDir}/web/git-ci-runner/docker" \ - "${serverDataDir}/web/git-ci-runner/runner-state" \ - "${serverSecretsDir}/ci-runner" \ - "${serverDataDir}/web/reposilite" \ - "${serverDataDir}/web/devenv/config" \ - "${serverDataDir}/web/devenv/workspace" \ - "${serverDataDir}/ssh/generic" \ - "${serverDataDir}/ssh/nyaa" \ - "${serverDataDir}/mc/botm" \ - "${serverDataDir}/mc/rogule-dev" \ - "${serverDataDir}/mc/rogule-build" \ - "${serverDataDir}/mc/starwalker" \ - "${serverDataDir}/mc/tl" + ${mkdirSecretParents} + ${chownWritableDataDirs} - chown -R 1000:1000 \ - "${serverDataDir}/web/git" \ - "${serverDataDir}/web/reposilite" \ - "${serverDataDir}/web/devenv/config" \ - "${serverDataDir}/web/devenv/workspace" \ - "${serverDataDir}/core/mail" \ - "${serverDataDir}/ssh/generic" \ - "${serverDataDir}/mc/botm" \ - "${serverDataDir}/mc/rogule-dev" \ - "${serverDataDir}/mc/rogule-build" \ - "${serverDataDir}/mc/starwalker" \ - "${serverDataDir}/mc/tl" + for keytype in ed25519 rsa ecdsa; do + keyfile="${serverSecretsDir}/ssh_host_''${keytype}_key" + if [[ ! -f "$keyfile" ]]; then + ${lib.getExe' pkgs.openssh "ssh-keygen"} -q -t "$keytype" -f "$keyfile" -N "" + fi + done - for keytype in ed25519 rsa ecdsa; do - keyfile="${serverSecretsDir}/ssh_host_''${keytype}_key" - if [[ ! -f "$keyfile" ]]; then - ${lib.getExe' pkgs.openssh "ssh-keygen"} -q -t "$keytype" -f "$keyfile" -N "" - fi - done - - # stalwart rejects weak/dictionary passwords + ${installFakeCerts} + '' + + lib.optionalString (testContainers ? sshpiper) '' + mkdir -p "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}" + install -m 644 ${testSshKey}/key.pub \ + "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}/authorized_keys" + '' + + lib.optionalString (testContainers ? mail) '' for f in mail-admin-password mail-account-password; do target="${serverSecretsDir}/$f" if [[ ! -f "$target" ]]; then @@ -154,22 +157,14 @@ in chmod 600 "$target" fi done - - if [[ ! -f "${serverSecretsDir}/ci-runner/token" ]]; then - echo "TOKEN=test-registration-token" > "${serverSecretsDir}/ci-runner/token" - chmod 600 "${serverSecretsDir}/ci-runner/token" - fi - - if [[ ! -f "${serverSecretsDir}/code-server-env" ]]; then - echo "test-secret" > "${serverSecretsDir}/code-server-env" - chmod 600 "${serverSecretsDir}/code-server-env" - fi - - ${installFakeCerts} - mkdir -p "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}" - install -m 644 ${testSshKey}/key.pub \ - "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}/authorized_keys" ''; + } + // lib.optionalAttrs (testContainers ? nginx && nginxAcmeDomains != [ ]) { + containers.nginx.config = { + systemd.services = lib.genAttrs (map (d: "acme-order-renew-${d}") nginxAcmeDomains) (_: { + enable = lib.mkForce false; + }); + }; }; }; @@ -182,8 +177,6 @@ in TEST_SSH_KEY_STORE = "${testSshKey}/key" SSHPIPER_TEST_USER = "${sshpiperTestUser}" - HOST_EXPORTER_PORT = ${toString hostExporterPort} - HOST_CADVISOR_PORT = ${toString hostCadvisorPort} HOST_SSHD_PORT = ${toString hostSshdPort} '' + builtins.readFile ./integration.py; diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 328dafe..c8235cf 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -29,12 +29,15 @@ if TYPE_CHECKING: KIND: dict[str, str] = {} DEVICES: dict[str, list[str]] = {} EDGES: list[tuple[str, str, list[int], list[int]]] = [] + EXPOSED_TCP: dict[str, list[int]] = {} + HOST_ACCESS_TCP: dict[str, list[int]] = {} + FORWARD_TCP: list[tuple[int, str, int]] = [] + NGINX_BACKENDS: dict[str, int] = {} + NGINX_DOMAINS: dict[str, str] = {} SSH_KEY: str = "" SSH_OPTS: str = "" TEST_SSH_KEY_STORE: str = "" SSHPIPER_TEST_USER: str = "" - HOST_EXPORTER_PORT: int = 0 - HOST_CADVISOR_PORT: int = 0 HOST_SSHD_PORT: int = 0 SW = "/run/current-system/sw/bin" @@ -42,6 +45,12 @@ SH = f"{SW}/sh" PEER_EDGES = {(src, to) for src, to, _, _ in EDGES if to not in ("@host", "@internet")} +# pre-compute the set of all host-access ports (union across containers) +ALL_HOST_ACCESS_PORTS = {p for ps in HOST_ACCESS_TCP.values() for p in ps} + +# published host ports +FWD_HOST_PORTS = {hp for hp, _, _ in FORWARD_TCP} + def failed_units(node_arg=""): out = server.succeed(f"systemctl {node_arg} list-units --state=failed --no-legend --plain || true") @@ -144,49 +153,38 @@ with subtest("(isolation): internet egress follows the access list"): server.fail(net_cmd(name, f"{IP} route show default | grep -q .")) with subtest("(functional): container DNS still resolves"): + src = enterable[0] + dst = next(c for c in included if c != src) resolved = server.succeed( - net_cmd("git", f"{DIG} +short +time=3 @{HOST_ADDR['git']} nginx.containers") + net_cmd(src, f"{DIG} +short +time=3 @{HOST_ADDR[src]} {dst}.containers") ).strip() - assert resolved == ADDR["nginx"], f"nginx.containers resolved to {resolved!r}, expected {ADDR['nginx']}" + assert resolved == ADDR[dst], f"{dst}.containers resolved to {resolved!r}, expected {ADDR[dst]}" with subtest("(functional): web services respond"): - for name, port in [("devenv", 8080), ("maven", 8080), ("git", 3000)]: - server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2")) - -# TODO: re-enable mail -""" -with subtest("(functional): mail accepts connections"): - smtp_probe = net_cmd("mail", f"{SH} -c 'exec 3<>/dev/tcp/{ADDR['mail']}/25 && head -1 <&3'") - server.wait_until_succeeds(f"{smtp_probe} | grep -q '^220'", timeout=60) - - imap_probe = net_cmd( - "mail", - f"{TIMEOUT} 3 {OPENSSL} s_client -connect {ADDR['mail']}:993 -quiet -verify_quiet 2>/dev/null", - ) - - def imap_ready(_): - greeting = server.execute(f"echo | {imap_probe} | head -1")[1] - return greeting.strip().startswith("* OK") - - retry(imap_ready, timeout_seconds=60) -""" + for name, port in NGINX_BACKENDS.items(): + if name in enterable: + server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2")) with subtest("(functional): nginx proxies to backend containers"): - server.succeed( - net_cmd( - "nginx", - f"{CURL} -sfk --max-time 5 --resolve git.floweytf.com:443:{ADDR['nginx']} " - "https://git.floweytf.com/ >&2", - ) - ) + if "nginx" in enterable: + for domain, backend in NGINX_DOMAINS.items(): + if backend in included: + server.succeed( + net_cmd( + "nginx", + f"{CURL} -sfk --max-time 5 --resolve {domain}:443:{ADDR['nginx']} " + f"https://{domain}/ >&2", + ) + ) with subtest("(functional): sshpiper authenticates/routes properly"): - whoami = server.succeed(f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} whoami").strip() - assert whoami == "runner", f"expected to land as runner, got {whoami}" - hostname = server.succeed( - f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} hostname" - ).strip() - assert hostname == "ssh-admin", f"expected to land on ssh-admin, got {hostname}" + if "sshpiper" in included: + whoami = server.succeed(f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} whoami").strip() + assert whoami == "runner", f"expected to land as runner, got {whoami}" + hostname = server.succeed( + f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} hostname" + ).strip() + assert hostname == "ssh-admin", f"expected to land on ssh-admin, got {hostname}" with subtest("(isolation): every declared container edge connects"): for src, to, tcp_ports, _ in EDGES: @@ -196,89 +194,136 @@ with subtest("(isolation): every declared container edge connects"): server.succeed(ct_connect(src, ADDR[to], port)) with subtest("(isolation): undeclared container edges are refused"): - for src, dst, port in [ - ("devenv", "git", 3000), - ("git", "devenv", 8080), - ("maven", "git", 3000), - # ("nginx", "mail", 25), - ("ssh-botm", "ssh-admin", 22), - ("overseer", "git", 3000), - ]: - assert (src, dst) not in PEER_EDGES, f"{src} -> {dst} is a declared edge, not a negative case" - server.fail(ct_connect(src, ADDR[dst], port, timeout=2)) + tested = 0 + for src in enterable: + for dst in enterable: + if src == dst or (src, dst) in PEER_EDGES: + continue + ports = EXPOSED_TCP.get(dst, []) + if not ports: + continue + server.fail(ct_connect(src, ADDR[dst], ports[0], timeout=2)) + tested += 1 + assert tested > 0, "no negative edge cases to test" with subtest("(isolation): edges are one-way"): - assert ("nginx", "git") in PEER_EDGES - assert ("git", "nginx") not in PEER_EDGES - server.succeed(ct_connect("nginx", ADDR["git"], 3000)) - server.fail(ct_connect("git", ADDR["nginx"], 443, timeout=2)) + tested = 0 + for src, dst, tcp_ports, _ in EDGES: + if dst in ("@host", "@internet") or src not in enterable or dst not in enterable: + continue + if (dst, src) in PEER_EDGES or not tcp_ports: + continue + server.succeed(ct_connect(src, ADDR[dst], tcp_ports[0])) + tested += 1 + rev_ports = EXPOSED_TCP.get(src, []) + if rev_ports: + server.fail(ct_connect(dst, ADDR[src], rev_ports[0], timeout=2)) + assert tested > 0, "no one-way edge cases to test" with subtest("(isolation): host access allowlist"): - server.succeed(ct_connect("overseer", HOST_ADDR["overseer"], HOST_EXPORTER_PORT)) - for port in (HOST_EXPORTER_PORT, HOST_CADVISOR_PORT): - server.fail(ct_connect("git", HOST_ADDR["git"], port, timeout=2)) - server.fail(ct_connect("git", HOST_ADDR["overseer"], port, timeout=2)) + for name, ports in HOST_ACCESS_TCP.items(): + if name in enterable: + for port in ports: + server.succeed(ct_connect(name, HOST_ADDR[name], port)) + no_access = [c for c in enterable if c not in HOST_ACCESS_TCP] + if no_access and ALL_HOST_ACCESS_PORTS: + ct = no_access[0] + for port in ALL_HOST_ACCESS_PORTS: + server.fail(ct_connect(ct, HOST_ADDR[ct], port, timeout=2)) + # reaching the port through a different container's host addr is also blocked + for other in HOST_ACCESS_TCP: + if other in included: + server.fail(ct_connect(ct, HOST_ADDR[other], port, timeout=2)) with subtest("(isolation): host services outside the access list"): internet.succeed(connect(server_ip, HOST_SSHD_PORT)) - for name in ("git", "overseer"): - server.fail(ct_connect(name, HOST_ADDR[name], HOST_SSHD_PORT, timeout=2)) - server.fail(ct_connect(name, server_ip, HOST_SSHD_PORT, timeout=2)) + for name in enterable: + if HOST_SSHD_PORT not in HOST_ACCESS_TCP.get(name, []): + server.fail(ct_connect(name, HOST_ADDR[name], HOST_SSHD_PORT, timeout=2)) + server.fail(ct_connect(name, server_ip, HOST_SSHD_PORT, timeout=2)) with subtest("(isolation): source address spoofing is dropped"): + src = enterable[0] + other = enterable[1] + before = spoof_counter() - server.succeed(net_cmd("git", f"{PING} -c 1 -W 2 {HOST_ADDR['git']} >&2")) + server.succeed(net_cmd(src, f"{PING} -c 1 -W 2 {HOST_ADDR[src]} >&2")) assert spoof_counter() == before, "an unspoofed packet hit the source guard" - iface = IFACE["git"] - server.succeed(net_cmd("git", f"{IP} addr add {ADDR['nginx']}/32 dev {iface}")) + iface = IFACE[src] + server.succeed(net_cmd(src, f"{IP} addr add {ADDR[other]}/32 dev {iface}")) try: server.execute( - net_cmd("git", f"{PING} -c 2 -W 2 -I {ADDR['nginx']} {HOST_ADDR['git']}") + net_cmd(src, f"{PING} -c 2 -W 2 -I {ADDR[other]} {HOST_ADDR[src]}") ) assert spoof_counter() > before, "a forged source address was not dropped" finally: - server.succeed(net_cmd("git", f"{IP} addr del {ADDR['nginx']}/32 dev {iface}")) + server.succeed(net_cmd(src, f"{IP} addr del {ADDR[other]}/32 dev {iface}")) with subtest("(isolation): renaming the interface changes nothing"): - iface, host_addr = IFACE["nginx"], HOST_ADDR["nginx"] + # find a container with a declared peer edge + rename_src = rename_dst = rename_port = None + for src, dst, tcp_ports, _ in EDGES: + if dst in ("@host", "@internet") or src not in enterable or dst not in included: + continue + if tcp_ports: + rename_src, rename_dst, rename_port = src, dst, tcp_ports[0] + break + + assert rename_src is not None and rename_dst is not None and rename_port is not None, \ + "no peer edge to test interface rename" + + iface, host_addr = IFACE[rename_src], HOST_ADDR[rename_src] def rename(frm, to): - server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} down")) - server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} name {to}")) - server.succeed(net_cmd("nginx", f"{IP} link set dev {to} up")) + server.succeed(net_cmd(rename_src, f"{IP} link set dev {frm} down")) + server.succeed(net_cmd(rename_src, f"{IP} link set dev {frm} name {to}")) + server.succeed(net_cmd(rename_src, f"{IP} link set dev {to} up")) for route in (host_addr, f"10.0.0.0/8 via {host_addr}", f"default via {host_addr}"): - server.succeed(net_cmd("nginx", f"{IP} route replace {route} dev {to}")) + server.succeed(net_cmd(rename_src, f"{IP} route replace {route} dev {to}")) rename(iface, "evil0") try: - server.succeed(ct_connect("nginx", ADDR["git"], 3000)) - # server.fail(ct_connect("nginx", ADDR["mail"], 25, timeout=2)) + server.succeed(ct_connect(rename_src, ADDR[rename_dst], rename_port)) finally: rename("evil0", iface) with subtest("(isolation): the veth cannot be moved out of the namespace"): - server.fail(net_cmd("git", f"{IP} link set dev {IFACE['git']} netns 1")) + ct = enterable[0] + server.fail(net_cmd(ct, f"{IP} link set dev {IFACE[ct]} netns 1")) with subtest("(isolation): route bypass"): - iface = IFACE["hopper"] + bypass_ct = next((c for c in enterable if c in no_gateway), None) + if bypass_ct is None: + bypass_ct = next((c for c in enterable if c not in allow_internet), None) + assert bypass_ct is not None, "no container without internet to test route bypass" + + iface = IFACE[bypass_ct] server.succeed( - net_cmd("hopper", f"{IP} route replace default via {HOST_ADDR['hopper']} dev {iface} onlink") + net_cmd(bypass_ct, f"{IP} route replace default via {HOST_ADDR[bypass_ct]} dev {iface} onlink") ) - server.fail(net_cmd("hopper", f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) - server.fail(ct_connect("hopper", ADDR["git"], 3000, timeout=2)) + server.fail(net_cmd(bypass_ct, f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) + unreachable = next( + (c for c in included if c != bypass_ct and (bypass_ct, c) not in PEER_EDGES and EXPOSED_TCP.get(c, [])), + None, + ) + if unreachable: + server.fail(ct_connect(bypass_ct, ADDR[unreachable], EXPOSED_TCP[unreachable][0], timeout=2)) with subtest("(isolation): no IPv6 link-local addresses"): - for name in ["git", "overseer"]: + for name in enterable: assert not has_ipv6_link_local(name), f"{name} still has an IPv6 link-local address" with subtest("(isolation): published ports grant no reach between containers"): - for port in (80, 443, 25565, 22, 25): - server.fail(ct_connect("overseer", HOST_ADDR["overseer"], port, timeout=2)) - server.fail(ct_connect("git", server_ip, port, timeout=2)) + if FWD_HOST_PORTS: + ct = next((c for c in enterable if c not in HOST_ACCESS_TCP), enterable[0]) + for port in FWD_HOST_PORTS: + server.fail(ct_connect(ct, HOST_ADDR[ct], port, timeout=2)) + server.fail(ct_connect(ct, server_ip, next(iter(FWD_HOST_PORTS)), timeout=2)) with subtest("(isolation): privilege escalation / container escape"): - leader = container_pid("git") + ct = enterable[0] + leader = container_pid(ct) host_uid = server.succeed(f"awk '/^Uid:/{{print $2}}' /proc/{leader}/status").strip() assert host_uid != "0", "container's PID 1 is not idmapped away from host uid 0" @@ -287,28 +332,16 @@ with subtest("(isolation): privilege escalation / container escape"): container_pidns = server.succeed(f"readlink /proc/{leader}/ns/pid").strip() assert host_pidns != container_pidns, "container shares the host's PID namespace" - server.succeed( - f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " - f"{SH} -c 'echo x >> /var/lib/gitea/canary' >&2" - ) - - sshpiper_pid = container_pid("sshpiper") - server.fail( - f"{NSENTER} --mount=/proc/{sshpiper_pid}/ns/mnt --root=/proc/{sshpiper_pid}/root -- " - f"{SH} -c 'echo x >> /etc/ssh/ssh_host_ed25519_key' >&2" - ) - server.fail( - f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " - f"{SH} -c 'touch /nix/store/canary' >&2" - ) + server.fail(ct_cmd(ct, "touch /nix/store/canary")) server.fail( f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " "/run/current-system/sw/bin/test -d /persist >&2" ) with subtest("(functional): external clients reach published ports"): - for port in (80, 25565, 22): - internet.succeed(ext_connect(port)) + for hp, _, _ in FORWARD_TCP: + internet.succeed(ext_connect(hp)) + # a non-published port should be unreachable internet.fail(ext_connect(8080)) with subtest("(functional): no failing-reason restarts"): @@ -317,21 +350,27 @@ with subtest("(functional): no failing-reason restarts"): assert ( failed_units(f"-M {name}") == [] ), f"{name} has failed units: {failed_units(f'-M {name}')}" - restarts = server.succeed(f"systemctl show container@{name}.service -p NRestarts --value").strip() - assert restarts == "0", f"container@{name}.service restarted {restarts} times" + unit = "container" if KIND[name] == "nspawn" else "microvm" + restarts = server.succeed(f"systemctl show {unit}@{name}.service -p NRestarts --value").strip() + assert restarts == "0", f"{unit}@{name}.service restarted {restarts} times" with subtest("(functional): port forwards survive an nftables reload"): - server.succeed("systemctl reload nftables.service") - elements = server.succeed("nft list map ip container-portforward tcp-fwd") - assert re.search(r"\b80\s*:", elements), f"nftables reload emptied the map: {elements!r}" - internet.wait_until_succeeds(ext_connect(80), timeout=30) + if FORWARD_TCP: + hp, ct_name, _ = FORWARD_TCP[0] + server.succeed("systemctl reload nftables.service") + elements = server.succeed("nft list map ip container-portforward tcp-fwd") + assert re.search(rf"\b{hp}\s*:", elements), f"nftables reload emptied the map: {elements!r}" + internet.wait_until_succeeds(ext_connect(hp), timeout=30) with subtest("(functional): published ports follow the container's lifetime"): - server.succeed("systemctl stop container@nginx") - internet.fail(ext_connect(80)) - server.succeed("systemctl start container@nginx") - server.wait_for_unit("container@nginx.service") - internet.wait_until_succeeds(ext_connect(80), timeout=30) + if FORWARD_TCP: + hp, ct_name, _ = FORWARD_TCP[0] + unit = "container" if KIND[ct_name] == "nspawn" else "microvm" + server.succeed(f"systemctl stop {unit}@{ct_name}") + internet.fail(ext_connect(hp)) + server.succeed(f"systemctl start {unit}@{ct_name}") + server.wait_for_unit(f"{unit}@{ct_name}.service") + internet.wait_until_succeeds(ext_connect(hp), timeout=30) with subtest("(functional): declared devices are usable in-container"): for name, devices in DEVICES.items(): -- 2.55.0