From 3a878914863f2153d342113f95594ba43d7bc9d9 Mon Sep 17 00:00:00 2001 From: Floweynt <50057682+Floweynt@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:18:00 -0500 Subject: [PATCH 1/6] ci: try to fix vulnix --- .github/workflows/vulnix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vulnix.yml b/.github/workflows/vulnix.yml index 13a5440..60f797c 100644 --- a/.github/workflows/vulnix.yml +++ b/.github/workflows/vulnix.yml @@ -25,7 +25,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} API: ${{ github.api_url }} REPO: ${{ github.repository }} - run: nix run -L .#ci -- upsert-issue vulnix "vulnix: known vulnerabilities in system closure" "$RUNNER_TEMP/issue.md" + run: 'nix run -L .#ci -- upsert-issue vulnix "vulnix: known vulnerabilities in system closure" "$RUNNER_TEMP/issue.md"' - name: close resolved issue if: steps.scan.outputs.count == '0' From 0279df6b1128a33a8a89d5df0089d05dcce575d6 Mon Sep 17 00:00:00 2001 From: flowey Date: Fri, 11 Sep 2026 02:06:39 +0000 Subject: [PATCH 2/6] cI: improve cache management (#2) Co-authored-by: Floweynt <50057682+Floweynt@users.noreply.github.com> Reviewed-on: https://git.floweytf.com/server/server-config/pulls/2 --- containers/server-n0/git.nix | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/containers/server-n0/git.nix b/containers/server-n0/git.nix index 07786df..b7ac5ce 100644 --- a/containers/server-n0/git.nix +++ b/containers/server-n0/git.nix @@ -163,10 +163,9 @@ in trusted-public-keys = [ "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys=" ]; }; - nix.gc = { - automatic = true; - dates = "daily"; - options = "--delete-older-than 14d"; + nix.settings = { + min-free = 5 * 1024 * 1024 * 1024; + max-free = 10 * 1024 * 1024 * 1024; }; # nix.optimise.automatic = true; - doesn't work with microvm.writableStoreOverlay From 6b21be0088941c6c299a688ae427afb6f1ade559 Mon Sep 17 00:00:00 2001 From: Floweynt <50057682+Floweynt@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:09:44 -0500 Subject: [PATCH 3/6] ci: try to fix vulnix v2 --- ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.py b/ci.py index b83f958..7c47483 100644 --- a/ci.py +++ b/ci.py @@ -254,7 +254,7 @@ def format_vulnix(vulns): def vulnix(args): paths = list(build_hosts(args.ref).values()) - result = run("vulnix", "-j", *paths, capture_output=True) + result = run("vulnix", "-j", "-C", *paths, capture_output=True) if result.returncode not in (0, 2): print(result.stderr, file=sys.stderr) sys.exit(result.returncode) From 1447febbaf07b7ef5fe910327742ff5590be1835 Mon Sep 17 00:00:00 2001 From: autumn Date: Fri, 11 Sep 2026 07:31:32 +0000 Subject: [PATCH 4/6] frcc experiments (#1) Co-authored-by: Autumn Liu Co-authored-by: Floweynt <50057682+Floweynt@users.noreply.github.com> Reviewed-on: https://git.floweytf.com/server/server-config/pulls/1 Co-authored-by: autumn --- containers/server-n0/ssh.nix | 34 +++++++++++++++++++++++++++++++++- dsl.nix | 20 +++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 6b293a5..01a5dda 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -61,16 +61,29 @@ let net ? { }, resources ? { }, traits ? [ ], + isolation ? { }, }: mkContainer { inherit name; inherit net resources; svc.autoStart = true; - isolation.nixDaemon = nixDaemon; + isolation = { + inherit nixDaemon; + } + // isolation; traits = [ sshTrait ] ++ traits; packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; }; + + kvmTrait = { + containerConfig = + { pkgs, ... }: + { + users.users.runner.extraGroups = [ "kvm" ]; + environment.systemPackages = [ pkgs.qemu_kvm ]; + }; + }; in { ssh-admin = mkSsh { @@ -121,6 +134,25 @@ in mem = "16G"; cpu = 65; }; + traits = [ kvmTrait ]; + isolation.allowedDevices = [ + { + node = "/dev/kvm"; + modifier = "rw"; + } + { + node = "/dev/vhost-net"; + modifier = "rw"; + } + { + node = "/dev/net/tun"; + modifier = "rwm"; + } + ]; + isolation.additionalCapabilities = [ + "CAP_NET_ADMIN" + "CAP_MKNOD" + ]; routingInfo = [ { username = "autumn"; diff --git a/dsl.nix b/dsl.nix index 1da09a6..0def2fc 100644 --- a/dsl.nix +++ b/dsl.nix @@ -228,6 +228,17 @@ rec { : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. + `isolation.allowedDevices` + : List of `{ node; modifier; }` host devices to grant the container's + cgroup access to; see `containers..allowedDevices` in the NixOS + manual. nspawn bind-mounts a handful of devices (`/dev/kvm`, + `/dev/net/tun`, ...) into the container itself once the cgroup allows it - + this doesn't need a matching `resources.fs` bind mount. + + `isolation.additionalCapabilities` + : Extra Linux capabilities granted to the container; see + `containers..additionalCapabilities` in the NixOS manual. + `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. @@ -272,6 +283,8 @@ rec { bindMounts = resources.fs or { }; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; + allowedDevices = isolation.allowedDevices or [ ]; + additionalCapabilities = isolation.additionalCapabilities or [ ]; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; @@ -368,7 +381,12 @@ rec { kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { - inherit autoStart privateUsers; + inherit + autoStart + privateUsers + allowedDevices + additionalCapabilities + ; bindMounts = { }; extraFlags = bindMountExtraFlags; privateNetwork = true; From ba23f719ab8910c532d65f3c2e3e011ff2ec40f9 Mon Sep 17 00:00:00 2001 From: Floweynt Date: Fri, 11 Sep 2026 20:17:12 +0000 Subject: [PATCH 5/6] ci: fix vulnix GITHUB_OUTPUT format error --- ci.nix | 7 +++++++ ci.py | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci.nix b/ci.nix index 2a6d0d1..77fa121 100644 --- a/ci.nix +++ b/ci.nix @@ -1,10 +1,17 @@ { lib, pkgs }: +let + # vulnix's bundled nix-2.34 broke its own --experimental-features detection + nixConf = pkgs.writeText "ci-nix.conf" "experimental-features = nix-command flakes"; +in # tools must track ci.py's subprocess calls; nix stays ambient (uses the # runner's own nix, not a pinned one). pkgs.writers.writePython3Bin "ci" { flakeIgnore = [ "E501" ]; libraries = [ pkgs.python3Packages.subprocess-tee ]; makeWrapperArgs = [ + "--set-default" + "NIX_USER_CONF_FILES" + (toString nixConf) "--prefix" "PATH" ":" diff --git a/ci.py b/ci.py index 7c47483..86eafef 100644 --- a/ci.py +++ b/ci.py @@ -18,7 +18,8 @@ def run(*args, **kw): def nix(*args): - return subprocess_tee.run(("nix", "--extra-experimental-features", "nix-command flakes", "-L", *args), check=True, text=True).stdout + # tee to stderr - stdout leaks into GITHUB_OUTPUT $() captures + return subprocess_tee.run(("nix", "--extra-experimental-features", "nix-command flakes", "-L", *args), check=True, text=True, stdout=sys.stderr).stdout def git(*args, check=True): @@ -254,7 +255,7 @@ def format_vulnix(vulns): def vulnix(args): paths = list(build_hosts(args.ref).values()) - result = run("vulnix", "-j", "-C", *paths, capture_output=True) + result = run("vulnix", "-j", *paths, capture_output=True) if result.returncode not in (0, 2): print(result.stderr, file=sys.stderr) sys.exit(result.returncode) From 1bf51e9a9ea8012bb297e2cad3c3cefb9ba1b58f Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Fri, 11 Sep 2026 14:43:43 -0500 Subject: [PATCH 6/6] update --- containers/server-n0/ssh.nix | 35 +++++++++++----------------------- dsl.nix | 35 +++++++++++++++------------------- tests/server-n0/integration.py | 25 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 01a5dda..bf67c51 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -76,14 +76,6 @@ let miscProperties = self._sshpiperRegister routingInfo; }; - kvmTrait = { - containerConfig = - { pkgs, ... }: - { - users.users.runner.extraGroups = [ "kvm" ]; - environment.systemPackages = [ pkgs.qemu_kvm ]; - }; - }; in { ssh-admin = mkSsh { @@ -126,32 +118,27 @@ in ssh-generic = mkSsh { name = "ssh-generic"; net.access = [ { to = "@internet"; } ]; + # /dev/kvm keeps its host ownership, so the kvm gid must not be shifted. + isolation.privateUsers = "identity"; resources = { fs."/home/runner/data" = { hostPath = "${data}/ssh/generic"; isReadOnly = false; }; + devices = [ "/dev/kvm" ]; mem = "16G"; cpu = 65; }; - traits = [ kvmTrait ]; - isolation.allowedDevices = [ + packages = with pkgs; [ qemu_kvm ]; + traits = [ { - node = "/dev/kvm"; - modifier = "rw"; + containerConfig = + { config, ... }: + { + users.groups.kvm.gid = config.ids.gids.kvm; + users.users.runner.extraGroups = [ "kvm" ]; + }; } - { - node = "/dev/vhost-net"; - modifier = "rw"; - } - { - node = "/dev/net/tun"; - modifier = "rwm"; - } - ]; - isolation.additionalCapabilities = [ - "CAP_NET_ADMIN" - "CAP_MKNOD" ]; routingInfo = [ { diff --git a/dsl.nix b/dsl.nix index 0def2fc..475c5a8 100644 --- a/dsl.nix +++ b/dsl.nix @@ -221,6 +221,13 @@ rec { `resources.fs` : Bind mounts, keyed by mount point. + `resources.devices` + : Host device nodes to expose, as paths. Bound into the container's `/dev` + and added to the unit's `DeviceAllow=` - the bind alone leaves the cgroup + denying `open()`. Nodes keep their host ownership, so a consumer inside + needs `isolation.privateUsers = "identity"` for the owning gid to mean + anything. + `isolation.nixDaemon` : Whether the Nix daemon and CLI are available inside the container. @@ -228,17 +235,6 @@ rec { : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. - `isolation.allowedDevices` - : List of `{ node; modifier; }` host devices to grant the container's - cgroup access to; see `containers..allowedDevices` in the NixOS - manual. nspawn bind-mounts a handful of devices (`/dev/kvm`, - `/dev/net/tun`, ...) into the container itself once the cgroup allows it - - this doesn't need a matching `resources.fs` bind mount. - - `isolation.additionalCapabilities` - : Extra Linux capabilities granted to the container; see - `containers..additionalCapabilities` in the NixOS manual. - `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. @@ -281,10 +277,9 @@ rec { cpuQuota = "${toString (cpu * 100)}%"; memoryMax = resources.mem or "512M"; bindMounts = resources.fs or { }; + devices = resources.devices or [ ]; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; - allowedDevices = isolation.allowedDevices or [ ]; - additionalCapabilities = isolation.additionalCapabilities or [ ]; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; @@ -298,6 +293,7 @@ rec { + lib.optionalString useIdmap ":idmap" ) allBindMounts; bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts); + deviceFlags = map (d: "--bind=${d}") devices; inherit (deriveIdentity ifPrefix.nspawn name) iface mac @@ -376,19 +372,15 @@ rec { hostAddr bindMountHostPaths includedInTests + devices ; bindMounts = allBindMounts; kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { - inherit - autoStart - privateUsers - allowedDevices - additionalCapabilities - ; + inherit autoStart privateUsers; bindMounts = { }; - extraFlags = bindMountExtraFlags; + extraFlags = bindMountExtraFlags ++ deviceFlags; privateNetwork = true; extraVeths = linkVeths; config = @@ -521,6 +513,7 @@ rec { ; inherit needsGateway; bindMountHostPaths = map (s: s.source) shares; + devices = [ ]; kind = "vm"; miscProperties = flattenMiscProp self miscProperties; microvmConfig = { @@ -957,6 +950,7 @@ rec { startLimitBurst = 5; startLimitIntervalSec = 120; unitConfig.RequiresMountsFor = c.bindMountHostPaths; + serviceConfig.DeviceAllow = map (d: "${d} rw") c.devices; }; # the host end of a link, for both kinds. nixos-containers only configures @@ -1316,6 +1310,7 @@ rec { HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)} IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)} KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)} + DEVICES = ${pyDict pyList (lib.mapAttrs (_: c: c.devices) autoStartTestContainers)} # only nspawn guests can be entered with nsenter; a vm has no host-side # netns, so every probe that runs *inside* a guest is limited to these enterable = [c for c in included if KIND[c] == "nspawn"] diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 74e003f..8920514 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: HOST_ADDR: dict[str, str] = {} IFACE: dict[str, str] = {} KIND: dict[str, str] = {} + DEVICES: dict[str, list[str]] = {} EDGES: list[tuple[str, str, list[int], list[int]]] = [] SSH_KEY: str = "" SSH_OPTS: str = "" @@ -55,6 +56,11 @@ def container_pid(name): return server.succeed(f"machinectl show {name} -p Leader --value").strip() +def ct_cmd(name, cmd): + pid = container_pid(name) + return f"{NSENTER} --mount=/proc/{pid}/ns/mnt --root=/proc/{pid}/root -- {SH} -c '{cmd}'" + + def net_cmd(name, cmd): pid = container_pid(name) return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}" @@ -323,3 +329,22 @@ with subtest("(functional): published ports follow the container's lifetime"): server.succeed("systemctl start container@nginx") server.wait_for_unit("container@nginx.service") internet.wait_until_succeeds(ext_connect(80), timeout=30) + +with subtest("(functional): declared devices are usable in-container"): + for name, devices in DEVICES.items(): + for dev in devices: + # opened, not stat'd: without DeviceAllow= the node is still visible + server.succeed(ct_cmd(name, f"exec 3<> {dev}")) + + +with subtest("(isolation): devices reach only the container that declared them"): + for name in enterable: + for dev in {d for ds in DEVICES.values() for d in ds} - set(DEVICES.get(name, [])): + server.fail(ct_cmd(name, f"test -e {dev}")) + + +with subtest("(isolation): identity uid mapping keeps the user namespace"): + for name in (n for n, ds in DEVICES.items() if ds): + host_ns = server.succeed("readlink /proc/1/ns/user").strip() + ct_ns = server.succeed(f"readlink /proc/{container_pid(name)}/ns/user").strip() + assert host_ns != ct_ns, f"{name} shares the host user namespace"