From fa39169feede76279e201f57cf7dcbf17c3bc1d5 Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Fri, 11 Sep 2026 14:43:43 -0500 Subject: [PATCH] update --- containers/server-n0/ssh.nix | 14 ++++++++++++++ dsl.nix | 15 ++++++++++++++- tests/server-n0/integration.py | 25 +++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 6b293a5..cf0327f 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -113,14 +113,28 @@ in ssh-generic = mkSsh { name = "ssh-generic"; net.access = [ { to = "@internet"; } ]; + # /dev/kvm keeps its host ownership, so the kvm gid must not be shifted. + isolation.privateUsers = "identity"; resources = { fs."/home/runner/data" = { hostPath = "${data}/ssh/generic"; isReadOnly = false; }; + devices = [ "/dev/kvm" ]; mem = "16G"; cpu = 65; }; + packages = with pkgs; [ qemu_kvm ]; + traits = [ + { + containerConfig = + { config, ... }: + { + users.groups.kvm.gid = config.ids.gids.kvm; + users.users.runner.extraGroups = [ "kvm" ]; + }; + } + ]; routingInfo = [ { username = "autumn"; diff --git a/dsl.nix b/dsl.nix index ab4be0e..8edf4a9 100644 --- a/dsl.nix +++ b/dsl.nix @@ -221,6 +221,13 @@ rec { `resources.fs` : Bind mounts, keyed by mount point. + `resources.devices` + : Host device nodes to expose, as paths. Bound into the container's `/dev` + and added to the unit's `DeviceAllow=` - the bind alone leaves the cgroup + denying `open()`. Nodes keep their host ownership, so a consumer inside + needs `isolation.privateUsers = "identity"` for the owning gid to mean + anything. + `isolation.nixDaemon` : Whether the Nix daemon and CLI are available inside the container. @@ -270,6 +277,7 @@ rec { cpuQuota = "${toString (cpu * 100)}%"; memoryMax = resources.mem or "512M"; bindMounts = resources.fs or { }; + devices = resources.devices or [ ]; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; @@ -285,6 +293,7 @@ rec { + lib.optionalString useIdmap ":idmap" ) allBindMounts; bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts); + deviceFlags = map (d: "--bind=${d}") devices; inherit (deriveIdentity ifPrefix.nspawn name) iface mac @@ -363,6 +372,7 @@ rec { hostAddr bindMountHostPaths includedInTests + devices ; bindMounts = allBindMounts; kind = "nspawn"; @@ -370,7 +380,7 @@ rec { nixosContainer = { inherit autoStart privateUsers; bindMounts = { }; - extraFlags = bindMountExtraFlags; + extraFlags = bindMountExtraFlags ++ deviceFlags; privateNetwork = true; extraVeths = linkVeths; config = @@ -503,6 +513,7 @@ rec { ; inherit needsGateway; bindMountHostPaths = map (s: s.source) shares; + devices = [ ]; kind = "vm"; miscProperties = flattenMiscProp self miscProperties; microvmConfig = { @@ -934,6 +945,7 @@ rec { startLimitBurst = 5; startLimitIntervalSec = 120; unitConfig.RequiresMountsFor = c.bindMountHostPaths; + serviceConfig.DeviceAllow = map (d: "${d} rw") c.devices; }; # the host end of a link, for both kinds. nixos-containers only configures @@ -1293,6 +1305,7 @@ rec { HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)} IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)} KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)} + DEVICES = ${pyDict pyList (lib.mapAttrs (_: c: c.devices) autoStartTestContainers)} # only nspawn guests can be entered with nsenter; a vm has no host-side # netns, so every probe that runs *inside* a guest is limited to these enterable = [c for c in included if KIND[c] == "nspawn"] diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 265ccf9..10bc9a5 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: HOST_ADDR: dict[str, str] = {} IFACE: dict[str, str] = {} KIND: dict[str, str] = {} + DEVICES: dict[str, list[str]] = {} EDGES: list[tuple[str, str, list[int], list[int]]] = [] SSH_KEY: str = "" SSH_OPTS: str = "" @@ -55,6 +56,11 @@ def container_pid(name): return server.succeed(f"machinectl show {name} -p Leader --value").strip() +def ct_cmd(name, cmd): + pid = container_pid(name) + return f"{NSENTER} --mount=/proc/{pid}/ns/mnt --root=/proc/{pid}/root -- {SH} -c '{cmd}'" + + def net_cmd(name, cmd): pid = container_pid(name) return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}" @@ -328,3 +334,22 @@ with subtest("(functional): published ports follow the container's lifetime"): server.succeed("systemctl start container@nginx") server.wait_for_unit("container@nginx.service") internet.wait_until_succeeds(ext_connect(80), timeout=30) + +with subtest("(functional): declared devices are usable in-container"): + for name, devices in DEVICES.items(): + for dev in devices: + # opened, not stat'd: without DeviceAllow= the node is still visible + server.succeed(ct_cmd(name, f"exec 3<> {dev}")) + + +with subtest("(isolation): devices reach only the container that declared them"): + for name in enterable: + for dev in {d for ds in DEVICES.values() for d in ds} - set(DEVICES.get(name, [])): + server.fail(ct_cmd(name, f"test -e {dev}")) + + +with subtest("(isolation): identity uid mapping keeps the user namespace"): + for name in (n for n, ds in DEVICES.items() if ds): + host_ns = server.succeed("readlink /proc/1/ns/user").strip() + ct_ns = server.succeed(f"readlink /proc/{container_pid(name)}/ns/user").strip() + assert host_ns != ct_ns, f"{name} shares the host user namespace"