From a6ff6536dcb9da4a85282917b6109e78e08d273e Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Fri, 11 Sep 2026 14:43:43 -0500 Subject: [PATCH] update --- containers/server-n0/ssh.nix | 36 ++++++++++----------------------- dsl.nix | 37 ++++++++++++++++------------------ tests/server-n0/integration.py | 25 +++++++++++++++++++++++ 3 files changed, 53 insertions(+), 45 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 01a5dda..0cd0888 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -75,15 +75,6 @@ let packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; }; - - kvmTrait = { - containerConfig = - { pkgs, ... }: - { - users.users.runner.extraGroups = [ "kvm" ]; - environment.systemPackages = [ pkgs.qemu_kvm ]; - }; - }; in { ssh-admin = mkSsh { @@ -126,32 +117,27 @@ in ssh-generic = mkSsh { name = "ssh-generic"; net.access = [ { to = "@internet"; } ]; + # /dev/kvm keeps its host ownership, so the kvm gid must not be shifted. + isolation.privateUsers = "identity"; resources = { fs."/home/runner/data" = { hostPath = "${data}/ssh/generic"; isReadOnly = false; }; + devices = [ "/dev/kvm" ]; mem = "16G"; cpu = 65; }; - traits = [ kvmTrait ]; - isolation.allowedDevices = [ + packages = with pkgs; [ qemu_kvm ]; + traits = [ { - node = "/dev/kvm"; - modifier = "rw"; + containerConfig = + { config, ... }: + { + users.groups.kvm.gid = config.ids.gids.kvm; + users.users.runner.extraGroups = [ "kvm" ]; + }; } - { - node = "/dev/vhost-net"; - modifier = "rw"; - } - { - node = "/dev/net/tun"; - modifier = "rwm"; - } - ]; - isolation.additionalCapabilities = [ - "CAP_NET_ADMIN" - "CAP_MKNOD" ]; routingInfo = [ { diff --git a/dsl.nix b/dsl.nix index 0def2fc..0a0035f 100644 --- a/dsl.nix +++ b/dsl.nix @@ -221,6 +221,15 @@ rec { `resources.fs` : Bind mounts, keyed by mount point. + `resources.devices` + : Host device nodes to expose, as paths, added to the unit's + `DeviceAllow=`. nspawn populates a handful of privileged devices + (`/dev/kvm`, `/dev/net/tun`, ...) into the container's own `/dev` once + the cgroup allows it, and silently skips ones the host doesn't have - + no `resources.fs` bind mount needed. Nodes keep their host ownership, + so a consumer inside needs `isolation.privateUsers = "identity"` for + the owning gid to mean anything. + `isolation.nixDaemon` : Whether the Nix daemon and CLI are available inside the container. @@ -228,17 +237,6 @@ rec { : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. - `isolation.allowedDevices` - : List of `{ node; modifier; }` host devices to grant the container's - cgroup access to; see `containers..allowedDevices` in the NixOS - manual. nspawn bind-mounts a handful of devices (`/dev/kvm`, - `/dev/net/tun`, ...) into the container itself once the cgroup allows it - - this doesn't need a matching `resources.fs` bind mount. - - `isolation.additionalCapabilities` - : Extra Linux capabilities granted to the container; see - `containers..additionalCapabilities` in the NixOS manual. - `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. @@ -281,10 +279,9 @@ rec { cpuQuota = "${toString (cpu * 100)}%"; memoryMax = resources.mem or "512M"; bindMounts = resources.fs or { }; + devices = resources.devices or [ ]; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; - allowedDevices = isolation.allowedDevices or [ ]; - additionalCapabilities = isolation.additionalCapabilities or [ ]; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; @@ -298,6 +295,7 @@ rec { + lib.optionalString useIdmap ":idmap" ) allBindMounts; bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts); + deviceFlags = map (d: "--bind=${d}") devices; inherit (deriveIdentity ifPrefix.nspawn name) iface mac @@ -376,19 +374,15 @@ rec { hostAddr bindMountHostPaths includedInTests + devices ; bindMounts = allBindMounts; kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { - inherit - autoStart - privateUsers - allowedDevices - additionalCapabilities - ; + inherit autoStart privateUsers; bindMounts = { }; - extraFlags = bindMountExtraFlags; + extraFlags = bindMountExtraFlags ++ deviceFlags; privateNetwork = true; extraVeths = linkVeths; config = @@ -521,6 +515,7 @@ rec { ; inherit needsGateway; bindMountHostPaths = map (s: s.source) shares; + devices = [ ]; kind = "vm"; miscProperties = flattenMiscProp self miscProperties; microvmConfig = { @@ -957,6 +952,7 @@ rec { startLimitBurst = 5; startLimitIntervalSec = 120; unitConfig.RequiresMountsFor = c.bindMountHostPaths; + serviceConfig.DeviceAllow = map (d: "${d} rw") c.devices; }; # the host end of a link, for both kinds. nixos-containers only configures @@ -1316,6 +1312,7 @@ rec { HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)} IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)} KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)} + DEVICES = ${pyDict pyList (lib.mapAttrs (_: c: c.devices) autoStartTestContainers)} # only nspawn guests can be entered with nsenter; a vm has no host-side # netns, so every probe that runs *inside* a guest is limited to these enterable = [c for c in included if KIND[c] == "nspawn"] diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 74e003f..8920514 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: HOST_ADDR: dict[str, str] = {} IFACE: dict[str, str] = {} KIND: dict[str, str] = {} + DEVICES: dict[str, list[str]] = {} EDGES: list[tuple[str, str, list[int], list[int]]] = [] SSH_KEY: str = "" SSH_OPTS: str = "" @@ -55,6 +56,11 @@ def container_pid(name): return server.succeed(f"machinectl show {name} -p Leader --value").strip() +def ct_cmd(name, cmd): + pid = container_pid(name) + return f"{NSENTER} --mount=/proc/{pid}/ns/mnt --root=/proc/{pid}/root -- {SH} -c '{cmd}'" + + def net_cmd(name, cmd): pid = container_pid(name) return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}" @@ -323,3 +329,22 @@ with subtest("(functional): published ports follow the container's lifetime"): server.succeed("systemctl start container@nginx") server.wait_for_unit("container@nginx.service") internet.wait_until_succeeds(ext_connect(80), timeout=30) + +with subtest("(functional): declared devices are usable in-container"): + for name, devices in DEVICES.items(): + for dev in devices: + # opened, not stat'd: without DeviceAllow= the node is still visible + server.succeed(ct_cmd(name, f"exec 3<> {dev}")) + + +with subtest("(isolation): devices reach only the container that declared them"): + for name in enterable: + for dev in {d for ds in DEVICES.values() for d in ds} - set(DEVICES.get(name, [])): + server.fail(ct_cmd(name, f"test -e {dev}")) + + +with subtest("(isolation): identity uid mapping keeps the user namespace"): + for name in (n for n, ds in DEVICES.items() if ds): + host_ns = server.succeed("readlink /proc/1/ns/user").strip() + ct_ns = server.succeed(f"readlink /proc/{container_pid(name)}/ns/user").strip() + assert host_ns != ct_ns, f"{name} shares the host user namespace"