From 8190b0080d009ec251d22e33df822ecd353323be Mon Sep 17 00:00:00 2001 From: autumn Date: Sat, 12 Sep 2026 06:40:11 +0000 Subject: [PATCH] containerization (#3) Co-authored-by: Floweynt <50057682+Floweynt@users.noreply.github.com> Co-authored-by: Autumn Liu Reviewed-on: https://git.floweytf.com/server/server-config/pulls/3 Co-authored-by: autumn --- containers/server-n0/ssh.nix | 36 +++++++++++----------------------- dsl.nix | 36 +++++++++++++++------------------- tests/server-n0/integration.py | 30 ++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 01a5dda..0cd0888 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -75,15 +75,6 @@ let packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; }; - - kvmTrait = { - containerConfig = - { pkgs, ... }: - { - users.users.runner.extraGroups = [ "kvm" ]; - environment.systemPackages = [ pkgs.qemu_kvm ]; - }; - }; in { ssh-admin = mkSsh { @@ -126,32 +117,27 @@ in ssh-generic = mkSsh { name = "ssh-generic"; net.access = [ { to = "@internet"; } ]; + # /dev/kvm keeps its host ownership, so the kvm gid must not be shifted. + isolation.privateUsers = "identity"; resources = { fs."/home/runner/data" = { hostPath = "${data}/ssh/generic"; isReadOnly = false; }; + devices = [ "/dev/kvm" ]; mem = "16G"; cpu = 65; }; - traits = [ kvmTrait ]; - isolation.allowedDevices = [ + packages = with pkgs; [ qemu_kvm ]; + traits = [ { - node = "/dev/kvm"; - modifier = "rw"; + containerConfig = + { config, ... }: + { + users.groups.kvm.gid = config.ids.gids.kvm; + users.users.runner.extraGroups = [ "kvm" ]; + }; } - { - node = "/dev/vhost-net"; - modifier = "rw"; - } - { - node = "/dev/net/tun"; - modifier = "rwm"; - } - ]; - isolation.additionalCapabilities = [ - "CAP_NET_ADMIN" - "CAP_MKNOD" ]; routingInfo = [ { diff --git a/dsl.nix b/dsl.nix index 0def2fc..4745527 100644 --- a/dsl.nix +++ b/dsl.nix @@ -221,6 +221,15 @@ rec { `resources.fs` : Bind mounts, keyed by mount point. + `resources.devices` + : Host device nodes to expose, as paths, added to the unit's + `DeviceAllow=`. nspawn populates a handful of privileged devices + (`/dev/kvm`, `/dev/net/tun`, ...) into the container's own `/dev` once + the cgroup allows it, and silently skips ones the host doesn't have - + no `resources.fs` bind mount needed. Nodes keep their host ownership, + so a consumer inside needs `isolation.privateUsers = "identity"` for + the owning gid to mean anything. + `isolation.nixDaemon` : Whether the Nix daemon and CLI are available inside the container. @@ -228,17 +237,6 @@ rec { : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. - `isolation.allowedDevices` - : List of `{ node; modifier; }` host devices to grant the container's - cgroup access to; see `containers..allowedDevices` in the NixOS - manual. nspawn bind-mounts a handful of devices (`/dev/kvm`, - `/dev/net/tun`, ...) into the container itself once the cgroup allows it - - this doesn't need a matching `resources.fs` bind mount. - - `isolation.additionalCapabilities` - : Extra Linux capabilities granted to the container; see - `containers..additionalCapabilities` in the NixOS manual. - `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. @@ -281,16 +279,15 @@ rec { cpuQuota = "${toString (cpu * 100)}%"; memoryMax = resources.mem or "512M"; bindMounts = resources.fs or { }; + devices = resources.devices or [ ]; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; - allowedDevices = isolation.allowedDevices or [ ]; - additionalCapabilities = isolation.additionalCapabilities or [ ]; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; userModules = map mkUser (lib.concatMap (t: t.users or [ ]) traits ++ users); allPorts = ports ++ map (p: (normalizePort p).container) hostPorts; - useIdmap = privateUsers != "no"; + useIdmap = privateUsers == "pick" || (builtins.isInt privateUsers && privateUsers > 0); bindMountExtraFlags = lib.mapAttrsToList ( mountPoint: bm: (if bm.isReadOnly then "--bind-ro=" else "--bind=") @@ -376,17 +373,13 @@ rec { hostAddr bindMountHostPaths includedInTests + devices ; bindMounts = allBindMounts; kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { - inherit - autoStart - privateUsers - allowedDevices - additionalCapabilities - ; + inherit autoStart privateUsers; bindMounts = { }; extraFlags = bindMountExtraFlags; privateNetwork = true; @@ -521,6 +514,7 @@ rec { ; inherit needsGateway; bindMountHostPaths = map (s: s.source) shares; + devices = [ ]; kind = "vm"; miscProperties = flattenMiscProp self miscProperties; microvmConfig = { @@ -957,6 +951,7 @@ rec { startLimitBurst = 5; startLimitIntervalSec = 120; unitConfig.RequiresMountsFor = c.bindMountHostPaths; + serviceConfig.DeviceAllow = map (d: "${d} rw") c.devices; }; # the host end of a link, for both kinds. nixos-containers only configures @@ -1316,6 +1311,7 @@ rec { HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)} IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)} KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)} + DEVICES = ${pyDict pyList (lib.mapAttrs (_: c: c.devices) autoStartTestContainers)} # only nspawn guests can be entered with nsenter; a vm has no host-side # netns, so every probe that runs *inside* a guest is limited to these enterable = [c for c in included if KIND[c] == "nspawn"] diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 74e003f..edd7d41 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: HOST_ADDR: dict[str, str] = {} IFACE: dict[str, str] = {} KIND: dict[str, str] = {} + DEVICES: dict[str, list[str]] = {} EDGES: list[tuple[str, str, list[int], list[int]]] = [] SSH_KEY: str = "" SSH_OPTS: str = "" @@ -55,6 +56,11 @@ def container_pid(name): return server.succeed(f"machinectl show {name} -p Leader --value").strip() +def ct_cmd(name, cmd): + pid = container_pid(name) + return f"{NSENTER} --mount=/proc/{pid}/ns/mnt --root=/proc/{pid}/root -- {SH} -c '{cmd}'" + + def net_cmd(name, cmd): pid = container_pid(name) return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}" @@ -323,3 +329,27 @@ with subtest("(functional): published ports follow the container's lifetime"): server.succeed("systemctl start container@nginx") server.wait_for_unit("container@nginx.service") internet.wait_until_succeeds(ext_connect(80), timeout=30) + +with subtest("(functional): declared devices are usable in-container"): + for name, devices in DEVICES.items(): + for dev in devices: + if server.execute(f"test -e {dev}")[0] != 0: + # host lacks the device (e.g. /dev/kvm without nested + # virtualization, as in this test's own VM) - nspawn skips + # populating it in-container, so there's nothing to check. + continue + # opened, not stat'd: without DeviceAllow= the node is still visible + server.succeed(ct_cmd(name, f"exec 3<> {dev}")) + + +with subtest("(isolation): devices reach only the container that declared them"): + for name in enterable: + for dev in {d for ds in DEVICES.values() for d in ds} - set(DEVICES.get(name, [])): + server.fail(ct_cmd(name, f"test -e {dev}")) + + +with subtest("(isolation): identity uid mapping keeps the user namespace"): + for name in (n for n, ds in DEVICES.items() if ds): + host_ns = server.succeed("readlink /proc/1/ns/user").strip() + ct_ns = server.succeed(f"readlink /proc/{container_pid(name)}/ns/user").strip() + assert host_ns != ct_ns, f"{name} shares the host user namespace"