From 33ccab9ad46bc816cbb2e09b5b16ae9fb26eb876 Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Wed, 9 Sep 2026 17:58:14 -0500 Subject: [PATCH 1/4] frcc experiments --- containers/server-n0/ssh.nix | 32 +++++++++++++++++++++++++++++++- dsl.nix | 20 +++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 6b293a5..dde4a13 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -61,16 +61,27 @@ let net ? { }, resources ? { }, traits ? [ ], + isolation ? { }, }: mkContainer { inherit name; inherit net resources; svc.autoStart = true; - isolation.nixDaemon = nixDaemon; + isolation = { nixDaemon = nixDaemon; } // isolation; traits = [ sshTrait ] ++ traits; packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; }; + + kvmTrait = { + containerConfig = + { pkgs, ... }: + { + users.users.runner.extraGroups = [ "wheel" ]; + security.sudo.wheelNeedsPassword = false; + environment.systemPackages = [ pkgs.qemu_kvm ]; + }; + }; in { ssh-admin = mkSsh { @@ -121,6 +132,25 @@ in mem = "16G"; cpu = 65; }; + traits = [ kvmTrait ]; + isolation.allowedDevices = [ + { + node = "/dev/kvm"; + modifier = "rw"; + } + { + node = "/dev/vhost-net"; + modifier = "rw"; + } + { + node = "/dev/net/tun"; + modifier = "rwm"; + } + ]; + isolation.additionalCapabilities = [ + "CAP_NET_ADMIN" + "CAP_MKNOD" + ]; routingInfo = [ { username = "autumn"; diff --git a/dsl.nix b/dsl.nix index ab4be0e..1c6dcf0 100644 --- a/dsl.nix +++ b/dsl.nix @@ -228,6 +228,17 @@ rec { : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. + `isolation.allowedDevices` + : List of `{ node; modifier; }` host devices to grant the container's + cgroup access to; see `containers..allowedDevices` in the NixOS + manual. nspawn bind-mounts a handful of devices (`/dev/kvm`, + `/dev/net/tun`, ...) into the container itself once the cgroup allows it - + this doesn't need a matching `resources.fs` bind mount. + + `isolation.additionalCapabilities` + : Extra Linux capabilities granted to the container; see + `containers..additionalCapabilities` in the NixOS manual. + `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. @@ -272,6 +283,8 @@ rec { bindMounts = resources.fs or { }; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; + allowedDevices = isolation.allowedDevices or [ ]; + additionalCapabilities = isolation.additionalCapabilities or [ ]; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; @@ -368,7 +381,12 @@ rec { kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { - inherit autoStart privateUsers; + inherit + autoStart + privateUsers + allowedDevices + additionalCapabilities + ; bindMounts = { }; extraFlags = bindMountExtraFlags; privateNetwork = true; -- 2.55.0 From 096908ac39609d4b284859a14694f4d961334dff Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Wed, 9 Sep 2026 17:58:14 -0500 Subject: [PATCH 2/4] frcc experiments --- containers/server-n0/ssh.nix | 32 +++++++++++++++++++++++++++++++- dsl.nix | 20 +++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 6b293a5..dde4a13 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -61,16 +61,27 @@ let net ? { }, resources ? { }, traits ? [ ], + isolation ? { }, }: mkContainer { inherit name; inherit net resources; svc.autoStart = true; - isolation.nixDaemon = nixDaemon; + isolation = { nixDaemon = nixDaemon; } // isolation; traits = [ sshTrait ] ++ traits; packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; }; + + kvmTrait = { + containerConfig = + { pkgs, ... }: + { + users.users.runner.extraGroups = [ "wheel" ]; + security.sudo.wheelNeedsPassword = false; + environment.systemPackages = [ pkgs.qemu_kvm ]; + }; + }; in { ssh-admin = mkSsh { @@ -121,6 +132,25 @@ in mem = "16G"; cpu = 65; }; + traits = [ kvmTrait ]; + isolation.allowedDevices = [ + { + node = "/dev/kvm"; + modifier = "rw"; + } + { + node = "/dev/vhost-net"; + modifier = "rw"; + } + { + node = "/dev/net/tun"; + modifier = "rwm"; + } + ]; + isolation.additionalCapabilities = [ + "CAP_NET_ADMIN" + "CAP_MKNOD" + ]; routingInfo = [ { username = "autumn"; diff --git a/dsl.nix b/dsl.nix index 1da09a6..0def2fc 100644 --- a/dsl.nix +++ b/dsl.nix @@ -228,6 +228,17 @@ rec { : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. + `isolation.allowedDevices` + : List of `{ node; modifier; }` host devices to grant the container's + cgroup access to; see `containers..allowedDevices` in the NixOS + manual. nspawn bind-mounts a handful of devices (`/dev/kvm`, + `/dev/net/tun`, ...) into the container itself once the cgroup allows it - + this doesn't need a matching `resources.fs` bind mount. + + `isolation.additionalCapabilities` + : Extra Linux capabilities granted to the container; see + `containers..additionalCapabilities` in the NixOS manual. + `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. @@ -272,6 +283,8 @@ rec { bindMounts = resources.fs or { }; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; + allowedDevices = isolation.allowedDevices or [ ]; + additionalCapabilities = isolation.additionalCapabilities or [ ]; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; @@ -368,7 +381,12 @@ rec { kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { - inherit autoStart privateUsers; + inherit + autoStart + privateUsers + allowedDevices + additionalCapabilities + ; bindMounts = { }; extraFlags = bindMountExtraFlags; privateNetwork = true; -- 2.55.0 From bb79660386a7e84c2651013b25ba2f4858789aae Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Fri, 11 Sep 2026 01:48:02 -0500 Subject: [PATCH 3/4] meow --- containers/server-n0/ssh.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index dde4a13..0a095b2 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -67,7 +67,7 @@ let inherit name; inherit net resources; svc.autoStart = true; - isolation = { nixDaemon = nixDaemon; } // isolation; + isolation = { inherit nixDaemon; } // isolation; traits = [ sshTrait ] ++ traits; packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; @@ -77,8 +77,7 @@ let containerConfig = { pkgs, ... }: { - users.users.runner.extraGroups = [ "wheel" ]; - security.sudo.wheelNeedsPassword = false; + users.users.runner.extraGroups = [ "kvm" ]; environment.systemPackages = [ pkgs.qemu_kvm ]; }; }; -- 2.55.0 From 3605fc36ef319615e22114880f3913013a41528b Mon Sep 17 00:00:00 2001 From: Autumn Liu Date: Fri, 11 Sep 2026 02:08:07 -0500 Subject: [PATCH 4/4] meow2 --- containers/server-n0/ssh.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix index 0a095b2..01a5dda 100644 --- a/containers/server-n0/ssh.nix +++ b/containers/server-n0/ssh.nix @@ -67,7 +67,10 @@ let inherit name; inherit net resources; svc.autoStart = true; - isolation = { inherit nixDaemon; } // isolation; + isolation = { + inherit nixDaemon; + } + // isolation; traits = [ sshTrait ] ++ traits; packages = with pkgs; [ tmux ] ++ packages; miscProperties = self._sshpiperRegister routingInfo; -- 2.55.0