{ lib, microvm }: let hexDigit = lib.listToAttrs ( lib.imap0 (i: c: lib.nameValuePair c i) (lib.stringToCharacters "0123456789abcdef") ); hexToInt = s: lib.foldl' (acc: c: acc * 16 + hexDigit.${c}) 0 (lib.stringToCharacters s); deriveMac = name: "02:" + lib.concatStringsSep ":" ( lib.genList (i: lib.substring (i * 2) 2 (builtins.hashString "sha256" name)) 5 ); deriveIfName = prefix: name: lib.substring 0 15 "${prefix}-${builtins.hashString "sha256" name}"; # IFNAMSIZ - 1 deriveIp4AddrAt = offset: name: let h = builtins.hashString "sha256" name; octet = i: toString (hexToInt (lib.substring (offset + i * 2) 2 h)); in "10.${octet 0}.${octet 1}.${octet 2}"; ifPrefix = { nspawn = "vh"; vm = "vm"; }; unitPrefix = { nspawn = "container"; vm = "microvm"; }; deploymentIfGlobs = lib.mapAttrsToList (_: p: "${p}-*") ifPrefix; ofKind = k: lib.filterAttrs (_: c: c.kind == k); deriveIdentity = prefix: name: { iface = deriveIfName prefix name; mac = deriveMac name; addr = deriveIp4AddrAt 12 name; hostAddr = deriveIp4AddrAt 24 name; }; specialTargets = [ "@host" "@internet" ]; isSpecialTarget = to: lib.elem to specialTargets; hasInternetAccess = access: lib.any (a: a.to == "@internet") access; substrateNet = { address = "10.0.0.0"; prefixLength = 8; }; needsGatewayFor = access: hostPorts: hostUdpPorts: hasInternetAccess access || hostPorts != [ ] || hostUdpPorts != [ ]; flattenMiscProp = self: props: if lib.isAttrs props then props else if lib.isList props then lib.mergeAttrsList (map (flattenMiscProp self) props) else props self; mkUser = { name, uid, home ? "/home/${name}", extraUserConfig ? { }, }: { users.users.${name} = { isNormalUser = true; inherit uid home; group = name; } // extraUserConfig; users.groups.${name} = { }; }; parseMemMiB = s: let unit = lib.toUpper (lib.substring (lib.stringLength s - 1) 1 s); n = lib.toInt (lib.substring 0 (lib.stringLength s - 1) s); in if unit == "G" then n * 1024 else if unit == "M" then n else lib.toInt s; in rec { /** Expands a port spec. # Type ``` normalizePort :: (Int | { host :: Int; container :: Int; }) -> { host :: Int; container :: Int; } ``` */ normalizePort = p: if builtins.isInt p then { host = p; container = p; } else p; /** Reads all `{name}.nix` files in `dir` and associates to { "${name}" = f name path } # Type ``` listMap :: Path -> (String -> Path -> a) -> AttrSet ``` */ listMap = dir: f: lib.mapAttrs' ( fileName: _: let name = lib.removeSuffix ".nix" fileName; in lib.nameValuePair name (f name (dir + "/${fileName}")) ) (lib.filterAttrs (n: t: t == "regular" && lib.hasSuffix ".nix" n) (builtins.readDir dir)); /** Creates configuration for a hardened systemd service. # Type ``` mkService :: AttrSet -> AttrSet ``` */ mkService = { description, execStart, wantedBy ? [ "multi-user.target" ], after ? [ ], wants ? [ ], networkOnline ? false, workingDir ? null, serviceConfig ? { }, unitConfig ? { }, path ? [ ], }: { inherit description wantedBy path unitConfig ; after = after ++ lib.optional networkOnline "network-online.target"; wants = wants ++ lib.optional networkOnline "network-online.target"; serviceConfig = { ExecStart = execStart; FailureAction = "exit-force"; NoNewPrivileges = true; SystemCallFilter = "@system-service"; SystemCallArchitectures = "native"; ProtectSystem = "strict"; } // lib.optionalAttrs (workingDir != null) { WorkingDirectory = workingDir; } // serviceConfig; }; /** Declares a systemd-nspawn container: networking, resource limits, bind mounts, and the NixOS config that runs inside it. # Inputs `traits` : Reusable partial configs (own `resources.fs`/`users`/`containerConfig`). `users` : List of user (`{ name; uid; home?; extraUserConfig?; }`) specs `net.access` : Outbound edges this container is allowed to open, as `{ to; tcp ? [ ]; udp ? [ ]; }`. `to` is another deployment's name, or one of the `@`-prefixed specials `"@host"` and `"@internet"` - the prefix keeps them out of the deployment namespace. Everything not listed is denied, and an edge is one-way: the peer needs its own entry to talk back. Ports are mandatory except for `"@internet"`. DNS and ICMP to the host are always allowed and are not declarable. `net.expose.tcp`, `net.expose.udp` : Ports opened on the container's own firewall. `net.forward.tcp`, `net.forward.udp` : Host ports forwarded into the container; see `normalizePort`. `svc.autoStart`, `svc.depends` : Whether the container starts automatically, and other deployments it depends on. `resources.cpu` : CPU core count; converted to `CPUQuota` (`cpu * 100%`). `resources.mem` : `MemoryMax`, e.g. `"512M"`. `resources.fs` : Bind mounts, keyed by mount point. `isolation.nixDaemon` : Whether the Nix daemon and CLI are available inside the container. `isolation.privateUsers` : systemd-nspawn user namespacing; see `--private-users=` in systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. `miscProperties` : Arbitrary data attached to the result. Used for cross-container discovery via `collectContainers`. `includedInTests` : Whether integration VM tests should includes this container. Ignored for real deployment. Set `false` for containers the test VM shouldn't run. # Type ``` mkContainer :: AttrSet -> Deployment ``` */ mkContainer = { name, net ? { }, svc ? { }, resources ? { }, isolation ? { }, traits ? [ ], users ? [ ], packages ? [ ], service ? null, containerConfig ? (_: { }), miscProperties ? { }, includedInTests ? true, }: let access = net.access or [ ]; ports = net.expose.tcp or [ ]; udpPorts = net.expose.udp or [ ]; hostPorts = net.forward.tcp or [ ]; hostUdpPorts = net.forward.udp or [ ]; autoStart = svc.autoStart or false; depends = svc.depends or [ ]; cpu = resources.cpu or 1; cpuQuota = "${toString (cpu * 100)}%"; memoryMax = resources.mem or "512M"; bindMounts = resources.fs or { }; allowsNixDaemon = isolation.nixDaemon or false; privateUsers = isolation.privateUsers or "pick"; allowInternet = hasInternetAccess access; allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; userModules = map mkUser (lib.concatMap (t: t.users or [ ]) traits ++ users); allPorts = ports ++ map (p: (normalizePort p).container) hostPorts; useIdmap = privateUsers != "no"; bindMountExtraFlags = lib.mapAttrsToList ( mountPoint: bm: (if bm.isReadOnly then "--bind-ro=" else "--bind=") + "${bm.hostPath}:${mountPoint}" + lib.optionalString useIdmap ":idmap" ) allBindMounts; bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts); inherit (deriveIdentity ifPrefix.nspawn name) iface mac addr hostAddr ; needsGateway = needsGatewayFor access hostPorts hostUdpPorts; linkVeths.${iface} = { }; mergedConfig = args@{ pkgs, lib, ... }: let traitConfigs = map (t: (t.containerConfig or (_: { })) args) traits; pkgModule = lib.optional (packages != [ ]) { environment.systemPackages = packages; }; nixModule = lib.optional allowsNixDaemon { environment.systemPackages = [ pkgs.nix ]; nix.settings.experimental-features = [ "nix-command" "flakes" ]; }; svcModule = lib.optional (service != null) { systemd.services.${name} = lib.mkMerge [ (service args) { after = [ "container-ready.target" ]; wants = [ "container-ready.target" ]; } ]; }; portModule = lib.optional (allPorts != [ ]) { networking.firewall.allowedTCPPorts = allPorts; } ++ lib.optional (udpPorts != [ ]) { networking.firewall.allowedUDPPorts = udpPorts; }; readyTarget = { systemd.targets.container-ready = { description = "Container prerequisites satisfied"; after = lib.optional allowInternet "network-online.target"; wants = lib.optional allowInternet "network-online.target"; wantedBy = [ "multi-user.target" ]; }; }; in lib.mkMerge ( traitConfigs ++ userModules ++ pkgModule ++ nixModule ++ svcModule ++ [ readyTarget ] ++ [ (containerConfig args) ] ++ portModule ); in lib.fix (self: { inherit name access autoStart depends ports hostPorts hostUdpPorts memoryMax cpuQuota privateUsers allowInternet needsGateway iface mac addr hostAddr bindMountHostPaths includedInTests ; bindMounts = allBindMounts; kind = "nspawn"; miscProperties = flattenMiscProp self miscProperties; nixosContainer = { inherit autoStart privateUsers; bindMounts = { }; extraFlags = bindMountExtraFlags; privateNetwork = true; extraVeths = linkVeths; config = { pkgs, config, lib, ... }: lib.mkMerge [ { networking.enableIPv6 = false; networking.useHostResolvConf = lib.mkForce false; networking.nameservers = [ hostAddr ]; networking.interfaces.${iface} = { ipv4.addresses = [ { address = addr; prefixLength = 32; } ]; ipv4.routes = [ { address = hostAddr; prefixLength = 32; } ( substrateNet // { via = hostAddr; } ) ]; }; systemd.services."network-addresses-${iface}".before = [ "network-online.target" ]; } (lib.optionalAttrs needsGateway { networking.defaultGateway = { address = hostAddr; interface = iface; }; }) (mergedConfig { inherit pkgs config lib; }) ]; }; }); /** Declares a microvm: same networking interface as `mkContainer` (`access`, `hostPorts` and friends all work the same way), virtiofs/9p shares, and the NixOS config that runs inside it. The VM counterpart to `mkContainer` - same shared interface, different backend. # Inputs `resources.fs` : Passed straight through to `microvm.shares`. `guestConfig` : NixOS module for the VM's own system; same role as `mkContainer`'s `containerConfig`; gets full module args (`pkgs`, `utils`, ...) since it's spliced in via `imports`, not called manually. `resources.cpu`, `resources.mem`, `hypervisor` : Passed through to the corresponding `microvm.*` options. `resources.cpu` is vcpu count. `resources.mem` is required, a string like `"8G"`, converted to MiB via `parseMemMiB`. `net.access`, `net.forward.tcp`, `net.forward.udp`, `svc.autoStart`, `svc.depends` : See `mkContainer`. `miscProperties`, `includedInTests` : See `mkContainer`. # Type ``` mkVm :: AttrSet -> Deployment ``` */ mkVm = { name, net ? { }, svc ? { }, resources ? { }, hypervisor ? "qemu", guestConfig ? (_: { }), miscProperties ? { }, includedInTests ? true, }: let access = net.access or [ ]; hostPorts = net.forward.tcp or [ ]; hostUdpPorts = net.forward.udp or [ ]; autoStart = svc.autoStart or false; depends = svc.depends or [ ]; vcpu = resources.cpu or 1; mem = parseMemMiB (resources.mem or (throw "mkVm ${name}: resources.mem is required")); shares = resources.fs or [ ]; allowInternet = hasInternetAccess access; needsGateway = needsGatewayFor access hostPorts hostUdpPorts; inherit (deriveIdentity ifPrefix.vm name) iface mac addr hostAddr ; in lib.fix (self: { inherit name access autoStart depends hostPorts hostUdpPorts allowInternet iface mac addr hostAddr shares vcpu mem hypervisor includedInTests ; inherit needsGateway; bindMountHostPaths = map (s: s.source) shares; kind = "vm"; miscProperties = flattenMiscProp self miscProperties; microvmConfig = { autostart = autoStart; config = _: { imports = [ microvm.nixosModules.microvm guestConfig ]; nixpkgs.overlays = [ microvm.overlay ]; nix.settings.experimental-features = [ "nix-command" "flakes" ]; networking.hostName = name; system.stateVersion = "26.05"; microvm = { inherit vcpu mem hypervisor shares ; interfaces = [ { type = "tap"; id = iface; inherit mac; } ]; }; networking.useNetworkd = true; networking.useDHCP = false; systemd.network.networks."10-main" = { matchConfig.MACAddress = mac; address = [ "${addr}/32" ]; routes = [ { Destination = "${hostAddr}/32"; Scope = "link"; } { Destination = "${substrateNet.address}/${toString substrateNet.prefixLength}"; Gateway = hostAddr; } ] ++ lib.optional needsGateway { Destination = "0.0.0.0/0"; Gateway = hostAddr; }; networkConfig.DNS = [ hostAddr ]; networkConfig.Domains = [ "~." "containers" ]; linkConfig.RequiredForOnline = if allowInternet then "routable" else "degraded"; }; networking.enableIPv6 = false; networking.nameservers = [ hostAddr ]; }; }; }); /** Subset of `svcs`, with `access` edges pointing at dropped deployments removed. Filtering the attrset alone leaves those edges dangling, and `mkNetworking` rejects an edge whose target it cannot resolve. # Type ``` filterDeployments :: (Deployment -> Bool) -> AttrSet -> AttrSet ``` */ filterDeployments = pred: svcs: let kept = lib.filterAttrs (_: pred) svcs; in lib.mapAttrs ( _: c: c // { access = lib.filter (a: isSpecialTarget a.to || kept ? ${a.to}) c.access; } ) kept; /** Finds every container that set `miscProperties.${key}`, paired with the value. Used for cross-container discovery. # Type ``` collectContainers :: String -> AttrSet -> [ { container :: Container; data :: Any; } ] ``` */ collectContainers = key: containers: lib.mapAttrsToList (_: v: { container = v; data = v.miscProperties.${key}; }) (lib.filterAttrs (_: v: v.miscProperties ? ${key}) containers); /** Prepares networking for all deployments: dnsmasq for name resolution and the nftables tables that carry isolation, NAT and port forwarding. Every link is point-to-point, so `iifname` identifies the peer and nothing reaches anything it has no `access` edge for. NixOS container builtin port forwarding can't be used because we set up our own networking tables. # Type ``` mkNetworking :: AttrSet -> AttrSet ``` */ mkNetworking = { svcs, }: let deploymentList = lib.attrValues svcs; quote = lib.concatMapStringsSep ", " (v: "\"${v}\""); deploymentIfs = quote deploymentIfGlobs; netAccessEdges = lib.concatMap ( deployment: map ( a: let portsOf = proto: a.${proto} or [ ]; protos = lib.filter (p: portsOf p != [ ]) [ "tcp" "udp" ]; bad = msg: throw "${deployment.name}: access edge to \"${a.to}\" ${msg}"; in if a.to != "@internet" && protos == [ ] then bad "${deployment.name} no tcp/udp ports" else if isSpecialTarget a.to || svcs ? ${a.to} then { src = deployment; inherit (a) to; inherit protos portsOf; } else bad "${deployment.name}: access target does not name a container, \"@host\" or \"@internet\"" ) deployment.access ) deploymentList; netEdgeRules = pred: prefixOf: lib.concatMapStrings ( e: lib.concatMapStrings ( proto: "${prefixOf e} ${proto} dport { ${ lib.concatMapStringsSep ", " toString (e.portsOf proto) } } accept\n" ) e.protos ) (lib.filter pred netAccessEdges); peerRules = netEdgeRules (e: !isSpecialTarget e.to) ( e: "iifname \"${e.src.iface}\" oifname \"${svcs.${e.to}.iface}\"" ); hostRules = netEdgeRules (e: e.to == "@host") ( e: "iifname \"${e.src.iface}\" ip daddr ${e.src.hostAddr}" ); srcGuard = lib.concatMapStrings (c: '' iifname "${c.iface}" ip saddr != ${c.addr} counter name "spoofed" drop '') deploymentList; internetIfs = map (c: c.iface) (lib.filter (c: c.allowInternet) deploymentList); uplinkIfs = quote [ "en*" "eth*" "wl*" ]; egressRules = lib.optionalString (internetIfs != [ ]) '' iifname { ${quote internetIfs} } oifname { ${uplinkIfs} } accept ''; masqRules = lib.optionalString (internetIfs != [ ]) '' iifname { ${quote internetIfs} } oifname { ${uplinkIfs} } masquerade ''; fwdElems = sel: lib.concatMapStringsSep ", " (f: "${toString f.host} : ${f.addr} . ${toString f.container}") ( lib.concatMap (c: map (p: normalizePort p // { inherit (c) addr; }) (sel c)) deploymentList ); tcpElems = fwdElems (c: c.hostPorts); udpElems = fwdElems (c: c.hostUdpPorts); elemsOf = e: lib.optionalString (e != "") "elements = { ${e} }"; in { boot.kernel.sysctl = { "net.ipv4.ip_forward" = 1; } # nftables drops all container IPv6, but the host end of each link still # autoconfs a fe80:: - a peerless link-local to aim at if a rule regresses. # leading "-": the veth doesn't exist at the boot sysctl run, so ignore it # there; systemd's net-device udev rule reapplies this per-iface at creation. // lib.listToAttrs ( map (c: lib.nameValuePair "-net.ipv6.conf.${c.iface}.disable_ipv6" 1) deploymentList ); networking.dhcpcd.denyInterfaces = deploymentIfGlobs; networking.firewall.trustedInterfaces = map (c: c.iface) deploymentList; services.dnsmasq = { enable = true; settings = { interface = [ "lo" ] ++ map (c: c.iface) deploymentList; local = "/containers/"; no-resolv = true; server = [ "1.1.1.1" "8.8.8.8" ]; host-record = map (c: "${c.name},${c.name}.containers,${c.addr}") deploymentList; }; }; networking.nftables.enable = true; networking.nftables.tables = { container-ip = { family = "inet"; content = '' counter spoofed { } chain prerouting { type filter hook prerouting priority -300; policy accept; ${srcGuard} } chain deployment-input { meta nfproto ipv6 drop ct state established,related accept meta l4proto icmp accept udp dport 53 accept tcp dport 53 accept ${hostRules} drop } chain input { type filter hook input priority filter - 10; policy accept; iifname { ${deploymentIfs} } jump deployment-input } chain forward { type filter hook forward priority 0; policy accept; meta nfproto ipv6 drop ct state established,related accept ${peerRules} ct status dnat accept ${egressRules} reject with icmp type host-unreachable } chain postrouting { type nat hook postrouting priority srcnat; policy accept; ${masqRules} } ''; }; container-portforward = { family = "ip"; content = '' map tcp-fwd { type inet_service : ipv4_addr . inet_service ${elemsOf tcpElems} } map udp-fwd { type inet_service : ipv4_addr . inet_service ${elemsOf udpElems} } chain prerouting { type nat hook prerouting priority dstnat; policy accept; iifname { ${deploymentIfs} } return fib daddr type local dnat to tcp dport map @tcp-fwd fib daddr type local dnat to udp dport map @udp-fwd } chain output { type nat hook output priority dstnat; policy accept; fib daddr type local dnat to tcp dport map @tcp-fwd fib daddr type local dnat to udp dport map @udp-fwd } ''; }; }; }; /** Throws if any two deployments derive the same interface name or address, or if one lands on a reserved address. All three are hashes of the container name, so a name collision is silent otherwise: two containers would share a veth name, or an address two routes would claim. Set `address` on one of them to resolve. # Type ``` checkIdentities :: String -> AttrSet -> AttrSet ``` */ checkIdentities = mainGateway: svcs: let svcList = lib.attrValues svcs; reservedAddrs = [ mainGateway "10.0.0.0" "10.255.255.255" ]; # addr and hostAddr live in one routing namespace: a container addr equal to # another's hostAddr makes the host claim it locally instead of routing to # the container, so they must be checked against each other, not per-field. ipEntries = map (a: { holder = "reserved"; value = a; }) reservedAddrs ++ lib.concatMap (c: [ { holder = "${c.name}.addr"; value = c.addr; } { holder = "${c.name}.hostAddr"; value = c.hostAddr; } ]) svcList; plainEntries = field: map (c: { holder = c.name; value = c.${field}; }) svcList; dupsOf = label: entries: lib.mapAttrsToList (v: es: " ${label} ${v}: ${lib.concatMapStringsSep ", " (e: e.holder) es}") ( lib.filterAttrs (_: es: lib.length es > 1) (lib.groupBy (e: e.value) entries) ); problems = dupsOf "ip" ipEntries ++ dupsOf "iface" (plainEntries "iface") ++ dupsOf "mac" (plainEntries "mac"); in if problems == [ ] then svcs else throw "derived identity collision: \n${lib.concatStringsSep "\n" problems}"; checkForwardPorts = svcs: let duplicateFor = proto: selector: let entries = lib.concatMap ( c: map (p: { holder = c.name; port = (normalizePort p).host; }) (selector c) ) (lib.attrValues svcs); duplicates = lib.filterAttrs (_: es: lib.length es > 1) (lib.groupBy (e: toString e.port) entries); in lib.mapAttrsToList ( port: es: " ${proto}/${port}: ${lib.concatMapStringsSep ", " (e: e.holder) es}" ) duplicates; problems = duplicateFor "tcp" (c: c.hostPorts) ++ duplicateFor "udp" (c: c.hostUdpPorts); in if problems == [ ] then svcs else throw "duplicate published ports:\n${lib.concatStringsSep "\n" problems}"; /** Whether a value is a `mkContainer` or `mkVm` result. # Type ``` isDeployment :: Any -> Bool ``` */ isDeployment = c: lib.isAttrs c && c ? kind; /** The systemd unit name a deployment runs as, without the `.service` suffix. # Type ``` serviceUnitName :: Deployment -> String ``` */ serviceUnitName = c: "${unitPrefix.${c.kind}}@${c.name}"; /** Turns a set of containers into the actual NixOS configuration. # Type ``` mkContainerSet :: { svcs :: AttrSet; } -> AttrSet ``` */ mkContainerSet = { svcs }: let nspawnSvcs = ofKind "nspawn" svcs; vmSvcs = ofKind "vm" svcs; netUnitsCommon = [ "nftables.service" ]; mkDep = c: let hasDeps = c.depends != [ ]; depUnits = map (d: "${serviceUnitName d}.service") c.depends; in lib.nameValuePair (serviceUnitName c) { wants = lib.optionals hasDeps depUnits; after = lib.optionals hasDeps depUnits ++ netUnitsCommon; requires = netUnitsCommon; startLimitBurst = 5; startLimitIntervalSec = 120; unitConfig.RequiresMountsFor = c.bindMountHostPaths; }; # the host end of a link, for both kinds. nixos-containers only configures # an extraVeth's host end in the container unit's postStart, which by # definition runs after the container is fully booted - so every service # inside it sees a peerless, carrier-less link until then. Declaring the # interface instead gets `network-addresses-`, which udev triggers off # the device unit the moment the veth appears. mkLink = c: { ipv4.addresses = [ { address = c.hostAddr; prefixLength = 32; } ]; ipv4.routes = [ { address = c.addr; prefixLength = 32; } ]; }; in { containers = lib.mapAttrs (_: c: c.nixosContainer) nspawnSvcs; microvm.vms = lib.mapAttrs (_: c: c.microvmConfig) vmSvcs; networking.interfaces = lib.mapAttrs' (_: c: lib.nameValuePair c.iface (mkLink c)) svcs; systemd.services = lib.listToAttrs (map mkDep (lib.attrValues svcs)); }; /** Prometheus recording rules for each nspawn container's resource limits (`container_limit_memory_bytes`, `container_limit_cpu_cores`). Used by `overseer`. VM deployments are skipped - their `mem`/`vcpu` are an allocation, not a cgroup ceiling, so they don't map onto the same metric; not covered by these dashboards yet. # Type ``` mkContainerLimitsRuleFile :: AttrSet -> AttrSet -> Derivation ``` */ mkContainerLimitsRuleFile = pkgs: deployments: let containers = ofKind "nspawn" deployments; parseMemBytes = s: let unit = lib.toUpper (lib.substring (lib.stringLength s - 1) 1 s); n = lib.toInt (lib.substring 0 (lib.stringLength s - 1) s); in if unit == "G" then n * 1024 * 1024 * 1024 else if unit == "M" then n * 1024 * 1024 else if unit == "K" then n * 1024 else lib.toInt s; in pkgs.writeText "container-limits.yml" ( builtins.toJSON { groups = [ { name = "container-limits"; rules = lib.concatLists ( lib.mapAttrsToList (cname: c: [ { record = "container_limit_memory_bytes"; expr = toString (parseMemBytes c.memoryMax); labels = { container = cname; }; } { record = "container_limit_cpu_cores"; expr = "${toString (lib.toInt (lib.removeSuffix "%" c.cpuQuota))}e-2"; labels = { container = cname; }; } ]) containers ); } ]; } ); /** Loads a host's containers, as a attrset of name -> container. Does not build NixOS config built yet. Use `mkHostFromContainers` for a real host. # Inputs `containerDirs` : Directories to scan for `*.nix` container files (e.g. `containers/common`, `containers/`). `ctx` : Extra values passed into every container file's own top-level args (`pkgs`, `hostInfo`, `inputs`, `customPkgs`, ...). # Type ``` loadContainers :: { hostName :: String; containerDirs :: [ Path ]; ctx :: AttrSet; } -> AttrSet ``` */ loadContainers = { hostName, containerDirs, ctx, }: let dslCtx = { inherit listMap mkService mkContainer mkVm collectContainers mkNetworking isDeployment mkContainerSet mkContainerLimitsRuleFile mkHostFromContainers ; }; fullCtx = ctx // { inherit containers; dsl = dslCtx; }; loadWith = extra: let perDir = map (dir: listMap dir (name: path: import path (fullCtx // extra // { inherit name; }))) ( lib.filter builtins.pathExists containerDirs ); nameCounts = lib.foldl' (acc: n: acc // { ${n} = (acc.${n} or 0) + 1; }) { } ( lib.concatMap lib.attrNames perDir ); duplicateNames = lib.attrNames (lib.filterAttrs (_: count: count > 1) nameCounts); in if duplicateNames != [ ] then throw "loadContainers ${hostName}: container name(s) defined in more than one of containerDirs: ${toString duplicateNames}" else lib.foldl' ( acc: dirAttrs: lib.foldl' ( acc2: name: let m = dirAttrs.${name}; in acc2 // (if isDeployment m then { ${name} = m; } else m) ) acc (lib.attrNames dirAttrs) ) { } perDir; assertFn = pred: x: m: if pred x then x else throw m; containers = lib.mapAttrs (k: v: assertFn isDeployment v "${k} is not a deployment") ( lib.filterAttrs (n: _: !lib.strings.hasPrefix "_" n) (lib.fix (self: loadWith { inherit self; })) ); in containers; /** Builds a host's NixOS config from container configuration. # Type ``` mkHostFromContainers :: { containers :: AttrSet; ctx :: AttrSet; } -> Module ``` */ mkHostFromContainers = { containers, ctx }: let checked = checkForwardPorts (checkIdentities ctx.hostInfo.mainGateway containers); hostConfigs = lib.concatMap (c: lib.optional (c ? hostConfig) c.hostConfig) ( lib.attrValues checked ); nspawnContainers = ofKind "nspawn" checked; in { config = lib.mkMerge ( [ (mkContainerSet { svcs = checked; }) (mkNetworking { svcs = checked; }) { containers = lib.mapAttrs (_: _: { config = { nixpkgs.pkgs = lib.mkDefault ctx.pkgs; system.stateVersion = lib.mkDefault "26.05"; }; }) nspawnContainers; } { systemd.services = lib.mapAttrs' ( _: c: lib.nameValuePair (serviceUnitName c) { serviceConfig = { MemoryMax = c.memoryMax; CPUQuota = c.cpuQuota; }; } ) nspawnContainers; } ] ++ hostConfigs ); }; /** Host-independent boilerplate for a NixOS VM integration test. # Type ``` mkIntegrationTest :: { host :: AttrSet; testContainers :: AttrSet; pkgs :: AttrSet; } -> AttrSet ``` */ mkIntegrationTest = { host, testContainers, pkgs, }: let testHostInfo = host.specialArgs.hostInfo // { isVm = true; }; allTestContainerNames = lib.attrNames testContainers; autoStartTestContainers = lib.filterAttrs (_: c: c.autoStart) testContainers; includedContainerNames = lib.attrNames autoStartTestContainers; nspawnNames = lib.attrNames (ofKind "nspawn" testContainers); vmNames = lib.attrNames (ofKind "vm" testContainers); allowInternetContainerNames = lib.attrNames ( lib.filterAttrs (_: c: c.allowInternet) autoStartTestContainers ); gatewayContainerNames = lib.attrNames ( lib.filterAttrs (_: c: c.needsGateway) autoStartTestContainers ); pyList = names: "[" + lib.concatMapStringsSep ", " (n: "\"${n}\"") names + "]"; pyDict = render: attrs: "{" + lib.concatStringsSep ", " (lib.mapAttrsToList (k: v: "\"${k}\": ${render v}") attrs) + "}"; pyStr = v: "\"${v}\""; pyInts = ps: "[" + lib.concatMapStringsSep ", " toString ps + "]"; pyEdges = lib.concatStringsSep ", " ( lib.concatMap ( c: map (a: "(\"${c.name}\", \"${a.to}\", ${pyInts (a.tcp or [ ])}, ${pyInts (a.udp or [ ])})") c.access ) (lib.attrValues autoStartTestContainers) ); bin = lib.getExe'; testTools = [ pkgs.curl pkgs.iproute2 pkgs.iputils pkgs.util-linux pkgs.openssh pkgs.openssl pkgs.coreutils pkgs.dnsutils pkgs.python3 pkgs.strace pkgs.tcpdump ]; in { inherit testHostInfo; containerNames = { all = allTestContainerNames; included = includedContainerNames; allowInternet = allowInternetContainerNames; gateway = gatewayContainerNames; }; internetNode = _: { networking.firewall.allowedTCPPorts = [ 80 ]; systemd.services.dummy-http = { wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = "${bin pkgs.python3 "python3"} -m http.server 80"; DynamicUser = true; AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; }; }; }; serverModule = { _module.args = host.specialArgs // { hostInfo = testHostInfo; }; networking.hostName = host.hostName; # containers don't forward journal to console by default containers = lib.genAttrs nspawnNames (_: { timeoutStartSec = "10min"; config.services.journald.extraConfig = "ForwardToConsole=yes"; }); # a test VM has neither the memory to hand out nor the cores to divide. # optimize.enable re-overrides qemu with nixosTestRunner; hydra caches # that build of plain qemu but not of qemu_kvm, so naming it here is # what keeps a nested guest from compiling qemu before it can boot microvm.vms = lib.genAttrs vmNames (_: { config.microvm = { mem = lib.mkForce 512; vcpu = lib.mkForce 1; qemu.package = lib.mkForce pkgs.qemu_test; }; }); systemd.services = lib.genAttrs (map serviceUnitName (lib.attrValues testContainers)) (_: { serviceConfig = { Restart = lib.mkForce "no"; RestartForceExitStatus = lib.mkForce ""; }; }) // { # erofs /nix/store is read-only, breaks nix-store --load-db register-nix-paths.enable = false; }; virtualisation.fileSystems."/persist" = { device = "none"; fsType = "tmpfs"; # needed before persist.mount runs (activation depends on it) neededForBoot = true; }; services.dnsmasq.settings.server = lib.mkForce [ ]; virtualisation.memorySize = lib.mkDefault 4096; virtualisation.cores = lib.mkDefault 4; virtualisation.useNixStoreImage = true; virtualisation.additionalPaths = testTools; }; testScriptPreamble = '' included = ${pyList includedContainerNames} ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.addr) autoStartTestContainers)} HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)} IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)} KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)} # only nspawn guests can be entered with nsenter; a vm has no host-side # netns, so every probe that runs *inside* a guest is limited to these enterable = [c for c in included if KIND[c] == "nspawn"] allow_internet = [c for c in enterable if c in ${pyList allowInternetContainerNames}] has_gateway = [c for c in enterable if c in ${pyList gatewayContainerNames}] not_allow_internet = [c for c in enterable if c not in allow_internet] no_gateway = [c for c in enterable if c not in has_gateway] # (src, to, tcp ports, udp ports); `to` may be "@host" or "@internet" EDGES = [${pyEdges}] NSENTER = "${bin pkgs.util-linux "nsenter"}" CURL = "${bin pkgs.curl "curl"}" IP = "${bin pkgs.iproute2 "ip"}" PING = "${bin pkgs.iputils "ping"}" DIG = "${bin pkgs.dnsutils "dig"}" SSH = "${bin pkgs.openssh "ssh"}" OPENSSL = "${bin pkgs.openssl "openssl"}" TIMEOUT = "${bin pkgs.coreutils "timeout"}" ''; }; }