1319 lines
38 KiB
Nix
1319 lines
38 KiB
Nix
{ lib, microvm }:
|
|
let
|
|
hexDigit = lib.listToAttrs (
|
|
lib.imap0 (i: c: lib.nameValuePair c i) (lib.stringToCharacters "0123456789abcdef")
|
|
);
|
|
hexToInt = s: lib.foldl' (acc: c: acc * 16 + hexDigit.${c}) 0 (lib.stringToCharacters s);
|
|
|
|
deriveMac =
|
|
name:
|
|
"02:"
|
|
+ lib.concatStringsSep ":" (
|
|
lib.genList (i: lib.substring (i * 2) 2 (builtins.hashString "sha256" name)) 5
|
|
);
|
|
deriveIfName = prefix: name: lib.substring 0 15 "${prefix}-${builtins.hashString "sha256" name}"; # IFNAMSIZ - 1
|
|
|
|
deriveIp4AddrAt =
|
|
offset: name:
|
|
let
|
|
h = builtins.hashString "sha256" name;
|
|
octet = i: toString (hexToInt (lib.substring (offset + i * 2) 2 h));
|
|
in
|
|
"10.${octet 0}.${octet 1}.${octet 2}";
|
|
|
|
ifPrefix = {
|
|
nspawn = "vh";
|
|
vm = "vm";
|
|
};
|
|
|
|
unitPrefix = {
|
|
nspawn = "container";
|
|
vm = "microvm";
|
|
};
|
|
|
|
deploymentIfGlobs = lib.mapAttrsToList (_: p: "${p}-*") ifPrefix;
|
|
|
|
ofKind = k: lib.filterAttrs (_: c: c.kind == k);
|
|
|
|
deriveIdentity = prefix: name: {
|
|
iface = deriveIfName prefix name;
|
|
mac = deriveMac name;
|
|
addr = deriveIp4AddrAt 12 name;
|
|
hostAddr = deriveIp4AddrAt 24 name;
|
|
};
|
|
|
|
specialTargets = [
|
|
"@host"
|
|
"@internet"
|
|
];
|
|
|
|
isSpecialTarget = to: lib.elem to specialTargets;
|
|
|
|
hasInternetAccess = access: lib.any (a: a.to == "@internet") access;
|
|
|
|
substrateNet = {
|
|
address = "10.0.0.0";
|
|
prefixLength = 8;
|
|
};
|
|
|
|
needsGatewayFor =
|
|
access: hostPorts: hostUdpPorts:
|
|
hasInternetAccess access || hostPorts != [ ] || hostUdpPorts != [ ];
|
|
|
|
flattenMiscProp =
|
|
self: props:
|
|
if lib.isAttrs props then
|
|
props
|
|
else if lib.isList props then
|
|
lib.mergeAttrsList (map (flattenMiscProp self) props)
|
|
else
|
|
props self;
|
|
|
|
mkUser =
|
|
{
|
|
name,
|
|
uid,
|
|
home ? "/home/${name}",
|
|
extraUserConfig ? { },
|
|
}:
|
|
{
|
|
users.users.${name} = {
|
|
isNormalUser = true;
|
|
inherit uid home;
|
|
group = name;
|
|
}
|
|
// extraUserConfig;
|
|
users.groups.${name} = { };
|
|
};
|
|
|
|
parseMemMiB =
|
|
s:
|
|
let
|
|
unit = lib.toUpper (lib.substring (lib.stringLength s - 1) 1 s);
|
|
n = lib.toInt (lib.substring 0 (lib.stringLength s - 1) s);
|
|
in
|
|
if unit == "G" then
|
|
n * 1024
|
|
else if unit == "M" then
|
|
n
|
|
else
|
|
lib.toInt s;
|
|
in
|
|
rec {
|
|
/**
|
|
Expands a port spec.
|
|
|
|
# Type
|
|
|
|
```
|
|
normalizePort :: (Int | { host :: Int; container :: Int; }) -> { host :: Int; container :: Int; }
|
|
```
|
|
*/
|
|
normalizePort =
|
|
p:
|
|
if builtins.isInt p then
|
|
{
|
|
host = p;
|
|
container = p;
|
|
}
|
|
else
|
|
p;
|
|
|
|
/**
|
|
Reads all `{name}.nix` files in `dir` and associates to { "${name}" = f name path }
|
|
|
|
# Type
|
|
|
|
```
|
|
listMap :: Path -> (String -> Path -> a) -> AttrSet
|
|
```
|
|
*/
|
|
listMap =
|
|
dir: f:
|
|
lib.mapAttrs' (
|
|
fileName: _:
|
|
let
|
|
name = lib.removeSuffix ".nix" fileName;
|
|
in
|
|
lib.nameValuePair name (f name (dir + "/${fileName}"))
|
|
) (lib.filterAttrs (n: t: t == "regular" && lib.hasSuffix ".nix" n) (builtins.readDir dir));
|
|
|
|
/**
|
|
Creates configuration for a hardened systemd service.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkService :: AttrSet -> AttrSet
|
|
```
|
|
*/
|
|
mkService =
|
|
{
|
|
description,
|
|
execStart,
|
|
wantedBy ? [ "multi-user.target" ],
|
|
after ? [ ],
|
|
wants ? [ ],
|
|
networkOnline ? false,
|
|
workingDir ? null,
|
|
serviceConfig ? { },
|
|
unitConfig ? { },
|
|
path ? [ ],
|
|
}:
|
|
{
|
|
inherit
|
|
description
|
|
wantedBy
|
|
path
|
|
unitConfig
|
|
;
|
|
after = after ++ lib.optional networkOnline "network-online.target";
|
|
wants = wants ++ lib.optional networkOnline "network-online.target";
|
|
serviceConfig = {
|
|
ExecStart = execStart;
|
|
FailureAction = "exit-force";
|
|
NoNewPrivileges = true;
|
|
SystemCallFilter = "@system-service";
|
|
SystemCallArchitectures = "native";
|
|
ProtectSystem = "strict";
|
|
}
|
|
// lib.optionalAttrs (workingDir != null) { WorkingDirectory = workingDir; }
|
|
// serviceConfig;
|
|
};
|
|
|
|
/**
|
|
Declares a systemd-nspawn container: networking, resource limits, bind
|
|
mounts, and the NixOS config that runs inside it.
|
|
|
|
# Inputs
|
|
|
|
`traits`
|
|
: Reusable partial configs (own `resources.fs`/`users`/`containerConfig`).
|
|
|
|
`users`
|
|
: List of user (`{ name; uid; home?; extraUserConfig?; }`) specs
|
|
|
|
`net.access`
|
|
: Outbound edges this container is allowed to open, as
|
|
`{ to; tcp ? [ ]; udp ? [ ]; }`. `to` is another deployment's name, or one
|
|
of the `@`-prefixed specials `"@host"` and `"@internet"` - the prefix
|
|
keeps them out of the deployment namespace. Everything not listed is
|
|
denied, and an edge is one-way: the peer needs its own entry to talk back.
|
|
Ports are mandatory except for `"@internet"`. DNS and ICMP to the host are
|
|
always allowed and are not declarable.
|
|
|
|
`net.expose.tcp`, `net.expose.udp`
|
|
: Ports opened on the container's own firewall.
|
|
|
|
`net.forward.tcp`, `net.forward.udp`
|
|
: Host ports forwarded into the container; see `normalizePort`.
|
|
|
|
`svc.autoStart`, `svc.depends`
|
|
: Whether the container starts automatically, and other deployments it
|
|
depends on.
|
|
|
|
`resources.cpu`
|
|
: CPU core count; converted to `CPUQuota` (`cpu * 100%`).
|
|
|
|
`resources.mem`
|
|
: `MemoryMax`, e.g. `"512M"`.
|
|
|
|
`resources.fs`
|
|
: Bind mounts, keyed by mount point.
|
|
|
|
`isolation.nixDaemon`
|
|
: Whether the Nix daemon and CLI are available inside the container.
|
|
|
|
`isolation.privateUsers`
|
|
: systemd-nspawn user namespacing; see `--private-users=` in
|
|
systemd-nspawn(1). Bind mounts always go through idmapped `--bind`.
|
|
|
|
`miscProperties`
|
|
: Arbitrary data attached to the result. Used for cross-container discovery
|
|
via `collectContainers`.
|
|
|
|
`includedInTests`
|
|
: Whether integration VM tests should includes this container. Ignored
|
|
for real deployment. Set `false` for containers the test VM shouldn't
|
|
run.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkContainer :: AttrSet -> Deployment
|
|
```
|
|
*/
|
|
mkContainer =
|
|
{
|
|
name,
|
|
net ? { },
|
|
svc ? { },
|
|
resources ? { },
|
|
isolation ? { },
|
|
traits ? [ ],
|
|
users ? [ ],
|
|
packages ? [ ],
|
|
service ? null,
|
|
containerConfig ? (_: { }),
|
|
miscProperties ? { },
|
|
includedInTests ? true,
|
|
}:
|
|
let
|
|
access = net.access or [ ];
|
|
ports = net.expose.tcp or [ ];
|
|
udpPorts = net.expose.udp or [ ];
|
|
hostPorts = net.forward.tcp or [ ];
|
|
hostUdpPorts = net.forward.udp or [ ];
|
|
autoStart = svc.autoStart or false;
|
|
depends = svc.depends or [ ];
|
|
cpu = resources.cpu or 1;
|
|
cpuQuota = "${toString (cpu * 100)}%";
|
|
memoryMax = resources.mem or "512M";
|
|
bindMounts = resources.fs or { };
|
|
allowsNixDaemon = isolation.nixDaemon or false;
|
|
privateUsers = isolation.privateUsers or "pick";
|
|
|
|
allowInternet = hasInternetAccess access;
|
|
allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts;
|
|
userModules = map mkUser (lib.concatMap (t: t.users or [ ]) traits ++ users);
|
|
allPorts = ports ++ map (p: (normalizePort p).container) hostPorts;
|
|
useIdmap = privateUsers != "no";
|
|
bindMountExtraFlags = lib.mapAttrsToList (
|
|
mountPoint: bm:
|
|
(if bm.isReadOnly then "--bind-ro=" else "--bind=")
|
|
+ "${bm.hostPath}:${mountPoint}"
|
|
+ lib.optionalString useIdmap ":idmap"
|
|
) allBindMounts;
|
|
bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts);
|
|
inherit (deriveIdentity ifPrefix.nspawn name)
|
|
iface
|
|
mac
|
|
addr
|
|
hostAddr
|
|
;
|
|
needsGateway = needsGatewayFor access hostPorts hostUdpPorts;
|
|
|
|
linkVeths.${iface} = { };
|
|
|
|
mergedConfig =
|
|
args@{
|
|
pkgs,
|
|
lib,
|
|
...
|
|
}:
|
|
let
|
|
traitConfigs = map (t: (t.containerConfig or (_: { })) args) traits;
|
|
pkgModule = lib.optional (packages != [ ]) { environment.systemPackages = packages; };
|
|
nixModule = lib.optional allowsNixDaemon {
|
|
environment.systemPackages = [ pkgs.nix ];
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
};
|
|
svcModule = lib.optional (service != null) {
|
|
systemd.services.${name} = lib.mkMerge [
|
|
(service args)
|
|
{
|
|
after = [ "container-ready.target" ];
|
|
wants = [ "container-ready.target" ];
|
|
}
|
|
];
|
|
};
|
|
portModule =
|
|
lib.optional (allPorts != [ ]) { networking.firewall.allowedTCPPorts = allPorts; }
|
|
++ lib.optional (udpPorts != [ ]) { networking.firewall.allowedUDPPorts = udpPorts; };
|
|
readyTarget = {
|
|
systemd.targets.container-ready = {
|
|
description = "Container prerequisites satisfied";
|
|
after = lib.optional allowInternet "network-online.target";
|
|
wants = lib.optional allowInternet "network-online.target";
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
};
|
|
in
|
|
lib.mkMerge (
|
|
traitConfigs
|
|
++ userModules
|
|
++ pkgModule
|
|
++ nixModule
|
|
++ svcModule
|
|
++ [ readyTarget ]
|
|
++ [ (containerConfig args) ]
|
|
++ portModule
|
|
);
|
|
in
|
|
lib.fix (self: {
|
|
inherit
|
|
name
|
|
access
|
|
autoStart
|
|
depends
|
|
ports
|
|
hostPorts
|
|
hostUdpPorts
|
|
memoryMax
|
|
cpuQuota
|
|
privateUsers
|
|
allowInternet
|
|
needsGateway
|
|
iface
|
|
mac
|
|
addr
|
|
hostAddr
|
|
bindMountHostPaths
|
|
includedInTests
|
|
;
|
|
bindMounts = allBindMounts;
|
|
kind = "nspawn";
|
|
miscProperties = flattenMiscProp self miscProperties;
|
|
nixosContainer = {
|
|
inherit autoStart privateUsers;
|
|
bindMounts = { };
|
|
extraFlags = bindMountExtraFlags;
|
|
privateNetwork = true;
|
|
extraVeths = linkVeths;
|
|
config =
|
|
{
|
|
pkgs,
|
|
config,
|
|
lib,
|
|
...
|
|
}:
|
|
lib.mkMerge [
|
|
{
|
|
networking.enableIPv6 = false;
|
|
networking.useHostResolvConf = lib.mkForce false;
|
|
networking.nameservers = [ hostAddr ];
|
|
networking.interfaces.${iface} = {
|
|
ipv4.addresses = [
|
|
{
|
|
address = addr;
|
|
prefixLength = 32;
|
|
}
|
|
];
|
|
ipv4.routes = [
|
|
{
|
|
address = hostAddr;
|
|
prefixLength = 32;
|
|
}
|
|
(
|
|
substrateNet
|
|
// {
|
|
via = hostAddr;
|
|
}
|
|
)
|
|
];
|
|
};
|
|
systemd.services."network-addresses-${iface}".before = [ "network-online.target" ];
|
|
}
|
|
(lib.optionalAttrs needsGateway {
|
|
networking.defaultGateway = {
|
|
address = hostAddr;
|
|
interface = iface;
|
|
};
|
|
})
|
|
(mergedConfig { inherit pkgs config lib; })
|
|
];
|
|
};
|
|
});
|
|
|
|
/**
|
|
Declares a microvm: same networking interface as `mkContainer` (`access`,
|
|
`hostPorts` and friends all work the same way), virtiofs/9p shares, and the
|
|
NixOS config that runs inside it. The VM counterpart to `mkContainer` -
|
|
same shared interface, different backend.
|
|
|
|
# Inputs
|
|
|
|
`resources.fs`
|
|
: Passed straight through to `microvm.shares`.
|
|
|
|
`guestConfig`
|
|
: NixOS module for the VM's own system; same role as `mkContainer`'s
|
|
`containerConfig`; gets full module args (`pkgs`, `utils`, ...) since
|
|
it's spliced in via `imports`, not called manually.
|
|
|
|
`resources.cpu`, `resources.mem`, `hypervisor`
|
|
: Passed through to the corresponding `microvm.*` options. `resources.cpu`
|
|
is vcpu count. `resources.mem` is required, a string like `"8G"`,
|
|
converted to MiB via `parseMemMiB`.
|
|
|
|
`net.access`, `net.forward.tcp`, `net.forward.udp`, `svc.autoStart`,
|
|
`svc.depends`
|
|
: See `mkContainer`.
|
|
|
|
`miscProperties`, `includedInTests`
|
|
: See `mkContainer`.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkVm :: AttrSet -> Deployment
|
|
```
|
|
*/
|
|
mkVm =
|
|
{
|
|
name,
|
|
net ? { },
|
|
svc ? { },
|
|
resources ? { },
|
|
hypervisor ? "qemu",
|
|
guestConfig ? (_: { }),
|
|
miscProperties ? { },
|
|
includedInTests ? true,
|
|
}:
|
|
let
|
|
access = net.access or [ ];
|
|
hostPorts = net.forward.tcp or [ ];
|
|
hostUdpPorts = net.forward.udp or [ ];
|
|
autoStart = svc.autoStart or false;
|
|
depends = svc.depends or [ ];
|
|
vcpu = resources.cpu or 1;
|
|
mem = parseMemMiB (resources.mem or (throw "mkVm ${name}: resources.mem is required"));
|
|
shares = resources.fs or [ ];
|
|
|
|
allowInternet = hasInternetAccess access;
|
|
needsGateway = needsGatewayFor access hostPorts hostUdpPorts;
|
|
inherit (deriveIdentity ifPrefix.vm name)
|
|
iface
|
|
mac
|
|
addr
|
|
hostAddr
|
|
;
|
|
in
|
|
lib.fix (self: {
|
|
inherit
|
|
name
|
|
access
|
|
autoStart
|
|
depends
|
|
hostPorts
|
|
hostUdpPorts
|
|
allowInternet
|
|
iface
|
|
mac
|
|
addr
|
|
hostAddr
|
|
shares
|
|
vcpu
|
|
mem
|
|
hypervisor
|
|
includedInTests
|
|
;
|
|
inherit needsGateway;
|
|
bindMountHostPaths = map (s: s.source) shares;
|
|
kind = "vm";
|
|
miscProperties = flattenMiscProp self miscProperties;
|
|
microvmConfig = {
|
|
autostart = autoStart;
|
|
config = _: {
|
|
imports = [
|
|
microvm.nixosModules.microvm
|
|
guestConfig
|
|
];
|
|
nixpkgs.overlays = [ microvm.overlay ];
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
networking.hostName = name;
|
|
system.stateVersion = "26.05";
|
|
microvm = {
|
|
inherit
|
|
vcpu
|
|
mem
|
|
hypervisor
|
|
shares
|
|
;
|
|
interfaces = [
|
|
{
|
|
type = "tap";
|
|
id = iface;
|
|
inherit mac;
|
|
}
|
|
];
|
|
};
|
|
networking.useNetworkd = true;
|
|
networking.useDHCP = false;
|
|
systemd.network.networks."10-main" = {
|
|
matchConfig.MACAddress = mac;
|
|
address = [ "${addr}/32" ];
|
|
routes = [
|
|
{
|
|
Destination = "${hostAddr}/32";
|
|
Scope = "link";
|
|
}
|
|
{
|
|
Destination = "${substrateNet.address}/${toString substrateNet.prefixLength}";
|
|
Gateway = hostAddr;
|
|
}
|
|
]
|
|
++ lib.optional needsGateway {
|
|
Destination = "0.0.0.0/0";
|
|
Gateway = hostAddr;
|
|
};
|
|
networkConfig.DNS = [ hostAddr ];
|
|
linkConfig.RequiredForOnline = if allowInternet then "routable" else "degraded";
|
|
};
|
|
networking.enableIPv6 = false;
|
|
};
|
|
};
|
|
});
|
|
|
|
/**
|
|
Subset of `svcs`, with `access` edges pointing at dropped deployments
|
|
removed. Filtering the attrset alone leaves those edges dangling, and
|
|
`mkNetworking` rejects an edge whose target it cannot resolve.
|
|
|
|
# Type
|
|
|
|
```
|
|
filterDeployments :: (Deployment -> Bool) -> AttrSet -> AttrSet
|
|
```
|
|
*/
|
|
filterDeployments =
|
|
pred: svcs:
|
|
let
|
|
kept = lib.filterAttrs (_: pred) svcs;
|
|
in
|
|
lib.mapAttrs (
|
|
_: c:
|
|
c
|
|
// {
|
|
access = lib.filter (a: isSpecialTarget a.to || kept ? ${a.to}) c.access;
|
|
}
|
|
) kept;
|
|
|
|
/**
|
|
Finds every container that set `miscProperties.${key}`, paired with the
|
|
value. Used for cross-container discovery.
|
|
|
|
# Type
|
|
|
|
```
|
|
collectContainers :: String -> AttrSet -> [ { container :: Container; data :: Any; } ]
|
|
```
|
|
*/
|
|
collectContainers =
|
|
key: containers:
|
|
lib.mapAttrsToList (_: v: {
|
|
container = v;
|
|
data = v.miscProperties.${key};
|
|
}) (lib.filterAttrs (_: v: v.miscProperties ? ${key}) containers);
|
|
|
|
/**
|
|
Prepares networking for all deployments: dnsmasq for name resolution and
|
|
the nftables tables that carry isolation, NAT and port forwarding. Every
|
|
link is point-to-point, so `iifname` identifies the peer and nothing
|
|
reaches anything it has no `access` edge for. NixOS container builtin port
|
|
forwarding can't be used because we set up our own networking tables.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkNetworking :: AttrSet -> AttrSet
|
|
```
|
|
*/
|
|
mkNetworking =
|
|
{
|
|
svcs,
|
|
}:
|
|
let
|
|
deploymentList = lib.attrValues svcs;
|
|
quote = lib.concatMapStringsSep ", " (v: "\"${v}\"");
|
|
|
|
deploymentIfs = quote deploymentIfGlobs;
|
|
|
|
netAccessEdges = lib.concatMap (
|
|
deployment:
|
|
map (
|
|
a:
|
|
let
|
|
portsOf = proto: a.${proto} or [ ];
|
|
protos = lib.filter (p: portsOf p != [ ]) [
|
|
"tcp"
|
|
"udp"
|
|
];
|
|
bad = msg: throw "${deployment.name}: access edge to \"${a.to}\" ${msg}";
|
|
in
|
|
if a.to != "@internet" && protos == [ ] then
|
|
bad "${deployment.name} no tcp/udp ports"
|
|
else if isSpecialTarget a.to || svcs ? ${a.to} then
|
|
{
|
|
src = deployment;
|
|
inherit (a) to;
|
|
inherit protos portsOf;
|
|
}
|
|
else
|
|
bad "${deployment.name}: access target does not name a container, \"@host\" or \"@internet\""
|
|
) deployment.access
|
|
) deploymentList;
|
|
|
|
netEdgeRules =
|
|
pred: prefixOf:
|
|
lib.concatMapStrings (
|
|
e:
|
|
lib.concatMapStrings (
|
|
proto:
|
|
"${prefixOf e} ${proto} dport { ${
|
|
lib.concatMapStringsSep ", " toString (e.portsOf proto)
|
|
} } accept\n"
|
|
) e.protos
|
|
) (lib.filter pred netAccessEdges);
|
|
|
|
peerRules = netEdgeRules (e: !isSpecialTarget e.to) (
|
|
e: "iifname \"${e.src.iface}\" oifname \"${svcs.${e.to}.iface}\""
|
|
);
|
|
|
|
hostRules = netEdgeRules (e: e.to == "@host") (
|
|
e: "iifname \"${e.src.iface}\" ip daddr ${e.src.hostAddr}"
|
|
);
|
|
|
|
srcGuard = lib.concatMapStrings (c: ''
|
|
iifname "${c.iface}" ip saddr != ${c.addr} counter name "spoofed" drop
|
|
'') deploymentList;
|
|
|
|
internetIfs = map (c: c.iface) (lib.filter (c: c.allowInternet) deploymentList);
|
|
|
|
uplinkIfs = quote [
|
|
"en*"
|
|
"eth*"
|
|
"wl*"
|
|
];
|
|
|
|
egressRules = lib.optionalString (internetIfs != [ ]) ''
|
|
iifname { ${quote internetIfs} } oifname { ${uplinkIfs} } accept
|
|
'';
|
|
|
|
masqRules = lib.optionalString (internetIfs != [ ]) ''
|
|
iifname { ${quote internetIfs} } oifname { ${uplinkIfs} } masquerade
|
|
'';
|
|
|
|
fwdElems =
|
|
sel:
|
|
lib.concatMapStringsSep ", " (f: "${toString f.host} : ${f.addr} . ${toString f.container}") (
|
|
lib.concatMap (c: map (p: normalizePort p // { inherit (c) addr; }) (sel c)) deploymentList
|
|
);
|
|
tcpElems = fwdElems (c: c.hostPorts);
|
|
udpElems = fwdElems (c: c.hostUdpPorts);
|
|
elemsOf = e: lib.optionalString (e != "") "elements = { ${e} }";
|
|
in
|
|
{
|
|
boot.kernel.sysctl = {
|
|
"net.ipv4.ip_forward" = 1;
|
|
}
|
|
# nftables drops all container IPv6, but the host end of each link still
|
|
# autoconfs a fe80:: - a peerless link-local to aim at if a rule regresses.
|
|
# leading "-": the veth doesn't exist at the boot sysctl run, so ignore it
|
|
# there; systemd's net-device udev rule reapplies this per-iface at creation.
|
|
// lib.listToAttrs (
|
|
map (c: lib.nameValuePair "-net.ipv6.conf.${c.iface}.disable_ipv6" 1) deploymentList
|
|
);
|
|
networking.dhcpcd.denyInterfaces = deploymentIfGlobs;
|
|
|
|
networking.firewall.trustedInterfaces = map (c: c.iface) deploymentList;
|
|
|
|
services.dnsmasq = {
|
|
enable = true;
|
|
settings = {
|
|
interface = [ "lo" ] ++ map (c: c.iface) deploymentList;
|
|
local = "/containers/";
|
|
no-resolv = true;
|
|
server = [
|
|
"1.1.1.1"
|
|
"8.8.8.8"
|
|
];
|
|
host-record = map (c: "${c.name},${c.name}.containers,${c.addr}") deploymentList;
|
|
};
|
|
};
|
|
|
|
networking.nftables.enable = true;
|
|
networking.nftables.tables = {
|
|
container-ip = {
|
|
family = "inet";
|
|
content = ''
|
|
counter spoofed { }
|
|
chain prerouting {
|
|
type filter hook prerouting priority -300; policy accept;
|
|
${srcGuard}
|
|
}
|
|
chain deployment-input {
|
|
meta nfproto ipv6 drop
|
|
ct state established,related accept
|
|
meta l4proto icmp accept
|
|
udp dport 53 accept
|
|
tcp dport 53 accept
|
|
${hostRules}
|
|
drop
|
|
}
|
|
chain input {
|
|
type filter hook input priority filter - 10; policy accept;
|
|
iifname { ${deploymentIfs} } jump deployment-input
|
|
}
|
|
chain forward {
|
|
type filter hook forward priority 0; policy accept;
|
|
meta nfproto ipv6 drop
|
|
ct state established,related accept
|
|
${peerRules}
|
|
ct status dnat accept
|
|
${egressRules}
|
|
reject with icmp type host-unreachable
|
|
}
|
|
chain postrouting {
|
|
type nat hook postrouting priority srcnat; policy accept;
|
|
${masqRules}
|
|
}
|
|
'';
|
|
};
|
|
container-portforward = {
|
|
family = "ip";
|
|
content = ''
|
|
map tcp-fwd {
|
|
type inet_service : ipv4_addr . inet_service
|
|
${elemsOf tcpElems}
|
|
}
|
|
map udp-fwd {
|
|
type inet_service : ipv4_addr . inet_service
|
|
${elemsOf udpElems}
|
|
}
|
|
chain prerouting {
|
|
type nat hook prerouting priority dstnat; policy accept;
|
|
iifname { ${deploymentIfs} } return
|
|
fib daddr type local dnat to tcp dport map @tcp-fwd
|
|
fib daddr type local dnat to udp dport map @udp-fwd
|
|
}
|
|
chain output {
|
|
type nat hook output priority dstnat; policy accept;
|
|
fib daddr type local dnat to tcp dport map @tcp-fwd
|
|
fib daddr type local dnat to udp dport map @udp-fwd
|
|
}
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
/**
|
|
Throws if any two deployments derive the same interface name or address, or
|
|
if one lands on a reserved address. All three are hashes of the container
|
|
name, so a name collision is silent otherwise: two containers would share a
|
|
veth name, or an address two routes would claim. Set `address` on one of
|
|
them to resolve.
|
|
|
|
# Type
|
|
|
|
```
|
|
checkIdentities :: String -> AttrSet -> AttrSet
|
|
```
|
|
*/
|
|
checkIdentities =
|
|
mainGateway: svcs:
|
|
let
|
|
svcList = lib.attrValues svcs;
|
|
|
|
reservedAddrs = [
|
|
mainGateway
|
|
"10.0.0.0"
|
|
"10.255.255.255"
|
|
];
|
|
|
|
# addr and hostAddr live in one routing namespace: a container addr equal to
|
|
# another's hostAddr makes the host claim it locally instead of routing to
|
|
# the container, so they must be checked against each other, not per-field.
|
|
ipEntries =
|
|
map (a: {
|
|
holder = "reserved";
|
|
value = a;
|
|
}) reservedAddrs
|
|
++ lib.concatMap (c: [
|
|
{
|
|
holder = "${c.name}.addr";
|
|
value = c.addr;
|
|
}
|
|
{
|
|
holder = "${c.name}.hostAddr";
|
|
value = c.hostAddr;
|
|
}
|
|
]) svcList;
|
|
|
|
plainEntries =
|
|
field:
|
|
map (c: {
|
|
holder = c.name;
|
|
value = c.${field};
|
|
}) svcList;
|
|
|
|
dupsOf =
|
|
label: entries:
|
|
lib.mapAttrsToList (v: es: " ${label} ${v}: ${lib.concatMapStringsSep ", " (e: e.holder) es}") (
|
|
lib.filterAttrs (_: es: lib.length es > 1) (lib.groupBy (e: e.value) entries)
|
|
);
|
|
|
|
problems =
|
|
dupsOf "ip" ipEntries ++ dupsOf "iface" (plainEntries "iface") ++ dupsOf "mac" (plainEntries "mac");
|
|
in
|
|
if problems == [ ] then
|
|
svcs
|
|
else
|
|
throw "derived identity collision: \n${lib.concatStringsSep "\n" problems}";
|
|
|
|
checkForwardPorts =
|
|
svcs:
|
|
let
|
|
duplicateFor =
|
|
proto: selector:
|
|
let
|
|
entries = lib.concatMap (
|
|
c:
|
|
map (p: {
|
|
holder = c.name;
|
|
port = (normalizePort p).host;
|
|
}) (selector c)
|
|
) (lib.attrValues svcs);
|
|
duplicates = lib.filterAttrs (_: es: lib.length es > 1) (lib.groupBy (e: toString e.port) entries);
|
|
in
|
|
lib.mapAttrsToList (
|
|
port: es: " ${proto}/${port}: ${lib.concatMapStringsSep ", " (e: e.holder) es}"
|
|
) duplicates;
|
|
problems = duplicateFor "tcp" (c: c.hostPorts) ++ duplicateFor "udp" (c: c.hostUdpPorts);
|
|
in
|
|
if problems == [ ] then
|
|
svcs
|
|
else
|
|
throw "duplicate published ports:\n${lib.concatStringsSep "\n" problems}";
|
|
|
|
/**
|
|
Whether a value is a `mkContainer` or `mkVm` result.
|
|
|
|
# Type
|
|
|
|
```
|
|
isDeployment :: Any -> Bool
|
|
```
|
|
*/
|
|
isDeployment = c: lib.isAttrs c && c ? kind;
|
|
|
|
/**
|
|
The systemd unit name a deployment runs as, without the `.service` suffix.
|
|
|
|
# Type
|
|
|
|
```
|
|
serviceUnitName :: Deployment -> String
|
|
```
|
|
*/
|
|
serviceUnitName = c: "${unitPrefix.${c.kind}}@${c.name}";
|
|
|
|
/**
|
|
Turns a set of containers into the actual NixOS configuration.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkContainerSet :: { svcs :: AttrSet; } -> AttrSet
|
|
```
|
|
*/
|
|
mkContainerSet =
|
|
{ svcs }:
|
|
let
|
|
nspawnSvcs = ofKind "nspawn" svcs;
|
|
vmSvcs = ofKind "vm" svcs;
|
|
|
|
netUnitsCommon = [ "nftables.service" ];
|
|
|
|
mkDep =
|
|
c:
|
|
let
|
|
hasDeps = c.depends != [ ];
|
|
depUnits = map (d: "${serviceUnitName d}.service") c.depends;
|
|
in
|
|
lib.nameValuePair (serviceUnitName c) {
|
|
wants = lib.optionals hasDeps depUnits;
|
|
after = lib.optionals hasDeps depUnits ++ netUnitsCommon;
|
|
requires = netUnitsCommon;
|
|
startLimitBurst = 5;
|
|
startLimitIntervalSec = 120;
|
|
unitConfig.RequiresMountsFor = c.bindMountHostPaths;
|
|
};
|
|
|
|
# the host end of a link, for both kinds. nixos-containers only configures
|
|
# an extraVeth's host end in the container unit's postStart, which by
|
|
# definition runs after the container is fully booted - so every service
|
|
# inside it sees a peerless, carrier-less link until then. Declaring the
|
|
# interface instead gets `network-addresses-<iface>`, which udev triggers off
|
|
# the device unit the moment the veth appears.
|
|
mkLink = c: {
|
|
ipv4.addresses = [
|
|
{
|
|
address = c.hostAddr;
|
|
prefixLength = 32;
|
|
}
|
|
];
|
|
ipv4.routes = [
|
|
{
|
|
address = c.addr;
|
|
prefixLength = 32;
|
|
}
|
|
];
|
|
};
|
|
in
|
|
{
|
|
containers = lib.mapAttrs (_: c: c.nixosContainer) nspawnSvcs;
|
|
microvm.vms = lib.mapAttrs (_: c: c.microvmConfig) vmSvcs;
|
|
networking.interfaces = lib.mapAttrs' (_: c: lib.nameValuePair c.iface (mkLink c)) svcs;
|
|
systemd.services = lib.listToAttrs (map mkDep (lib.attrValues svcs));
|
|
};
|
|
|
|
/**
|
|
Prometheus recording rules for each nspawn container's resource limits
|
|
(`container_limit_memory_bytes`, `container_limit_cpu_cores`). Used by
|
|
`overseer`. VM deployments are skipped - their `mem`/`vcpu` are an
|
|
allocation, not a cgroup ceiling, so they don't map onto the same
|
|
metric; not covered by these dashboards yet.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkContainerLimitsRuleFile :: AttrSet -> AttrSet -> Derivation
|
|
```
|
|
*/
|
|
mkContainerLimitsRuleFile =
|
|
pkgs: deployments:
|
|
let
|
|
containers = ofKind "nspawn" deployments;
|
|
parseMemBytes =
|
|
s:
|
|
let
|
|
unit = lib.toUpper (lib.substring (lib.stringLength s - 1) 1 s);
|
|
n = lib.toInt (lib.substring 0 (lib.stringLength s - 1) s);
|
|
in
|
|
if unit == "G" then
|
|
n * 1024 * 1024 * 1024
|
|
else if unit == "M" then
|
|
n * 1024 * 1024
|
|
else if unit == "K" then
|
|
n * 1024
|
|
else
|
|
lib.toInt s;
|
|
in
|
|
pkgs.writeText "container-limits.yml" (
|
|
builtins.toJSON {
|
|
groups = [
|
|
{
|
|
name = "container-limits";
|
|
rules = lib.concatLists (
|
|
lib.mapAttrsToList (cname: c: [
|
|
{
|
|
record = "container_limit_memory_bytes";
|
|
expr = toString (parseMemBytes c.memoryMax);
|
|
labels = {
|
|
container = cname;
|
|
};
|
|
}
|
|
{
|
|
record = "container_limit_cpu_cores";
|
|
expr = "${toString (lib.toInt (lib.removeSuffix "%" c.cpuQuota))}e-2";
|
|
labels = {
|
|
container = cname;
|
|
};
|
|
}
|
|
]) containers
|
|
);
|
|
}
|
|
];
|
|
}
|
|
);
|
|
|
|
/**
|
|
Loads a host's containers, as a attrset of name -> container. Does not
|
|
build NixOS config built yet. Use `mkHostFromContainers` for a real host.
|
|
|
|
# Inputs
|
|
|
|
`containerDirs`
|
|
: Directories to scan for `*.nix` container files (e.g.
|
|
`containers/common`, `containers/<hostName>`).
|
|
|
|
`ctx`
|
|
: Extra values passed into every container file's own top-level args
|
|
(`pkgs`, `hostInfo`, `inputs`, `customPkgs`, ...).
|
|
|
|
# Type
|
|
|
|
```
|
|
loadContainers :: { hostName :: String; containerDirs :: [ Path ]; ctx :: AttrSet; } -> AttrSet
|
|
```
|
|
*/
|
|
loadContainers =
|
|
{
|
|
hostName,
|
|
containerDirs,
|
|
ctx,
|
|
}:
|
|
let
|
|
dslCtx = {
|
|
inherit
|
|
listMap
|
|
mkService
|
|
mkContainer
|
|
mkVm
|
|
collectContainers
|
|
mkNetworking
|
|
isDeployment
|
|
mkContainerSet
|
|
mkContainerLimitsRuleFile
|
|
mkHostFromContainers
|
|
;
|
|
};
|
|
|
|
fullCtx = ctx // {
|
|
inherit containers;
|
|
dsl = dslCtx;
|
|
};
|
|
|
|
loadWith =
|
|
extra:
|
|
let
|
|
perDir = map (dir: listMap dir (name: path: import path (fullCtx // extra // { inherit name; }))) (
|
|
lib.filter builtins.pathExists containerDirs
|
|
);
|
|
|
|
nameCounts = lib.foldl' (acc: n: acc // { ${n} = (acc.${n} or 0) + 1; }) { } (
|
|
lib.concatMap lib.attrNames perDir
|
|
);
|
|
duplicateNames = lib.attrNames (lib.filterAttrs (_: count: count > 1) nameCounts);
|
|
in
|
|
if duplicateNames != [ ] then
|
|
throw "loadContainers ${hostName}: container name(s) defined in more than one of containerDirs: ${toString duplicateNames}"
|
|
else
|
|
lib.foldl' (
|
|
acc: dirAttrs:
|
|
lib.foldl' (
|
|
acc2: name:
|
|
let
|
|
m = dirAttrs.${name};
|
|
in
|
|
acc2 // (if isDeployment m then { ${name} = m; } else m)
|
|
) acc (lib.attrNames dirAttrs)
|
|
) { } perDir;
|
|
|
|
assertFn =
|
|
pred: x: m:
|
|
if pred x then x else throw m;
|
|
|
|
containers = lib.mapAttrs (k: v: assertFn isDeployment v "${k} is not a deployment") (
|
|
lib.filterAttrs (n: _: !lib.strings.hasPrefix "_" n) (lib.fix (self: loadWith { inherit self; }))
|
|
);
|
|
in
|
|
containers;
|
|
|
|
/**
|
|
Builds a host's NixOS config from container configuration.
|
|
|
|
# Type
|
|
|
|
```
|
|
mkHostFromContainers :: { containers :: AttrSet; ctx :: AttrSet; } -> Module
|
|
```
|
|
*/
|
|
mkHostFromContainers =
|
|
{ containers, ctx }:
|
|
let
|
|
checked = checkForwardPorts (checkIdentities ctx.hostInfo.mainGateway containers);
|
|
hostConfigs = lib.concatMap (c: lib.optional (c ? hostConfig) c.hostConfig) (
|
|
lib.attrValues checked
|
|
);
|
|
nspawnContainers = ofKind "nspawn" checked;
|
|
in
|
|
{
|
|
config = lib.mkMerge (
|
|
[
|
|
(mkContainerSet { svcs = checked; })
|
|
(mkNetworking { svcs = checked; })
|
|
{
|
|
containers = lib.mapAttrs (_: _: {
|
|
config = {
|
|
nixpkgs.pkgs = lib.mkDefault ctx.pkgs;
|
|
system.stateVersion = lib.mkDefault "26.05";
|
|
};
|
|
}) nspawnContainers;
|
|
}
|
|
{
|
|
systemd.services = lib.mapAttrs' (
|
|
_: c:
|
|
lib.nameValuePair (serviceUnitName c) {
|
|
serviceConfig = {
|
|
MemoryMax = c.memoryMax;
|
|
CPUQuota = c.cpuQuota;
|
|
};
|
|
}
|
|
) nspawnContainers;
|
|
}
|
|
]
|
|
++ hostConfigs
|
|
);
|
|
};
|
|
|
|
/**
|
|
Host-independent boilerplate for a NixOS VM integration test.
|
|
# Type
|
|
|
|
```
|
|
mkIntegrationTest :: { host :: AttrSet; testContainers :: AttrSet; pkgs :: AttrSet; } -> AttrSet
|
|
```
|
|
*/
|
|
mkIntegrationTest =
|
|
{
|
|
host,
|
|
testContainers,
|
|
pkgs,
|
|
}:
|
|
let
|
|
testHostInfo = host.specialArgs.hostInfo // {
|
|
isVm = true;
|
|
};
|
|
|
|
allTestContainerNames = lib.attrNames testContainers;
|
|
autoStartTestContainers = lib.filterAttrs (_: c: c.autoStart) testContainers;
|
|
includedContainerNames = lib.attrNames autoStartTestContainers;
|
|
nspawnNames = lib.attrNames (ofKind "nspawn" testContainers);
|
|
vmNames = lib.attrNames (ofKind "vm" testContainers);
|
|
allowInternetContainerNames = lib.attrNames (
|
|
lib.filterAttrs (_: c: c.allowInternet) autoStartTestContainers
|
|
);
|
|
gatewayContainerNames = lib.attrNames (
|
|
lib.filterAttrs (_: c: c.needsGateway) autoStartTestContainers
|
|
);
|
|
|
|
pyList = names: "[" + lib.concatMapStringsSep ", " (n: "\"${n}\"") names + "]";
|
|
pyDict =
|
|
render: attrs:
|
|
"{" + lib.concatStringsSep ", " (lib.mapAttrsToList (k: v: "\"${k}\": ${render v}") attrs) + "}";
|
|
pyStr = v: "\"${v}\"";
|
|
pyInts = ps: "[" + lib.concatMapStringsSep ", " toString ps + "]";
|
|
|
|
pyEdges = lib.concatStringsSep ", " (
|
|
lib.concatMap (
|
|
c:
|
|
map (a: "(\"${c.name}\", \"${a.to}\", ${pyInts (a.tcp or [ ])}, ${pyInts (a.udp or [ ])})") c.access
|
|
) (lib.attrValues autoStartTestContainers)
|
|
);
|
|
|
|
bin = lib.getExe';
|
|
|
|
testTools = [
|
|
pkgs.curl
|
|
pkgs.iproute2
|
|
pkgs.iputils
|
|
pkgs.util-linux
|
|
pkgs.openssh
|
|
pkgs.openssl
|
|
pkgs.coreutils
|
|
pkgs.dnsutils
|
|
pkgs.python3
|
|
pkgs.strace
|
|
pkgs.tcpdump
|
|
];
|
|
in
|
|
{
|
|
inherit testHostInfo;
|
|
containerNames = {
|
|
all = allTestContainerNames;
|
|
included = includedContainerNames;
|
|
allowInternet = allowInternetContainerNames;
|
|
gateway = gatewayContainerNames;
|
|
};
|
|
|
|
internetNode = _: {
|
|
networking.firewall.allowedTCPPorts = [ 80 ];
|
|
systemd.services.dummy-http = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
ExecStart = "${bin pkgs.python3 "python3"} -m http.server 80";
|
|
DynamicUser = true;
|
|
AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
|
|
};
|
|
};
|
|
};
|
|
|
|
serverModule = {
|
|
_module.args = host.specialArgs // {
|
|
hostInfo = testHostInfo;
|
|
};
|
|
networking.hostName = host.hostName;
|
|
|
|
# containers don't forward journal to console by default
|
|
containers = lib.genAttrs nspawnNames (_: {
|
|
timeoutStartSec = "10min";
|
|
config.services.journald.extraConfig = "ForwardToConsole=yes";
|
|
});
|
|
|
|
# a test VM has neither the memory to hand out nor the cores to divide.
|
|
# optimize.enable re-overrides qemu with nixosTestRunner; hydra caches
|
|
# that build of plain qemu but not of qemu_kvm, so naming it here is
|
|
# what keeps a nested guest from compiling qemu before it can boot
|
|
microvm.vms = lib.genAttrs vmNames (_: {
|
|
config.microvm = {
|
|
mem = lib.mkForce 512;
|
|
vcpu = lib.mkForce 1;
|
|
qemu.package = lib.mkForce pkgs.qemu_test;
|
|
};
|
|
});
|
|
|
|
systemd.services =
|
|
lib.genAttrs (map serviceUnitName (lib.attrValues testContainers)) (_: {
|
|
serviceConfig = {
|
|
Restart = lib.mkForce "no";
|
|
RestartForceExitStatus = lib.mkForce "";
|
|
};
|
|
})
|
|
// {
|
|
# erofs /nix/store is read-only, breaks nix-store --load-db
|
|
register-nix-paths.enable = false;
|
|
};
|
|
|
|
virtualisation.fileSystems."/persist" = {
|
|
device = "none";
|
|
fsType = "tmpfs";
|
|
# needed before persist.mount runs (activation depends on it)
|
|
neededForBoot = true;
|
|
};
|
|
services.dnsmasq.settings.server = lib.mkForce [ ];
|
|
|
|
virtualisation.memorySize = lib.mkDefault 4096;
|
|
virtualisation.cores = lib.mkDefault 4;
|
|
virtualisation.useNixStoreImage = true;
|
|
virtualisation.additionalPaths = testTools;
|
|
};
|
|
|
|
testScriptPreamble = ''
|
|
included = ${pyList includedContainerNames}
|
|
|
|
ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.addr) autoStartTestContainers)}
|
|
HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)}
|
|
IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)}
|
|
KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)}
|
|
# only nspawn guests can be entered with nsenter; a vm has no host-side
|
|
# netns, so every probe that runs *inside* a guest is limited to these
|
|
enterable = [c for c in included if KIND[c] == "nspawn"]
|
|
|
|
allow_internet = [c for c in enterable if c in ${pyList allowInternetContainerNames}]
|
|
has_gateway = [c for c in enterable if c in ${pyList gatewayContainerNames}]
|
|
not_allow_internet = [c for c in enterable if c not in allow_internet]
|
|
no_gateway = [c for c in enterable if c not in has_gateway]
|
|
|
|
# (src, to, tcp ports, udp ports); `to` may be "@host" or "@internet"
|
|
EDGES = [${pyEdges}]
|
|
|
|
NSENTER = "${bin pkgs.util-linux "nsenter"}"
|
|
CURL = "${bin pkgs.curl "curl"}"
|
|
IP = "${bin pkgs.iproute2 "ip"}"
|
|
PING = "${bin pkgs.iputils "ping"}"
|
|
DIG = "${bin pkgs.dnsutils "dig"}"
|
|
SSH = "${bin pkgs.openssh "ssh"}"
|
|
OPENSSL = "${bin pkgs.openssl "openssl"}"
|
|
TIMEOUT = "${bin pkgs.coreutils "timeout"}"
|
|
'';
|
|
};
|
|
}
|