initial commit

.clang-format

@@ -0,0 +1,13 @@
+BasedOnStyle: microsoft
+AccessModifierOffset: -4
+AlignConsecutiveAssignments: false
+AlwaysBreakTemplateDeclarations: Yes
+ColumnLimit: 150
+NamespaceIndentation: All
+PointerAlignment: Left
+TabWidth: 4
+AllowShortBlocksOnASingleLine: true
+AllowShortFunctionsOnASingleLine: All
+UseTab: Never
+AlignAfterOpenBracket: BlockIndent
+AllowAllArgumentsOnNextLine: true

.clang-tidy

@@ -0,0 +1,34 @@
+Checks: >
+    bugprone*,
+    cert*,
+    cppcore*,
+    misc*,
+    modernize*,
+    performance*,
+    portability*,
+    readability*,
+    -modernize-avoid-c-arrays,
+    -cppcoreguidelines-avoid-c-arrays,
+    -cppcoreguidelines-avoid-const-or-ref-data-members,
+    -cert-dcl58-cpp,
+    -misc-non-private-member-variables-in-classes,
+    -cppcoreguidelines-macro-usage,
+    -cppcoreguidelines-non-private-member-variables-in-classes,
+    -cppcoreguidelines-avoid-do-while
+    -cert-dcl21-cpp,
+    -cert-dcl51-cpp,
+    -cppcoreguidelines-avoid-magic-numbers,
+    -cert-dcl37-c,
+    -bugprone-reserved-identifier
+FormatStyle: file
+CheckOptions:
+  - key: readability-implicit-bool-conversion.AllowIntegerConditions
+    value: "1"
+  - key: readability-identifier-length.IgnoredVariableNames
+    value: ch
+  - key: readability-identifier-length.IgnoredParameterNames
+    value: ch
+  - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
+    value: true
+  - key: readability-magic-numbers.IgnorePowersOf2IntegerValues
+    value: true

.clangd

@@ -0,0 +1,4 @@
+Diagnostics:
+  UnusedIncludes: Strict
+  MissingIncludes: None
+

.github/workflows/diff-pr.yml

@@ -0,0 +1,45 @@
+name: diff-pr
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: diff-pr-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  dix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: cachix/install-nix-action@v31
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: worktree base for diff
+        run: git worktree add "$RUNNER_TEMP/base" "${{ github.event.pull_request.base.sha }}"
+
+      - name: build closures and render diff
+        run: |
+          nix run .#ci -- diff "$RUNNER_TEMP/base" . \
+            "### System closure diff vs \`${{ github.event.pull_request.base.ref }}\`" \
+            > "$RUNNER_TEMP/comment.md"
+          nix run .#ci -- inspect-diff "$RUNNER_TEMP/base" . \
+            "### Inspection diff vs \`${{ github.event.pull_request.base.ref }}\`" \
+            >> "$RUNNER_TEMP/comment.md"
+
+      - name: post/update PR comment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          API: ${{ github.api_url }}
+          REPO: ${{ github.repository }}
+          PR: ${{ github.event.pull_request.number }}
+        run: nix run .#ci -- post-comment "<!-- dix-diff -->" "$RUNNER_TEMP/comment.md"

.github/workflows/diff-tag.yml

@@ -0,0 +1,37 @@
+name: diff-tag
+
+on:
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  dix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: cachix/install-nix-action@v31
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: diff latest tag against HEAD
+        run: |
+          tag=$(git tag --sort=-creatordate | head -n1)
+          if [ -z "$tag" ]; then
+            echo "No tags yet, skipping diff." | tee -a "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+          git worktree add "$RUNNER_TEMP/old" "$tag"
+          head=$(git rev-parse --short HEAD)
+          {
+            nix run .#ci -- diff "$RUNNER_TEMP/old" . \
+              "# System closure diff: \`$tag\` -> \`$head\`"
+            nix run .#ci -- inspect-diff "$RUNNER_TEMP/old" . \
+              "# Inspection diff: \`$tag\` -> \`$head\`"
+          } | tee -a "$GITHUB_STEP_SUMMARY"

.github/workflows/flake-check.yml

@@ -0,0 +1,26 @@
+name: flake-check
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: flake-check-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  flake-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: cachix/install-nix-action@v31
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - run: nix flake check -L

.github/workflows/flake-update.yml

@@ -0,0 +1,33 @@
+name: flake-update
+
+on:
+  schedule:
+    - cron: "17 4 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: cachix/install-nix-action@v31
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - run: nix flake update
+
+      - name: open/update PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          API: ${{ github.api_url }}
+          REPO: ${{ github.repository }}
+        run: |
+          printf 'Automated `nix flake update`.\n' > "$RUNNER_TEMP/body.md"
+          nix run .#ci -- open-pr \
+            flake-update "$GITHUB_REF_NAME" "flake.lock: update inputs" "$RUNNER_TEMP/body.md" automated

.github/workflows/lint.yml

@@ -0,0 +1,41 @@
+name: lint
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: lint-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: cachix/install-nix-action@v31
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: pick reporter
+        run: |
+          if [ "${{ github.event_name }}" = pull_request ]; then
+            echo "REPORTER=github-pr-review" >> "$GITHUB_ENV"
+          else
+            echo "REPORTER=local" >> "$GITHUB_ENV"
+          fi
+
+      - name: run linters
+        run: nix run .#ci -- lint . "$RUNNER_TEMP/findings"
+
+      - name: report findings
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: nix run .#ci -- lint-report "$REPORTER" "$RUNNER_TEMP/findings"

.github/workflows/vulnix.yml

@@ -0,0 +1,41 @@
+name: vulnix
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: cachix/install-nix-action@v31
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+
+      - name: scan host closures
+        id: scan
+        run: echo "count=$(nix run .#ci -- vulnix . "$RUNNER_TEMP/issue.md")" >> "$GITHUB_OUTPUT"
+
+      - name: file/update issue
+        if: steps.scan.outputs.count != '0'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          API: ${{ github.api_url }}
+          REPO: ${{ github.repository }}
+        run: nix run .#ci -- upsert-issue vulnix "vulnix: known vulnerabilities in system closure" "$RUNNER_TEMP/issue.md"
+
+      - name: close resolved issue
+        if: steps.scan.outputs.count == '0'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          API: ${{ github.api_url }}
+          REPO: ${{ github.repository }}
+        run: nix run .#ci -- close-issue vulnix

.gitignore

@@ -0,0 +1,4 @@
+result
+result-*
+out.txt
+__pycache__/

ci.nix

@@ -0,0 +1,23 @@
+{ lib, pkgs }:
+# tools must track ci.py's subprocess calls; nix stays ambient (uses the
+# runner's own nix, not a pinned one).
+pkgs.writers.writePython3Bin "ci" {
+  flakeIgnore = [ "E501" ];
+  makeWrapperArgs = [
+    "--prefix"
+    "PATH"
+    ":"
+    (lib.makeBinPath (
+      with pkgs;
+      [
+        dix
+        vulnix
+        nixfmt
+        statix
+        deadnix
+        reviewdog
+        git
+      ]
+    ))
+  ];
+} (builtins.readFile ./ci.py)

ci.py

@@ -0,0 +1,451 @@
+import argparse
+import base64
+import datetime
+import difflib
+import json
+import os
+import re
+import subprocess
+import sys
+import urllib.request
+from collections import Counter
+from pathlib import Path
+
+
+def run(*args, **kw):
+    return subprocess.run(args, text=True, **kw)
+
+
+def nix(*args):
+    return run("nix", "--extra-experimental-features", "nix-command flakes", *args, check=True, capture_output=True).stdout
+
+
+def git(*args, check=True):
+    return run("git", *args, check=check, capture_output=True)
+
+
+def gitea(method, path, body=None):
+    url = f'{os.environ["API"]}/repos/{os.environ["REPO"]}{path}'
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, method=method)
+    req.add_header("Authorization", f'token {os.environ["GH_TOKEN"]}')
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    with urllib.request.urlopen(req) as resp:
+        raw = resp.read()
+    return json.loads(raw) if raw else None
+
+
+def find_open_issue(label):
+    issues = gitea("GET", f"/issues?state=open&labels={label}")
+    return issues[0] if issues else None
+
+
+def build_hosts(ref):
+    hosts = json.loads(nix("eval", "--json", f"{ref}#nixosConfigurations", "--apply", "builtins.attrNames"))
+    return {
+        host: nix("build", f"{ref}#nixosConfigurations.{host}.config.system.build.toplevel",
+                  "--no-link", "--print-out-paths").strip()
+        for host in hosts
+    }
+
+
+def diff(args):
+    old, new = build_hosts(args.old), build_hosts(args.new)
+
+    def section(host):
+        if host not in old:
+            return f"## {host}\n\n_new host_"
+        if host not in new:
+            return f"## {host}\n\n_host removed_"
+        rendered = run("dix", "--color", "never", old[host], new[host], capture_output=True).stdout
+        return f"## {host}\n\n```\n{rendered}```"
+
+    parts = ([args.title] if args.title else []) + [section(h) for h in sorted(set(old) | set(new))]
+    print("\n\n".join(parts))
+
+
+# meta.toplevel is the closure path dix already covers; nftables.ruleset/tables
+# are duplicated by nftables.rendered (the assembled ruleset we keep and diff).
+# Pruned so inspect-diff reports only source config topology, once.
+INSPECT_PRUNE = [
+    ("meta", "toplevel"),
+    ("nftables", "ruleset"),
+    ("nftables", "tables"),
+]
+
+
+def compact(value):
+    return json.dumps(value, sort_keys=True)
+
+
+def scalar(value):
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+def fmt(value, indent=0):
+    pad = "  " * indent
+    lines = []
+    if isinstance(value, dict):
+        for key in sorted(value):
+            child = value[key]
+            if isinstance(child, (dict, list)) and child:
+                lines.append(f"{pad}{key}:")
+                lines += fmt(child, indent + 1)
+            else:
+                lines.append(f"{pad}{key}: {scalar(child)}")
+    elif isinstance(value, list):
+        for item in value:
+            if isinstance(item, (dict, list)) and item:
+                body = fmt(item)
+                lines.append(f"{pad}- {body[0]}")
+                lines += [f"{pad}  {line}" for line in body[1:]]
+            else:
+                lines.append(f"{pad}- {scalar(item)}")
+    else:
+        lines.append(f"{pad}{scalar(value)}")
+    return lines
+
+
+STORE_HASH = re.compile(r"/nix/store/[a-z0-9]{32}-")
+
+
+def strip_store(value):
+    # closure hashes churn on any rebuild and are dix's job; drop them so exec
+    # commands / paths read as "<name>/bin/x" and diff only on real changes
+    if isinstance(value, dict):
+        return {k: strip_store(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [strip_store(v) for v in value]
+    if isinstance(value, str):
+        return STORE_HASH.sub("", value)
+    return value
+
+
+def inspect_hosts(ref):
+    try:
+        names = json.loads(nix("eval", "--json", f"{ref}#inspect", "--apply", "builtins.attrNames"))
+    except subprocess.CalledProcessError as e:
+        if "does not provide attribute" in e.stderr:
+            return None
+        raise RuntimeError(f"inspect eval failed for {ref}:\n{e.stderr}") from e
+    return {
+        host: nix("build", f"{ref}#inspect.{host}",
+                  "--no-link", "--print-out-paths").strip()
+        for host in names
+    }
+
+
+def read_inspect(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+def inspect_data(ref):
+    paths = inspect_hosts(ref)
+    if paths is None:
+        return None
+    reports = {host: read_inspect(path) for host, path in paths.items()}
+    for report in reports.values():
+        for parent, key in INSPECT_PRUNE:
+            report.get(parent, {}).pop(key, None)
+    return {host: strip_store(report) for host, report in reports.items()}
+
+
+def tree_changes(old, new, path=()):
+    if old == new:
+        return
+    if isinstance(old, dict) and isinstance(new, dict):
+        for key in sorted(old.keys() | new.keys()):
+            if key not in old:
+                yield path + (key,), "add", new[key]
+            elif key not in new:
+                yield path + (key,), "del", old[key]
+            else:
+                yield from tree_changes(old[key], new[key], path + (key,))
+    elif isinstance(old, list) and isinstance(new, list):
+        # order-independent: a reordered list is not a change
+        okeys, nkeys = list(map(compact, old)), list(map(compact, new))
+        values = dict(zip(okeys, old)) | dict(zip(nkeys, new))
+        for key in (Counter(okeys) - Counter(nkeys)).elements():
+            yield path, "delitem", values[key]
+        for key in (Counter(nkeys) - Counter(okeys)).elements():
+            yield path, "additem", values[key]
+    else:
+        yield path, "change", (old, new)
+
+
+def render_change(path, kind, payload):
+    label = ".".join(map(str, path[1:])) or path[0]
+    if kind in ("add", "del", "additem", "delitem"):
+        sign = "+" if kind in ("add", "additem") else "-"
+        value = [payload] if kind in ("additem", "delitem") else payload
+        if isinstance(value, (dict, list)) and value:
+            return [f"{sign} {label}:"] + [f"{sign} {line}" for line in fmt(value, 1)]
+        return [f"{sign} {label}: {scalar(value)}"]
+    old, new = payload
+    if isinstance(old, str) and isinstance(new, str) and "\n" in old + new:
+        body = difflib.unified_diff(old.splitlines(), new.splitlines(), lineterm="")
+        return [f"  {label}:"] + [ln for ln in body if not ln.startswith(("---", "+++"))]
+    if isinstance(old, (dict, list)) or isinstance(new, (dict, list)):
+        return render_change(path, "del", old) + render_change(path, "add", new)
+    return [f"- {label}: {scalar(old)}", f"+ {label}: {scalar(new)}"]
+
+
+def host_diff(old, new):
+    lines = []
+    attr = None
+    for path, kind, payload in tree_changes(old, new):
+        if path[0] != attr:
+            attr = path[0]
+            lines.append(f"@@ {attr} @@")
+        lines += render_change(path, kind, payload)
+    return "```diff\n" + "\n".join(lines) + "\n```" if lines else None
+
+
+def host_added(report):
+    return "```diff\n" + "\n".join(f"+ {line}" for line in fmt(report)) + "\n```"
+
+
+def inspect_diff(args):
+    old, new = inspect_data(args.old), inspect_data(args.new)
+    if old is None or new is None:
+        return
+
+    def block(host):
+        if host not in old:
+            return f"## {host} (new)\n\n{host_added(new[host])}"
+        if host not in new:
+            return f"## {host}\n\n_host removed_"
+        body = host_diff(old[host], new[host])
+        return f"## {host}\n\n{body}" if body else None
+
+    blocks = [b for b in map(block, sorted(set(old) | set(new))) if b]
+    if blocks:
+        print("\n\n".join(([args.title] if args.title else []) + blocks))
+
+
+def format_vulnix(vulns):
+    rows = []
+    for v in vulns:
+        for cve in v["affected_by"]:
+            score = v.get("cvssv3_basescore", {}).get(cve)
+            desc = v.get("description", {}).get(cve, "")
+            if len(desc) > 120:
+                desc = desc[:117] + "..."
+            rows.append((v["pname"], v["version"], cve, score, desc))
+    rows.sort(key=lambda r: -(r[3] or 0))
+
+    lines = [
+        "| Package | Version | CVE | CVSS | Description |",
+        "|---------|---------|-----|------|-------------|",
+    ]
+    for pname, version, cve, score, desc in rows:
+        score_str = f"{score:.1f}" if score else "?"
+        cve_link = f"[{cve}](https://nvd.nist.gov/vuln/detail/{cve})"
+        lines.append(f"| {pname} | {version} | {cve_link} | {score_str} | {desc} |")
+    return "\n".join(lines)
+
+
+def vulnix(args):
+    paths = list(build_hosts(args.ref).values())
+    result = run("vulnix", "-j", *paths, capture_output=True)
+    if result.returncode not in (0, 2):
+        print(result.stderr, file=sys.stderr)
+        sys.exit(result.returncode)
+    vulns = json.loads(result.stdout or "[]")
+    if vulns:
+        cve_count = sum(len(v["affected_by"]) for v in vulns)
+        date = datetime.date.today().isoformat()
+        with open(args.body_out, "w") as f:
+            f.write(f"vulnix scan ({date}): **{cve_count}** CVEs across **{len(vulns)}** packages.\n\n")
+            f.write(format_vulnix(vulns))
+            f.write("\n")
+    print(len(vulns))
+
+
+def find_cycles(edges):
+    adj = {}
+    for e in edges:
+        if e["kind"] == "after":
+            adj.setdefault(e["from"], []).append(e["to"])
+
+    visited, on_path = set(), set()
+    path = []
+    cycles = []
+
+    def dfs(node):
+        visited.add(node)
+        path.append(node)
+        on_path.add(node)
+        for nb in adj.get(node, []):
+            if nb in on_path:
+                i = path.index(nb)
+                cycles.append(path[i:] + [nb])
+            elif nb not in visited:
+                dfs(nb)
+        path.pop()
+        on_path.discard(node)
+
+    for node in adj:
+        if node not in visited:
+            dfs(node)
+    return cycles
+
+
+def lint(args):
+    out = os.path.abspath(args.out)
+    target = os.path.abspath(args.target)
+    os.makedirs(out, exist_ok=True)
+    os.chdir(target)
+
+    files = [f for f in git("ls-files", "-z", "*.nix").stdout.split("\0") if f]
+    if files:
+        run("nixfmt", *files)
+    with open(f"{out}/nixfmt.diff", "w") as f:
+        f.write(git("diff").stdout)
+    git("checkout", "--", ".")
+
+    with open(f"{out}/statix.errfmt", "w") as f:
+        f.write(run("statix", "check", ".", "-o", "errfmt", capture_output=True).stdout)
+
+    dead = run("deadnix", "-o", "json", ".", capture_output=True).stdout
+    with open(f"{out}/deadnix.rdjsonl", "w") as f:
+        for line in filter(str.strip, dead.splitlines()):
+            report = json.loads(line)
+            for r in report["results"]:
+                loc = {"path": report["file"]}
+                if "line" in r:
+                    loc["range"] = {
+                        "start": {"line": r["line"], "column": r["column"]},
+                        "end": {"line": r["line"], "column": r["endColumn"]},
+                    }
+                rec = {
+                    "message": r["message"],
+                    "location": loc,
+                    "severity": "ERROR",
+                }
+                json.dump(rec, f)
+                f.write("\n")
+
+    paths = inspect_hosts(target)
+    with open(f"{out}/cycles.rdjsonl", "w") as f:
+        for host, path in (paths or {}).items():
+            report = read_inspect(path)
+            for cycle in find_cycles(report["dependencies"]["edges"]):
+                chain = " → ".join(u.split(":", 1)[1] for u in cycle)
+                rec = {
+                    "message": f"[{host}] systemd ordering cycle: {chain}",
+                    "location": {"path": "flake.nix"},
+                    "severity": "ERROR",
+                }
+                json.dump(rec, f)
+                f.write("\n")
+
+
+LINTERS = [
+    ("nixfmt.diff", ["-f=diff", "-f.diff.strip=1", "-name=nixfmt", "-level=error"]),
+    ("statix.errfmt", ["-efm=%f>%l:%c:%*[A-Z]:%*[0-9]:%m", "-name=statix", "-level=error"]),
+    ("deadnix.rdjsonl", ["-f=rdjsonl", "-name=deadnix"]),
+    ("cycles.rdjsonl", ["-f=rdjsonl", "-name=systemd-cycles", "-filter-mode=nofilter"]),
+]
+
+
+def lint_report(args):
+    def reviewdog(path, flags):
+        with open(path) as f:
+            return run("reviewdog", *flags, f"-reporter={args.reporter}", "-fail-level=error", stdin=f).returncode
+
+    codes = [reviewdog(f"{args.dir}/{name}", flags)
+             for name, flags in LINTERS
+             if os.path.getsize(f"{args.dir}/{name}") > 0]
+    sys.exit(1 if any(codes) else 0)
+
+
+def post_comment(args):
+    body = args.marker + "\n" + Path(args.body_file).read_text()
+    pr = os.environ["PR"]
+    comments = gitea("GET", f"/issues/{pr}/comments")
+    existing = next((c for c in comments if c["body"].startswith(args.marker)), None)
+    if existing:
+        gitea("PATCH", f"/issues/comments/{existing['id']}", {"body": body})
+    else:
+        gitea("POST", f"/issues/{pr}/comments", {"body": body})
+
+
+def upsert_issue(args):
+    body = Path(args.body_file).read_text()
+    issue = find_open_issue(args.label)
+    if issue:
+        gitea("PATCH", f"/issues/{issue['number']}", {"body": body})
+    else:
+        gitea("POST", "/issues", {"title": args.title, "body": body, "labels": [label_id(args.label)]})
+
+
+def close_issue(args):
+    issue = find_open_issue(args.label)
+    if issue:
+        gitea("PATCH", f"/issues/{issue['number']}", {"state": "closed"})
+
+
+def label_id(name):
+    match = next((x for x in gitea("GET", "/labels") if x["name"] == name), None)
+    return match["id"] if match else gitea("POST", "/labels", {"name": name, "color": "#ededed"})["id"]
+
+
+def open_pr(args):
+    if git("diff", "--quiet", check=False).returncode == 0:
+        print("no changes; nothing to do")
+        return
+    git("switch", "-C", args.branch)
+    git("add", "-A")
+    git("-c", "user.name=gitea-actions[bot]", "-c", "user.email=gitea-actions@noreply.local",
+        "commit", "-m", args.title)
+
+    host = os.environ["API"].removesuffix("/api/v1")
+    auth = base64.b64encode(f'token:{os.environ["GH_TOKEN"]}'.encode()).decode()
+    git("-c", f"http.extraHeader=Authorization: Basic {auth}",
+        "push", "-f", f'{host}/{os.environ["REPO"]}.git', f"HEAD:{args.branch}")
+
+    owner = os.environ["REPO"].split("/")[0]
+    if gitea("GET", f"/pulls?state=open&head={owner}:{args.branch}"):
+        return
+    labels = [label_id(args.label)] if args.label else []
+    gitea("POST", "/pulls",
+          {"title": args.title, "head": args.branch, "base": args.base,
+           "body": Path(args.body_file).read_text(), "labels": labels})
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="ci")
+    sub = parser.add_subparsers(required=True)
+
+    def command(name, fn, *positional):
+        p = sub.add_parser(name)
+        for arg in positional:
+            if arg.endswith("?"):
+                p.add_argument(arg[:-1], nargs="?")
+            else:
+                p.add_argument(arg)
+        p.set_defaults(fn=fn)
+
+    command("diff", diff, "old", "new", "title?")
+    command("inspect-diff", inspect_diff, "old", "new", "title?")
+    command("vulnix", vulnix, "ref", "body_out")
+    command("lint", lint, "target", "out")
+    command("lint-report", lint_report, "reporter", "dir")
+    command("post-comment", post_comment, "marker", "body_file")
+    command("upsert-issue", upsert_issue, "label", "title", "body_file")
+    command("close-issue", close_issue, "label")
+    command("open-pr", open_pr, "branch", "base", "title", "body_file", "label?")
+
+    args = parser.parse_args()
+    args.fn(args)
+
+
+if __name__ == "__main__":
+    main()

configuration.nix

@@ -0,0 +1,141 @@
+{
+  config,
+  pkgs,
+  lib,
+  hostInfo,
+  customPkgs,
+  ...
+}:
+let
+  inherit (hostInfo) paths;
+in
+{
+  assertions =
+    let
+      isMounted =
+        path:
+        lib.any (
+          # rootfs may be subject to erasure so it's not persistent
+          fs: fs.mountPoint != "/" && (fs.mountPoint == path || lib.hasPrefix "${fs.mountPoint}/" path)
+        ) (lib.attrValues config.fileSystems);
+
+      isMountedEarly = path: (config.fileSystems.${path} or { neededForBoot = false; }).neededForBoot;
+
+      mountAssertion = path: {
+        assertion = isMounted path;
+        message = "configuration.nix needs '${path}' to be a mounted path, but no fileSystems entry covers it.";
+      };
+
+      earlyMountAssertion = path: {
+        assertion = isMountedEarly path;
+        message = "configuration.nix needs '${path}' to be mounted early (neededForBoot = true), but its fileSystems entry doesn't set that.";
+      };
+
+      strictPaths = lib.optionals (!hostInfo.isVm) [
+        "/nix"
+        "/var/log"
+      ];
+    in
+    map mountAssertion (
+      [
+        paths.server
+        paths.secrets
+      ]
+      ++ strictPaths
+    )
+    ++ lib.optionals (!hostInfo.isVm) [ (earlyMountAssertion "/var/log") ];
+
+  time.timeZone = "UTC";
+  system.stateVersion = "26.05";
+
+  nix = {
+    settings = {
+      experimental-features = [
+        "nix-command"
+        "flakes"
+      ];
+    };
+    nrBuildUsers = 80;
+  };
+
+  powerManagement.cpuFreqGovernor = "performance";
+
+  boot.kernelParams = [ "transparent_hugepage=madvise" ];
+
+  services = {
+    fstrim = {
+      enable = true;
+      interval = "weekly";
+    };
+
+    openssh = {
+      enable = true;
+      ports = [ 5952 ];
+      hostKeys =
+        lib.map
+          (type: {
+            inherit type;
+            path = "${paths.secrets}/ssh_host_${type}_key";
+          })
+          [
+            "ed25519"
+            "ecdsa"
+            "rsa"
+          ];
+      settings.PasswordAuthentication = false;
+    };
+  };
+
+  users.mutableUsers = false;
+  users.users.server = {
+    uid = 1000;
+    isNormalUser = true;
+    extraGroups = [
+      "wheel"
+      "systemd-journal"
+    ];
+    hashedPassword = "!";
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPaGpTX3EzfezVc2nl3MgsKVamYYP2h2VJaRny8pZkvi flowey@arch"
+    ];
+  };
+
+  security.sudo.wheelNeedsPassword = false;
+
+  security.polkit.extraConfig = ''
+    polkit.addRule(function(action, subject) {
+      if (action.id.startsWith("org.freedesktop.machine1.") &&
+          subject.user === "server") {
+        return polkit.Result.YES;
+      }
+    });
+  '';
+
+  systemd.tmpfiles.rules = [
+    "L /home/server/config     - - - - /persist/@configuration"
+    "L /home/server/containers - - - - /persist/@containers"
+    "L /home/server/secrets    - - - - /persist/@secrets"
+  ];
+
+  environment.etc."machine-id".text = "${
+    builtins.substring 0 32 (builtins.hashString "sha256" "machine-id-for-${hostInfo.hostname}")
+  }\n";
+  environment.systemPackages = [
+    customPkgs.ct
+    customPkgs.server-rebuild
+    customPkgs.server-status
+    customPkgs.server-kexec
+    customPkgs.kitty-terminfo
+    pkgs.git
+    pkgs.wezterm.terminfo
+  ];
+
+  networking.firewall.allowedTCPPorts = [ 5952 ];
+
+  system.activationScripts.server-keypair = ''
+    if [[ ! -f "${paths.server}/keypair/id_ed25519" ]]; then
+      mkdir -p "${paths.server}/keypair"
+      ${lib.getExe' pkgs.openssh "ssh-keygen"} -t ed25519 -f "${paths.server}/keypair/id_ed25519" -N ""
+    fi
+  '';
+}

containers/common/overseer.nix

@@ -0,0 +1,386 @@
+{
+  name,
+  hostInfo,
+  dsl,
+  pkgs,
+  lib,
+  containers,
+  ...
+}:
+let
+  inherit (dsl) mkContainer;
+  inherit (hostInfo.paths) data;
+
+  nodePort = 9100;
+  cadvPort = 9200;
+
+  containerNetworkRulesFile =
+    let
+      mkRules =
+        metric: record:
+        lib.mapAttrsToList (cname: c: {
+          inherit record;
+          expr = "rate(${metric}{device=\"${c.iface}\"}[5m])";
+          labels = {
+            container = cname;
+          };
+        }) containers;
+    in
+    pkgs.writeText "container-network.yml" (
+      builtins.toJSON {
+        groups = [
+          {
+            name = "container-network";
+            rules =
+              mkRules "node_network_receive_bytes_total" "container_network_receive_rate"
+              ++ mkRules "node_network_transmit_bytes_total" "container_network_transmit_rate";
+          }
+        ];
+      }
+    );
+
+  containerDefinedRulesFile = pkgs.writeText "container-defined.yml" (
+    builtins.toJSON {
+      groups = [
+        {
+          name = "container-defined";
+          rules = lib.mapAttrsToList (cname: _: {
+            record = "container_defined";
+            expr = "1";
+            labels = {
+              container = cname;
+            };
+          }) containers;
+        }
+      ];
+    }
+  );
+
+  alloyConfig = pkgs.writeText "config.alloy" ''
+    loki.source.journal "journal" {
+      max_age    = "12h"
+      path       = "/var/log/journal"
+      labels     = { job = "systemd-journal" }
+      relabel_rules = loki.relabel.journal.rules
+      forward_to    = [loki.write.local.receiver]
+    }
+
+    loki.relabel "journal" {
+      forward_to = []
+
+      rule {
+        source_labels = ["__journal__systemd_unit"]
+        target_label  = "unit"
+      }
+      rule {
+        source_labels = ["__journal__hostname"]
+        target_label  = "host"
+      }
+      rule {
+        source_labels = ["__journal_syslog_identifier"]
+        target_label  = "syslog_id"
+      }
+      rule {
+        source_labels = ["__journal_priority_keyword"]
+        target_label  = "level"
+      }
+      rule {
+        source_labels = ["__journal__systemd_machine_name"]
+        target_label  = "container"
+      }
+    }
+
+    loki.write "local" {
+      endpoint {
+        url = "http://127.0.0.1:3100/loki/api/v1/push"
+      }
+    }
+  '';
+
+  dashboards = lib.filterAttrs (name: _: lib.hasSuffix "-dashboard" name) (
+    dsl.listMap ./overseer (_: path: import path { inherit pkgs lib; })
+  );
+
+  dashboardDir = pkgs.linkFarm "grafana-dashboards" (
+    lib.mapAttrsToList (name: dashboard: {
+      name = "${lib.removeSuffix "-dashboard" name}.json";
+      path = dashboard;
+    }) dashboards
+  );
+
+  landingRoot = pkgs.writeTextDir "index.html" ''
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+      <meta charset="utf-8">
+      <title>Overseer</title>
+      <style>
+        body { font-family: sans-serif; max-width: 480px; margin: 4rem auto; }
+        ul   { padding: 0; list-style: none; }
+        li   { margin: .5rem 0; }
+      </style>
+    </head>
+    <body>
+      <h1>Overseer</h1>
+      <ul>
+        <li><a data-host="stats.localhost">stats.localhost</a> — Grafana</li>
+        <li><a data-host="prometheus.localhost">prometheus.localhost</a> — Prometheus</li>
+      </ul>
+      <script>
+        const p = location.port && location.port !== "80" ? ":" + location.port : "";
+        document.querySelectorAll("a[data-host]").forEach(a => {
+          const h = a.dataset.host;
+          a.href = "http://" + h + p;
+          a.textContent = h + p;
+        });
+      </script>
+    </body>
+    </html>
+  '';
+in
+let
+  c =
+    mkContainer {
+      inherit name;
+      net.access = [
+        {
+          to = "@host";
+          tcp = [
+            nodePort
+            cadvPort
+          ];
+        }
+      ];
+      svc.autoStart = true;
+      resources.fs = {
+        "/var/lib/loki" = {
+          hostPath = "${data}/overseer/loki/data";
+          isReadOnly = false;
+        };
+        "/var/lib/prometheus2" = {
+          hostPath = "${data}/overseer/prometheus/data";
+          isReadOnly = false;
+        };
+        "/var/lib/grafana" = {
+          hostPath = "${data}/overseer/grafana";
+          isReadOnly = false;
+        };
+        "/var/log/journal" = {
+          hostPath = "/var/log/journal";
+          isReadOnly = true;
+        };
+      };
+      containerConfig = _: {
+        services.loki = {
+          enable = true;
+          configuration = {
+            auth_enabled = false;
+            server = {
+              http_listen_port = 3100;
+              grpc_listen_port = 9096;
+            };
+            common = {
+              instance_addr = "127.0.0.1";
+              path_prefix = "/var/lib/loki";
+              storage.filesystem = {
+                chunks_directory = "/var/lib/loki/chunks";
+                rules_directory = "/var/lib/loki/rules";
+              };
+              replication_factor = 1;
+              ring.kvstore.store = "inmemory";
+            };
+            schema_config.configs = [
+              {
+                from = "2020-10-24";
+                store = "tsdb";
+                object_store = "filesystem";
+                schema = "v13";
+                index = {
+                  prefix = "index_";
+                  period = "24h";
+                };
+              }
+            ];
+          };
+        };
+
+        services.alloy = {
+          enable = true;
+          configPath = alloyConfig;
+        };
+
+        services.prometheus = {
+          enable = true;
+          port = 9090;
+          globalConfig.scrape_interval = "15s";
+          scrapeConfigs = [
+            {
+              job_name = "node";
+              static_configs = [ { targets = [ "${hostAddr}:${toString nodePort}" ]; } ];
+            }
+            {
+              job_name = "cadvisor";
+              static_configs = [ { targets = [ "${hostAddr}:${toString cadvPort}" ]; } ];
+            }
+            {
+              job_name = "prometheus";
+              static_configs = [ { targets = [ "localhost:9090" ]; } ];
+            }
+          ];
+          ruleFiles = [
+            containerNetworkRulesFile
+            containerDefinedRulesFile
+            (dsl.mkContainerLimitsRuleFile pkgs containers)
+          ];
+          rules = [
+            ''
+              groups:
+                - name: containers
+                  rules:
+                    - alert: ContainerFailed
+                      expr: node_systemd_unit_state{name=~"container@.*",state="failed"} == 1
+                      for: 1m
+                      labels:
+                        severity: critical
+                      annotations:
+                        summary: "{{ $labels.name }} entered failed state"
+                    - alert: ContainerOOMKill
+                      expr: increase(container_oom_events_total{id=~"/machine.slice/container@.+\\.service.*"}[5m]) > 0
+                      labels:
+                        severity: warning
+                      annotations:
+                        summary: "OOM kill in {{ $labels.id }}"
+            ''
+          ];
+        };
+
+        services.grafana = {
+          enable = true;
+          dataDir = "/var/lib/grafana";
+          settings = {
+            server.http_port = 3000;
+            server.http_addr = "127.0.0.1";
+            # WORKAROUND: Grafana 10+ requires an explicit secret_key; omitting it
+            # causes auth session tokens to be regenerated on every restart
+            security.secret_key = "SW2YcwTIb9zpOOhoPsMm";
+            "auth".disable_login_form = true;
+            "auth.anonymous" = {
+              enabled = true;
+              org_role = "Admin";
+            };
+          };
+          provision = {
+            enable = true;
+            datasources.settings = {
+              apiVersion = 1;
+              datasources = [
+                {
+                  name = "Prometheus";
+                  type = "prometheus";
+                  uid = "prometheus";
+                  access = "proxy";
+                  url = "http://localhost:9090";
+                  isDefault = true;
+                  jsonData.timeInterval = "15s";
+                }
+                {
+                  name = "Loki";
+                  type = "loki";
+                  uid = "loki";
+                  access = "proxy";
+                  url = "http://127.0.0.1:3100";
+                }
+              ];
+            };
+            dashboards.settings = {
+              apiVersion = 1;
+              providers = [
+                {
+                  name = "default";
+                  type = "file";
+                  disableDeletion = true;
+                  options.path = toString dashboardDir;
+                }
+              ];
+            };
+          };
+        };
+
+        services.nginx = {
+          enable = true;
+          virtualHosts."_" = {
+            listen = [
+              {
+                addr = "0.0.0.0";
+                port = 80;
+                ssl = false;
+                extraParameters = [ "default_server" ];
+              }
+            ];
+            root = "${landingRoot}";
+          };
+          virtualHosts."stats.localhost" = {
+            listen = [
+              {
+                addr = "0.0.0.0";
+                port = 80;
+                ssl = false;
+              }
+            ];
+            locations."/" = {
+              proxyPass = "http://127.0.0.1:3000";
+              proxyWebsockets = true;
+              extraConfig = "proxy_set_header Host $host;";
+            };
+          };
+          virtualHosts."prometheus.localhost" = {
+            listen = [
+              {
+                addr = "0.0.0.0";
+                port = 80;
+                ssl = false;
+              }
+            ];
+            locations."/" = {
+              proxyPass = "http://127.0.0.1:9090";
+            };
+          };
+        };
+
+        systemd.services = lib.genAttrs [ "loki" "alloy" "prometheus" "grafana" "nginx" ] (_: {
+          serviceConfig.FailureAction = "exit-force";
+        });
+      };
+      net.expose.tcp = [ 80 ];
+      resources.mem = "2G";
+    }
+    // {
+      hostConfig = {
+        services.prometheus.exporters.node = {
+          enable = true;
+          listenAddress = "0.0.0.0";
+          port = nodePort;
+          enabledCollectors = [
+            "systemd"
+            "processes"
+          ];
+          disabledCollectors = [
+            "netstat"
+            "softnet"
+          ];
+          # radeon sometimes breaks on kexec
+          extraFlags = [
+            "--collector.systemd.enable-task-metrics"
+            "--collector.hwmon.chip-exclude=radeon"
+          ];
+        };
+        services.cadvisor = {
+          enable = true;
+          listenAddress = "0.0.0.0";
+          port = cadvPort;
+        };
+      };
+    };
+  hostAddr = c.hostAddr;
+in
+c

containers/common/overseer/access-logs-dashboard.nix

@@ -0,0 +1,61 @@
+{ pkgs, lib }:
+let
+  inherit (import ./helpers.nix { inherit pkgs lib; }) mkRow loki mkDashboard;
+
+  mkLogs =
+    {
+      title,
+      expr,
+      w ? 24,
+      h ? 10,
+    }:
+    {
+      inherit title;
+      type = "logs";
+      datasource = loki;
+      gridPos = { inherit w h; };
+      options = {
+        showTime = true;
+        showLabels = true;
+        sortOrder = "Descending";
+        wrapLogMessage = true;
+        enableLogDetails = true;
+      };
+      targets = [
+        {
+          datasource = loki;
+          inherit expr;
+        }
+      ];
+    };
+in
+mkDashboard {
+  name = "access-logs.json";
+  title = "Access Logs";
+  uid = "access-logs";
+  from = "now-6h";
+  panels = [
+    (mkRow "HTTP")
+    (mkLogs {
+      title = "nginx (access + error)";
+      expr = ''{syslog_id="nginx"}'';
+    })
+    (mkRow "SSH")
+    (mkLogs {
+      title = "sshpiper (all connections)";
+      expr = ''{container="sshpiper"}'';
+    })
+    (mkLogs {
+      title = "Failed / invalid auth attempts (all backends, incl. host)";
+      expr = ''{syslog_id=~"sshd.*"} |~ "Failed|Invalid|error:"'';
+    })
+    (mkLogs {
+      title = "sshd auth events (all backends, incl. host)";
+      expr = ''{syslog_id=~"sshd.*"}'';
+    })
+    (mkLogs {
+      title = "Host sshd (port 5952)";
+      expr = ''{syslog_id=~"sshd.*", container=""}'';
+    })
+  ];
+}

containers/common/overseer/containers-dashboard.nix

@@ -0,0 +1,251 @@
+{ pkgs, lib }:
+let
+  inherit (import ./helpers.nix { inherit pkgs lib; }) prom mkDashboard;
+in
+mkDashboard {
+  name = "containers.json";
+  title = "Containers";
+  uid = "containers";
+  from = "now-3h";
+  panels = [
+    {
+      title = "Container States";
+      type = "piechart";
+      gridPos = {
+        h = 8;
+        w = 8;
+      };
+      datasource = prom;
+      options = {
+        pieType = "donut";
+        legend = {
+          displayMode = "table";
+          placement = "right";
+          values = [
+            "value"
+            "percent"
+          ];
+        };
+        tooltip.mode = "single";
+      };
+      fieldConfig.overrides = [
+        {
+          matcher = {
+            id = "byName";
+            options = "active";
+          };
+          properties = [
+            {
+              id = "color";
+              value = {
+                mode = "fixed";
+                fixedColor = "green";
+              };
+            }
+          ];
+        }
+        {
+          matcher = {
+            id = "byName";
+            options = "failed";
+          };
+          properties = [
+            {
+              id = "color";
+              value = {
+                mode = "fixed";
+                fixedColor = "red";
+              };
+            }
+          ];
+        }
+        {
+          matcher = {
+            id = "byName";
+            options = "inactive";
+          };
+          properties = [
+            {
+              id = "color";
+              value = {
+                mode = "fixed";
+                fixedColor = "blue";
+              };
+            }
+          ];
+        }
+        {
+          matcher = {
+            id = "byName";
+            options = "activating";
+          };
+          properties = [
+            {
+              id = "color";
+              value = {
+                mode = "fixed";
+                fixedColor = "yellow";
+              };
+            }
+          ];
+        }
+        {
+          matcher = {
+            id = "byName";
+            options = "unknown";
+          };
+          properties = [
+            {
+              id = "color";
+              value = {
+                mode = "fixed";
+                fixedColor = "gray";
+              };
+            }
+          ];
+        }
+      ];
+      targets = [
+        {
+          datasource = prom;
+          expr = ''count by (state) ((label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service")) or on(container) label_replace(container_defined, "state", "unknown", "container", ".+"))'';
+          instant = true;
+          legendFormat = "{{state}}";
+        }
+      ];
+    }
+    {
+      title = "Container States";
+      type = "table";
+      gridPos = {
+        h = 16;
+        w = 24;
+      };
+      datasource = prom;
+      options.footer.show = false;
+      fieldConfig = {
+        defaults = { };
+        overrides = [
+          {
+            matcher = {
+              id = "byName";
+              options = "State";
+            };
+            properties = [
+              {
+                id = "custom.displayMode";
+                value = "color-text";
+              }
+              {
+                id = "mappings";
+                value = [
+                  {
+                    type = "value";
+                    options.active = {
+                      text = "active";
+                      color = "green";
+                      index = 0;
+                    };
+                  }
+                  {
+                    type = "value";
+                    options.failed = {
+                      text = "failed";
+                      color = "red";
+                      index = 1;
+                    };
+                  }
+                  {
+                    type = "value";
+                    options.activating = {
+                      text = "activating";
+                      color = "yellow";
+                      index = 2;
+                    };
+                  }
+                  {
+                    type = "value";
+                    options.inactive = {
+                      text = "inactive";
+                      color = "blue";
+                      index = 3;
+                    };
+                  }
+                  {
+                    type = "value";
+                    options.unknown = {
+                      text = "unknown";
+                      color = "gray";
+                      index = 4;
+                    };
+                  }
+                ];
+              }
+            ];
+          }
+        ];
+      };
+      targets = [
+        {
+          datasource = prom;
+          expr = ''
+            (label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service"))
+            or on(container)
+            label_replace(container_defined, "state", "unknown", "container", ".+")
+          '';
+          instant = true;
+          format = "table";
+          legendFormat = "__auto";
+        }
+      ];
+      transformations = [
+        {
+          id = "organize";
+          options = {
+            excludeByName = {
+              Time = true;
+              Value = true;
+              "__name__" = true;
+              instance = true;
+              job = true;
+              name = true;
+            };
+            renameByName = {
+              container = "Container";
+              state = "State";
+            };
+          };
+        }
+      ];
+    }
+    {
+      title = "Container State (1 = running, 0 = stopped/failed)";
+      type = "timeseries";
+      gridPos = {
+        h = 8;
+        w = 24;
+      };
+      datasource = prom;
+      fieldConfig.defaults = {
+        custom.lineWidth = 2;
+        color.mode = "palette-classic";
+        min = 0;
+        max = 1;
+      };
+      options = {
+        tooltip.mode = "multi";
+        legend = {
+          displayMode = "table";
+          placement = "bottom";
+        };
+      };
+      targets = [
+        {
+          datasource = prom;
+          expr = ''label_replace(node_systemd_unit_state{name=~"container@.*",state="active"}, "container", "$1", "name", "container@(.+).service")'';
+          legendFormat = "{{container}}";
+        }
+      ];
+    }
+  ];
+}

containers/common/overseer/helpers.nix

@@ -0,0 +1,228 @@
+{ pkgs, lib }:
+let
+  prom = {
+    type = "prometheus";
+    uid = "prometheus";
+  };
+  loki = {
+    type = "loki";
+    uid = "loki";
+  };
+
+  autoLayout =
+    panels:
+    let
+      step =
+        acc: panel:
+        let
+          w = panel.gridPos.w;
+          h = panel.gridPos.h;
+          wrap = acc.x + w > 24;
+          x = if wrap then 0 else acc.x;
+          y = if wrap then acc.y + acc.rowH else acc.y;
+          rowH = if wrap then h else lib.max acc.rowH h;
+        in
+        {
+          x = x + w;
+          inherit y rowH;
+          result = acc.result ++ [
+            (
+              panel
+              // {
+                gridPos = panel.gridPos // {
+                  inherit x y;
+                };
+              }
+            )
+          ];
+        };
+    in
+    (lib.foldl' step {
+      x = 0;
+      y = 0;
+      rowH = 0;
+      result = [ ];
+    } panels).result;
+
+  T = expr: legendFormat: {
+    datasource = prom;
+    inherit expr legendFormat;
+  };
+
+  mkRow = title: {
+    inherit title;
+    type = "row";
+    collapsed = false;
+    panels = [ ];
+    gridPos = {
+      w = 24;
+      h = 1;
+    };
+  };
+
+  mkStat =
+    {
+      title,
+      w ? 4,
+      h ? 4,
+      expr,
+      unit ? "short",
+      decimals ? 2,
+    }:
+    {
+      inherit title;
+      type = "stat";
+      datasource = prom;
+      gridPos = { inherit w h; };
+      fieldConfig.defaults = {
+        inherit unit decimals;
+        color.mode = "thresholds";
+        thresholds = {
+          mode = "absolute";
+          steps = [
+            {
+              color = "blue";
+              value = null;
+            }
+          ];
+        };
+      };
+      options = {
+        reduceOptions.calcs = [ "lastNotNull" ];
+        colorMode = "value";
+        textMode = "value";
+      };
+      targets = [
+        {
+          datasource = prom;
+          inherit expr;
+          instant = true;
+          legendFormat = "";
+        }
+      ];
+    };
+
+  mkGauge =
+    {
+      title,
+      w ? 4,
+      h ? 4,
+      expr,
+      unit ? "percent",
+      min ? 0,
+      max ? 100,
+    }:
+    {
+      inherit title;
+      type = "gauge";
+      datasource = prom;
+      gridPos = { inherit w h; };
+      fieldConfig.defaults = {
+        inherit unit min max;
+        thresholds = {
+          mode = "absolute";
+          steps = [
+            {
+              color = "green";
+              value = null;
+            }
+            {
+              color = "yellow";
+              value = 70;
+            }
+            {
+              color = "red";
+              value = 90;
+            }
+          ];
+        };
+      };
+      options = {
+        reduceOptions.calcs = [ "lastNotNull" ];
+        orientation = "auto";
+        showThresholdLabels = false;
+        showThresholdMarkers = true;
+      };
+      targets = [
+        {
+          datasource = prom;
+          inherit expr;
+          instant = true;
+          legendFormat = "";
+        }
+      ];
+    };
+
+  mkTs =
+    {
+      title,
+      w ? 24,
+      h ? 8,
+      unit ? "short",
+      targets,
+    }:
+    {
+      inherit title targets;
+      type = "timeseries";
+      datasource = prom;
+      gridPos = { inherit w h; };
+      fieldConfig.defaults = {
+        inherit unit;
+        custom = {
+          lineWidth = 1;
+          fillOpacity = 5;
+        };
+        color.mode = "palette-classic";
+      };
+      options = {
+        tooltip.mode = "multi";
+        legend = {
+          displayMode = "table";
+          placement = "bottom";
+          calcs = [
+            "mean"
+            "max"
+            "lastNotNull"
+          ];
+        };
+      };
+    };
+
+  mkDashboard =
+    {
+      name,
+      title,
+      uid,
+      from ? "now-1h",
+      panels,
+      templating ? [ ],
+    }:
+    pkgs.writeText name (
+      builtins.toJSON (
+        {
+          inherit title uid;
+          schemaVersion = 37;
+          refresh = "30s";
+          time = {
+            inherit from;
+            to = "now";
+          };
+          panels = lib.imap1 (id: p: p // { inherit id; }) (autoLayout panels);
+        }
+        // lib.optionalAttrs (templating != [ ]) { templating.list = templating; }
+      )
+    );
+in
+{
+  inherit
+    prom
+    loki
+    autoLayout
+    T
+    mkRow
+    mkStat
+    mkGauge
+    mkTs
+    mkDashboard
+    ;
+}

containers/common/overseer/host-dashboard.nix

@@ -0,0 +1,167 @@
+{ pkgs, lib }:
+let
+  inherit (import ./helpers.nix { inherit pkgs lib; })
+    T
+    mkRow
+    mkStat
+    mkGauge
+    mkTs
+    mkDashboard
+    ;
+in
+mkDashboard {
+  name = "host.json";
+  title = "Host";
+  uid = "host";
+  panels = [
+
+    (mkRow "Overview")
+    (mkGauge {
+      title = "CPU";
+      expr = ''100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])))'';
+    })
+    (mkGauge {
+      title = "Memory used";
+      expr = "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)";
+    })
+    (mkStat {
+      title = "Load (1m)";
+      unit = "short";
+      decimals = 2;
+      expr = "node_load1";
+    })
+    (mkStat {
+      title = "Load (5m)";
+      unit = "short";
+      decimals = 2;
+      expr = "node_load5";
+    })
+    (mkStat {
+      title = "Uptime";
+      unit = "dtdurations";
+      decimals = 0;
+      expr = "node_time_seconds - node_boot_time_seconds";
+    })
+
+    (mkRow "CPU")
+    (mkTs {
+      title = "CPU usage (%)";
+      unit = "percent";
+      targets = [
+        (T ''avg(rate(node_cpu_seconds_total{mode="user"}[5m]))    * 100'' "user")
+        (T ''avg(rate(node_cpu_seconds_total{mode="system"}[5m]))  * 100'' "system")
+        (T ''avg(rate(node_cpu_seconds_total{mode="iowait"}[5m]))  * 100'' "iowait")
+        (T ''avg(rate(node_cpu_seconds_total{mode="steal"}[5m]))   * 100'' "steal")
+        (T ''avg(rate(node_cpu_seconds_total{mode="softirq"}[5m])) * 100'' "softirq")
+      ];
+    })
+    (mkTs {
+      title = "Per-core usage (%)";
+      unit = "percent";
+      targets = [
+        (T ''(1 - rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100'' "cpu{{cpu}}")
+      ];
+    })
+    (mkTs {
+      title = "CPU temperatures";
+      unit = "celsius";
+      targets = [
+        (T "node_hwmon_temp_celsius" "{{chip}} {{sensor}}")
+      ];
+    })
+    (mkTs {
+      title = "Load average";
+      targets = [
+        (T "node_load1" "1m")
+        (T "node_load5" "5m")
+        (T "node_load15" "15m")
+      ];
+    })
+
+    (mkRow "Memory")
+    (mkTs {
+      title = "Memory";
+      unit = "bytes";
+      targets = [
+        (T "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes" "used")
+        (T "node_memory_Cached_bytes + node_memory_Buffers_bytes" "cache + buffers")
+        (T "node_memory_MemAvailable_bytes" "available")
+        (T "node_memory_MemTotal_bytes" "total")
+      ];
+    })
+    (mkTs {
+      title = "Swap";
+      unit = "bytes";
+      targets = [
+        (T "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes" "used")
+        (T "node_memory_SwapTotal_bytes" "total")
+      ];
+    })
+
+    (mkRow "Disk")
+    (mkTs {
+      title = "Filesystem usage (%)";
+      unit = "percent";
+      targets = [
+        (T ''100 * (1 - node_filesystem_avail_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"} / node_filesystem_size_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"})'' "{{mountpoint}}")
+      ];
+    })
+    (mkTs {
+      title = "Disk IO (bytes/s)";
+      unit = "Bps";
+      targets = [
+        (T ''rate(node_disk_read_bytes_total{device!~"loop.*"}[5m])'' "{{device}} read")
+        (T ''rate(node_disk_written_bytes_total{device!~"loop.*"}[5m])'' "{{device}} write")
+      ];
+    })
+    (mkTs {
+      title = "Disk IO utilisation (%)";
+      unit = "percent";
+      targets = [
+        (T ''rate(node_disk_io_time_seconds_total{device!~"loop.*"}[5m]) * 100'' "{{device}}")
+      ];
+    })
+
+    (mkRow "Network")
+    (mkTs {
+      title = "Network (bytes/s)";
+      unit = "Bps";
+      targets = [
+        (T ''rate(node_network_receive_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx")
+        (T ''rate(node_network_transmit_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx")
+      ];
+    })
+    (mkTs {
+      title = "Network errors + drops (pps)";
+      targets = [
+        (T ''rate(node_network_receive_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx err")
+        (T ''rate(node_network_transmit_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx err")
+        (T ''rate(node_network_receive_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx drop")
+        (T ''rate(node_network_transmit_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx drop")
+      ];
+    })
+
+    (mkRow "System")
+    (mkTs {
+      title = "Context switches + forks (ops/s)";
+      targets = [
+        (T "rate(node_context_switches_total[5m])" "context switches")
+        (T "rate(node_forks_total[5m])" "forks")
+      ];
+    })
+    (mkTs {
+      title = "File descriptors";
+      targets = [
+        (T "node_filefd_allocated" "allocated")
+        (T "node_filefd_maximum" "limit")
+      ];
+    })
+    (mkTs {
+      title = "Processes";
+      targets = [
+        (T "node_procs_running" "running")
+        (T "node_procs_blocked" "blocked on IO")
+      ];
+    })
+  ];
+}

containers/common/overseer/resources-dashboard.nix

@@ -0,0 +1,203 @@
+{ pkgs, lib }:
+let
+  inherit (import ./helpers.nix { inherit pkgs lib; })
+    T
+    mkRow
+    mkStat
+    mkTs
+    prom
+    mkDashboard
+    ;
+
+  ctid = "id=~\"/machine.slice/container@.+\\\\.service/payload/.+\"";
+  ctsel = "id=~\"/machine.slice/container@$container\\\\.service/payload/.+\"";
+
+  ctall = "id=~\"/machine.slice/container@.+\\\\.service.*\"";
+  ctnet = "id=~\"/machine.slice/container@$container\\\\.service.*\"";
+
+  labeled =
+    m:
+    "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service/payload.+\")";
+  byContainer = m: "sum by (container) (${labeled m})";
+
+  labeledAll =
+    m:
+    "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service.*\")";
+  byContainerAll = m: "sum by (container) (${labeledAll m})";
+
+  nmSvc = m: "label_replace(${m}, \"container\", \"$1\", \"name\", \"container@(.+)\\\\.service\")";
+
+  orZero = m: fb: "(${m}) or on(container) (${fb} * 0)";
+
+  cpuAll =
+    rate:
+    orZero (byContainer "rate(container_cpu_usage_seconds_total{${ctid}}[${rate}])") "container_limit_cpu_cores";
+  memAll = orZero (byContainerAll "container_memory_rss{${ctall}}") "container_limit_memory_bytes";
+  taskAll = nmSvc "node_systemd_unit_tasks_current{name=~\"container@.+\"}";
+in
+mkDashboard {
+  name = "resources.json";
+  title = "Container Resources";
+  uid = "container-resources";
+
+  templating = [
+    {
+      name = "container";
+      type = "query";
+      datasource = prom;
+      query = "label_values(container_limit_cpu_cores, container)";
+      refresh = 2;
+      sort = 1;
+      includeAll = true;
+      multi = true;
+      current = {
+        selected = true;
+        text = [ "All" ];
+        value = [ "$__all" ];
+      };
+    }
+  ];
+
+  panels = [
+    (mkRow "Overview")
+    (mkStat {
+      title = "Running";
+      decimals = 0;
+      expr = "count(node_systemd_unit_state{name=~\"container@.*\",state=\"active\"} == 1) or vector(0)";
+    })
+    (mkStat {
+      title = "CPU used";
+      decimals = 2;
+      expr = "sum(${cpuAll "5m"}) or vector(0)";
+    })
+    (mkStat {
+      title = "CPU limit";
+      decimals = 2;
+      expr = "sum(container_limit_cpu_cores) or vector(0)";
+    })
+    (mkStat {
+      title = "Mem RSS";
+      decimals = 1;
+      unit = "bytes";
+      expr = "sum(${memAll}) or vector(0)";
+    })
+    (mkStat {
+      title = "Mem limit";
+      decimals = 1;
+      unit = "bytes";
+      expr = "sum(container_limit_memory_bytes) or vector(0)";
+    })
+    (mkStat {
+      title = "Tasks";
+      decimals = 0;
+      expr = "sum(${taskAll}) or vector(0)";
+    })
+
+    (mkRow "All Containers")
+    (mkTs {
+      title = "CPU (cores)";
+      h = 24;
+      targets = [ (T (cpuAll "5m") "{{container}}") ];
+    })
+    (mkTs {
+      title = "Memory RSS";
+      unit = "bytes";
+      h = 24;
+      targets = [ (T memAll "{{container}}") ];
+    })
+    (mkTs {
+      title = "Tasks";
+      h = 24;
+      targets = [ (T taskAll "{{container}}") ];
+    })
+    (mkTs {
+      title = "CPU throttled %";
+      unit = "percent";
+      h = 12;
+      targets = [
+        (T "(${byContainer "rate(container_cpu_cfs_throttled_periods_total{${ctid}}[5m])"} / ${byContainer "rate(container_cpu_cfs_periods_total{${ctid}}[5m])"}) * 100 or on(container) (container_limit_cpu_cores * 0)" "{{container}}")
+      ];
+    })
+    (mkTs {
+      title = "Network egress (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [ (T "sum by (container) (container_network_receive_rate)" "{{container}}") ];
+    })
+    (mkTs {
+      title = "Network ingress (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [ (T "sum by (container) (container_network_transmit_rate)" "{{container}}") ];
+    })
+    (mkTs {
+      title = "Disk read (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [ (T (byContainer "rate(container_fs_reads_bytes_total{${ctid}}[5m])") "{{container}}") ];
+    })
+    (mkTs {
+      title = "Disk write (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [
+        (T (byContainer "rate(container_fs_writes_bytes_total{${ctid}}[5m])") "{{container}}")
+      ];
+    })
+
+    {
+      type = "row";
+      title = "$container";
+      repeat = "container";
+      collapsed = true;
+      panels = [ ];
+      gridPos = {
+        w = 24;
+        h = 1;
+      };
+    }
+    (mkTs {
+      title = "CPU (cores)";
+      targets = [
+        (T "sum(rate(container_cpu_usage_seconds_total{${ctsel}}[5m]))" "used")
+        (T "container_limit_cpu_cores{container=\"$container\"}" "limit")
+      ];
+    })
+    (mkTs {
+      title = "CPU throttled %";
+      unit = "percent";
+      targets = [
+        (T "(sum(rate(container_cpu_cfs_throttled_periods_total{${ctsel}}[5m])) / sum(rate(container_cpu_cfs_periods_total{${ctsel}}[5m]))) * 100 or vector(0)" "throttled %")
+      ];
+    })
+    (mkTs {
+      title = "Memory";
+      unit = "bytes";
+      targets = [
+        (T "sum(container_memory_rss{${ctnet}})" "RSS")
+        (T "sum(container_memory_working_set_bytes{${ctnet}})" "working set")
+        (T "container_limit_memory_bytes{container=\"$container\"}" "limit")
+      ];
+    })
+    (mkTs {
+      title = "Network (bytes/s)";
+      unit = "Bps";
+      targets = [
+        (T "container_network_receive_rate{container=\"$container\"}" "{{device}} ↑")
+        (T "container_network_transmit_rate{container=\"$container\"}" "{{device}} ↓")
+      ];
+    })
+    (mkTs {
+      title = "Disk IO (bytes/s)";
+      unit = "Bps";
+      targets = [
+        (T "sum(rate(container_fs_reads_bytes_total{${ctsel}}[5m]))" "read")
+        (T "sum(rate(container_fs_writes_bytes_total{${ctsel}}[5m]))" "write")
+      ];
+    })
+    (mkTs {
+      title = "Tasks";
+      targets = [ (T "node_systemd_unit_tasks_current{name=\"container@$container.service\"}" "tasks") ];
+    })
+  ];
+}

containers/server-n0/devenv.nix

@@ -0,0 +1,54 @@
+{
+  name,
+  hostInfo,
+  dsl,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService;
+  inherit (hostInfo.paths) data secrets;
+in
+mkContainer {
+  inherit name;
+  net.access = [ { to = "@internet"; } ];
+  svc.autoStart = true;
+  resources.fs = {
+    "/config" = {
+      hostPath = "${data}/web/devenv/config";
+      isReadOnly = false;
+    };
+    "/workspace" = {
+      hostPath = "${data}/web/devenv/workspace";
+      isReadOnly = false;
+    };
+    "/run/secrets/code-server-env" = {
+      hostPath = "${secrets}/code-server-env";
+      isReadOnly = true;
+    };
+  };
+  users = [
+    {
+      name = "coder";
+      uid = 1000;
+      home = "/config";
+    }
+  ];
+  service =
+    { pkgs, lib, ... }:
+    mkService {
+      description = "code-server";
+      execStart = "${lib.getExe' pkgs.code-server "code-server"} --bind-addr 0.0.0.0:8080 --user-data-dir /config/data --extensions-dir /config/extensions --auth password /workspace";
+      networkOnline = true;
+      serviceConfig = {
+        EnvironmentFile = "/run/secrets/code-server-env";
+        User = "coder";
+        ReadWritePaths = [
+          "/config"
+          "/workspace"
+        ];
+      };
+    };
+  net.expose.tcp = [ 8080 ];
+  resources.mem = "4G";
+  resources.cpu = 2;
+}

containers/server-n0/git.nix

@@ -0,0 +1,178 @@
+{
+  name,
+  hostInfo,
+  dsl,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkVm;
+  inherit (hostInfo.paths) data secrets;
+
+  ciRunnerData = "${data}/web/git-ci-runner";
+  ciRunnerSecrets = "${secrets}/ci-runner";
+in
+{
+  ${name} = mkContainer {
+    inherit name;
+    net.access = [ { to = "@internet"; } ];
+    net.forward.tcp = [
+      {
+        host = 23;
+        container = 8023;
+      }
+    ];
+    svc.autoStart = true;
+    resources.fs."/var/lib/gitea" = {
+      hostPath = "${data}/web/git";
+      isReadOnly = false;
+    };
+    containerConfig = _: {
+      services.gitea = {
+        enable = true;
+        stateDir = "/var/lib/gitea";
+        settings = {
+          server = {
+            HTTP_ADDR = "0.0.0.0";
+            HTTP_PORT = 3000;
+            DOMAIN = "git.floweytf.com";
+            ROOT_URL = "https://git.floweytf.com/";
+            START_SSH_SERVER = true;
+            BUILTIN_SSH_SERVER_USER = "git";
+            SSH_PORT = 8023;
+          };
+          service.DISABLE_REGISTRATION = true;
+        };
+      };
+      users.users.gitea.uid = 1000;
+      users.groups.gitea.gid = 1000;
+      time.timeZone = "UTC";
+      systemd.services.gitea.serviceConfig.FailureAction = "exit-force";
+    };
+    net.expose.tcp = [ 3000 ];
+  };
+
+  ci-runner = mkVm {
+    name = "ci-runner";
+    net.access = [
+      {
+        to = "git";
+        tcp = [ 3000 ];
+      }
+      { to = "@internet"; }
+    ];
+    svc.autoStart = true;
+    resources.cpu = 4;
+    resources.mem = "8G";
+    resources.fs = [
+      {
+        proto = "virtiofs";
+        tag = "nix-rw-store";
+        source = "${ciRunnerData}/nix-rw-store";
+        mountPoint = "/nix/.rw-store";
+      }
+      {
+        proto = "virtiofs";
+        tag = "docker-data";
+        source = "${ciRunnerData}/docker";
+        mountPoint = "/var/lib/docker";
+      }
+      {
+        proto = "virtiofs";
+        tag = "runner-state";
+        source = "${ciRunnerData}/runner-state";
+        mountPoint = "/var/lib/gitea-runner";
+      }
+      {
+        proto = "virtiofs";
+        tag = "runner-token";
+        source = ciRunnerSecrets;
+        mountPoint = "/run/ci-runner-secrets";
+        readOnly = true;
+      }
+    ];
+    guestConfig =
+      {
+        pkgs,
+        utils,
+        lib,
+        ...
+      }:
+      let
+        runnerInstanceName = "ci-runner";
+
+        ciJobImage = pkgs.dockerTools.buildLayeredImage {
+          name = "ci-job-image";
+          tag = "latest";
+          contents = with pkgs; [
+            bashInteractive
+            coreutils
+            gitMinimal
+            nixVersions.latest
+            nodejs
+            cacert
+          ];
+          config = {
+            Cmd = [ "/bin/sh" ];
+            Env = [
+              "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+              "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+              "NIX_REMOTE=daemon"
+            ];
+          };
+        };
+      in
+      {
+        boot.kernelModules = [ "kvm-intel" ];
+        boot.extraModprobeConfig = "options kvm_intel nested=1";
+
+        microvm.writableStoreOverlay = "/nix/.rw-store";
+
+        nix.settings = {
+          experimental-features = [
+            "nix-command"
+            "flakes"
+          ];
+          max-jobs = "auto";
+        };
+
+        nix.gc = {
+          automatic = true;
+          dates = "daily";
+          options = "--delete-older-than 14d";
+        };
+        # nix.optimise.automatic = true; - doesn't work with microvm.writableStoreOverlay
+
+        virtualisation.docker.enable = true;
+
+        systemd.services.load-ci-job-image = {
+          description = "Load the CI job Docker image";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "docker.service" ];
+          requires = [ "docker.service" ];
+          before = [ "gitea-runner-${utils.escapeSystemdPath runnerInstanceName}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStart = "${lib.getExe' pkgs.docker "docker"} load -i ${ciJobImage}";
+          };
+        };
+
+        services.gitea-actions-runner.instances.${runnerInstanceName} = {
+          enable = true;
+          name = runnerInstanceName;
+          url = "http://git:3000/";
+          tokenFile = "/run/ci-runner-secrets/token";
+          labels = [
+            "nix:docker://ci-job-image:latest"
+          ];
+          settings.container = {
+            options = "--device=/dev/kvm --volume=/nix/store:/nix/store:ro --volume=/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket";
+            valid_volumes = [
+              "/nix/store"
+              "/nix/var/nix/daemon-socket"
+            ];
+          };
+        };
+      };
+  };
+}

containers/server-n0/hopper.nix

@@ -0,0 +1,79 @@
+{
+  name,
+  pkgs,
+  lib,
+  dsl,
+  containers,
+  customPkgs,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService collectContainers;
+
+  mkHopperConfig =
+    { routes }:
+    let
+      header = ''
+        listen = "0.0.0.0:25565"
+
+        [routing.routes]
+      '';
+      mkRoute = { domain, ip }: ''
+        "${domain}" = { ip = "${ip}" }
+      '';
+    in
+    header + lib.concatMapStrings mkRoute routes;
+
+  backends = collectContainers "hopperRegister" containers;
+
+  mcRoutes = map ({ container, data }: {
+    domain = "${data.hopperSubdomain}.mc.floweytf.com";
+    ip = "${container.name}:${toString data.hopperPort}";
+  }) backends;
+
+  hopperCfg = pkgs.writeText "Config.toml" (mkHopperConfig {
+    routes = mcRoutes;
+  });
+in
+{
+  ${name} = mkContainer {
+    inherit name;
+    net.access = map ({ container, data }: {
+      to = container.name;
+      tcp = [ data.hopperPort ];
+    }) backends;
+    svc.autoStart = true;
+    net.expose.tcp = [ 25565 ];
+    net.forward.tcp = [ 25565 ];
+    resources.fs."/Config.toml" = {
+      hostPath = toString hopperCfg;
+      isReadOnly = true;
+    };
+    service =
+      _:
+      mkService {
+        description = "Hopper Minecraft reverse proxy";
+        execStart = lib.getExe' customPkgs.hopper-rs "hopper";
+        workingDir = "/";
+        networkOnline = true;
+        serviceConfig = {
+          DynamicUser = true;
+          PrivateTmp = true;
+          PrivateDevices = true;
+          CapabilityBoundingSet = "";
+          RestrictAddressFamilies = "AF_INET AF_INET6";
+        };
+      };
+  };
+  _hopperRegister =
+    {
+      hopperSubdomain ? null,
+      hopperPort ? 25565,
+    }:
+    self: {
+      hopperRegister = {
+        hopperSubdomain = if hopperSubdomain == null then self.name else hopperSubdomain;
+        inherit hopperPort;
+      };
+    };
+}

containers/server-n0/mail.nix

@@ -0,0 +1,264 @@
+{
+  name,
+  pkgs,
+  lib,
+  hostInfo,
+  dsl,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService;
+  inherit (hostInfo.paths) data secrets;
+
+  domain = "mail.floweytf.com";
+  accountName = "flowey";
+  stateDir = "/var/lib/stalwart";
+  acmeDir = "/etc/mail-acme";
+
+  taggingScript = ''
+    require ["envelope", "variables", "imap4flags", "fileinto"];
+
+    if envelope :localpart :matches "to" "*" {
+      addflag "''${1}";
+    }
+
+    fileinto "INBOX";
+  '';
+
+  cli = lib.getExe' pkgs.stalwart-cli "stalwart-cli";
+  jq = lib.getExe pkgs.jq;
+  curl = lib.getExe pkgs.curl;
+  grep = lib.getExe' pkgs.gnugrep "grep";
+  paste = lib.getExe' pkgs.coreutils "paste";
+
+  # use sqlite b/c rocksdb segfaults
+  configJsonTemplate = pkgs.writeText "stalwart-config.json" (
+    builtins.toJSON {
+      "@type" = "Sqlite";
+      path = "${stateDir}/data.sqlite3";
+    }
+  );
+
+  staticPlan = pkgs.writeText "mail-static-plan.json" (
+    builtins.concatStringsSep "\n" (
+      map builtins.toJSON [
+        {
+          "@type" = "upsert";
+          object = "Domain";
+          matchOn = [ "name" ];
+          value."domain-mail" = {
+            name = domain;
+            catchAllAddress = "${accountName}@${domain}";
+            isEnabled = true;
+          };
+        }
+        {
+          "@type" = "upsert";
+          object = "SieveSystemScript";
+          matchOn = [ "name" ];
+          value."sieve-tag" = {
+            name = "tag-by-recipient";
+            isActive = true;
+            contents = taggingScript;
+          };
+        }
+      ]
+    )
+    + "\n"
+  );
+
+  stalwartAuthEnv = ''
+    export STALWART_URL="http://127.0.0.1:8080"
+    export STALWART_USER="admin"
+    export STALWART_PASSWORD="$(cat "$CREDENTIALS_DIRECTORY/admin_password")"
+  '';
+
+  readyGate = pkgs.writeShellScript "mail-ready-gate" ''
+    set -uo pipefail
+    until ${curl} -sS -o /dev/null "http://127.0.0.1:8080/api/account"; do
+      if ! systemctl -q is-active stalwart.service; then
+        echo "mail: stalwart is not running" >&2
+        exit 1
+      fi
+      sleep 1
+    done
+  '';
+
+  bootstrapScript = pkgs.writeShellScript "mail-bootstrap" ''
+    set -uo pipefail
+    ${stalwartAuthEnv}
+
+    out="$(${cli} update Bootstrap singleton \
+      --field defaultDomain=${domain} \
+      --field serverHostname=${domain} \
+      --field generateDkimKeys=true \
+      --field requestTlsCertificate=false \
+      --field 'directory={"@type":"Internal"}' 2>&1)"
+    status=$?
+
+    if [ "$status" -eq 0 ]; then
+      echo "mail-bootstrap: first-time setup complete, restarting stalwart into normal mode" >&2
+      systemctl restart stalwart.service
+      exit 0
+    fi
+
+    if ${grep} -qi 'bootstrap mode' <<<"$out"; then
+      exit 0
+    fi
+
+    echo "mail-bootstrap: $out" >&2
+    exit 1
+  '';
+
+  certCredCert = "/run/credentials/stalwart.service/mail_tls_cert";
+  certCredKey = "/run/credentials/stalwart.service/mail_tls_key";
+
+  retryOnFailure = {
+    Restart = "on-failure";
+    RestartSec = 5;
+  };
+
+  retryLimit = {
+    StartLimitBurst = 5;
+    StartLimitIntervalSec = 120;
+  };
+
+  provisionScript = pkgs.writeShellScript "mail-provision" ''
+    set -euo pipefail
+    ${stalwartAuthEnv}
+    export account_pass="$(cat "$CREDENTIALS_DIRECTORY/account_password")"
+
+    account_entry="$(${jq} -nc --arg name "${accountName}" \
+      '{"@type":"upsert","object":"Account","matchOn":["name"],"value":{("account-"+$name):{"@type":"User","name":$name,"domainId":"#domain-mail","credentials":{"0":{"@type":"Password","secret":env.account_pass}}}}}')"
+
+    { cat ${staticPlan}; echo "$account_entry"; } | ${cli} apply --stdin
+
+    existing_cert_ids="$(${cli} query Certificate --json --fields id | ${jq} -r .id | ${paste} -sd, -)"
+    if [ -n "$existing_cert_ids" ]; then
+      ${cli} delete Certificate --ids "$existing_cert_ids"
+    fi
+    ${cli} create Certificate \
+      --field 'certificate={"@type":"File","filePath":"${certCredCert}"}' \
+      --field 'privateKey={"@type":"File","filePath":"${certCredKey}"}'
+
+    systemctl restart stalwart.service
+  '';
+in
+mkContainer {
+  inherit name;
+  net.access = [ { to = "@internet"; } ];
+  svc.autoStart = true;
+  net.forward.tcp = [
+    25
+    993
+  ];
+  resources.fs = {
+    "${stateDir}" = {
+      hostPath = "${data}/core/mail";
+      isReadOnly = false;
+    };
+    "${acmeDir}" = {
+      hostPath = "${data}/core/nginx/acme/${domain}";
+      isReadOnly = true;
+    };
+    "/run/secrets/mail-admin-password" = {
+      hostPath = "${secrets}/mail-admin-password";
+      isReadOnly = true;
+    };
+    "/run/secrets/mail-account-password" = {
+      hostPath = "${secrets}/mail-account-password";
+      isReadOnly = true;
+    };
+  };
+  users = [
+    {
+      name = "stalwart";
+      uid = 1000;
+      home = stateDir;
+    }
+  ];
+  containerConfig = _: {
+    systemd.tmpfiles.rules = [
+      "Z ${stateDir} 0750 stalwart stalwart -"
+      "C ${stateDir}/config.json 0640 stalwart stalwart - ${configJsonTemplate}"
+    ];
+
+    systemd.services.stalwart = mkService {
+      description = "Stalwart Mail Server";
+      execStart = pkgs.writeShellScript "stalwart-run" ''
+        set -euo pipefail
+        export STALWART_RECOVERY_ADMIN="admin:$(cat "$CREDENTIALS_DIRECTORY/admin_password")"
+        exec ${lib.getExe' pkgs.stalwart_0_16 "stalwart"} -c ${stateDir}/config.json
+      '';
+      serviceConfig = {
+        User = "stalwart";
+        Group = "stalwart";
+        AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+        ReadWritePaths = [ stateDir ];
+        LoadCredential = [
+          "admin_password:/run/secrets/mail-admin-password"
+          "mail_tls_cert:${acmeDir}/fullchain.pem"
+          "mail_tls_key:${acmeDir}/key.pem"
+        ];
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+    };
+
+    systemd.paths.mail-cert-renew = {
+      wantedBy = [ "multi-user.target" ];
+      pathConfig = {
+        PathChanged = "${acmeDir}/key.pem";
+        Unit = "mail-provision.service";
+      };
+    };
+
+    systemd.services.mail-ready = mkService {
+      description = "Wait for the Stalwart management API";
+      execStart = readyGate;
+      after = [ "stalwart.service" ];
+      wants = [ "stalwart.service" ];
+      wantedBy = [ ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        TimeoutStartSec = 60;
+      };
+    };
+
+    systemd.services.mail-bootstrap = mkService {
+      description = "One-time Stalwart bootstrap (server identity + admin account)";
+      execStart = bootstrapScript;
+      after = [ "mail-ready.service" ];
+      wants = [ "mail-ready.service" ];
+      unitConfig = retryLimit;
+      serviceConfig = {
+        Type = "oneshot";
+        LoadCredential = [ "admin_password:/run/secrets/mail-admin-password" ];
+      }
+      // retryOnFailure;
+    };
+
+    systemd.services.mail-provision = mkService {
+      description = "Provision mail.floweytf.com domain + catch-all account in Stalwart";
+      execStart = provisionScript;
+      after = [
+        "mail-ready.service"
+        "mail-bootstrap.service"
+      ];
+      wants = [
+        "mail-ready.service"
+        "mail-bootstrap.service"
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        LoadCredential = [
+          "admin_password:/run/secrets/mail-admin-password"
+          "account_password:/run/secrets/mail-account-password"
+        ];
+      }
+      // retryOnFailure;
+      unitConfig = retryLimit;
+    };
+  };
+}

containers/server-n0/maven.nix

@@ -0,0 +1,41 @@
+{
+  name,
+  hostInfo,
+  dsl,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService;
+  inherit (hostInfo.paths) data;
+in
+mkContainer {
+  inherit name;
+  svc.autoStart = true;
+  resources.fs."/app/data" = {
+    hostPath = "${data}/web/reposilite";
+    isReadOnly = false;
+  };
+  users = [
+    {
+      name = "reposilite";
+      uid = 1000;
+      home = "/app";
+    }
+  ];
+  service =
+    { pkgs, lib, ... }:
+    mkService {
+      description = "Reposilite";
+      execStart = "${lib.getExe' pkgs.reposilite "reposilite"} --working-directory /app/data";
+      workingDir = "/app/data";
+      serviceConfig = {
+        User = "reposilite";
+        PrivateTmp = true;
+        ReadWritePaths = [ "/app/data" ];
+      };
+    }
+    // {
+      environment.JAVA_OPTS = "-Xmx128M";
+    };
+  net.expose.tcp = [ 8080 ];
+}

containers/server-n0/mc.nix

@@ -0,0 +1,205 @@
+{
+  lib,
+  pkgs,
+  hostInfo,
+  dsl,
+  self,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService;
+  inherit (hostInfo.paths) data;
+
+  mkMcTrait =
+    {
+      jdk,
+      dataPath,
+      extraPackages ? [ ],
+    }:
+    {
+      resources.fs."/data" = {
+        hostPath = dataPath;
+        isReadOnly = false;
+      };
+      users = [
+        {
+          name = "runner";
+          uid = 1000;
+        }
+      ];
+      containerConfig =
+        {
+          pkgs,
+          ...
+        }:
+        {
+          environment.systemPackages = [
+            jdk
+            pkgs.bash
+          ]
+          ++ extraPackages;
+          systemd.services.minecraft = mkService {
+            description = "Minecraft server";
+            wantedBy = [ "multi-user.target" ];
+            after = [
+              "local-fs.target"
+              "container-ready.target"
+            ];
+            wants = [ "container-ready.target" ];
+            path = [
+              jdk
+              pkgs.bash
+            ];
+            execStart = "${lib.getExe' pkgs.bash "bash"} -c ${
+              lib.escapeShellArg (
+                "exec ${lib.getExe' pkgs.dtach "dtach"} -N /run/ct-attach/attach.sock "
+                + "${lib.getExe' pkgs.util-linux "script"} -qfc /data/launch.sh "
+                + ">(exec ${lib.getExe' pkgs.systemd "systemd-cat"} -t minecraft)"
+              )
+            }";
+            serviceConfig = {
+              WorkingDirectory = "/data";
+              User = "runner";
+              RuntimeDirectory = "ct-attach";
+              RuntimeDirectoryMode = "0700";
+              StandardInput = "null";
+              StandardOutput = "journal";
+              StandardError = "journal";
+              ReadWritePaths = [ "/data" ];
+            };
+          };
+        };
+    };
+
+  extractAttrs =
+    l: s: lib.mergeAttrsList (map (f: lib.optionalAttrs (s ? ${f}) { ${f} = s.${f}; }) l);
+
+  mkMcSet =
+    { dataDir }:
+    servers:
+    lib.listToAttrs (
+      map (
+        args@{
+          name,
+          jdk,
+          autoStart ? false,
+          allowInternet ? true,
+          dataPath ? "${dataDir}/${name}",
+          extraPackages ? [ ],
+          mem ? "6G",
+          cpu ? 2,
+          ...
+        }:
+        lib.nameValuePair "mc-${name}" (mkContainer {
+          name = "mc-${name}";
+          net.access = lib.optional allowInternet { to = "@internet"; };
+          net.expose.tcp = [ (args.hopperPort or 25565) ];
+          svc.autoStart = autoStart;
+          resources = { inherit cpu mem; };
+          traits = [ (mkMcTrait { inherit jdk dataPath extraPackages; }) ];
+          miscProperties = self._hopperRegister (extractAttrs [ "hopperSubdomain" "hopperPort" ] args);
+          includedInTests = false;
+        })
+      ) servers
+    );
+
+  jdk8FontPkgs = with pkgs; [
+    freetype
+    fontconfig
+    dejavu_fonts
+  ];
+
+  mcServers = [
+    {
+      name = "nomifactory";
+      jdk = pkgs.jdk8;
+      extraPackages = jdk8FontPkgs;
+      hopperSubdomain = "nomi";
+      hopperPort = 25555;
+    }
+    {
+      name = "monifactory";
+      jdk = pkgs.jdk21;
+      hopperSubdomain = "moni";
+    }
+    {
+      name = "rad2";
+      jdk = pkgs.jdk8;
+      extraPackages = jdk8FontPkgs;
+    }
+    {
+      name = "smp";
+      jdk = pkgs.jdk17;
+    }
+    {
+      name = "rc4";
+      jdk = pkgs.jdk17;
+    }
+    {
+      name = "vh";
+      jdk = pkgs.jdk17;
+      hopperPort = 25555;
+    }
+    {
+      name = "gregblock";
+      jdk = pkgs.jdk8;
+      extraPackages = jdk8FontPkgs;
+    }
+    {
+      name = "tfg";
+      jdk = pkgs.jdk17;
+    }
+    {
+      name = "ratgame";
+      jdk = pkgs.jdk21;
+    }
+    {
+      name = "maidrpg";
+      jdk = pkgs.jdk17;
+    }
+    {
+      name = "maidsmp";
+      jdk = pkgs.jdk21;
+    }
+    {
+      name = "maid-event";
+      jdk = pkgs.jdk21;
+      hopperSubdomain = "maidevent";
+      dataPath = "${data}/mc/maidevent/speedrun";
+    }
+    {
+      name = "gtnh";
+      jdk = pkgs.jdk21;
+    }
+    {
+      name = "turing";
+      jdk = pkgs.jdk21;
+    }
+    {
+      name = "tfg2";
+      jdk = pkgs.jdk21;
+      autoStart = true;
+      mem = "10G";
+      cpu = 8;
+    }
+    {
+      name = "tl";
+      jdk = pkgs.jdk21;
+    }
+    {
+      name = "botm";
+      jdk = pkgs.jdk17;
+    }
+    {
+      name = "rogule-build";
+      jdk = pkgs.jdk21;
+    }
+    {
+      name = "rogule-dev";
+      jdk = pkgs.jdk21;
+    }
+  ];
+in
+mkMcSet {
+  dataDir = "${data}/mc";
+} mcServers

containers/server-n0/mtce.nix

@@ -0,0 +1,84 @@
+{
+  name,
+  pkgs,
+  lib,
+  hostInfo,
+  dsl,
+  inputs,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService;
+  inherit (hostInfo.paths) data secrets;
+  inherit (inputs) mtce;
+  dataDir = "/var/lib/mtce-bot";
+
+  threads = 64;
+  timeout = 1800;
+
+  configJson = pkgs.writeText "mtce-bot-config.json" (
+    builtins.toJSON {
+      inherit threads timeout;
+    }
+  );
+
+  mtce-engine = mtce.lib.mkMtce pkgs {
+    cpuFlags = [
+      "-O3"
+      "-march=cascadelake"
+    ];
+    vectorizedBitSize = 512;
+  };
+  mtce-bot = mtce.lib.mkBot pkgs;
+in
+mkContainer {
+  inherit name;
+  svc.autoStart = true;
+  resources.fs = {
+    "${dataDir}/data.json" = {
+      hostPath = "${data}/misc/mtce-discord-bot/data.json";
+      isReadOnly = false;
+    };
+    "${dataDir}/audit" = {
+      hostPath = "${data}/misc/mtce-discord-bot/audit";
+      isReadOnly = false;
+    };
+    "${dataDir}/config.json" = {
+      hostPath = "${configJson}";
+      isReadOnly = true;
+    };
+    "${dataDir}/mtce" = {
+      hostPath = lib.getExe' mtce-engine "mtce";
+      isReadOnly = true;
+    };
+    "/run/secrets/mtce-discord-bot-env" = {
+      hostPath = "${secrets}/mtce-discord-bot-env";
+      isReadOnly = true;
+    };
+  };
+  users = [
+    {
+      name = "bot";
+      uid = 1000;
+    }
+  ];
+  service =
+    _:
+    mkService {
+      description = "Discord bot (monumenta-charm-eval)";
+      execStart = lib.getExe' mtce-bot "mtce-bot";
+      workingDir = dataDir;
+      networkOnline = true;
+      serviceConfig = {
+        User = "bot";
+        EnvironmentFile = "/run/secrets/mtce-discord-bot-env";
+        PrivateTmp = true;
+        CapabilityBoundingSet = "";
+        ReadWritePaths = [ dataDir ];
+      };
+    };
+  resources.mem = "2G";
+  resources.cpu = 64;
+  net.access = [ { to = "@internet"; } ];
+  includedInTests = false;
+}

containers/server-n0/nginx.nix

@@ -0,0 +1,215 @@
+{
+  name,
+  hostInfo,
+  dsl,
+  lib,
+  containers,
+  ...
+}:
+let
+  inherit (dsl) mkContainer;
+  inherit (hostInfo.paths) data;
+
+  mkNginxConfig =
+    {
+      email,
+      vhosts,
+      resolver,
+    }:
+    let
+      mkVhostEntries =
+        {
+          domain,
+          upstream ? null,
+          websockets ? true,
+          redirect ? true,
+          listenHttp ? false,
+          extraLocations ? { },
+        }:
+        let
+          httpsListen = {
+            addr = "0.0.0.0";
+            port = 443;
+            ssl = true;
+          };
+          httpListen = {
+            addr = "0.0.0.0";
+            port = 80;
+            ssl = false;
+          };
+          defaultLoc = lib.optionalAttrs (upstream != null) {
+            "/" = {
+              proxyWebsockets = websockets;
+              # avoid startup error when containers aren't registered yet
+              # proxyPass is left unset for that reason, so recommendedProxySettings
+              # won't auto-include its headers here - set them manually to match
+              extraConfig = ''
+                set $backend "${upstream.container.name}:${toString upstream.port}";
+                proxy_pass http://$backend;
+                proxy_set_header Host $host;
+                proxy_set_header X-Real-IP $remote_addr;
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                proxy_set_header X-Forwarded-Proto $scheme;
+                proxy_set_header X-Forwarded-Host $host;
+                proxy_set_header X-Forwarded-Server $hostname;
+              '';
+            };
+          };
+          acmePrimaryOnMain = !redirect && listenHttp;
+          mainVhost = lib.nameValuePair domain (
+            {
+              listen = [ httpsListen ] ++ lib.optional listenHttp httpListen;
+              locations = defaultLoc // extraLocations;
+            }
+            // (
+              if acmePrimaryOnMain then
+                {
+                  addSSL = true;
+                  enableACME = true;
+                }
+              else
+                {
+                  onlySSL = true;
+                  useACMEHost = domain;
+                }
+            )
+          );
+          redirectVhost = lib.nameValuePair "${domain}-redirect" {
+            serverName = domain;
+            enableACME = true;
+            listen = [ httpListen ];
+            locations = {
+              "/" = {
+                return = "301 https://$host$request_uri";
+              };
+            };
+          };
+        in
+        [ mainVhost ] ++ lib.optional redirect redirectVhost;
+    in
+    {
+      access = map (v: {
+        to = v.upstream.container.name;
+        tcp = [ v.upstream.port ];
+      }) (lib.filter (v: v.upstream or null != null) vhosts);
+
+      containerConfig =
+        {
+          lib,
+          config,
+          ...
+        }:
+        {
+          services.nginx = {
+            enable = true;
+            recommendedTlsSettings = true;
+            recommendedOptimisation = true;
+            recommendedGzipSettings = true;
+            recommendedProxySettings = true;
+            clientMaxBodySize = "100M";
+            appendHttpConfig = ''
+              access_log syslog:server=unix:/dev/log,tag=nginx combined;
+            '';
+            resolver = {
+              addresses = [ resolver ];
+              ipv6 = false;
+              valid = "30s";
+            };
+            virtualHosts = lib.listToAttrs (lib.concatLists (map mkVhostEntries vhosts)) // {
+              "_" = {
+                listen = [
+                  {
+                    addr = "0.0.0.0";
+                    port = 443;
+                    extraParameters = [
+                      "ssl"
+                      "default_server"
+                    ];
+                  }
+                  {
+                    addr = "0.0.0.0";
+                    port = 80;
+                    extraParameters = [ "default_server" ];
+                  }
+                ];
+                extraConfig = "ssl_reject_handshake on;";
+                locations."/".return = "404";
+              };
+            };
+          };
+          security.acme = {
+            acceptTerms = true;
+            defaults.email = email;
+          };
+          systemd.services =
+            lib.mapAttrs' (
+              cert: _:
+              lib.nameValuePair "acme-${cert}" {
+                after = [ "container-ready.target" ];
+                wants = [ "container-ready.target" ];
+              }
+            ) config.security.acme.certs
+            // {
+              nginx.serviceConfig.FailureAction = "exit-force";
+            };
+        };
+    };
+
+  up = container: port: { inherit container port; };
+
+  nginxSetup =
+    hostAddr:
+    mkNginxConfig {
+      email = "floweydev@gmail.com";
+      resolver = hostAddr;
+      vhosts = [
+        {
+          domain = "devenv.floweytf.com";
+          upstream = up containers.devenv 8080;
+        }
+        {
+          domain = "maven.floweytf.com";
+          upstream = up containers.maven 8080;
+        }
+        {
+          domain = "git.floweytf.com";
+          upstream = up containers.git 3000;
+        }
+        {
+          domain = "utils.floweytf.com";
+          redirect = false;
+          listenHttp = true;
+          extraLocations = {
+            "/static/" = {
+              alias = "/usr/share/nginx/static/";
+              extraConfig = "autoindex on;";
+            };
+          };
+        }
+        {
+          # no upstream, vhost only exists for certs
+          domain = "mail.floweytf.com";
+        }
+      ];
+    };
+in
+let
+  setup = nginxSetup hostAddr;
+  c = mkContainer {
+    inherit name;
+    net.access = setup.access ++ [ { to = "@internet"; } ];
+    svc.autoStart = true;
+    svc.depends = lib.attrVals [ "devenv" "maven" "git" ] containers;
+    net.forward.tcp = [
+      80
+      443
+    ];
+    resources.fs."/var/lib/acme" = {
+      hostPath = "${data}/core/nginx/acme";
+      isReadOnly = false;
+    };
+    inherit (setup) containerConfig;
+  };
+  hostAddr = c.hostAddr;
+in
+c

containers/server-n0/pihole.nix

@@ -0,0 +1,64 @@
+{
+  name,
+  hostInfo,
+  dsl,
+  ...
+}:
+let
+  inherit (dsl) mkContainer;
+  inherit (hostInfo.paths) data;
+in
+mkContainer {
+  inherit name;
+  net.access = [ { to = "@internet"; } ];
+  svc.autoStart = false;
+  net.expose.tcp = [ 53 ];
+  # TODO: figure out how to have this be the external dns resolver only
+  resources.fs."/var/lib/pihole" = {
+    hostPath = "${data}/misc/pihole";
+    isReadOnly = false;
+  };
+  containerConfig = _: {
+    services.pihole-ftl = {
+      enable = true;
+      stateDirectory = "/var/lib/pihole";
+      openFirewallDNS = true;
+      settings.dns.upstreams = [
+        "1.1.1.1"
+        "8.8.8.8"
+      ];
+      lists = [
+        {
+          url = "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts";
+          type = "block";
+          description = "StevenBlack unified hosts";
+          enabled = true;
+        }
+        {
+          url = "https://big.oisd.nl/domainswild";
+          type = "block";
+          description = "OISD Big";
+          enabled = true;
+        }
+        {
+          url = "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@latest/domains/pro.txt";
+          type = "block";
+          description = "HaGeZi Multi Pro";
+          enabled = true;
+        }
+        {
+          url = "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@latest/domains/tif.txt";
+          type = "block";
+          description = "HaGeZi Threat Intelligence Feeds";
+          enabled = true;
+        }
+      ];
+    };
+    services.pihole-web = {
+      enable = true;
+      ports = [ "443s" ];
+    };
+    systemd.services.pihole-ftl.serviceConfig.FailureAction = "exit-force";
+  };
+  resources.mem = "512M";
+}

containers/server-n0/ssh.nix

@@ -0,0 +1,186 @@
+{
+  pkgs,
+  hostInfo,
+  dsl,
+  self,
+  lib,
+  customPkgs,
+  ...
+}:
+let
+  inherit (dsl) mkContainer;
+  inherit (hostInfo.paths) server data;
+
+  sshTrait = {
+    resources.fs."/etc/ssh/authorized_keys" = {
+      hostPath = "${server}/keypair/id_ed25519.pub";
+      isReadOnly = true;
+    };
+    users = [
+      {
+        name = "runner";
+        uid = 1000;
+      }
+    ];
+    containerConfig =
+      {
+        lib,
+        pkgs,
+        ...
+      }:
+      lib.mkMerge [
+        {
+          services.openssh = {
+            enable = true;
+            authorizedKeysFiles = [ "/etc/ssh/authorized_keys" ];
+            settings = {
+              PasswordAuthentication = false;
+              UseDns = false;
+              PrintMotd = false;
+            };
+          };
+          systemd.services.sshd.serviceConfig.FailureAction = "exit-force";
+        }
+        {
+          environment.systemPackages = [
+            customPkgs.kitty-terminfo
+            pkgs.wezterm.terminfo
+          ];
+        }
+      ];
+  };
+
+  mkSsh =
+    {
+      name,
+      packages ? [ ],
+      nixDaemon ? true,
+      routingInfo ? {
+        username = lib.strings.removePrefix "ssh-" name;
+      },
+      net ? { },
+      resources ? { },
+      traits ? [ ],
+    }:
+    mkContainer {
+      inherit name;
+      inherit net resources;
+      svc.autoStart = true;
+      isolation.nixDaemon = nixDaemon;
+      traits = [ sshTrait ] ++ traits;
+      packages = with pkgs; [ tmux ] ++ packages;
+      miscProperties = self._sshpiperRegister routingInfo;
+    };
+in
+{
+  ssh-admin = mkSsh {
+    name = "ssh-admin";
+    net.access = [
+      {
+        to = "overseer";
+        tcp = [ 80 ];
+      }
+    ];
+    resources.fs = {
+      "/home/runner/data" = {
+        hostPath = server;
+        isReadOnly = false;
+      };
+      "/run/dbus/system_bus_socket" = {
+        hostPath = "/run/dbus/system_bus_socket";
+        isReadOnly = false;
+      };
+    };
+    packages = with pkgs; [
+      git
+      systemd
+    ];
+  };
+
+  ssh-botm = mkSsh {
+    name = "ssh-botm";
+    resources.fs."/home/runner/botm" = {
+      hostPath = "${data}/mc/botm";
+      isReadOnly = false;
+    };
+    packages = with pkgs; [
+      python3
+      jdk17
+      tmux
+    ];
+  };
+
+  ssh-generic = mkSsh {
+    name = "ssh-generic";
+    net.access = [ { to = "@internet"; } ];
+    resources = {
+      fs."/home/runner/data" = {
+        hostPath = "${data}/ssh/generic";
+        isReadOnly = false;
+      };
+      mem = "16G";
+      cpu = 65;
+    };
+    routingInfo = [
+      {
+        username = "autumn";
+        keysName = "generic";
+      }
+    ];
+  };
+
+  ssh-proxy = mkSsh {
+    name = "ssh-proxy";
+    resources.fs."/home/runner/data" = {
+      hostPath = "${data}/ssh/nyaa";
+      isReadOnly = true;
+    };
+    packages = with pkgs; [ python3 ];
+    routingInfo = [
+      {
+        username = "nyaa";
+        keysName = "nyaa";
+      }
+    ];
+  };
+
+  ssh-rogule = mkSsh {
+    name = "ssh-rogule";
+    resources.fs = {
+      "/home/runner/rogule-dev" = {
+        hostPath = "${data}/mc/rogule-dev";
+        isReadOnly = false;
+      };
+      "/home/runner/rogule-build" = {
+        hostPath = "${data}/mc/rogule-build";
+        isReadOnly = false;
+      };
+    };
+    packages = with pkgs; [
+      python3
+      jdk21
+    ];
+  };
+
+  ssh-starwalker = mkSsh {
+    name = "ssh-starwalker";
+    resources.fs."/home/runner/starwalker" = {
+      hostPath = "${data}/mc/starwalker";
+      isReadOnly = false;
+    };
+    packages = with pkgs; [
+      jdk21
+    ];
+  };
+
+  ssh-tl = mkSsh {
+    name = "ssh-tl";
+    resources.fs."/home/runner/tl" = {
+      hostPath = "${data}/mc/tl";
+      isReadOnly = false;
+    };
+    packages = with pkgs; [
+      jdk21
+    ];
+  };
+}

containers/server-n0/sshpiper.nix

@@ -0,0 +1,122 @@
+{
+  name,
+  pkgs,
+  hostInfo,
+  dsl,
+  lib,
+  containers,
+  customPkgs,
+  ...
+}:
+let
+  inherit (dsl) mkContainer mkService collectContainers;
+  inherit (hostInfo.paths) server secrets;
+  dataDir = hostInfo.paths.data;
+
+  mkSshpiperConfig =
+    {
+      pipes,
+      privateKey ? "/common_id",
+    }:
+    let
+      header = ''
+        version: "1.0"
+        pipes:
+      '';
+      mkPipe =
+        {
+          username,
+          host,
+          keysName ? username,
+        }:
+        ''
+          - from:
+              - username: "${username}"
+                authorized_keys: /etc/sshpiperd/keys/user_${keysName}/authorized_keys
+            to:
+                host: ${host}:22
+                username: "runner"
+                private_key: "${privateKey}"
+                ignore_hostkey: true
+        '';
+    in
+    header + lib.concatMapStrings mkPipe pipes;
+
+  sshPipes = lib.concatMap (
+    { container, data }: map (lib.mergeAttrs { host = container.name; }) data
+  ) (collectContainers "sshpiperRoutes" containers);
+
+  sshpiperYaml = pkgs.writeText "sshpiper.yaml" (mkSshpiperConfig {
+    pipes = sshPipes;
+  });
+in
+{
+  ${name} = mkContainer {
+    inherit name;
+    net.access = map (p: {
+      to = p.host;
+      tcp = [ 22 ];
+    }) sshPipes;
+    svc.autoStart = true;
+    net.forward.tcp = [
+      {
+        host = 22;
+        container = 2222;
+      }
+    ];
+    resources.fs = {
+      "/etc/sshpiperd/keys" = {
+        hostPath = "${dataDir}/core/sshpiper";
+        isReadOnly = true;
+      };
+      "/common_id" = {
+        hostPath = "${server}/keypair/id_ed25519";
+        isReadOnly = true;
+      };
+      "/etc/ssh/ssh_host_ed25519_key" = {
+        hostPath = "${secrets}/ssh_host_ed25519_key";
+        isReadOnly = true;
+      };
+      "/etc/ssh/ssh_host_rsa_key" = {
+        hostPath = "${secrets}/ssh_host_rsa_key";
+        isReadOnly = true;
+      };
+      "/etc/ssh/ssh_host_ecdsa_key" = {
+        hostPath = "${secrets}/ssh_host_ecdsa_key";
+        isReadOnly = true;
+      };
+    };
+    service =
+      { pkgs, ... }:
+      mkService {
+        description = "sshpiper SSH multiplexer";
+        execStart = "${lib.getExe' customPkgs.sshpiper "sshpiperd"} --log-level trace --server-key '/etc/ssh/ssh_host_*_key' ${lib.getExe' customPkgs.sshpiper "yaml"} --config /run/sshpiperd/config.yaml";
+        networkOnline = true;
+        serviceConfig = {
+          # sshpiperd rejects world-readable configs; copy store (444) -> runtime (600)
+          ExecStartPre = "${lib.getExe' pkgs.coreutils "install"} -m 600 ${sshpiperYaml} /run/sshpiperd/config.yaml";
+          RuntimeDirectory = "sshpiperd";
+          PrivateTmp = true;
+          RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX";
+        };
+      };
+  };
+
+  _sshpiperRegister =
+    let
+      doTranslate =
+        self:
+        {
+          username,
+          keysName ? username,
+        }:
+        {
+          inherit username keysName;
+          host = self.name;
+        };
+    in
+    args: self: {
+      sshpiperRoutes =
+        if lib.isAttrs args then [ (doTranslate self args) ] else lib.map (doTranslate self) args;
+    };
+}

flake.lock

@@ -0,0 +1,121 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "microvm": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "spectrum": "spectrum"
+      },
+      "locked": {
+        "lastModified": 1786300091,
+        "narHash": "sha256-4UFJOVGpaYtHW5yasSv80MaJxTBYdk2zyf3jhKtt0wA=",
+        "owner": "microvm-nix",
+        "repo": "microvm.nix",
+        "rev": "71beea0076cd46dafcee97a5a2e7d00cbba5bd2f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "microvm-nix",
+        "repo": "microvm.nix",
+        "type": "github"
+      }
+    },
+    "mtce": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1782711974,
+        "narHash": "sha256-ju0/qU6ZGqRxc5yDhuljrprG0BSPgguc8Kz3txktSjI=",
+        "owner": "Floweynt",
+        "repo": "monumenta-charm-eval",
+        "rev": "1aa8b532975679787f7c5bed180fd7fc0da6b905",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Floweynt",
+        "ref": "master",
+        "repo": "monumenta-charm-eval",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1786384358,
+        "narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "microvm": "microvm",
+        "mtce": "mtce",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "spectrum": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1785761586,
+        "narHash": "sha256-MWOMVqJJERwjQGgRIv8d6rlEBqbLM3VZWxHkNKIIZNw=",
+        "ref": "refs/heads/main",
+        "rev": "a7762d6f54b40560dd5255ce902e6e6a5d980fe9",
+        "revCount": 1416,
+        "type": "git",
+        "url": "https://spectrum-os.org/git/spectrum"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://spectrum-os.org/git/spectrum"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,154 @@
+{
+  description = "Server infrastructure";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    mtce.url = "github:Floweynt/monumenta-charm-eval/master";
+    mtce.inputs.nixpkgs.follows = "nixpkgs";
+    microvm.url = "github:microvm-nix/microvm.nix";
+    microvm.inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs =
+    {
+      nixpkgs,
+      mtce,
+      microvm,
+      ...
+    }:
+    let
+      system = "x86_64-linux";
+      inherit (nixpkgs) lib;
+      pkgs = import nixpkgs { inherit system; };
+
+      customPkgs = {
+        hopper-rs = pkgs.callPackage ./packages/hopper-rs.nix { };
+        sshpiper = pkgs.callPackage ./packages/sshpiper.nix { };
+        kitty-terminfo = pkgs.callPackage ./packages/kitty-terminfo.nix { };
+        ct = pkgs.callPackage ./packages/ct.nix { };
+        server-rebuild = pkgs.callPackage ./packages/server-rebuild.nix { };
+        server-status = pkgs.callPackage ./packages/server-status.nix { };
+        server-kexec = pkgs.callPackage ./packages/server-kexec.nix { };
+      };
+
+      ci = import ./ci.nix { inherit lib pkgs; };
+
+      inputs = { inherit mtce microvm; };
+
+      dsl = import ./dsl.nix { inherit lib microvm; };
+      introspect = import ./introspect.nix { inherit lib dsl; };
+
+      hostInfoFor = hostName: {
+        hostname = hostName;
+        mainGateway = "10.0.0.1";
+        isVm = false;
+        paths = {
+          server = "/persist/@containers";
+          data = "/persist/@containers/data";
+          secrets = "/persist/@secrets/";
+        };
+      };
+
+      specialArgsFor = hostName: {
+        hostInfo = hostInfoFor hostName;
+        inherit inputs customPkgs;
+      };
+
+      hostDefFor =
+        hostName:
+        import (./hosts + "/${hostName}.nix") {
+          inherit lib;
+          hostInfo = hostInfoFor hostName;
+        };
+
+      commonModule = import ./configuration.nix;
+
+      prepareHost =
+        hostName:
+        let
+          hostDef = hostDefFor hostName;
+          containerDirs = [
+            ./containers/common
+            (./containers + "/${hostName}")
+          ];
+          specialArgs = specialArgsFor hostName;
+          ctx = specialArgs // {
+            inherit pkgs lib;
+          };
+          containers = dsl.loadContainers {
+            inherit
+              hostName
+              containerDirs
+              ctx
+              ;
+          };
+
+          self = {
+            inherit
+              hostName
+              hostDef
+              containerDirs
+              specialArgs
+              ctx
+              containers
+              ;
+          };
+
+          hostCfg = lib.nixosSystem {
+            inherit system specialArgs;
+            modules = [
+              commonModule
+              microvm.nixosModules.host
+              hostDef.module
+              (dsl.mkHostFromContainers { inherit containers ctx; })
+            ];
+          };
+
+          testDir = ./tests + "/${hostName}";
+
+          tests =
+            if builtins.pathExists testDir then
+              lib.mapAttrs' (testName: testCfg: lib.nameValuePair "${testName}-${hostName}" testCfg) (
+                dsl.listMap testDir (
+                  _name: path:
+                  pkgs.testers.nixosTest (
+                    import path {
+                      inherit lib dsl commonModule;
+                      host = self;
+                    }
+                  )
+                )
+              )
+            else
+              { };
+        in
+        self // { inherit hostCfg tests; };
+
+      allHosts = lib.genAttrs (lib.attrNames (dsl.listMap ./hosts (hostName: _: hostName))) prepareHost;
+    in
+    {
+      nixosConfigurations = lib.mapAttrs (_: host: host.hostCfg) allHosts;
+
+      inspect = lib.mapAttrs
+        (name: host:
+          pkgs.writeText "inspect-${name}.json"
+            (builtins.toJSON (introspect.hostReport { inherit host; })))
+        allHosts;
+
+      packages.${system} = customPkgs // {
+        inherit ci;
+      };
+
+      formatter.${system} = pkgs.nixfmt;
+
+      checks.${system} = lib.foldl' (acc: host: acc // host.tests) { } (lib.attrValues allHosts) // {
+        inherit ci;
+        inherit (customPkgs)
+          ct
+          server-rebuild
+          server-status
+          server-kexec
+          ;
+      };
+    };
+}

hosts/example.nix

@@ -0,0 +1,38 @@
+_: {
+  # New host: copy this file to hosts/<name>.nix. Replace fileSystems/boot
+  # below with the real hardware (see hosts/server-n0.nix for a bigger
+  # example). Add a containers/<name>/ directory for its containers.  
+
+  module =
+    { lib, hostInfo, ... }:
+    {
+      networking.hostName = hostInfo.hostname;
+
+      fileSystems."/" = {
+        device = "/dev/disk/by-label/NIXOS_ROOT";
+        fsType = "ext4";
+      };
+
+      fileSystems."/persist" = {
+        device = "/dev/disk/by-label/NIXOS_PERSIST";
+        fsType = "ext4";
+      };
+
+      fileSystems."/nix" = {
+        device = "/dev/disk/by-label/NIXOS_NIX";
+        fsType = "ext4";
+      };
+
+      fileSystems."/var/log" = {
+        device = "/dev/disk/by-label/NIXOS_VARLOG";
+        fsType = "ext4";
+        neededForBoot = true;
+      };
+
+      boot.loader.grub.device = "nodev";
+      boot.loader.grub.efiSupport = true;
+      boot.loader.efi.canTouchEfiVariables = true;
+
+      nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+    };
+}

hosts/server-n0-darling-erasure.sh

@@ -0,0 +1,23 @@
+mkdir -p /mnt
+
+echo "--- erasing system state begin ---"
+# We first mount the btrfs root to /mnt
+# so we can manipulate btrfs subvolumes.
+mount -o subvol=/ /dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820 /mnt
+
+# back up the old root (important!!)
+BOOTID=$(date +"@boot-%Y_%m_%d-%H_%M_%S-%s")
+echo "backing up old root: /mnt/@root/ -> /mnt/@old-roots/$BOOTID..."
+echo "note that the timestamp is of *this* boot, not of the boot that created the old root!"
+mv /mnt/@root /mnt/@old-roots/$BOOTID
+btrfs property set -f -ts /mnt/@old-roots/$BOOTID ro true
+
+echo "restoring blank /@root subvolume..."
+btrfs subvolume snapshot /mnt/@root-blank /mnt/@root
+
+# Once we're done rolling back to a blank snapshot,
+# we can unmount /mnt and continue on the boot process.
+echo "--- erasing system state end ---"
+
+umount /mnt
+

hosts/server-n0.nix

@@ -0,0 +1,109 @@
+_: {
+  module =
+    {
+      config,
+      lib,
+      pkgs,
+      modulesPath,
+      hostInfo,
+      ...
+    }:
+    {
+      imports = [
+        (modulesPath + "/installer/scan/not-detected.nix")
+      ];
+
+      networking.hostName = hostInfo.hostname;
+
+      fileSystems."/" = {
+        device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820";
+        fsType = "btrfs";
+        options = [
+          "subvol=@root"
+          "noatime"
+          "compress=zstd"
+        ];
+      };
+
+      fileSystems."/nix" = {
+        device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820";
+        fsType = "btrfs";
+        options = [
+          "subvol=@nix"
+          "noatime"
+          "compress=zstd"
+        ];
+      };
+
+      fileSystems."/persist" = {
+        device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820";
+        fsType = "btrfs";
+        options = [
+          "subvol=@persist"
+          "noatime"
+          "compress=zstd"
+        ];
+      };
+
+      fileSystems."/btrfs_root" = {
+        device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820";
+        fsType = "btrfs";
+        options = [
+          "noatime"
+          "ro"
+          "compress=zstd"
+        ];
+      };
+
+      fileSystems."/var/log" = {
+        device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820";
+        fsType = "btrfs";
+        options = [
+          "subvol=@logs"
+          "noatime"
+          "compress=zstd"
+        ];
+        neededForBoot = true;
+      };
+
+      fileSystems."/boot" = {
+        device = "/dev/disk/by-uuid/7155-EF89";
+        fsType = "vfat";
+        options = [
+          "fmask=0022"
+          "dmask=0022"
+        ];
+      };
+
+      boot = {
+        loader.limine.enable = true;
+
+        kernelPackages = pkgs.linuxPackages_latest;
+        kernelModules = [ "kvm-intel" ];
+        extraModprobeConfig = "options kvm_intel nested=1";
+        extraModulePackages = [ ];
+        supportedFilesystems = [ "btrfs" ];
+        initrd = {
+          compressorArgs = [
+            "-22"
+            "-T0"
+            "--long"
+            "--ultra"
+          ];
+          availableKernelModules = [
+            "nvme"
+            "xhci_pci"
+            "ahci"
+            "usbhid"
+          ];
+          kernelModules = [ ];
+          systemd.enable = false;
+          postDeviceCommands = lib.mkBefore (builtins.readFile ./server-n0-darling-erasure.sh);
+        };
+      };
+
+      nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+      hardware.enableRedistributableFirmware = true;
+      hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+    };
+}

introspect.nix

@@ -0,0 +1,399 @@
+{ lib, dsl }:
+let
+  plain =
+    v:
+    if lib.isFunction v then
+      null
+    else if lib.isDerivation v then
+      "${v}"
+    else if lib.isList v then
+      map plain v
+    else if lib.isAttrs v then
+      lib.mapAttrs (_: plain) v
+    else if builtins.isPath v then
+      toString v
+    else
+      v;
+
+  relations = [
+    {
+      field = "after";
+      kind = "after";
+      reverse = false;
+    }
+    {
+      field = "before";
+      kind = "after";
+      reverse = true;
+    }
+    {
+      field = "wants";
+      kind = "wants";
+      reverse = false;
+    }
+    {
+      field = "wantedBy";
+      kind = "wants";
+      reverse = true;
+    }
+    {
+      field = "requires";
+      kind = "requires";
+      reverse = false;
+    }
+    {
+      field = "requiredBy";
+      kind = "requires";
+      reverse = true;
+    }
+    {
+      field = "upholds";
+      kind = "upholds";
+      reverse = false;
+    }
+    {
+      field = "upheldBy";
+      kind = "upholds";
+      reverse = true;
+    }
+    {
+      field = "bindsTo";
+      kind = "bindsTo";
+      reverse = false;
+    }
+    {
+      field = "partOf";
+      kind = "partOf";
+      reverse = false;
+    }
+    {
+      field = "requisite";
+      kind = "requisite";
+      reverse = false;
+    }
+    {
+      field = "conflicts";
+      kind = "conflicts";
+      reverse = false;
+    }
+    {
+      field = "onFailure";
+      kind = "onFailure";
+      reverse = false;
+    }
+    {
+      field = "onSuccess";
+      kind = "onSuccess";
+      reverse = false;
+    }
+  ];
+
+  relationFields = map (r: r.field) relations;
+
+  # raw NixOS command options, mirrors every generated Exec* wrapper
+  commandFields = [
+    "script"
+    "scriptArgs"
+    "preStart"
+    "postStart"
+    "reload"
+    "preStop"
+    "postStop"
+  ];
+
+  allUnits =
+    c:
+    lib.concatMap lib.attrValues [
+      c.systemd.paths
+      c.systemd.services
+      c.systemd.slices
+      c.systemd.sockets
+      c.systemd.targets
+      c.systemd.timers
+    ]
+    ++ c.systemd.mounts
+    ++ c.systemd.automounts;
+
+  unitSummary =
+    u:
+    {
+      inherit (u) name;
+      description = u.description or "";
+      enable = u.enable or true;
+      unitConfig = u.unitConfig or { };
+    }
+    // lib.filterAttrs (_: v: v != [ ]) (lib.genAttrs relationFields (f: u.${f} or [ ]))
+    // lib.optionalAttrs (u ? serviceConfig) (
+      {
+        inherit (u) serviceConfig;
+        environment = u.environment or { };
+        path = u.path or [ ];
+        restartIfChanged = u.restartIfChanged or null;
+        stopIfChanged = u.stopIfChanged or null;
+      }
+      # raw command text: the real logic behind the store-path Exec* wrappers
+      // lib.filterAttrs (_: v: v != "") (lib.genAttrs commandFields (f: u.${f} or ""))
+    )
+    // lib.optionalAttrs (u ? timerConfig) { inherit (u) timerConfig; }
+    // lib.optionalAttrs (u ? socketConfig) {
+      inherit (u) socketConfig;
+      listenStreams = u.listenStreams or [ ];
+      listenDatagrams = u.listenDatagrams or [ ];
+    }
+    // lib.optionalAttrs (u ? pathConfig) { inherit (u) pathConfig; }
+    // lib.optionalAttrs (u ? what) {
+      inherit (u) what;
+      where = u.where or "";
+      type = u.type or "";
+      options = u.options or "";
+    };
+
+  unitEdges =
+    node: u:
+    let
+      ref = n: "${node}:${n}";
+    in
+    lib.concatMap (
+      r:
+      map (
+        peer:
+        {
+          inherit (r) kind;
+        }
+        // (
+          if r.reverse then
+            {
+              from = ref peer;
+              to = ref u.name;
+            }
+          else
+            {
+              from = ref u.name;
+              to = ref peer;
+            }
+        )
+      ) (u.${r.field} or [ ])
+    ) relations;
+
+  fileSystemsOf =
+    c:
+    lib.mapAttrsToList (mountPoint: fs: {
+      inherit mountPoint;
+      inherit (fs)
+        device
+        fsType
+        options
+        neededForBoot
+        depends
+        ;
+    }) c.fileSystems;
+
+  firewallOf =
+    c:
+    let
+      fw = c.networking.firewall;
+      portsOf = f: {
+        inherit (f)
+          allowedTCPPorts
+          allowedUDPPorts
+          allowedTCPPortRanges
+          allowedUDPPortRanges
+          ;
+      };
+    in
+    {
+      inherit (fw) enable trustedInterfaces;
+      interfaces = lib.mapAttrs (_: portsOf) fw.interfaces;
+    }
+    // portsOf fw;
+
+  isBuildUser = n: builtins.match "nixbld[0-9]+" n != null;
+
+  usersOf = c: {
+    inherit (c.users) mutableUsers;
+    groups = lib.mapAttrs (_: g: {
+      inherit (g) gid;
+      members = lib.filter (m: !isBuildUser m) g.members;
+    }) c.users.groups;
+    users = lib.mapAttrs (_: u: {
+      inherit (u)
+        uid
+        group
+        home
+        shell
+        description
+        createHome
+        isNormalUser
+        isSystemUser
+        extraGroups
+        ;
+      authorizedKeys = u.openssh.authorizedKeys.keys;
+      passwordSet = lib.any (v: v != null) [
+        u.hashedPassword
+        u.hashedPasswordFile
+        u.password
+        u.initialPassword
+        u.initialHashedPassword
+      ];
+    }) (lib.filterAttrs (n: _: !isBuildUser n) c.users.users);
+  };
+in
+{
+  hostReport =
+    { host }:
+    let
+      cfg = host.hostCfg.config;
+      deployments = host.containers;
+      depList = lib.attrValues deployments;
+
+      configOf =
+        c:
+        if c.kind == "nspawn" then
+          cfg.containers.${c.name}.config
+        else
+          cfg.microvm.vms.${c.name}.config.config;
+      unitOf = c: "${dsl.serviceUnitName c}.service";
+
+      nodes = {
+        "@host" = cfg;
+      }
+      // lib.mapAttrs (_: configOf) deployments;
+
+      forward =
+        proto: c: p:
+        let
+          q = dsl.normalizePort p;
+        in
+        {
+          inherit proto;
+          hostPort = q.host;
+          target = c.name;
+          targetPort = q.container;
+          targetAddr = c.addr;
+        };
+
+      mountsOf =
+        c:
+        if c.kind == "nspawn" then
+          lib.mapAttrsToList (mountPoint: bm: {
+            inherit mountPoint;
+            inherit (bm) hostPath isReadOnly;
+          }) c.bindMounts
+        else
+          map (s: {
+            mountPoint = s.mountPoint or null;
+            hostPath = s.source;
+            isReadOnly = s.readOnly or false;
+            inherit (s) proto tag;
+          }) c.shares;
+    in
+    plain {
+      meta = {
+        host = host.hostName;
+        system = cfg.nixpkgs.hostPlatform.system;
+        inherit (cfg.system) stateVersion;
+        inherit (host.specialArgs.hostInfo) paths mainGateway;
+        toplevel = cfg.system.build.toplevel;
+      };
+
+      deployments = lib.mapAttrs (_: c: {
+        inherit (c)
+          name
+          kind
+          autoStart
+          access
+          allowInternet
+          needsGateway
+          includedInTests
+          ;
+        # names only; c.depends holds whole deployment objects
+        depends = map (d: d.name) c.depends;
+        unit = unitOf c;
+        limits =
+          if c.kind == "nspawn" then
+            { inherit (c) memoryMax cpuQuota privateUsers; }
+          else
+            { inherit (c) vcpu mem hypervisor; };
+        inherit (c) miscProperties;
+      }) deployments;
+
+      access = {
+        specials = [
+          "@host"
+          "@internet"
+        ];
+        edges = lib.concatMap (
+          c:
+          map (a: {
+            from = c.name;
+            inherit (a) to;
+            tcp = a.tcp or [ ];
+            udp = a.udp or [ ];
+          }) c.access
+        ) depList;
+      };
+
+      kernel = {
+        sysctl = cfg.boot.kernel.sysctl;
+        params = cfg.boot.kernelParams;
+        modules = cfg.boot.kernelModules;
+      };
+
+      network = {
+        inherit (cfg.networking) nameservers;
+        links = lib.mapAttrs (_: c: {
+          inherit (c)
+            kind
+            iface
+            mac
+            addr
+            hostAddr
+            allowInternet
+            needsGateway
+            ;
+        }) deployments;
+        hostInterfaces = lib.mapAttrs (_: i: { inherit (i) ipv4; }) cfg.networking.interfaces;
+        dnsmasq = cfg.services.dnsmasq.settings;
+      };
+
+      ports = {
+        forwards = lib.concatMap (
+          c: map (forward "tcp" c) c.hostPorts ++ map (forward "udp" c) c.hostUdpPorts
+        ) depList;
+        declared = lib.mapAttrs (_: c: c.ports or [ ]) deployments;
+        firewall = lib.mapAttrs (_: firewallOf) nodes;
+      };
+
+      mounts = {
+        fileSystems = lib.mapAttrs (_: fileSystemsOf) nodes;
+        declared = lib.mapAttrs (_: mountsOf) deployments;
+      };
+
+      users = lib.mapAttrs (_: usersOf) nodes;
+
+      units = lib.mapAttrs (
+        _: c: lib.listToAttrs (map (u: lib.nameValuePair u.name (unitSummary u)) (allUnits c))
+      ) nodes;
+
+      dependencies = {
+        edges =
+          lib.concatLists (lib.mapAttrsToList (node: c: lib.concatMap (unitEdges node) (allUnits c)) nodes)
+          ++ map (c: {
+            from = "@host:${unitOf c}";
+            to = "${c.name}:multi-user.target";
+            kind = "spawns";
+          }) depList;
+      };
+
+      nftables = {
+        inherit (cfg.networking.nftables) enable ruleset;
+        tables = lib.mapAttrs (n: t: {
+          name = t.name or n;
+          inherit (t) family enable content;
+        }) cfg.networking.nftables.tables;
+        rendered = builtins.readFile (
+          builtins.elemAt cfg.systemd.services.nftables.serviceConfig.ExecStart 1
+        );
+      };
+    };
+}

mount.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DEV=/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820
+EFI=/dev/disk/by-uuid/7155-EF89
+
+mount -o subvol=@root    "$DEV" /mnt
+mkdir -p /mnt/{nix,persist,var/log,btrfs_root,boot}
+mount -o subvol=@nix     "$DEV" /mnt/nix
+mount -o subvol=@persist "$DEV" /mnt/persist
+mount -o subvol=@logs    "$DEV" /mnt/var/log
+mount -o subvolid=5      "$DEV" /mnt/btrfs_root
+mount "$EFI" /mnt/boot
+
+btrfs subvolume create /mnt/persist/@secrets

packages/ct.nix

@@ -0,0 +1,8 @@
+{
+  writeShellApplication,
+  dtach,
+}:
+import ./mk-script.nix { inherit writeShellApplication; } {
+  name = "ct";
+  runtimeInputs = [ dtach ];
+}

packages/hopper-lazy-resolve.patch

@@ -0,0 +1,92 @@
+diff --git a/src/config/router.rs b/src/config/router.rs
+index cfee4f8..9247cc3 100644
+--- a/src/config/router.rs
++++ b/src/config/router.rs
+@@ -58,11 +58,10 @@ impl Router for RouterConfig {
+             .or(self.default.as_ref())
+             .ok_or(RouterError::NoServer)?;
+ 
+-        let address: SocketAddr = match route.ip {
+-            RouteType::Simple(address) => address.into(),
+-            RouteType::Balanced(ref list) => {
+-                let hash = client.hash();
+-                list.get(hash as usize)
++        let address: SocketAddr = match &route.ip {
++            RouteType::Simple(addr) => addr.resolve().ok_or(RouterError::Unavailable)?,
++            RouteType::Balanced(list) => {
++                list.get(client.hash() as usize).ok_or(RouterError::Unavailable)?
+             }
+         };
+ 
+diff --git a/src/config/router/balancer.rs b/src/config/router/balancer.rs
+index 770a01b..b2e2491 100644
+--- a/src/config/router/balancer.rs
++++ b/src/config/router/balancer.rs
+@@ -22,7 +22,7 @@ pub struct Balanced {
+ // }
+ 
+ impl Balanced {
+-    pub(super) fn get(&self, n: usize) -> SocketAddr {
+-        self.servers[n & self.servers.len()].into()
++    pub(super) fn get(&self, n: usize) -> Option<SocketAddr> {
++        self.servers[n % self.servers.len()].resolve()
+     }
+ }
+diff --git a/src/config/router/resolver.rs b/src/config/router/resolver.rs
+index b66a16a..8fd3b0d 100644
+--- a/src/config/router/resolver.rs
++++ b/src/config/router/resolver.rs
+@@ -2,26 +2,26 @@ use std::net::{SocketAddr, ToSocketAddrs};
+ 
+ use serde::{Deserialize, Deserializer};
+ 
+-#[derive(Deserialize, Debug, Clone, Copy)]
+-pub(super) struct ResolvableAddr(#[serde(deserialize_with = "resolve_hostname")] SocketAddr);
++#[derive(Deserialize, Debug, Clone)]
++pub(super) struct ResolvableAddr(#[serde(deserialize_with = "parse_addr_string")] String);
+ 
+-fn resolve_hostname<'de, D>(deserializer: D) -> Result<SocketAddr, D::Error>
++fn parse_addr_string<'de, D>(deserializer: D) -> Result<String, D::Error>
+ where
+     D: Deserializer<'de>,
+ {
+     use serde::de::Error;
+ 
+-    let inner = <String>::deserialize(deserializer)?;
+-
+-    let mut addr = inner
+-        .to_socket_addrs()
+-        .map_err(|err| Error::custom(format!("invalid hostname format: {err}")))?;
+-
+-    addr.next().ok_or_else(|| Error::missing_field("address"))
++    let s = String::deserialize(deserializer)?;
++    if !s.contains(':') {
++        return Err(Error::custom(format!(
++            "invalid socket address syntax, expected host:port: {s}"
++        )));
++    }
++    Ok(s)
+ }
+ 
+-impl From<ResolvableAddr> for SocketAddr {
+-    fn from(addr: ResolvableAddr) -> Self {
+-        addr.0
++impl ResolvableAddr {
++    pub fn resolve(&self) -> Option<SocketAddr> {
++        self.0.to_socket_addrs().ok()?.next()
+     }
+ }
+diff --git a/src/server/router.rs b/src/server/router.rs
+index 4aae8e7..ad2849b 100644
+--- a/src/server/router.rs
++++ b/src/server/router.rs
+@@ -7,6 +7,9 @@ use thiserror::Error;
+ pub enum RouterError {
+     #[error("no server with such hostname has been found")]
+     NoServer,
++
++    #[error("backend server is currently unavailable")]
++    Unavailable,
+ }
+ 
+ // #[async_trait::async_trait]

packages/hopper-rs.nix

@@ -0,0 +1,29 @@
+{
+  lib,
+  fetchFromGitHub,
+  rustPlatform,
+}:
+rustPlatform.buildRustPackage {
+  pname = "hopper-rs";
+  version = "v0.6.2-2-g73cb171";
+
+  src = fetchFromGitHub {
+    owner = "bra1l0r";
+    repo = "hopper-rs";
+    rev = "73cb171e203cef6c9c444b69c43ceefa2da213e4";
+    hash = "sha256-QhDH03KHKtsh7Ch44TE02sr42i/CPD9BGK7QQ6PwHKc=";
+  };
+
+  patches = [ ./hopper-lazy-resolve.patch ];
+
+  cargoHash = "sha256-bC5BlP3opYYHkK/mauF9qdTczTdmGQD7yEOCOMylNOc=";
+
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Lightweight Minecraft reverse proxy";
+    homepage = "https://github.com/bra1l0r/hopper-rs";
+    license = licenses.gpl3Only;
+    mainProgram = "hopper";
+  };
+}

packages/kitty-terminfo.nix

@@ -0,0 +1,9 @@
+{
+  runCommand,
+  ncurses,
+  kitty,
+}:
+runCommand "kitty-terminfo" { nativeBuildInputs = [ ncurses ]; } ''
+  mkdir -p $out/share/terminfo
+  tic -x -o $out/share/terminfo ${kitty.src}/terminfo/kitty.terminfo
+''

packages/mk-script.nix

@@ -0,0 +1,9 @@
+{ writeShellApplication }:
+{
+  name,
+  runtimeInputs ? [ ],
+}:
+writeShellApplication {
+  inherit name runtimeInputs;
+  text = builtins.readFile (../scripts + "/${name}.sh");
+}

packages/server-kexec.nix

@@ -0,0 +1,8 @@
+{
+  writeShellApplication,
+  kexec-tools,
+}:
+import ./mk-script.nix { inherit writeShellApplication; } {
+  name = "server-kexec";
+  runtimeInputs = [ kexec-tools ];
+}

packages/server-rebuild.nix

@@ -0,0 +1,2 @@
+{ writeShellApplication }:
+import ./mk-script.nix { inherit writeShellApplication; } { name = "server-rebuild"; }

packages/server-status.nix

@@ -0,0 +1,2 @@
+{ writeShellApplication }:
+import ./mk-script.nix { inherit writeShellApplication; } { name = "server-status"; }

packages/sshpiper.nix

@@ -0,0 +1,33 @@
+{
+  lib,
+  buildGoModule,
+  fetchFromGitHub,
+}:
+buildGoModule {
+  pname = "sshpiper";
+  version = "unstable-2024";
+
+  src = fetchFromGitHub {
+    owner = "tg123";
+    repo = "sshpiper";
+    rev = "e5c5b904cfa57f7d60e37cb38cdb0bbcaadb022e";
+    hash = "sha256-8Lqf7pCcAWAgz7tclKiV0DyHhbaaBule6/X30Fzm8DY=";
+    fetchSubmodules = true;
+  };
+
+  vendorHash = "sha256-6FU1r6vY6AorbP5z+C3zJpHQzpqzkzK9iQFnxDmYv8c=";
+
+  subPackages = [
+    "cmd/sshpiperd"
+    "plugin/yaml"
+  ];
+
+  tags = [ "full" ];
+
+  meta = with lib; {
+    description = "SSH reverse proxy / multiplexer";
+    homepage = "https://github.com/tg123/sshpiper";
+    license = licenses.mit;
+    mainProgram = "sshpiperd";
+  };
+}

scripts/ct.sh

@@ -0,0 +1,136 @@
+cmd=${1:-help}
+ct_name=${2:-}
+
+require_name() {
+  if [ -z "$ct_name" ]; then
+    echo "usage: ct $cmd <container>" >&2
+    exit 1
+  fi
+}
+
+case "$cmd" in
+  list)
+    sep() { printf '%0.s-' {1..60}; printf '\n'; }
+    uptime_str() {
+      awk '{s=int($1);d=int(s/86400);h=int((s%86400)/3600);m=int((s%3600)/60);
+        if(d>0)printf "%d days %d hours %d minutes",d,h,m;
+        else if(h>0)printf "%d hours %d minutes",h,m;
+        else printf "%d minutes",m}' /proc/uptime
+    }
+
+    printf 'HOST: %s  UP: %s\n' "$(hostname)" "$(uptime_str)"
+    sep
+
+    printf 'CONTAINERS\n'
+    mapfile -t all_units < <({
+      systemctl list-unit-files --no-legend --no-pager --plain 2>/dev/null
+      systemctl list-units --all --no-legend --no-pager --plain 2>/dev/null
+    } | awk '$1 ~ /^container@.+\.service$/ {print $1}' | sort -u)
+
+    if [ ${#all_units[@]} -gt 0 ]; then
+      systemctl show "${all_units[@]}" \
+        --property=Id,UnitFileState,ActiveState --no-pager 2>/dev/null \
+      | awk '
+        /^Id=container@/ { n=$0; sub(/^Id=container@/,"",n); sub(/\.service$/,"",n) }
+        /^UnitFileState=/ { f=$0; sub(/^UnitFileState=/,"",f) }
+        /^ActiveState=/   { a=$0; sub(/^ActiveState=/,"",a) }
+        /^[[:space:]]*$/ { if (n!="") { _print(); n=f=a="" } }
+        END              { if (n!="") _print() }
+        function _print() {
+          start = (f=="enabled") ? "auto" : "manual"
+          if      (a=="active")   st="running"
+          else if (a=="failed")   st="FAILED"
+          else if (a=="inactive") st="stopped"
+          else                    st=(a!="") ? a : "not-loaded"
+          mid = "-"
+          if (a == "active") {
+            midcmd = "machinectl show " n " -p Id --value 2>/dev/null"
+            got = ""
+            midcmd | getline got
+            close(midcmd)
+            if (got != "") mid = substr(got,1,12) "..."
+          }
+          printf "  %-35s %-8s %-10s %s\n", n, start, st, mid
+        }
+      '
+    else
+      printf '  none\n'
+    fi
+
+    sep
+
+    printf 'FAILED UNITS (host)\n'
+    failed=$(systemctl list-units --state=failed --no-legend --no-pager --plain 2>/dev/null \
+      | awk 'NF>0{print $1}')
+    if [ -z "$failed" ]; then printf '  none\n'
+    else printf '%s\n' "$failed" | sed 's/^/  /'; fi
+
+    sep
+
+    printf 'FAILED UNITS (containers)\n'
+    any_failed=0
+    machines=$(machinectl list --no-legend --no-pager 2>/dev/null | awk 'NF>0{print $1}')
+    for m in $machines; do
+      ct_failed=$(sudo systemctl -M "$m" list-units --state=failed \
+        --no-legend --no-pager --plain 2>/dev/null \
+        | awk 'NF>0 && $1!~/^container@/{print $1}' || true)
+      if [ -n "$ct_failed" ]; then
+        printf '  [%s]\n' "$m"
+        printf '%s\n' "$ct_failed" | sed 's/^/    /'
+        any_failed=1
+      fi
+    done
+    [ "$any_failed" -eq 0 ] && printf '  none\n'
+
+    sep
+
+    df -h /persist | awk 'NR==2{printf "DISK  %s used / %s total (%s)\n",$3,$2,$5}'
+    free -h | awk '/^Mem:/{printf "MEM   %s used / %s total\n",$3,$2}'
+    ;;
+  start)
+    require_name
+    sudo systemctl start "container@$ct_name"
+    ;;
+  stop)
+    require_name
+    sudo systemctl stop "container@$ct_name"
+    ;;
+  restart)
+    require_name
+    sudo systemctl restart "container@$ct_name"
+    ;;
+  status)
+    require_name
+    machinectl status "$ct_name"
+    ;;
+  logs)
+    require_name
+    shift 2
+    sudo journalctl -M "$ct_name" --no-hostname "$@"
+    ;;
+  shell)
+    require_name
+    exec sudo machinectl shell "$ct_name"
+    ;;
+  attach)
+    require_name
+    leader=$(machinectl show "$ct_name" -p Leader --value 2>/dev/null)
+    if [ -z "$leader" ] || [ "$leader" = "0" ]; then
+      printf 'error: container %s is not running\n' "$ct_name" >&2
+      exit 1
+    fi
+    exec sudo nsenter -m -t "$leader" -- dtach -a -r none /run/ct-attach/attach.sock
+    ;;
+  *)
+    printf 'usage: ct <cmd> [container]\n\n'
+    printf '  %-9s %-13s %s\n' \
+      "list"    ""             "list all containers with state (running/stopped/FAILED)" \
+      "start"   "<container>"  "start a container" \
+      "stop"    "<container>"  "stop a container" \
+      "restart" "<container>"  "restart a container" \
+      "status"  "<container>"  "show container status" \
+      "logs"    "<container>"  "view logs (extra args forwarded to journalctl)" \
+      "shell"   "<container>"  "open a shell" \
+      "attach"  "<container>"  "attach to Minecraft console (detach: Ctrl+\\)"
+    ;;
+esac

scripts/server-kexec.sh

@@ -0,0 +1,17 @@
+SYSTEM=${1:-/nix/var/nix/profiles/system}
+
+if [[ ! -e "$SYSTEM/kernel" ]]; then
+  echo "error: no kernel found at $SYSTEM" >&2
+  exit 1
+fi
+
+echo "kexec: loading $SYSTEM"
+echo "kernel-params: $(cat "$SYSTEM/kernel-params")"
+
+sudo kexec \
+  --load "$SYSTEM/kernel" \
+  --initrd="$SYSTEM/initrd" \
+  --command-line="$(cat "$SYSTEM/kernel-params") init=$SYSTEM/init"
+
+echo "kexec: executing (systemd will stop services first)"
+sudo systemctl kexec

scripts/server-rebuild.sh

@@ -0,0 +1 @@
+exec sudo nixos-rebuild switch --flake "/persist/@configuration#$(hostname)" "$@"

scripts/server-status.sh

@@ -0,0 +1,21 @@
+state=$(systemctl is-system-running 2>/dev/null || true)
+uptime_str() {
+  awk '{s=int($1);d=int(s/86400);h=int((s%86400)/3600);m=int((s%3600)/60);
+    if(d>0)printf "%d days %d hours",d,h;
+    else if(h>0)printf "%d hours %d min",h,m;
+    else printf "%d min",m}' /proc/uptime
+}
+printf 'HOST: %s  UP: %s  STATE: %s\n' "$(hostname)" "$(uptime_str)" "$state"
+printf '%0.s-' {1..60}; printf '\n'
+
+systemctl list-units --all --type=service \
+  --state=active,failed --no-legend --no-pager --plain 2>/dev/null \
+| awk '
+  NF >= 4 {
+    unit=$1; active=$3; sub=$4
+    if (unit ~ /^(getty@|console-getty\.service)/) next
+    if (unit ~ /^systemd-(tmpfiles|binfmt|modules-load|sysctl|ask-password|remount|random-seed|update-done|hibernate|sleep|suspend|reboot|poweroff)/) next
+    state = (active == "failed") ? "FAILED" : sub
+    printf "  %-50s %s\n", unit, state
+  }
+'

statix.toml

@@ -0,0 +1,2 @@
+disabled = ["repeated_keys"]
+ignore = [".direnv"]

tests/server-n0/integration.nix

@@ -0,0 +1,190 @@
+{
+  lib,
+  host,
+  dsl,
+  commonModule,
+}:
+pkgs:
+let
+  inherit (host.specialArgs.hostInfo) paths;
+  serverDataDir = paths.data;
+  serverSecretsDir = paths.secrets;
+
+  testContainers = dsl.filterDeployments (c: c.includedInTests) host.containers;
+
+  it = dsl.mkIntegrationTest { inherit host testContainers pkgs; };
+
+  nginxAcmeDomains = [
+    "devenv.floweytf.com"
+    "maven.floweytf.com"
+    "git.floweytf.com"
+    "utils.floweytf.com"
+    "mail.floweytf.com"
+  ];
+
+  nginxAcmeDir = domain: "${serverDataDir}/core/nginx/acme/${domain}";
+
+  fakeCert =
+    domain:
+    pkgs.runCommand "fake-cert-${domain}"
+      {
+        nativeBuildInputs = [ pkgs.openssl ];
+      }
+      ''
+        mkdir -p "$out"
+        openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
+          -subj "/CN=${domain}" \
+          -keyout "$out/key.pem" -out "$out/fullchain.pem" 2>/dev/null
+      '';
+
+  mkdirAcmeDirs = lib.concatMapStrings (
+    domain: ''mkdir -p "${nginxAcmeDir domain}"'' + "\n"
+  ) nginxAcmeDomains;
+
+  installFakeCerts = lib.concatMapStrings (
+    domain:
+    let
+      dir = nginxAcmeDir domain;
+      cert = fakeCert domain;
+    in
+    ''
+      if [[ ! -f "${dir}/fullchain.pem" ]]; then
+        install -m 644 ${cert}/fullchain.pem "${dir}/fullchain.pem"
+        install -m 600 ${cert}/key.pem "${dir}/key.pem"
+      fi
+    ''
+  ) nginxAcmeDomains;
+
+  # used in ssh-admin
+  testSshKey =
+    pkgs.runCommand "test-ssh-key"
+      {
+        nativeBuildInputs = [ pkgs.openssh ];
+      }
+      ''
+        mkdir -p "$out"
+        ssh-keygen -q -t ed25519 -N "" -f "$out/key" -C test
+      '';
+  sshpiperTestUser = "admin";
+
+  hostExporterPort = 9100;
+  hostCadvisorPort = 9200;
+  hostSshdPort = 5952;
+in
+{
+  name = "integration-${host.hostName}";
+
+  nodes = {
+    internet = it.internetNode;
+
+    server =
+      { pkgs, lib, ... }:
+      {
+        imports = [
+          commonModule
+          host.specialArgs.inputs.microvm.nixosModules.host
+          (dsl.mkHostFromContainers {
+            containers = testContainers;
+            inherit (host) ctx;
+          })
+          it.serverModule
+        ];
+
+        containers.nginx.config = {
+          systemd.services = lib.genAttrs (map (d: "acme-order-renew-${d}") nginxAcmeDomains) (_: {
+            enable = lib.mkForce false;
+          });
+        };
+
+        system.activationScripts.test-secrets =
+          # bash
+          ''
+            mkdir -p /nix/var/nix/db /nix/var/nix/daemon-socket
+
+            mkdir -p /var/log/journal
+
+            ${mkdirAcmeDirs}
+            mkdir -p "${serverSecretsDir}" \
+              "${serverDataDir}/core/sshpiper" \
+              "${serverDataDir}/core/mail" \
+              "${serverDataDir}/overseer/loki/data" \
+              "${serverDataDir}/overseer/prometheus/data" \
+              "${serverDataDir}/overseer/grafana" \
+              "${serverDataDir}/web/git" \
+              "${serverDataDir}/web/git-ci-runner/nix-rw-store" \
+              "${serverDataDir}/web/git-ci-runner/docker" \
+              "${serverDataDir}/web/git-ci-runner/runner-state" \
+              "${serverSecretsDir}/ci-runner" \
+              "${serverDataDir}/web/reposilite" \
+              "${serverDataDir}/web/devenv/config" \
+              "${serverDataDir}/web/devenv/workspace" \
+              "${serverDataDir}/ssh/generic" \
+              "${serverDataDir}/ssh/nyaa" \
+              "${serverDataDir}/mc/botm" \
+              "${serverDataDir}/mc/rogule-dev" \
+              "${serverDataDir}/mc/rogule-build" \
+              "${serverDataDir}/mc/starwalker" \
+              "${serverDataDir}/mc/tl"
+
+            chown -R 1000:1000 \
+              "${serverDataDir}/web/git" \
+              "${serverDataDir}/web/reposilite" \
+              "${serverDataDir}/web/devenv/config" \
+              "${serverDataDir}/web/devenv/workspace" \
+              "${serverDataDir}/core/mail" \
+              "${serverDataDir}/ssh/generic" \
+              "${serverDataDir}/mc/botm" \
+              "${serverDataDir}/mc/rogule-dev" \
+              "${serverDataDir}/mc/rogule-build" \
+              "${serverDataDir}/mc/starwalker" \
+              "${serverDataDir}/mc/tl"
+
+            for keytype in ed25519 rsa ecdsa; do
+              keyfile="${serverSecretsDir}/ssh_host_''${keytype}_key"
+              if [[ ! -f "$keyfile" ]]; then
+                ${lib.getExe' pkgs.openssh "ssh-keygen"} -q -t "$keytype" -f "$keyfile" -N ""
+              fi
+            done
+
+            # stalwart rejects weak/dictionary passwords
+            for f in mail-admin-password mail-account-password; do
+              target="${serverSecretsDir}/$f"
+              if [[ ! -f "$target" ]]; then
+                head -c 32 /dev/urandom | base64 | tr -d '\n' > "$target"
+                chmod 600 "$target"
+              fi
+            done
+
+            if [[ ! -f "${serverSecretsDir}/ci-runner/token" ]]; then
+              echo "TOKEN=test-registration-token" > "${serverSecretsDir}/ci-runner/token"
+              chmod 600 "${serverSecretsDir}/ci-runner/token"
+            fi
+
+            if [[ ! -f "${serverSecretsDir}/code-server-env" ]]; then
+              echo "test-secret" > "${serverSecretsDir}/code-server-env"
+              chmod 600 "${serverSecretsDir}/code-server-env"
+            fi
+
+            ${installFakeCerts}
+            mkdir -p "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}"
+            install -m 644 ${testSshKey}/key.pub \
+              "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}/authorized_keys"
+          '';
+      };
+  };
+
+  testScript =
+    it.testScriptPreamble
+    + ''
+      # store paths are world-readable; ssh refuses a key with those permissions
+      SSH_KEY = "/root/test-ssh-key"
+      SSH_OPTS = f"-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -i {SSH_KEY}"
+      TEST_SSH_KEY_STORE = "${testSshKey}/key"
+
+      SSHPIPER_TEST_USER = "${sshpiperTestUser}"
+      HOST_EXPORTER_PORT = ${toString hostExporterPort}
+      HOST_CADVISOR_PORT = ${toString hostCadvisorPort}
+      HOST_SSHD_PORT = ${toString hostSshdPort}
+    ''
+    + builtins.readFile ./integration.py;
+}

tests/server-n0/integration.py

@@ -0,0 +1,330 @@
+import json
+import re
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    server: Any = None
+    internet: Any = None
+    start_all: Any = None
+    subtest: Any = None
+    retry: Any = None
+
+    NSENTER: str = ""
+    CURL: str = ""
+    IP: str = ""
+    PING: str = ""
+    DIG: str = ""
+    SSH: str = ""
+    OPENSSL: str = ""
+    TIMEOUT: str = ""
+    included: list[str] = []
+    enterable: list[str] = []
+    allow_internet: list[str] = []
+    not_allow_internet: list[str] = []
+    has_gateway: list[str] = []
+    no_gateway: list[str] = []
+    ADDR: dict[str, str] = {}
+    HOST_ADDR: dict[str, str] = {}
+    IFACE: dict[str, str] = {}
+    KIND: dict[str, str] = {}
+    EDGES: list[tuple[str, str, list[int], list[int]]] = []
+    SSH_KEY: str = ""
+    SSH_OPTS: str = ""
+    TEST_SSH_KEY_STORE: str = ""
+    SSHPIPER_TEST_USER: str = ""
+    HOST_EXPORTER_PORT: int = 0
+    HOST_CADVISOR_PORT: int = 0
+    HOST_SSHD_PORT: int = 0
+
+SW = "/run/current-system/sw/bin"
+SH = f"{SW}/sh"
+
+PEER_EDGES = {(src, to) for src, to, _, _ in EDGES if to not in ("@host", "@internet")}
+
+
+def failed_units(node_arg=""):
+    out = server.succeed(f"systemctl {node_arg} list-units --state=failed --no-legend --plain || true")
+    return [
+        line.split()[0]
+        for line in out.splitlines()
+        if line.strip() and not line.split()[0].startswith("acme-")
+    ]
+
+
+def container_pid(name):
+    return server.succeed(f"machinectl show {name} -p Leader --value").strip()
+
+
+def net_cmd(name, cmd):
+    pid = container_pid(name)
+    return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}"
+
+
+def container_ip(name):
+    out = server.succeed(net_cmd(name, f"{IP} -4 -o addr show scope global"))
+    return out.split()[3].split("/")[0]
+
+
+def has_ipv6_link_local(name):
+    out = server.succeed(net_cmd(name, f"{IP} -6 -o addr show scope link"))
+    return out.strip() != ""
+
+
+def container_iface(name):
+    out = server.succeed(net_cmd(name, f"{IP} -j link show"))
+    for link in json.loads(out):
+        if link.get("link_type") == "ether" and "LOOPBACK" not in link.get("flags", []):
+            return link["ifname"]
+    raise Exception(f"no ethernet link found for {name}")
+
+
+def connect(host, port, timeout=5):
+    """Open a TCP connection and close it again, using only tools every node has."""
+    return f"{SW}/timeout {timeout} {SH} -c 'exec 3<>/dev/tcp/{host}/{port} && exec 3<&-'"
+
+
+def ct_connect(name, host, port, timeout=5):
+    return net_cmd(name, connect(host, port, timeout))
+
+
+def ext_connect(port, timeout=5):
+    """A published port, probed the way a published port is actually used.
+
+    Probing it from a container measures a different path - containers are
+    excluded from DNAT entirely.
+    """
+    return connect(server_ip, port, timeout)
+
+
+def spoof_counter():
+    out = server.succeed("nft -j list counter inet container-ip spoofed")
+    return json.loads(out)["nftables"][1]["counter"]["packets"]
+
+
+start_all()
+server.succeed(f"install -m 600 {TEST_SSH_KEY_STORE} {SSH_KEY}")
+
+with subtest("(functional): containers autostart on boot with no errors"):
+    server.wait_for_unit("multi-user.target")
+    for name in included:
+        unit = "container" if KIND[name] == "nspawn" else "microvm"
+        server.wait_for_unit(f"{unit}@{name}.service")
+        server.succeed(f"systemctl is-active {unit}@{name}.service")
+
+    assert failed_units() == [], f"host has failed units: {failed_units()}"
+    for name in enterable:
+        assert failed_units(f"-M {name}") == [], f"{name} has failed units: {failed_units(f'-M {name}')}"
+
+internet.wait_for_unit("dummy-http.service")
+internet_ip = internet.succeed("ip -4 -o addr show dev eth1 scope global").split()[3].split("/")[0]
+server_ip = server.succeed("ip -4 -o addr show dev eth1 scope global").split()[3].split("/")[0]
+
+with subtest("(functional): derived addressing matches what the ruleset assumes"):
+    for name in enterable:
+        assert container_ip(name) == ADDR[name], f"{name} has {container_ip(name)}, ruleset says {ADDR[name]}"
+        assert container_iface(name) == IFACE[name], f"{name} is on {container_iface(name)}, ruleset says {IFACE[name]}"
+    for name in [c for c in included if KIND[c] == "vm"]:
+        server.succeed(f"{IP} link show {IFACE[name]}")
+        server.wait_until_succeeds(f"{PING} -c 1 -W 2 {ADDR[name]}", timeout=180)
+
+with subtest("(isolation): internet egress follows the access list"):
+    for name in allow_internet:
+        server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2"))
+    for name in not_allow_internet:
+        server.fail(net_cmd(name, f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2"))
+    for name in has_gateway:
+        server.succeed(net_cmd(name, f"{IP} route show default | grep -q ."))
+    for name in no_gateway:
+        server.fail(net_cmd(name, f"{IP} route show default | grep -q ."))
+
+with subtest("(functional): container DNS still resolves"):
+    resolved = server.succeed(
+        net_cmd("git", f"{DIG} +short +time=3 @{HOST_ADDR['git']} nginx.containers")
+    ).strip()
+    assert resolved == ADDR["nginx"], f"nginx.containers resolved to {resolved!r}, expected {ADDR['nginx']}"
+
+with subtest("(functional): web services respond"):
+    for name, port in [("devenv", 8080), ("maven", 8080), ("git", 3000)]:
+        server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2"))
+
+with subtest("(functional): mail accepts connections"):
+    smtp_probe = net_cmd("mail", f"{SH} -c 'exec 3<>/dev/tcp/{ADDR['mail']}/25 && head -1 <&3'")
+    server.wait_until_succeeds(f"{smtp_probe} | grep -q '^220'", timeout=60)
+
+    imap_probe = net_cmd(
+        "mail",
+        f"{TIMEOUT} 3 {OPENSSL} s_client -connect {ADDR['mail']}:993 -quiet -verify_quiet 2>/dev/null",
+    )
+
+    def imap_ready(_):
+        greeting = server.execute(f"echo | {imap_probe} | head -1")[1]
+        return greeting.strip().startswith("* OK")
+
+    retry(imap_ready, timeout_seconds=60)
+
+with subtest("(functional): nginx proxies to backend containers"):
+    server.succeed(
+        net_cmd(
+            "nginx",
+            f"{CURL} -sfk --max-time 5 --resolve git.floweytf.com:443:{ADDR['nginx']} "
+            "https://git.floweytf.com/ >&2",
+        )
+    )
+
+with subtest("(functional): sshpiper authenticates/routes properly"):
+    whoami = server.succeed(f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} whoami").strip()
+    assert whoami == "runner", f"expected to land as runner, got {whoami}"
+    hostname = server.succeed(
+        f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} hostname"
+    ).strip()
+    assert hostname == "ssh-admin", f"expected to land on ssh-admin, got {hostname}"
+
+with subtest("(isolation): every declared container edge connects"):
+    for src, to, tcp_ports, _ in EDGES:
+        if to in ("@host", "@internet") or src not in enterable or to not in included:
+            continue
+        for port in tcp_ports:
+            server.succeed(ct_connect(src, ADDR[to], port))
+
+with subtest("(isolation): undeclared container edges are refused"):
+    for src, dst, port in [
+        ("devenv", "git", 3000),
+        ("git", "devenv", 8080),
+        ("maven", "git", 3000),
+        ("nginx", "mail", 25),
+        ("ssh-botm", "ssh-admin", 22),
+        ("overseer", "git", 3000),
+    ]:
+        assert (src, dst) not in PEER_EDGES, f"{src} -> {dst} is a declared edge, not a negative case"
+        server.fail(ct_connect(src, ADDR[dst], port, timeout=2))
+
+with subtest("(isolation): edges are one-way"):
+    assert ("nginx", "git") in PEER_EDGES
+    assert ("git", "nginx") not in PEER_EDGES
+    server.succeed(ct_connect("nginx", ADDR["git"], 3000))
+    server.fail(ct_connect("git", ADDR["nginx"], 443, timeout=2))
+
+with subtest("(isolation): host access allowlist"):
+    server.succeed(
+        net_cmd(
+            "overseer",
+            f"{CURL} -sf --max-time 5 http://{HOST_ADDR['overseer']}:{HOST_EXPORTER_PORT}/metrics >&2",
+        )
+    )
+    for port in (HOST_EXPORTER_PORT, HOST_CADVISOR_PORT):
+        server.fail(ct_connect("git", HOST_ADDR["git"], port, timeout=2))
+        server.fail(ct_connect("git", HOST_ADDR["overseer"], port, timeout=2))
+
+with subtest("(isolation): host services outside the access list"):
+    internet.succeed(connect(server_ip, HOST_SSHD_PORT))
+    for name in ("git", "overseer"):
+        server.fail(ct_connect(name, HOST_ADDR[name], HOST_SSHD_PORT, timeout=2))
+        server.fail(ct_connect(name, server_ip, HOST_SSHD_PORT, timeout=2))
+
+with subtest("(isolation): source address spoofing is dropped"):
+    before = spoof_counter()
+    server.succeed(net_cmd("git", f"{PING} -c 1 -W 2 {HOST_ADDR['git']} >&2"))
+    assert spoof_counter() == before, "an unspoofed packet hit the source guard"
+
+    iface = IFACE["git"]
+    server.succeed(net_cmd("git", f"{IP} addr add {ADDR['nginx']}/32 dev {iface}"))
+    try:
+        server.execute(
+            net_cmd("git", f"{PING} -c 2 -W 2 -I {ADDR['nginx']} {HOST_ADDR['git']}")
+        )
+        assert spoof_counter() > before, "a forged source address was not dropped"
+    finally:
+        server.succeed(net_cmd("git", f"{IP} addr del {ADDR['nginx']}/32 dev {iface}"))
+
+with subtest("(isolation): renaming the interface changes nothing"):
+    iface, host_addr = IFACE["nginx"], HOST_ADDR["nginx"]
+
+    def rename(frm, to):
+        server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} down"))
+        server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} name {to}"))
+        server.succeed(net_cmd("nginx", f"{IP} link set dev {to} up"))
+        for route in (host_addr, f"10.0.0.0/8 via {host_addr}", f"default via {host_addr}"):
+            server.succeed(net_cmd("nginx", f"{IP} route replace {route} dev {to}"))
+
+    rename(iface, "evil0")
+    try:
+        server.succeed(ct_connect("nginx", ADDR["git"], 3000))
+        server.fail(ct_connect("nginx", ADDR["mail"], 25, timeout=2))
+    finally:
+        rename("evil0", iface)
+
+with subtest("(isolation): the veth cannot be moved out of the namespace"):
+    server.fail(net_cmd("git", f"{IP} link set dev {IFACE['git']} netns 1"))
+
+with subtest("(isolation): route bypass"):
+    iface = IFACE["hopper"]
+    server.succeed(
+        net_cmd("hopper", f"{IP} route replace default via {HOST_ADDR['hopper']} dev {iface} onlink")
+    )
+    server.fail(net_cmd("hopper", f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2"))
+    server.fail(ct_connect("hopper", ADDR["git"], 3000, timeout=2))
+
+with subtest("(isolation): no IPv6 link-local addresses"):
+    for name in ["git", "overseer"]:
+        assert not has_ipv6_link_local(name), f"{name} still has an IPv6 link-local address"
+
+with subtest("(isolation): published ports grant no reach between containers"):
+    for port in (80, 443, 25565, 22, 25):
+        server.fail(ct_connect("overseer", HOST_ADDR["overseer"], port, timeout=2))
+        server.fail(ct_connect("git", server_ip, port, timeout=2))
+
+with subtest("(isolation): privilege escalation / container escape"):
+    leader = container_pid("git")
+
+    host_uid = server.succeed(f"awk '/^Uid:/{{print $2}}' /proc/{leader}/status").strip()
+    assert host_uid != "0", "container's PID 1 is not idmapped away from host uid 0"
+
+    host_pidns = server.succeed("readlink /proc/1/ns/pid").strip()
+    container_pidns = server.succeed(f"readlink /proc/{leader}/ns/pid").strip()
+    assert host_pidns != container_pidns, "container shares the host's PID namespace"
+
+    server.succeed(
+        f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- "
+        f"{SH} -c 'echo x >> /var/lib/gitea/canary' >&2"
+    )
+
+    sshpiper_pid = container_pid("sshpiper")
+    server.fail(
+        f"{NSENTER} --mount=/proc/{sshpiper_pid}/ns/mnt --root=/proc/{sshpiper_pid}/root -- "
+        f"{SH} -c 'echo x >> /etc/ssh/ssh_host_ed25519_key' >&2"
+    )
+    server.fail(
+        f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- "
+        f"{SH} -c 'touch /nix/store/canary' >&2"
+    )
+    server.fail(
+        f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- "
+        "/run/current-system/sw/bin/test -d /persist >&2"
+    )
+
+with subtest("(functional): external clients reach published ports"):
+    for port in (80, 25565, 22):
+        internet.succeed(ext_connect(port))
+    internet.fail(ext_connect(8080))
+
+with subtest("(functional): no failing-reason restarts"):
+    assert failed_units() == [], f"host has failed units: {failed_units()}"
+    for name in included:
+        assert (
+            failed_units(f"-M {name}") == []
+        ), f"{name} has failed units: {failed_units(f'-M {name}')}"
+        restarts = server.succeed(f"systemctl show container@{name}.service -p NRestarts --value").strip()
+        assert restarts == "0", f"container@{name}.service restarted {restarts} times"
+
+with subtest("(functional): port forwards survive an nftables reload"):
+    server.succeed("systemctl reload nftables.service")
+    elements = server.succeed("nft list map ip container-portforward tcp-fwd")
+    assert re.search(r"\b80\s*:", elements), f"nftables reload emptied the map: {elements!r}"
+    internet.wait_until_succeeds(ext_connect(80), timeout=30)
+
+with subtest("(functional): published ports follow the container's lifetime"):
+    server.succeed("systemctl stop container@nginx")
+    internet.fail(ext_connect(80))
+    server.succeed("systemctl start container@nginx")
+    server.wait_for_unit("container@nginx.service")
+    internet.wait_until_succeeds(ext_connect(80), timeout=30)