From 76fa38e34c8c52062a8413b496bb6e8519ffb143 Mon Sep 17 00:00:00 2001 From: Floweynt <50057682+Floweynt@users.noreply.github.com> Date: Mon, 31 Aug 2026 08:41:45 -0500 Subject: [PATCH] initial commit --- .clang-format | 13 + .clang-tidy | 34 + .clangd | 4 + .github/workflows/diff-pr.yml | 45 + .github/workflows/diff-tag.yml | 37 + .github/workflows/flake-check.yml | 26 + .github/workflows/flake-update.yml | 33 + .github/workflows/lint.yml | 41 + .github/workflows/vulnix.yml | 41 + .gitignore | 4 + ci.nix | 23 + ci.py | 451 ++++++ configuration.nix | 141 ++ containers/common/overseer.nix | 386 +++++ .../common/overseer/access-logs-dashboard.nix | 61 + .../common/overseer/containers-dashboard.nix | 251 ++++ containers/common/overseer/helpers.nix | 228 +++ containers/common/overseer/host-dashboard.nix | 167 +++ .../common/overseer/resources-dashboard.nix | 203 +++ containers/server-n0/devenv.nix | 54 + containers/server-n0/git.nix | 178 +++ containers/server-n0/hopper.nix | 79 + containers/server-n0/mail.nix | 264 ++++ containers/server-n0/maven.nix | 41 + containers/server-n0/mc.nix | 205 +++ containers/server-n0/mtce.nix | 84 ++ containers/server-n0/nginx.nix | 215 +++ containers/server-n0/pihole.nix | 64 + containers/server-n0/ssh.nix | 186 +++ containers/server-n0/sshpiper.nix | 122 ++ dsl.nix | 1318 +++++++++++++++++ flake.lock | 121 ++ flake.nix | 154 ++ hosts/example.nix | 38 + hosts/server-n0-darling-erasure.sh | 23 + hosts/server-n0.nix | 109 ++ introspect.nix | 399 +++++ mount.sh | 15 + packages/ct.nix | 8 + packages/hopper-lazy-resolve.patch | 92 ++ packages/hopper-rs.nix | 29 + packages/kitty-terminfo.nix | 9 + packages/mk-script.nix | 9 + packages/server-kexec.nix | 8 + packages/server-rebuild.nix | 2 + packages/server-status.nix | 2 + packages/sshpiper.nix | 33 + scripts/ct.sh | 136 ++ scripts/server-kexec.sh | 17 + scripts/server-rebuild.sh | 1 + scripts/server-status.sh | 21 + statix.toml | 2 + tests/server-n0/integration.nix | 190 +++ tests/server-n0/integration.py | 330 +++++ 54 files changed, 6747 insertions(+) create mode 100644 .clang-format create mode 100644 .clang-tidy create mode 100644 .clangd create mode 100644 .github/workflows/diff-pr.yml create mode 100644 .github/workflows/diff-tag.yml create mode 100644 .github/workflows/flake-check.yml create mode 100644 .github/workflows/flake-update.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/vulnix.yml create mode 100644 .gitignore create mode 100644 ci.nix create mode 100644 ci.py create mode 100644 configuration.nix create mode 100644 containers/common/overseer.nix create mode 100644 containers/common/overseer/access-logs-dashboard.nix create mode 100644 containers/common/overseer/containers-dashboard.nix create mode 100644 containers/common/overseer/helpers.nix create mode 100644 containers/common/overseer/host-dashboard.nix create mode 100644 containers/common/overseer/resources-dashboard.nix create mode 100644 containers/server-n0/devenv.nix create mode 100644 containers/server-n0/git.nix create mode 100644 containers/server-n0/hopper.nix create mode 100644 containers/server-n0/mail.nix create mode 100644 containers/server-n0/maven.nix create mode 100644 containers/server-n0/mc.nix create mode 100644 containers/server-n0/mtce.nix create mode 100644 containers/server-n0/nginx.nix create mode 100644 containers/server-n0/pihole.nix create mode 100644 containers/server-n0/ssh.nix create mode 100644 containers/server-n0/sshpiper.nix create mode 100644 dsl.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 hosts/example.nix create mode 100644 hosts/server-n0-darling-erasure.sh create mode 100644 hosts/server-n0.nix create mode 100644 introspect.nix create mode 100644 mount.sh create mode 100644 packages/ct.nix create mode 100644 packages/hopper-lazy-resolve.patch create mode 100644 packages/hopper-rs.nix create mode 100644 packages/kitty-terminfo.nix create mode 100644 packages/mk-script.nix create mode 100644 packages/server-kexec.nix create mode 100644 packages/server-rebuild.nix create mode 100644 packages/server-status.nix create mode 100644 packages/sshpiper.nix create mode 100644 scripts/ct.sh create mode 100644 scripts/server-kexec.sh create mode 100644 scripts/server-rebuild.sh create mode 100644 scripts/server-status.sh create mode 100644 statix.toml create mode 100644 tests/server-n0/integration.nix create mode 100644 tests/server-n0/integration.py diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..f71d3ea --- /dev/null +++ b/.clang-format @@ -0,0 +1,13 @@ +BasedOnStyle: microsoft +AccessModifierOffset: -4 +AlignConsecutiveAssignments: false +AlwaysBreakTemplateDeclarations: Yes +ColumnLimit: 150 +NamespaceIndentation: All +PointerAlignment: Left +TabWidth: 4 +AllowShortBlocksOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +UseTab: Never +AlignAfterOpenBracket: BlockIndent +AllowAllArgumentsOnNextLine: true diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..cd27135 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,34 @@ +Checks: > + bugprone*, + cert*, + cppcore*, + misc*, + modernize*, + performance*, + portability*, + readability*, + -modernize-avoid-c-arrays, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-const-or-ref-data-members, + -cert-dcl58-cpp, + -misc-non-private-member-variables-in-classes, + -cppcoreguidelines-macro-usage, + -cppcoreguidelines-non-private-member-variables-in-classes, + -cppcoreguidelines-avoid-do-while + -cert-dcl21-cpp, + -cert-dcl51-cpp, + -cppcoreguidelines-avoid-magic-numbers, + -cert-dcl37-c, + -bugprone-reserved-identifier +FormatStyle: file +CheckOptions: + - key: readability-implicit-bool-conversion.AllowIntegerConditions + value: "1" + - key: readability-identifier-length.IgnoredVariableNames + value: ch + - key: readability-identifier-length.IgnoredParameterNames + value: ch + - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor + value: true + - key: readability-magic-numbers.IgnorePowersOf2IntegerValues + value: true diff --git a/.clangd b/.clangd new file mode 100644 index 0000000..81fac8b --- /dev/null +++ b/.clangd @@ -0,0 +1,4 @@ +Diagnostics: + UnusedIncludes: Strict + MissingIncludes: None + diff --git a/.github/workflows/diff-pr.yml b/.github/workflows/diff-pr.yml new file mode 100644 index 0000000..7ba4bd7 --- /dev/null +++ b/.github/workflows/diff-pr.yml @@ -0,0 +1,45 @@ +name: diff-pr + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: diff-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + dix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: worktree base for diff + run: git worktree add "$RUNNER_TEMP/base" "${{ github.event.pull_request.base.sha }}" + + - name: build closures and render diff + run: | + nix run .#ci -- diff "$RUNNER_TEMP/base" . \ + "### System closure diff vs \`${{ github.event.pull_request.base.ref }}\`" \ + > "$RUNNER_TEMP/comment.md" + nix run .#ci -- inspect-diff "$RUNNER_TEMP/base" . \ + "### Inspection diff vs \`${{ github.event.pull_request.base.ref }}\`" \ + >> "$RUNNER_TEMP/comment.md" + + - name: post/update PR comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + API: ${{ github.api_url }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + run: nix run .#ci -- post-comment "" "$RUNNER_TEMP/comment.md" diff --git a/.github/workflows/diff-tag.yml b/.github/workflows/diff-tag.yml new file mode 100644 index 0000000..8224366 --- /dev/null +++ b/.github/workflows/diff-tag.yml @@ -0,0 +1,37 @@ +name: diff-tag + +on: + push: + branches: [master] + +permissions: + contents: read + +jobs: + dix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: diff latest tag against HEAD + run: | + tag=$(git tag --sort=-creatordate | head -n1) + if [ -z "$tag" ]; then + echo "No tags yet, skipping diff." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + git worktree add "$RUNNER_TEMP/old" "$tag" + head=$(git rev-parse --short HEAD) + { + nix run .#ci -- diff "$RUNNER_TEMP/old" . \ + "# System closure diff: \`$tag\` -> \`$head\`" + nix run .#ci -- inspect-diff "$RUNNER_TEMP/old" . \ + "# Inspection diff: \`$tag\` -> \`$head\`" + } | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml new file mode 100644 index 0000000..452cfe6 --- /dev/null +++ b/.github/workflows/flake-check.yml @@ -0,0 +1,26 @@ +name: flake-check + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +concurrency: + group: flake-check-${{ github.ref }} + cancel-in-progress: true + +jobs: + flake-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - run: nix flake check -L diff --git a/.github/workflows/flake-update.yml b/.github/workflows/flake-update.yml new file mode 100644 index 0000000..2e57a9e --- /dev/null +++ b/.github/workflows/flake-update.yml @@ -0,0 +1,33 @@ +name: flake-update + +on: + schedule: + - cron: "17 4 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - run: nix flake update + + - name: open/update PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + API: ${{ github.api_url }} + REPO: ${{ github.repository }} + run: | + printf 'Automated `nix flake update`.\n' > "$RUNNER_TEMP/body.md" + nix run .#ci -- open-pr \ + flake-update "$GITHUB_REF_NAME" "flake.lock: update inputs" "$RUNNER_TEMP/body.md" automated diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..89a6d6f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +name: lint + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + pull-requests: write + +concurrency: + group: lint-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: pick reporter + run: | + if [ "${{ github.event_name }}" = pull_request ]; then + echo "REPORTER=github-pr-review" >> "$GITHUB_ENV" + else + echo "REPORTER=local" >> "$GITHUB_ENV" + fi + + - name: run linters + run: nix run .#ci -- lint . "$RUNNER_TEMP/findings" + + - name: report findings + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: nix run .#ci -- lint-report "$REPORTER" "$RUNNER_TEMP/findings" diff --git a/.github/workflows/vulnix.yml b/.github/workflows/vulnix.yml new file mode 100644 index 0000000..aa7e347 --- /dev/null +++ b/.github/workflows/vulnix.yml @@ -0,0 +1,41 @@ +name: vulnix + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: cachix/install-nix-action@v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + + - name: scan host closures + id: scan + run: echo "count=$(nix run .#ci -- vulnix . "$RUNNER_TEMP/issue.md")" >> "$GITHUB_OUTPUT" + + - name: file/update issue + if: steps.scan.outputs.count != '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + API: ${{ github.api_url }} + REPO: ${{ github.repository }} + run: nix run .#ci -- upsert-issue vulnix "vulnix: known vulnerabilities in system closure" "$RUNNER_TEMP/issue.md" + + - name: close resolved issue + if: steps.scan.outputs.count == '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + API: ${{ github.api_url }} + REPO: ${{ github.repository }} + run: nix run .#ci -- close-issue vulnix diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bdba82e --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +result +result-* +out.txt +__pycache__/ diff --git a/ci.nix b/ci.nix new file mode 100644 index 0000000..7bc375b --- /dev/null +++ b/ci.nix @@ -0,0 +1,23 @@ +{ lib, pkgs }: +# tools must track ci.py's subprocess calls; nix stays ambient (uses the +# runner's own nix, not a pinned one). +pkgs.writers.writePython3Bin "ci" { + flakeIgnore = [ "E501" ]; + makeWrapperArgs = [ + "--prefix" + "PATH" + ":" + (lib.makeBinPath ( + with pkgs; + [ + dix + vulnix + nixfmt + statix + deadnix + reviewdog + git + ] + )) + ]; +} (builtins.readFile ./ci.py) diff --git a/ci.py b/ci.py new file mode 100644 index 0000000..e4a3d57 --- /dev/null +++ b/ci.py @@ -0,0 +1,451 @@ +import argparse +import base64 +import datetime +import difflib +import json +import os +import re +import subprocess +import sys +import urllib.request +from collections import Counter +from pathlib import Path + + +def run(*args, **kw): + return subprocess.run(args, text=True, **kw) + + +def nix(*args): + return run("nix", "--extra-experimental-features", "nix-command flakes", *args, check=True, capture_output=True).stdout + + +def git(*args, check=True): + return run("git", *args, check=check, capture_output=True) + + +def gitea(method, path, body=None): + url = f'{os.environ["API"]}/repos/{os.environ["REPO"]}{path}' + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f'token {os.environ["GH_TOKEN"]}') + if data is not None: + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req) as resp: + raw = resp.read() + return json.loads(raw) if raw else None + + +def find_open_issue(label): + issues = gitea("GET", f"/issues?state=open&labels={label}") + return issues[0] if issues else None + + +def build_hosts(ref): + hosts = json.loads(nix("eval", "--json", f"{ref}#nixosConfigurations", "--apply", "builtins.attrNames")) + return { + host: nix("build", f"{ref}#nixosConfigurations.{host}.config.system.build.toplevel", + "--no-link", "--print-out-paths").strip() + for host in hosts + } + + +def diff(args): + old, new = build_hosts(args.old), build_hosts(args.new) + + def section(host): + if host not in old: + return f"## {host}\n\n_new host_" + if host not in new: + return f"## {host}\n\n_host removed_" + rendered = run("dix", "--color", "never", old[host], new[host], capture_output=True).stdout + return f"## {host}\n\n```\n{rendered}```" + + parts = ([args.title] if args.title else []) + [section(h) for h in sorted(set(old) | set(new))] + print("\n\n".join(parts)) + + +# meta.toplevel is the closure path dix already covers; nftables.ruleset/tables +# are duplicated by nftables.rendered (the assembled ruleset we keep and diff). +# Pruned so inspect-diff reports only source config topology, once. +INSPECT_PRUNE = [ + ("meta", "toplevel"), + ("nftables", "ruleset"), + ("nftables", "tables"), +] + + +def compact(value): + return json.dumps(value, sort_keys=True) + + +def scalar(value): + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def fmt(value, indent=0): + pad = " " * indent + lines = [] + if isinstance(value, dict): + for key in sorted(value): + child = value[key] + if isinstance(child, (dict, list)) and child: + lines.append(f"{pad}{key}:") + lines += fmt(child, indent + 1) + else: + lines.append(f"{pad}{key}: {scalar(child)}") + elif isinstance(value, list): + for item in value: + if isinstance(item, (dict, list)) and item: + body = fmt(item) + lines.append(f"{pad}- {body[0]}") + lines += [f"{pad} {line}" for line in body[1:]] + else: + lines.append(f"{pad}- {scalar(item)}") + else: + lines.append(f"{pad}{scalar(value)}") + return lines + + +STORE_HASH = re.compile(r"/nix/store/[a-z0-9]{32}-") + + +def strip_store(value): + # closure hashes churn on any rebuild and are dix's job; drop them so exec + # commands / paths read as "/bin/x" and diff only on real changes + if isinstance(value, dict): + return {k: strip_store(v) for k, v in value.items()} + if isinstance(value, list): + return [strip_store(v) for v in value] + if isinstance(value, str): + return STORE_HASH.sub("", value) + return value + + +def inspect_hosts(ref): + try: + names = json.loads(nix("eval", "--json", f"{ref}#inspect", "--apply", "builtins.attrNames")) + except subprocess.CalledProcessError as e: + if "does not provide attribute" in e.stderr: + return None + raise RuntimeError(f"inspect eval failed for {ref}:\n{e.stderr}") from e + return { + host: nix("build", f"{ref}#inspect.{host}", + "--no-link", "--print-out-paths").strip() + for host in names + } + + +def read_inspect(path): + with open(path) as f: + return json.load(f) + + +def inspect_data(ref): + paths = inspect_hosts(ref) + if paths is None: + return None + reports = {host: read_inspect(path) for host, path in paths.items()} + for report in reports.values(): + for parent, key in INSPECT_PRUNE: + report.get(parent, {}).pop(key, None) + return {host: strip_store(report) for host, report in reports.items()} + + +def tree_changes(old, new, path=()): + if old == new: + return + if isinstance(old, dict) and isinstance(new, dict): + for key in sorted(old.keys() | new.keys()): + if key not in old: + yield path + (key,), "add", new[key] + elif key not in new: + yield path + (key,), "del", old[key] + else: + yield from tree_changes(old[key], new[key], path + (key,)) + elif isinstance(old, list) and isinstance(new, list): + # order-independent: a reordered list is not a change + okeys, nkeys = list(map(compact, old)), list(map(compact, new)) + values = dict(zip(okeys, old)) | dict(zip(nkeys, new)) + for key in (Counter(okeys) - Counter(nkeys)).elements(): + yield path, "delitem", values[key] + for key in (Counter(nkeys) - Counter(okeys)).elements(): + yield path, "additem", values[key] + else: + yield path, "change", (old, new) + + +def render_change(path, kind, payload): + label = ".".join(map(str, path[1:])) or path[0] + if kind in ("add", "del", "additem", "delitem"): + sign = "+" if kind in ("add", "additem") else "-" + value = [payload] if kind in ("additem", "delitem") else payload + if isinstance(value, (dict, list)) and value: + return [f"{sign} {label}:"] + [f"{sign} {line}" for line in fmt(value, 1)] + return [f"{sign} {label}: {scalar(value)}"] + old, new = payload + if isinstance(old, str) and isinstance(new, str) and "\n" in old + new: + body = difflib.unified_diff(old.splitlines(), new.splitlines(), lineterm="") + return [f" {label}:"] + [ln for ln in body if not ln.startswith(("---", "+++"))] + if isinstance(old, (dict, list)) or isinstance(new, (dict, list)): + return render_change(path, "del", old) + render_change(path, "add", new) + return [f"- {label}: {scalar(old)}", f"+ {label}: {scalar(new)}"] + + +def host_diff(old, new): + lines = [] + attr = None + for path, kind, payload in tree_changes(old, new): + if path[0] != attr: + attr = path[0] + lines.append(f"@@ {attr} @@") + lines += render_change(path, kind, payload) + return "```diff\n" + "\n".join(lines) + "\n```" if lines else None + + +def host_added(report): + return "```diff\n" + "\n".join(f"+ {line}" for line in fmt(report)) + "\n```" + + +def inspect_diff(args): + old, new = inspect_data(args.old), inspect_data(args.new) + if old is None or new is None: + return + + def block(host): + if host not in old: + return f"## {host} (new)\n\n{host_added(new[host])}" + if host not in new: + return f"## {host}\n\n_host removed_" + body = host_diff(old[host], new[host]) + return f"## {host}\n\n{body}" if body else None + + blocks = [b for b in map(block, sorted(set(old) | set(new))) if b] + if blocks: + print("\n\n".join(([args.title] if args.title else []) + blocks)) + + +def format_vulnix(vulns): + rows = [] + for v in vulns: + for cve in v["affected_by"]: + score = v.get("cvssv3_basescore", {}).get(cve) + desc = v.get("description", {}).get(cve, "") + if len(desc) > 120: + desc = desc[:117] + "..." + rows.append((v["pname"], v["version"], cve, score, desc)) + rows.sort(key=lambda r: -(r[3] or 0)) + + lines = [ + "| Package | Version | CVE | CVSS | Description |", + "|---------|---------|-----|------|-------------|", + ] + for pname, version, cve, score, desc in rows: + score_str = f"{score:.1f}" if score else "?" + cve_link = f"[{cve}](https://nvd.nist.gov/vuln/detail/{cve})" + lines.append(f"| {pname} | {version} | {cve_link} | {score_str} | {desc} |") + return "\n".join(lines) + + +def vulnix(args): + paths = list(build_hosts(args.ref).values()) + result = run("vulnix", "-j", *paths, capture_output=True) + if result.returncode not in (0, 2): + print(result.stderr, file=sys.stderr) + sys.exit(result.returncode) + vulns = json.loads(result.stdout or "[]") + if vulns: + cve_count = sum(len(v["affected_by"]) for v in vulns) + date = datetime.date.today().isoformat() + with open(args.body_out, "w") as f: + f.write(f"vulnix scan ({date}): **{cve_count}** CVEs across **{len(vulns)}** packages.\n\n") + f.write(format_vulnix(vulns)) + f.write("\n") + print(len(vulns)) + + +def find_cycles(edges): + adj = {} + for e in edges: + if e["kind"] == "after": + adj.setdefault(e["from"], []).append(e["to"]) + + visited, on_path = set(), set() + path = [] + cycles = [] + + def dfs(node): + visited.add(node) + path.append(node) + on_path.add(node) + for nb in adj.get(node, []): + if nb in on_path: + i = path.index(nb) + cycles.append(path[i:] + [nb]) + elif nb not in visited: + dfs(nb) + path.pop() + on_path.discard(node) + + for node in adj: + if node not in visited: + dfs(node) + return cycles + + +def lint(args): + out = os.path.abspath(args.out) + target = os.path.abspath(args.target) + os.makedirs(out, exist_ok=True) + os.chdir(target) + + files = [f for f in git("ls-files", "-z", "*.nix").stdout.split("\0") if f] + if files: + run("nixfmt", *files) + with open(f"{out}/nixfmt.diff", "w") as f: + f.write(git("diff").stdout) + git("checkout", "--", ".") + + with open(f"{out}/statix.errfmt", "w") as f: + f.write(run("statix", "check", ".", "-o", "errfmt", capture_output=True).stdout) + + dead = run("deadnix", "-o", "json", ".", capture_output=True).stdout + with open(f"{out}/deadnix.rdjsonl", "w") as f: + for line in filter(str.strip, dead.splitlines()): + report = json.loads(line) + for r in report["results"]: + loc = {"path": report["file"]} + if "line" in r: + loc["range"] = { + "start": {"line": r["line"], "column": r["column"]}, + "end": {"line": r["line"], "column": r["endColumn"]}, + } + rec = { + "message": r["message"], + "location": loc, + "severity": "ERROR", + } + json.dump(rec, f) + f.write("\n") + + paths = inspect_hosts(target) + with open(f"{out}/cycles.rdjsonl", "w") as f: + for host, path in (paths or {}).items(): + report = read_inspect(path) + for cycle in find_cycles(report["dependencies"]["edges"]): + chain = " → ".join(u.split(":", 1)[1] for u in cycle) + rec = { + "message": f"[{host}] systemd ordering cycle: {chain}", + "location": {"path": "flake.nix"}, + "severity": "ERROR", + } + json.dump(rec, f) + f.write("\n") + + +LINTERS = [ + ("nixfmt.diff", ["-f=diff", "-f.diff.strip=1", "-name=nixfmt", "-level=error"]), + ("statix.errfmt", ["-efm=%f>%l:%c:%*[A-Z]:%*[0-9]:%m", "-name=statix", "-level=error"]), + ("deadnix.rdjsonl", ["-f=rdjsonl", "-name=deadnix"]), + ("cycles.rdjsonl", ["-f=rdjsonl", "-name=systemd-cycles", "-filter-mode=nofilter"]), +] + + +def lint_report(args): + def reviewdog(path, flags): + with open(path) as f: + return run("reviewdog", *flags, f"-reporter={args.reporter}", "-fail-level=error", stdin=f).returncode + + codes = [reviewdog(f"{args.dir}/{name}", flags) + for name, flags in LINTERS + if os.path.getsize(f"{args.dir}/{name}") > 0] + sys.exit(1 if any(codes) else 0) + + +def post_comment(args): + body = args.marker + "\n" + Path(args.body_file).read_text() + pr = os.environ["PR"] + comments = gitea("GET", f"/issues/{pr}/comments") + existing = next((c for c in comments if c["body"].startswith(args.marker)), None) + if existing: + gitea("PATCH", f"/issues/comments/{existing['id']}", {"body": body}) + else: + gitea("POST", f"/issues/{pr}/comments", {"body": body}) + + +def upsert_issue(args): + body = Path(args.body_file).read_text() + issue = find_open_issue(args.label) + if issue: + gitea("PATCH", f"/issues/{issue['number']}", {"body": body}) + else: + gitea("POST", "/issues", {"title": args.title, "body": body, "labels": [label_id(args.label)]}) + + +def close_issue(args): + issue = find_open_issue(args.label) + if issue: + gitea("PATCH", f"/issues/{issue['number']}", {"state": "closed"}) + + +def label_id(name): + match = next((x for x in gitea("GET", "/labels") if x["name"] == name), None) + return match["id"] if match else gitea("POST", "/labels", {"name": name, "color": "#ededed"})["id"] + + +def open_pr(args): + if git("diff", "--quiet", check=False).returncode == 0: + print("no changes; nothing to do") + return + git("switch", "-C", args.branch) + git("add", "-A") + git("-c", "user.name=gitea-actions[bot]", "-c", "user.email=gitea-actions@noreply.local", + "commit", "-m", args.title) + + host = os.environ["API"].removesuffix("/api/v1") + auth = base64.b64encode(f'token:{os.environ["GH_TOKEN"]}'.encode()).decode() + git("-c", f"http.extraHeader=Authorization: Basic {auth}", + "push", "-f", f'{host}/{os.environ["REPO"]}.git', f"HEAD:{args.branch}") + + owner = os.environ["REPO"].split("/")[0] + if gitea("GET", f"/pulls?state=open&head={owner}:{args.branch}"): + return + labels = [label_id(args.label)] if args.label else [] + gitea("POST", "/pulls", + {"title": args.title, "head": args.branch, "base": args.base, + "body": Path(args.body_file).read_text(), "labels": labels}) + + +def main(): + parser = argparse.ArgumentParser(prog="ci") + sub = parser.add_subparsers(required=True) + + def command(name, fn, *positional): + p = sub.add_parser(name) + for arg in positional: + if arg.endswith("?"): + p.add_argument(arg[:-1], nargs="?") + else: + p.add_argument(arg) + p.set_defaults(fn=fn) + + command("diff", diff, "old", "new", "title?") + command("inspect-diff", inspect_diff, "old", "new", "title?") + command("vulnix", vulnix, "ref", "body_out") + command("lint", lint, "target", "out") + command("lint-report", lint_report, "reporter", "dir") + command("post-comment", post_comment, "marker", "body_file") + command("upsert-issue", upsert_issue, "label", "title", "body_file") + command("close-issue", close_issue, "label") + command("open-pr", open_pr, "branch", "base", "title", "body_file", "label?") + + args = parser.parse_args() + args.fn(args) + + +if __name__ == "__main__": + main() diff --git a/configuration.nix b/configuration.nix new file mode 100644 index 0000000..83f84f5 --- /dev/null +++ b/configuration.nix @@ -0,0 +1,141 @@ +{ + config, + pkgs, + lib, + hostInfo, + customPkgs, + ... +}: +let + inherit (hostInfo) paths; +in +{ + assertions = + let + isMounted = + path: + lib.any ( + # rootfs may be subject to erasure so it's not persistent + fs: fs.mountPoint != "/" && (fs.mountPoint == path || lib.hasPrefix "${fs.mountPoint}/" path) + ) (lib.attrValues config.fileSystems); + + isMountedEarly = path: (config.fileSystems.${path} or { neededForBoot = false; }).neededForBoot; + + mountAssertion = path: { + assertion = isMounted path; + message = "configuration.nix needs '${path}' to be a mounted path, but no fileSystems entry covers it."; + }; + + earlyMountAssertion = path: { + assertion = isMountedEarly path; + message = "configuration.nix needs '${path}' to be mounted early (neededForBoot = true), but its fileSystems entry doesn't set that."; + }; + + strictPaths = lib.optionals (!hostInfo.isVm) [ + "/nix" + "/var/log" + ]; + in + map mountAssertion ( + [ + paths.server + paths.secrets + ] + ++ strictPaths + ) + ++ lib.optionals (!hostInfo.isVm) [ (earlyMountAssertion "/var/log") ]; + + time.timeZone = "UTC"; + system.stateVersion = "26.05"; + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + }; + nrBuildUsers = 80; + }; + + powerManagement.cpuFreqGovernor = "performance"; + + boot.kernelParams = [ "transparent_hugepage=madvise" ]; + + services = { + fstrim = { + enable = true; + interval = "weekly"; + }; + + openssh = { + enable = true; + ports = [ 5952 ]; + hostKeys = + lib.map + (type: { + inherit type; + path = "${paths.secrets}/ssh_host_${type}_key"; + }) + [ + "ed25519" + "ecdsa" + "rsa" + ]; + settings.PasswordAuthentication = false; + }; + }; + + users.mutableUsers = false; + users.users.server = { + uid = 1000; + isNormalUser = true; + extraGroups = [ + "wheel" + "systemd-journal" + ]; + hashedPassword = "!"; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPaGpTX3EzfezVc2nl3MgsKVamYYP2h2VJaRny8pZkvi flowey@arch" + ]; + }; + + security.sudo.wheelNeedsPassword = false; + + security.polkit.extraConfig = '' + polkit.addRule(function(action, subject) { + if (action.id.startsWith("org.freedesktop.machine1.") && + subject.user === "server") { + return polkit.Result.YES; + } + }); + ''; + + systemd.tmpfiles.rules = [ + "L /home/server/config - - - - /persist/@configuration" + "L /home/server/containers - - - - /persist/@containers" + "L /home/server/secrets - - - - /persist/@secrets" + ]; + + environment.etc."machine-id".text = "${ + builtins.substring 0 32 (builtins.hashString "sha256" "machine-id-for-${hostInfo.hostname}") + }\n"; + environment.systemPackages = [ + customPkgs.ct + customPkgs.server-rebuild + customPkgs.server-status + customPkgs.server-kexec + customPkgs.kitty-terminfo + pkgs.git + pkgs.wezterm.terminfo + ]; + + networking.firewall.allowedTCPPorts = [ 5952 ]; + + system.activationScripts.server-keypair = '' + if [[ ! -f "${paths.server}/keypair/id_ed25519" ]]; then + mkdir -p "${paths.server}/keypair" + ${lib.getExe' pkgs.openssh "ssh-keygen"} -t ed25519 -f "${paths.server}/keypair/id_ed25519" -N "" + fi + ''; +} diff --git a/containers/common/overseer.nix b/containers/common/overseer.nix new file mode 100644 index 0000000..5ace706 --- /dev/null +++ b/containers/common/overseer.nix @@ -0,0 +1,386 @@ +{ + name, + hostInfo, + dsl, + pkgs, + lib, + containers, + ... +}: +let + inherit (dsl) mkContainer; + inherit (hostInfo.paths) data; + + nodePort = 9100; + cadvPort = 9200; + + containerNetworkRulesFile = + let + mkRules = + metric: record: + lib.mapAttrsToList (cname: c: { + inherit record; + expr = "rate(${metric}{device=\"${c.iface}\"}[5m])"; + labels = { + container = cname; + }; + }) containers; + in + pkgs.writeText "container-network.yml" ( + builtins.toJSON { + groups = [ + { + name = "container-network"; + rules = + mkRules "node_network_receive_bytes_total" "container_network_receive_rate" + ++ mkRules "node_network_transmit_bytes_total" "container_network_transmit_rate"; + } + ]; + } + ); + + containerDefinedRulesFile = pkgs.writeText "container-defined.yml" ( + builtins.toJSON { + groups = [ + { + name = "container-defined"; + rules = lib.mapAttrsToList (cname: _: { + record = "container_defined"; + expr = "1"; + labels = { + container = cname; + }; + }) containers; + } + ]; + } + ); + + alloyConfig = pkgs.writeText "config.alloy" '' + loki.source.journal "journal" { + max_age = "12h" + path = "/var/log/journal" + labels = { job = "systemd-journal" } + relabel_rules = loki.relabel.journal.rules + forward_to = [loki.write.local.receiver] + } + + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "host" + } + rule { + source_labels = ["__journal_syslog_identifier"] + target_label = "syslog_id" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + rule { + source_labels = ["__journal__systemd_machine_name"] + target_label = "container" + } + } + + loki.write "local" { + endpoint { + url = "http://127.0.0.1:3100/loki/api/v1/push" + } + } + ''; + + dashboards = lib.filterAttrs (name: _: lib.hasSuffix "-dashboard" name) ( + dsl.listMap ./overseer (_: path: import path { inherit pkgs lib; }) + ); + + dashboardDir = pkgs.linkFarm "grafana-dashboards" ( + lib.mapAttrsToList (name: dashboard: { + name = "${lib.removeSuffix "-dashboard" name}.json"; + path = dashboard; + }) dashboards + ); + + landingRoot = pkgs.writeTextDir "index.html" '' + + + + + Overseer + + + +

Overseer

+ + + + + ''; +in +let + c = + mkContainer { + inherit name; + net.access = [ + { + to = "@host"; + tcp = [ + nodePort + cadvPort + ]; + } + ]; + svc.autoStart = true; + resources.fs = { + "/var/lib/loki" = { + hostPath = "${data}/overseer/loki/data"; + isReadOnly = false; + }; + "/var/lib/prometheus2" = { + hostPath = "${data}/overseer/prometheus/data"; + isReadOnly = false; + }; + "/var/lib/grafana" = { + hostPath = "${data}/overseer/grafana"; + isReadOnly = false; + }; + "/var/log/journal" = { + hostPath = "/var/log/journal"; + isReadOnly = true; + }; + }; + containerConfig = _: { + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server = { + http_listen_port = 3100; + grpc_listen_port = 9096; + }; + common = { + instance_addr = "127.0.0.1"; + path_prefix = "/var/lib/loki"; + storage.filesystem = { + chunks_directory = "/var/lib/loki/chunks"; + rules_directory = "/var/lib/loki/rules"; + }; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + schema_config.configs = [ + { + from = "2020-10-24"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + }; + + services.alloy = { + enable = true; + configPath = alloyConfig; + }; + + services.prometheus = { + enable = true; + port = 9090; + globalConfig.scrape_interval = "15s"; + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ { targets = [ "${hostAddr}:${toString nodePort}" ]; } ]; + } + { + job_name = "cadvisor"; + static_configs = [ { targets = [ "${hostAddr}:${toString cadvPort}" ]; } ]; + } + { + job_name = "prometheus"; + static_configs = [ { targets = [ "localhost:9090" ]; } ]; + } + ]; + ruleFiles = [ + containerNetworkRulesFile + containerDefinedRulesFile + (dsl.mkContainerLimitsRuleFile pkgs containers) + ]; + rules = [ + '' + groups: + - name: containers + rules: + - alert: ContainerFailed + expr: node_systemd_unit_state{name=~"container@.*",state="failed"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "{{ $labels.name }} entered failed state" + - alert: ContainerOOMKill + expr: increase(container_oom_events_total{id=~"/machine.slice/container@.+\\.service.*"}[5m]) > 0 + labels: + severity: warning + annotations: + summary: "OOM kill in {{ $labels.id }}" + '' + ]; + }; + + services.grafana = { + enable = true; + dataDir = "/var/lib/grafana"; + settings = { + server.http_port = 3000; + server.http_addr = "127.0.0.1"; + # WORKAROUND: Grafana 10+ requires an explicit secret_key; omitting it + # causes auth session tokens to be regenerated on every restart + security.secret_key = "SW2YcwTIb9zpOOhoPsMm"; + "auth".disable_login_form = true; + "auth.anonymous" = { + enabled = true; + org_role = "Admin"; + }; + }; + provision = { + enable = true; + datasources.settings = { + apiVersion = 1; + datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + uid = "prometheus"; + access = "proxy"; + url = "http://localhost:9090"; + isDefault = true; + jsonData.timeInterval = "15s"; + } + { + name = "Loki"; + type = "loki"; + uid = "loki"; + access = "proxy"; + url = "http://127.0.0.1:3100"; + } + ]; + }; + dashboards.settings = { + apiVersion = 1; + providers = [ + { + name = "default"; + type = "file"; + disableDeletion = true; + options.path = toString dashboardDir; + } + ]; + }; + }; + }; + + services.nginx = { + enable = true; + virtualHosts."_" = { + listen = [ + { + addr = "0.0.0.0"; + port = 80; + ssl = false; + extraParameters = [ "default_server" ]; + } + ]; + root = "${landingRoot}"; + }; + virtualHosts."stats.localhost" = { + listen = [ + { + addr = "0.0.0.0"; + port = 80; + ssl = false; + } + ]; + locations."/" = { + proxyPass = "http://127.0.0.1:3000"; + proxyWebsockets = true; + extraConfig = "proxy_set_header Host $host;"; + }; + }; + virtualHosts."prometheus.localhost" = { + listen = [ + { + addr = "0.0.0.0"; + port = 80; + ssl = false; + } + ]; + locations."/" = { + proxyPass = "http://127.0.0.1:9090"; + }; + }; + }; + + systemd.services = lib.genAttrs [ "loki" "alloy" "prometheus" "grafana" "nginx" ] (_: { + serviceConfig.FailureAction = "exit-force"; + }); + }; + net.expose.tcp = [ 80 ]; + resources.mem = "2G"; + } + // { + hostConfig = { + services.prometheus.exporters.node = { + enable = true; + listenAddress = "0.0.0.0"; + port = nodePort; + enabledCollectors = [ + "systemd" + "processes" + ]; + disabledCollectors = [ + "netstat" + "softnet" + ]; + # radeon sometimes breaks on kexec + extraFlags = [ + "--collector.systemd.enable-task-metrics" + "--collector.hwmon.chip-exclude=radeon" + ]; + }; + services.cadvisor = { + enable = true; + listenAddress = "0.0.0.0"; + port = cadvPort; + }; + }; + }; + hostAddr = c.hostAddr; +in +c diff --git a/containers/common/overseer/access-logs-dashboard.nix b/containers/common/overseer/access-logs-dashboard.nix new file mode 100644 index 0000000..47eff52 --- /dev/null +++ b/containers/common/overseer/access-logs-dashboard.nix @@ -0,0 +1,61 @@ +{ pkgs, lib }: +let + inherit (import ./helpers.nix { inherit pkgs lib; }) mkRow loki mkDashboard; + + mkLogs = + { + title, + expr, + w ? 24, + h ? 10, + }: + { + inherit title; + type = "logs"; + datasource = loki; + gridPos = { inherit w h; }; + options = { + showTime = true; + showLabels = true; + sortOrder = "Descending"; + wrapLogMessage = true; + enableLogDetails = true; + }; + targets = [ + { + datasource = loki; + inherit expr; + } + ]; + }; +in +mkDashboard { + name = "access-logs.json"; + title = "Access Logs"; + uid = "access-logs"; + from = "now-6h"; + panels = [ + (mkRow "HTTP") + (mkLogs { + title = "nginx (access + error)"; + expr = ''{syslog_id="nginx"}''; + }) + (mkRow "SSH") + (mkLogs { + title = "sshpiper (all connections)"; + expr = ''{container="sshpiper"}''; + }) + (mkLogs { + title = "Failed / invalid auth attempts (all backends, incl. host)"; + expr = ''{syslog_id=~"sshd.*"} |~ "Failed|Invalid|error:"''; + }) + (mkLogs { + title = "sshd auth events (all backends, incl. host)"; + expr = ''{syslog_id=~"sshd.*"}''; + }) + (mkLogs { + title = "Host sshd (port 5952)"; + expr = ''{syslog_id=~"sshd.*", container=""}''; + }) + ]; +} diff --git a/containers/common/overseer/containers-dashboard.nix b/containers/common/overseer/containers-dashboard.nix new file mode 100644 index 0000000..c8d4f5d --- /dev/null +++ b/containers/common/overseer/containers-dashboard.nix @@ -0,0 +1,251 @@ +{ pkgs, lib }: +let + inherit (import ./helpers.nix { inherit pkgs lib; }) prom mkDashboard; +in +mkDashboard { + name = "containers.json"; + title = "Containers"; + uid = "containers"; + from = "now-3h"; + panels = [ + { + title = "Container States"; + type = "piechart"; + gridPos = { + h = 8; + w = 8; + }; + datasource = prom; + options = { + pieType = "donut"; + legend = { + displayMode = "table"; + placement = "right"; + values = [ + "value" + "percent" + ]; + }; + tooltip.mode = "single"; + }; + fieldConfig.overrides = [ + { + matcher = { + id = "byName"; + options = "active"; + }; + properties = [ + { + id = "color"; + value = { + mode = "fixed"; + fixedColor = "green"; + }; + } + ]; + } + { + matcher = { + id = "byName"; + options = "failed"; + }; + properties = [ + { + id = "color"; + value = { + mode = "fixed"; + fixedColor = "red"; + }; + } + ]; + } + { + matcher = { + id = "byName"; + options = "inactive"; + }; + properties = [ + { + id = "color"; + value = { + mode = "fixed"; + fixedColor = "blue"; + }; + } + ]; + } + { + matcher = { + id = "byName"; + options = "activating"; + }; + properties = [ + { + id = "color"; + value = { + mode = "fixed"; + fixedColor = "yellow"; + }; + } + ]; + } + { + matcher = { + id = "byName"; + options = "unknown"; + }; + properties = [ + { + id = "color"; + value = { + mode = "fixed"; + fixedColor = "gray"; + }; + } + ]; + } + ]; + targets = [ + { + datasource = prom; + expr = ''count by (state) ((label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service")) or on(container) label_replace(container_defined, "state", "unknown", "container", ".+"))''; + instant = true; + legendFormat = "{{state}}"; + } + ]; + } + { + title = "Container States"; + type = "table"; + gridPos = { + h = 16; + w = 24; + }; + datasource = prom; + options.footer.show = false; + fieldConfig = { + defaults = { }; + overrides = [ + { + matcher = { + id = "byName"; + options = "State"; + }; + properties = [ + { + id = "custom.displayMode"; + value = "color-text"; + } + { + id = "mappings"; + value = [ + { + type = "value"; + options.active = { + text = "active"; + color = "green"; + index = 0; + }; + } + { + type = "value"; + options.failed = { + text = "failed"; + color = "red"; + index = 1; + }; + } + { + type = "value"; + options.activating = { + text = "activating"; + color = "yellow"; + index = 2; + }; + } + { + type = "value"; + options.inactive = { + text = "inactive"; + color = "blue"; + index = 3; + }; + } + { + type = "value"; + options.unknown = { + text = "unknown"; + color = "gray"; + index = 4; + }; + } + ]; + } + ]; + } + ]; + }; + targets = [ + { + datasource = prom; + expr = '' + (label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service")) + or on(container) + label_replace(container_defined, "state", "unknown", "container", ".+") + ''; + instant = true; + format = "table"; + legendFormat = "__auto"; + } + ]; + transformations = [ + { + id = "organize"; + options = { + excludeByName = { + Time = true; + Value = true; + "__name__" = true; + instance = true; + job = true; + name = true; + }; + renameByName = { + container = "Container"; + state = "State"; + }; + }; + } + ]; + } + { + title = "Container State (1 = running, 0 = stopped/failed)"; + type = "timeseries"; + gridPos = { + h = 8; + w = 24; + }; + datasource = prom; + fieldConfig.defaults = { + custom.lineWidth = 2; + color.mode = "palette-classic"; + min = 0; + max = 1; + }; + options = { + tooltip.mode = "multi"; + legend = { + displayMode = "table"; + placement = "bottom"; + }; + }; + targets = [ + { + datasource = prom; + expr = ''label_replace(node_systemd_unit_state{name=~"container@.*",state="active"}, "container", "$1", "name", "container@(.+).service")''; + legendFormat = "{{container}}"; + } + ]; + } + ]; +} diff --git a/containers/common/overseer/helpers.nix b/containers/common/overseer/helpers.nix new file mode 100644 index 0000000..2a64f6b --- /dev/null +++ b/containers/common/overseer/helpers.nix @@ -0,0 +1,228 @@ +{ pkgs, lib }: +let + prom = { + type = "prometheus"; + uid = "prometheus"; + }; + loki = { + type = "loki"; + uid = "loki"; + }; + + autoLayout = + panels: + let + step = + acc: panel: + let + w = panel.gridPos.w; + h = panel.gridPos.h; + wrap = acc.x + w > 24; + x = if wrap then 0 else acc.x; + y = if wrap then acc.y + acc.rowH else acc.y; + rowH = if wrap then h else lib.max acc.rowH h; + in + { + x = x + w; + inherit y rowH; + result = acc.result ++ [ + ( + panel + // { + gridPos = panel.gridPos // { + inherit x y; + }; + } + ) + ]; + }; + in + (lib.foldl' step { + x = 0; + y = 0; + rowH = 0; + result = [ ]; + } panels).result; + + T = expr: legendFormat: { + datasource = prom; + inherit expr legendFormat; + }; + + mkRow = title: { + inherit title; + type = "row"; + collapsed = false; + panels = [ ]; + gridPos = { + w = 24; + h = 1; + }; + }; + + mkStat = + { + title, + w ? 4, + h ? 4, + expr, + unit ? "short", + decimals ? 2, + }: + { + inherit title; + type = "stat"; + datasource = prom; + gridPos = { inherit w h; }; + fieldConfig.defaults = { + inherit unit decimals; + color.mode = "thresholds"; + thresholds = { + mode = "absolute"; + steps = [ + { + color = "blue"; + value = null; + } + ]; + }; + }; + options = { + reduceOptions.calcs = [ "lastNotNull" ]; + colorMode = "value"; + textMode = "value"; + }; + targets = [ + { + datasource = prom; + inherit expr; + instant = true; + legendFormat = ""; + } + ]; + }; + + mkGauge = + { + title, + w ? 4, + h ? 4, + expr, + unit ? "percent", + min ? 0, + max ? 100, + }: + { + inherit title; + type = "gauge"; + datasource = prom; + gridPos = { inherit w h; }; + fieldConfig.defaults = { + inherit unit min max; + thresholds = { + mode = "absolute"; + steps = [ + { + color = "green"; + value = null; + } + { + color = "yellow"; + value = 70; + } + { + color = "red"; + value = 90; + } + ]; + }; + }; + options = { + reduceOptions.calcs = [ "lastNotNull" ]; + orientation = "auto"; + showThresholdLabels = false; + showThresholdMarkers = true; + }; + targets = [ + { + datasource = prom; + inherit expr; + instant = true; + legendFormat = ""; + } + ]; + }; + + mkTs = + { + title, + w ? 24, + h ? 8, + unit ? "short", + targets, + }: + { + inherit title targets; + type = "timeseries"; + datasource = prom; + gridPos = { inherit w h; }; + fieldConfig.defaults = { + inherit unit; + custom = { + lineWidth = 1; + fillOpacity = 5; + }; + color.mode = "palette-classic"; + }; + options = { + tooltip.mode = "multi"; + legend = { + displayMode = "table"; + placement = "bottom"; + calcs = [ + "mean" + "max" + "lastNotNull" + ]; + }; + }; + }; + + mkDashboard = + { + name, + title, + uid, + from ? "now-1h", + panels, + templating ? [ ], + }: + pkgs.writeText name ( + builtins.toJSON ( + { + inherit title uid; + schemaVersion = 37; + refresh = "30s"; + time = { + inherit from; + to = "now"; + }; + panels = lib.imap1 (id: p: p // { inherit id; }) (autoLayout panels); + } + // lib.optionalAttrs (templating != [ ]) { templating.list = templating; } + ) + ); +in +{ + inherit + prom + loki + autoLayout + T + mkRow + mkStat + mkGauge + mkTs + mkDashboard + ; +} diff --git a/containers/common/overseer/host-dashboard.nix b/containers/common/overseer/host-dashboard.nix new file mode 100644 index 0000000..b490a9b --- /dev/null +++ b/containers/common/overseer/host-dashboard.nix @@ -0,0 +1,167 @@ +{ pkgs, lib }: +let + inherit (import ./helpers.nix { inherit pkgs lib; }) + T + mkRow + mkStat + mkGauge + mkTs + mkDashboard + ; +in +mkDashboard { + name = "host.json"; + title = "Host"; + uid = "host"; + panels = [ + + (mkRow "Overview") + (mkGauge { + title = "CPU"; + expr = ''100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])))''; + }) + (mkGauge { + title = "Memory used"; + expr = "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)"; + }) + (mkStat { + title = "Load (1m)"; + unit = "short"; + decimals = 2; + expr = "node_load1"; + }) + (mkStat { + title = "Load (5m)"; + unit = "short"; + decimals = 2; + expr = "node_load5"; + }) + (mkStat { + title = "Uptime"; + unit = "dtdurations"; + decimals = 0; + expr = "node_time_seconds - node_boot_time_seconds"; + }) + + (mkRow "CPU") + (mkTs { + title = "CPU usage (%)"; + unit = "percent"; + targets = [ + (T ''avg(rate(node_cpu_seconds_total{mode="user"}[5m])) * 100'' "user") + (T ''avg(rate(node_cpu_seconds_total{mode="system"}[5m])) * 100'' "system") + (T ''avg(rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100'' "iowait") + (T ''avg(rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100'' "steal") + (T ''avg(rate(node_cpu_seconds_total{mode="softirq"}[5m])) * 100'' "softirq") + ]; + }) + (mkTs { + title = "Per-core usage (%)"; + unit = "percent"; + targets = [ + (T ''(1 - rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100'' "cpu{{cpu}}") + ]; + }) + (mkTs { + title = "CPU temperatures"; + unit = "celsius"; + targets = [ + (T "node_hwmon_temp_celsius" "{{chip}} {{sensor}}") + ]; + }) + (mkTs { + title = "Load average"; + targets = [ + (T "node_load1" "1m") + (T "node_load5" "5m") + (T "node_load15" "15m") + ]; + }) + + (mkRow "Memory") + (mkTs { + title = "Memory"; + unit = "bytes"; + targets = [ + (T "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes" "used") + (T "node_memory_Cached_bytes + node_memory_Buffers_bytes" "cache + buffers") + (T "node_memory_MemAvailable_bytes" "available") + (T "node_memory_MemTotal_bytes" "total") + ]; + }) + (mkTs { + title = "Swap"; + unit = "bytes"; + targets = [ + (T "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes" "used") + (T "node_memory_SwapTotal_bytes" "total") + ]; + }) + + (mkRow "Disk") + (mkTs { + title = "Filesystem usage (%)"; + unit = "percent"; + targets = [ + (T ''100 * (1 - node_filesystem_avail_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"} / node_filesystem_size_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"})'' "{{mountpoint}}") + ]; + }) + (mkTs { + title = "Disk IO (bytes/s)"; + unit = "Bps"; + targets = [ + (T ''rate(node_disk_read_bytes_total{device!~"loop.*"}[5m])'' "{{device}} read") + (T ''rate(node_disk_written_bytes_total{device!~"loop.*"}[5m])'' "{{device}} write") + ]; + }) + (mkTs { + title = "Disk IO utilisation (%)"; + unit = "percent"; + targets = [ + (T ''rate(node_disk_io_time_seconds_total{device!~"loop.*"}[5m]) * 100'' "{{device}}") + ]; + }) + + (mkRow "Network") + (mkTs { + title = "Network (bytes/s)"; + unit = "Bps"; + targets = [ + (T ''rate(node_network_receive_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx") + (T ''rate(node_network_transmit_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx") + ]; + }) + (mkTs { + title = "Network errors + drops (pps)"; + targets = [ + (T ''rate(node_network_receive_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx err") + (T ''rate(node_network_transmit_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx err") + (T ''rate(node_network_receive_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx drop") + (T ''rate(node_network_transmit_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx drop") + ]; + }) + + (mkRow "System") + (mkTs { + title = "Context switches + forks (ops/s)"; + targets = [ + (T "rate(node_context_switches_total[5m])" "context switches") + (T "rate(node_forks_total[5m])" "forks") + ]; + }) + (mkTs { + title = "File descriptors"; + targets = [ + (T "node_filefd_allocated" "allocated") + (T "node_filefd_maximum" "limit") + ]; + }) + (mkTs { + title = "Processes"; + targets = [ + (T "node_procs_running" "running") + (T "node_procs_blocked" "blocked on IO") + ]; + }) + ]; +} diff --git a/containers/common/overseer/resources-dashboard.nix b/containers/common/overseer/resources-dashboard.nix new file mode 100644 index 0000000..5f2ac38 --- /dev/null +++ b/containers/common/overseer/resources-dashboard.nix @@ -0,0 +1,203 @@ +{ pkgs, lib }: +let + inherit (import ./helpers.nix { inherit pkgs lib; }) + T + mkRow + mkStat + mkTs + prom + mkDashboard + ; + + ctid = "id=~\"/machine.slice/container@.+\\\\.service/payload/.+\""; + ctsel = "id=~\"/machine.slice/container@$container\\\\.service/payload/.+\""; + + ctall = "id=~\"/machine.slice/container@.+\\\\.service.*\""; + ctnet = "id=~\"/machine.slice/container@$container\\\\.service.*\""; + + labeled = + m: + "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service/payload.+\")"; + byContainer = m: "sum by (container) (${labeled m})"; + + labeledAll = + m: + "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service.*\")"; + byContainerAll = m: "sum by (container) (${labeledAll m})"; + + nmSvc = m: "label_replace(${m}, \"container\", \"$1\", \"name\", \"container@(.+)\\\\.service\")"; + + orZero = m: fb: "(${m}) or on(container) (${fb} * 0)"; + + cpuAll = + rate: + orZero (byContainer "rate(container_cpu_usage_seconds_total{${ctid}}[${rate}])") "container_limit_cpu_cores"; + memAll = orZero (byContainerAll "container_memory_rss{${ctall}}") "container_limit_memory_bytes"; + taskAll = nmSvc "node_systemd_unit_tasks_current{name=~\"container@.+\"}"; +in +mkDashboard { + name = "resources.json"; + title = "Container Resources"; + uid = "container-resources"; + + templating = [ + { + name = "container"; + type = "query"; + datasource = prom; + query = "label_values(container_limit_cpu_cores, container)"; + refresh = 2; + sort = 1; + includeAll = true; + multi = true; + current = { + selected = true; + text = [ "All" ]; + value = [ "$__all" ]; + }; + } + ]; + + panels = [ + (mkRow "Overview") + (mkStat { + title = "Running"; + decimals = 0; + expr = "count(node_systemd_unit_state{name=~\"container@.*\",state=\"active\"} == 1) or vector(0)"; + }) + (mkStat { + title = "CPU used"; + decimals = 2; + expr = "sum(${cpuAll "5m"}) or vector(0)"; + }) + (mkStat { + title = "CPU limit"; + decimals = 2; + expr = "sum(container_limit_cpu_cores) or vector(0)"; + }) + (mkStat { + title = "Mem RSS"; + decimals = 1; + unit = "bytes"; + expr = "sum(${memAll}) or vector(0)"; + }) + (mkStat { + title = "Mem limit"; + decimals = 1; + unit = "bytes"; + expr = "sum(container_limit_memory_bytes) or vector(0)"; + }) + (mkStat { + title = "Tasks"; + decimals = 0; + expr = "sum(${taskAll}) or vector(0)"; + }) + + (mkRow "All Containers") + (mkTs { + title = "CPU (cores)"; + h = 24; + targets = [ (T (cpuAll "5m") "{{container}}") ]; + }) + (mkTs { + title = "Memory RSS"; + unit = "bytes"; + h = 24; + targets = [ (T memAll "{{container}}") ]; + }) + (mkTs { + title = "Tasks"; + h = 24; + targets = [ (T taskAll "{{container}}") ]; + }) + (mkTs { + title = "CPU throttled %"; + unit = "percent"; + h = 12; + targets = [ + (T "(${byContainer "rate(container_cpu_cfs_throttled_periods_total{${ctid}}[5m])"} / ${byContainer "rate(container_cpu_cfs_periods_total{${ctid}}[5m])"}) * 100 or on(container) (container_limit_cpu_cores * 0)" "{{container}}") + ]; + }) + (mkTs { + title = "Network egress (bytes/s)"; + unit = "Bps"; + h = 24; + targets = [ (T "sum by (container) (container_network_receive_rate)" "{{container}}") ]; + }) + (mkTs { + title = "Network ingress (bytes/s)"; + unit = "Bps"; + h = 24; + targets = [ (T "sum by (container) (container_network_transmit_rate)" "{{container}}") ]; + }) + (mkTs { + title = "Disk read (bytes/s)"; + unit = "Bps"; + h = 24; + targets = [ (T (byContainer "rate(container_fs_reads_bytes_total{${ctid}}[5m])") "{{container}}") ]; + }) + (mkTs { + title = "Disk write (bytes/s)"; + unit = "Bps"; + h = 24; + targets = [ + (T (byContainer "rate(container_fs_writes_bytes_total{${ctid}}[5m])") "{{container}}") + ]; + }) + + { + type = "row"; + title = "$container"; + repeat = "container"; + collapsed = true; + panels = [ ]; + gridPos = { + w = 24; + h = 1; + }; + } + (mkTs { + title = "CPU (cores)"; + targets = [ + (T "sum(rate(container_cpu_usage_seconds_total{${ctsel}}[5m]))" "used") + (T "container_limit_cpu_cores{container=\"$container\"}" "limit") + ]; + }) + (mkTs { + title = "CPU throttled %"; + unit = "percent"; + targets = [ + (T "(sum(rate(container_cpu_cfs_throttled_periods_total{${ctsel}}[5m])) / sum(rate(container_cpu_cfs_periods_total{${ctsel}}[5m]))) * 100 or vector(0)" "throttled %") + ]; + }) + (mkTs { + title = "Memory"; + unit = "bytes"; + targets = [ + (T "sum(container_memory_rss{${ctnet}})" "RSS") + (T "sum(container_memory_working_set_bytes{${ctnet}})" "working set") + (T "container_limit_memory_bytes{container=\"$container\"}" "limit") + ]; + }) + (mkTs { + title = "Network (bytes/s)"; + unit = "Bps"; + targets = [ + (T "container_network_receive_rate{container=\"$container\"}" "{{device}} ↑") + (T "container_network_transmit_rate{container=\"$container\"}" "{{device}} ↓") + ]; + }) + (mkTs { + title = "Disk IO (bytes/s)"; + unit = "Bps"; + targets = [ + (T "sum(rate(container_fs_reads_bytes_total{${ctsel}}[5m]))" "read") + (T "sum(rate(container_fs_writes_bytes_total{${ctsel}}[5m]))" "write") + ]; + }) + (mkTs { + title = "Tasks"; + targets = [ (T "node_systemd_unit_tasks_current{name=\"container@$container.service\"}" "tasks") ]; + }) + ]; +} diff --git a/containers/server-n0/devenv.nix b/containers/server-n0/devenv.nix new file mode 100644 index 0000000..400238f --- /dev/null +++ b/containers/server-n0/devenv.nix @@ -0,0 +1,54 @@ +{ + name, + hostInfo, + dsl, + ... +}: +let + inherit (dsl) mkContainer mkService; + inherit (hostInfo.paths) data secrets; +in +mkContainer { + inherit name; + net.access = [ { to = "@internet"; } ]; + svc.autoStart = true; + resources.fs = { + "/config" = { + hostPath = "${data}/web/devenv/config"; + isReadOnly = false; + }; + "/workspace" = { + hostPath = "${data}/web/devenv/workspace"; + isReadOnly = false; + }; + "/run/secrets/code-server-env" = { + hostPath = "${secrets}/code-server-env"; + isReadOnly = true; + }; + }; + users = [ + { + name = "coder"; + uid = 1000; + home = "/config"; + } + ]; + service = + { pkgs, lib, ... }: + mkService { + description = "code-server"; + execStart = "${lib.getExe' pkgs.code-server "code-server"} --bind-addr 0.0.0.0:8080 --user-data-dir /config/data --extensions-dir /config/extensions --auth password /workspace"; + networkOnline = true; + serviceConfig = { + EnvironmentFile = "/run/secrets/code-server-env"; + User = "coder"; + ReadWritePaths = [ + "/config" + "/workspace" + ]; + }; + }; + net.expose.tcp = [ 8080 ]; + resources.mem = "4G"; + resources.cpu = 2; +} diff --git a/containers/server-n0/git.nix b/containers/server-n0/git.nix new file mode 100644 index 0000000..f4510d7 --- /dev/null +++ b/containers/server-n0/git.nix @@ -0,0 +1,178 @@ +{ + name, + hostInfo, + dsl, + ... +}: +let + inherit (dsl) mkContainer mkVm; + inherit (hostInfo.paths) data secrets; + + ciRunnerData = "${data}/web/git-ci-runner"; + ciRunnerSecrets = "${secrets}/ci-runner"; +in +{ + ${name} = mkContainer { + inherit name; + net.access = [ { to = "@internet"; } ]; + net.forward.tcp = [ + { + host = 23; + container = 8023; + } + ]; + svc.autoStart = true; + resources.fs."/var/lib/gitea" = { + hostPath = "${data}/web/git"; + isReadOnly = false; + }; + containerConfig = _: { + services.gitea = { + enable = true; + stateDir = "/var/lib/gitea"; + settings = { + server = { + HTTP_ADDR = "0.0.0.0"; + HTTP_PORT = 3000; + DOMAIN = "git.floweytf.com"; + ROOT_URL = "https://git.floweytf.com/"; + START_SSH_SERVER = true; + BUILTIN_SSH_SERVER_USER = "git"; + SSH_PORT = 8023; + }; + service.DISABLE_REGISTRATION = true; + }; + }; + users.users.gitea.uid = 1000; + users.groups.gitea.gid = 1000; + time.timeZone = "UTC"; + systemd.services.gitea.serviceConfig.FailureAction = "exit-force"; + }; + net.expose.tcp = [ 3000 ]; + }; + + ci-runner = mkVm { + name = "ci-runner"; + net.access = [ + { + to = "git"; + tcp = [ 3000 ]; + } + { to = "@internet"; } + ]; + svc.autoStart = true; + resources.cpu = 4; + resources.mem = "8G"; + resources.fs = [ + { + proto = "virtiofs"; + tag = "nix-rw-store"; + source = "${ciRunnerData}/nix-rw-store"; + mountPoint = "/nix/.rw-store"; + } + { + proto = "virtiofs"; + tag = "docker-data"; + source = "${ciRunnerData}/docker"; + mountPoint = "/var/lib/docker"; + } + { + proto = "virtiofs"; + tag = "runner-state"; + source = "${ciRunnerData}/runner-state"; + mountPoint = "/var/lib/gitea-runner"; + } + { + proto = "virtiofs"; + tag = "runner-token"; + source = ciRunnerSecrets; + mountPoint = "/run/ci-runner-secrets"; + readOnly = true; + } + ]; + guestConfig = + { + pkgs, + utils, + lib, + ... + }: + let + runnerInstanceName = "ci-runner"; + + ciJobImage = pkgs.dockerTools.buildLayeredImage { + name = "ci-job-image"; + tag = "latest"; + contents = with pkgs; [ + bashInteractive + coreutils + gitMinimal + nixVersions.latest + nodejs + cacert + ]; + config = { + Cmd = [ "/bin/sh" ]; + Env = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "NIX_REMOTE=daemon" + ]; + }; + }; + in + { + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModprobeConfig = "options kvm_intel nested=1"; + + microvm.writableStoreOverlay = "/nix/.rw-store"; + + nix.settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + max-jobs = "auto"; + }; + + nix.gc = { + automatic = true; + dates = "daily"; + options = "--delete-older-than 14d"; + }; + # nix.optimise.automatic = true; - doesn't work with microvm.writableStoreOverlay + + virtualisation.docker.enable = true; + + systemd.services.load-ci-job-image = { + description = "Load the CI job Docker image"; + wantedBy = [ "multi-user.target" ]; + after = [ "docker.service" ]; + requires = [ "docker.service" ]; + before = [ "gitea-runner-${utils.escapeSystemdPath runnerInstanceName}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${lib.getExe' pkgs.docker "docker"} load -i ${ciJobImage}"; + }; + }; + + services.gitea-actions-runner.instances.${runnerInstanceName} = { + enable = true; + name = runnerInstanceName; + url = "http://git:3000/"; + tokenFile = "/run/ci-runner-secrets/token"; + labels = [ + "nix:docker://ci-job-image:latest" + ]; + settings.container = { + options = "--device=/dev/kvm --volume=/nix/store:/nix/store:ro --volume=/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket"; + valid_volumes = [ + "/nix/store" + "/nix/var/nix/daemon-socket" + ]; + }; + }; + }; + }; +} diff --git a/containers/server-n0/hopper.nix b/containers/server-n0/hopper.nix new file mode 100644 index 0000000..77fca75 --- /dev/null +++ b/containers/server-n0/hopper.nix @@ -0,0 +1,79 @@ +{ + name, + pkgs, + lib, + dsl, + containers, + customPkgs, + ... +}: +let + inherit (dsl) mkContainer mkService collectContainers; + + mkHopperConfig = + { routes }: + let + header = '' + listen = "0.0.0.0:25565" + + [routing.routes] + ''; + mkRoute = { domain, ip }: '' + "${domain}" = { ip = "${ip}" } + ''; + in + header + lib.concatMapStrings mkRoute routes; + + backends = collectContainers "hopperRegister" containers; + + mcRoutes = map ({ container, data }: { + domain = "${data.hopperSubdomain}.mc.floweytf.com"; + ip = "${container.name}:${toString data.hopperPort}"; + }) backends; + + hopperCfg = pkgs.writeText "Config.toml" (mkHopperConfig { + routes = mcRoutes; + }); +in +{ + ${name} = mkContainer { + inherit name; + net.access = map ({ container, data }: { + to = container.name; + tcp = [ data.hopperPort ]; + }) backends; + svc.autoStart = true; + net.expose.tcp = [ 25565 ]; + net.forward.tcp = [ 25565 ]; + resources.fs."/Config.toml" = { + hostPath = toString hopperCfg; + isReadOnly = true; + }; + service = + _: + mkService { + description = "Hopper Minecraft reverse proxy"; + execStart = lib.getExe' customPkgs.hopper-rs "hopper"; + workingDir = "/"; + networkOnline = true; + serviceConfig = { + DynamicUser = true; + PrivateTmp = true; + PrivateDevices = true; + CapabilityBoundingSet = ""; + RestrictAddressFamilies = "AF_INET AF_INET6"; + }; + }; + }; + _hopperRegister = + { + hopperSubdomain ? null, + hopperPort ? 25565, + }: + self: { + hopperRegister = { + hopperSubdomain = if hopperSubdomain == null then self.name else hopperSubdomain; + inherit hopperPort; + }; + }; +} diff --git a/containers/server-n0/mail.nix b/containers/server-n0/mail.nix new file mode 100644 index 0000000..1c0309c --- /dev/null +++ b/containers/server-n0/mail.nix @@ -0,0 +1,264 @@ +{ + name, + pkgs, + lib, + hostInfo, + dsl, + ... +}: +let + inherit (dsl) mkContainer mkService; + inherit (hostInfo.paths) data secrets; + + domain = "mail.floweytf.com"; + accountName = "flowey"; + stateDir = "/var/lib/stalwart"; + acmeDir = "/etc/mail-acme"; + + taggingScript = '' + require ["envelope", "variables", "imap4flags", "fileinto"]; + + if envelope :localpart :matches "to" "*" { + addflag "''${1}"; + } + + fileinto "INBOX"; + ''; + + cli = lib.getExe' pkgs.stalwart-cli "stalwart-cli"; + jq = lib.getExe pkgs.jq; + curl = lib.getExe pkgs.curl; + grep = lib.getExe' pkgs.gnugrep "grep"; + paste = lib.getExe' pkgs.coreutils "paste"; + + # use sqlite b/c rocksdb segfaults + configJsonTemplate = pkgs.writeText "stalwart-config.json" ( + builtins.toJSON { + "@type" = "Sqlite"; + path = "${stateDir}/data.sqlite3"; + } + ); + + staticPlan = pkgs.writeText "mail-static-plan.json" ( + builtins.concatStringsSep "\n" ( + map builtins.toJSON [ + { + "@type" = "upsert"; + object = "Domain"; + matchOn = [ "name" ]; + value."domain-mail" = { + name = domain; + catchAllAddress = "${accountName}@${domain}"; + isEnabled = true; + }; + } + { + "@type" = "upsert"; + object = "SieveSystemScript"; + matchOn = [ "name" ]; + value."sieve-tag" = { + name = "tag-by-recipient"; + isActive = true; + contents = taggingScript; + }; + } + ] + ) + + "\n" + ); + + stalwartAuthEnv = '' + export STALWART_URL="http://127.0.0.1:8080" + export STALWART_USER="admin" + export STALWART_PASSWORD="$(cat "$CREDENTIALS_DIRECTORY/admin_password")" + ''; + + readyGate = pkgs.writeShellScript "mail-ready-gate" '' + set -uo pipefail + until ${curl} -sS -o /dev/null "http://127.0.0.1:8080/api/account"; do + if ! systemctl -q is-active stalwart.service; then + echo "mail: stalwart is not running" >&2 + exit 1 + fi + sleep 1 + done + ''; + + bootstrapScript = pkgs.writeShellScript "mail-bootstrap" '' + set -uo pipefail + ${stalwartAuthEnv} + + out="$(${cli} update Bootstrap singleton \ + --field defaultDomain=${domain} \ + --field serverHostname=${domain} \ + --field generateDkimKeys=true \ + --field requestTlsCertificate=false \ + --field 'directory={"@type":"Internal"}' 2>&1)" + status=$? + + if [ "$status" -eq 0 ]; then + echo "mail-bootstrap: first-time setup complete, restarting stalwart into normal mode" >&2 + systemctl restart stalwart.service + exit 0 + fi + + if ${grep} -qi 'bootstrap mode' <<<"$out"; then + exit 0 + fi + + echo "mail-bootstrap: $out" >&2 + exit 1 + ''; + + certCredCert = "/run/credentials/stalwart.service/mail_tls_cert"; + certCredKey = "/run/credentials/stalwart.service/mail_tls_key"; + + retryOnFailure = { + Restart = "on-failure"; + RestartSec = 5; + }; + + retryLimit = { + StartLimitBurst = 5; + StartLimitIntervalSec = 120; + }; + + provisionScript = pkgs.writeShellScript "mail-provision" '' + set -euo pipefail + ${stalwartAuthEnv} + export account_pass="$(cat "$CREDENTIALS_DIRECTORY/account_password")" + + account_entry="$(${jq} -nc --arg name "${accountName}" \ + '{"@type":"upsert","object":"Account","matchOn":["name"],"value":{("account-"+$name):{"@type":"User","name":$name,"domainId":"#domain-mail","credentials":{"0":{"@type":"Password","secret":env.account_pass}}}}}')" + + { cat ${staticPlan}; echo "$account_entry"; } | ${cli} apply --stdin + + existing_cert_ids="$(${cli} query Certificate --json --fields id | ${jq} -r .id | ${paste} -sd, -)" + if [ -n "$existing_cert_ids" ]; then + ${cli} delete Certificate --ids "$existing_cert_ids" + fi + ${cli} create Certificate \ + --field 'certificate={"@type":"File","filePath":"${certCredCert}"}' \ + --field 'privateKey={"@type":"File","filePath":"${certCredKey}"}' + + systemctl restart stalwart.service + ''; +in +mkContainer { + inherit name; + net.access = [ { to = "@internet"; } ]; + svc.autoStart = true; + net.forward.tcp = [ + 25 + 993 + ]; + resources.fs = { + "${stateDir}" = { + hostPath = "${data}/core/mail"; + isReadOnly = false; + }; + "${acmeDir}" = { + hostPath = "${data}/core/nginx/acme/${domain}"; + isReadOnly = true; + }; + "/run/secrets/mail-admin-password" = { + hostPath = "${secrets}/mail-admin-password"; + isReadOnly = true; + }; + "/run/secrets/mail-account-password" = { + hostPath = "${secrets}/mail-account-password"; + isReadOnly = true; + }; + }; + users = [ + { + name = "stalwart"; + uid = 1000; + home = stateDir; + } + ]; + containerConfig = _: { + systemd.tmpfiles.rules = [ + "Z ${stateDir} 0750 stalwart stalwart -" + "C ${stateDir}/config.json 0640 stalwart stalwart - ${configJsonTemplate}" + ]; + + systemd.services.stalwart = mkService { + description = "Stalwart Mail Server"; + execStart = pkgs.writeShellScript "stalwart-run" '' + set -euo pipefail + export STALWART_RECOVERY_ADMIN="admin:$(cat "$CREDENTIALS_DIRECTORY/admin_password")" + exec ${lib.getExe' pkgs.stalwart_0_16 "stalwart"} -c ${stateDir}/config.json + ''; + serviceConfig = { + User = "stalwart"; + Group = "stalwart"; + AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; + ReadWritePaths = [ stateDir ]; + LoadCredential = [ + "admin_password:/run/secrets/mail-admin-password" + "mail_tls_cert:${acmeDir}/fullchain.pem" + "mail_tls_key:${acmeDir}/key.pem" + ]; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + systemd.paths.mail-cert-renew = { + wantedBy = [ "multi-user.target" ]; + pathConfig = { + PathChanged = "${acmeDir}/key.pem"; + Unit = "mail-provision.service"; + }; + }; + + systemd.services.mail-ready = mkService { + description = "Wait for the Stalwart management API"; + execStart = readyGate; + after = [ "stalwart.service" ]; + wants = [ "stalwart.service" ]; + wantedBy = [ ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + TimeoutStartSec = 60; + }; + }; + + systemd.services.mail-bootstrap = mkService { + description = "One-time Stalwart bootstrap (server identity + admin account)"; + execStart = bootstrapScript; + after = [ "mail-ready.service" ]; + wants = [ "mail-ready.service" ]; + unitConfig = retryLimit; + serviceConfig = { + Type = "oneshot"; + LoadCredential = [ "admin_password:/run/secrets/mail-admin-password" ]; + } + // retryOnFailure; + }; + + systemd.services.mail-provision = mkService { + description = "Provision mail.floweytf.com domain + catch-all account in Stalwart"; + execStart = provisionScript; + after = [ + "mail-ready.service" + "mail-bootstrap.service" + ]; + wants = [ + "mail-ready.service" + "mail-bootstrap.service" + ]; + serviceConfig = { + Type = "oneshot"; + LoadCredential = [ + "admin_password:/run/secrets/mail-admin-password" + "account_password:/run/secrets/mail-account-password" + ]; + } + // retryOnFailure; + unitConfig = retryLimit; + }; + }; +} diff --git a/containers/server-n0/maven.nix b/containers/server-n0/maven.nix new file mode 100644 index 0000000..bfa00aa --- /dev/null +++ b/containers/server-n0/maven.nix @@ -0,0 +1,41 @@ +{ + name, + hostInfo, + dsl, + ... +}: +let + inherit (dsl) mkContainer mkService; + inherit (hostInfo.paths) data; +in +mkContainer { + inherit name; + svc.autoStart = true; + resources.fs."/app/data" = { + hostPath = "${data}/web/reposilite"; + isReadOnly = false; + }; + users = [ + { + name = "reposilite"; + uid = 1000; + home = "/app"; + } + ]; + service = + { pkgs, lib, ... }: + mkService { + description = "Reposilite"; + execStart = "${lib.getExe' pkgs.reposilite "reposilite"} --working-directory /app/data"; + workingDir = "/app/data"; + serviceConfig = { + User = "reposilite"; + PrivateTmp = true; + ReadWritePaths = [ "/app/data" ]; + }; + } + // { + environment.JAVA_OPTS = "-Xmx128M"; + }; + net.expose.tcp = [ 8080 ]; +} diff --git a/containers/server-n0/mc.nix b/containers/server-n0/mc.nix new file mode 100644 index 0000000..c247831 --- /dev/null +++ b/containers/server-n0/mc.nix @@ -0,0 +1,205 @@ +{ + lib, + pkgs, + hostInfo, + dsl, + self, + ... +}: +let + inherit (dsl) mkContainer mkService; + inherit (hostInfo.paths) data; + + mkMcTrait = + { + jdk, + dataPath, + extraPackages ? [ ], + }: + { + resources.fs."/data" = { + hostPath = dataPath; + isReadOnly = false; + }; + users = [ + { + name = "runner"; + uid = 1000; + } + ]; + containerConfig = + { + pkgs, + ... + }: + { + environment.systemPackages = [ + jdk + pkgs.bash + ] + ++ extraPackages; + systemd.services.minecraft = mkService { + description = "Minecraft server"; + wantedBy = [ "multi-user.target" ]; + after = [ + "local-fs.target" + "container-ready.target" + ]; + wants = [ "container-ready.target" ]; + path = [ + jdk + pkgs.bash + ]; + execStart = "${lib.getExe' pkgs.bash "bash"} -c ${ + lib.escapeShellArg ( + "exec ${lib.getExe' pkgs.dtach "dtach"} -N /run/ct-attach/attach.sock " + + "${lib.getExe' pkgs.util-linux "script"} -qfc /data/launch.sh " + + ">(exec ${lib.getExe' pkgs.systemd "systemd-cat"} -t minecraft)" + ) + }"; + serviceConfig = { + WorkingDirectory = "/data"; + User = "runner"; + RuntimeDirectory = "ct-attach"; + RuntimeDirectoryMode = "0700"; + StandardInput = "null"; + StandardOutput = "journal"; + StandardError = "journal"; + ReadWritePaths = [ "/data" ]; + }; + }; + }; + }; + + extractAttrs = + l: s: lib.mergeAttrsList (map (f: lib.optionalAttrs (s ? ${f}) { ${f} = s.${f}; }) l); + + mkMcSet = + { dataDir }: + servers: + lib.listToAttrs ( + map ( + args@{ + name, + jdk, + autoStart ? false, + allowInternet ? true, + dataPath ? "${dataDir}/${name}", + extraPackages ? [ ], + mem ? "6G", + cpu ? 2, + ... + }: + lib.nameValuePair "mc-${name}" (mkContainer { + name = "mc-${name}"; + net.access = lib.optional allowInternet { to = "@internet"; }; + net.expose.tcp = [ (args.hopperPort or 25565) ]; + svc.autoStart = autoStart; + resources = { inherit cpu mem; }; + traits = [ (mkMcTrait { inherit jdk dataPath extraPackages; }) ]; + miscProperties = self._hopperRegister (extractAttrs [ "hopperSubdomain" "hopperPort" ] args); + includedInTests = false; + }) + ) servers + ); + + jdk8FontPkgs = with pkgs; [ + freetype + fontconfig + dejavu_fonts + ]; + + mcServers = [ + { + name = "nomifactory"; + jdk = pkgs.jdk8; + extraPackages = jdk8FontPkgs; + hopperSubdomain = "nomi"; + hopperPort = 25555; + } + { + name = "monifactory"; + jdk = pkgs.jdk21; + hopperSubdomain = "moni"; + } + { + name = "rad2"; + jdk = pkgs.jdk8; + extraPackages = jdk8FontPkgs; + } + { + name = "smp"; + jdk = pkgs.jdk17; + } + { + name = "rc4"; + jdk = pkgs.jdk17; + } + { + name = "vh"; + jdk = pkgs.jdk17; + hopperPort = 25555; + } + { + name = "gregblock"; + jdk = pkgs.jdk8; + extraPackages = jdk8FontPkgs; + } + { + name = "tfg"; + jdk = pkgs.jdk17; + } + { + name = "ratgame"; + jdk = pkgs.jdk21; + } + { + name = "maidrpg"; + jdk = pkgs.jdk17; + } + { + name = "maidsmp"; + jdk = pkgs.jdk21; + } + { + name = "maid-event"; + jdk = pkgs.jdk21; + hopperSubdomain = "maidevent"; + dataPath = "${data}/mc/maidevent/speedrun"; + } + { + name = "gtnh"; + jdk = pkgs.jdk21; + } + { + name = "turing"; + jdk = pkgs.jdk21; + } + { + name = "tfg2"; + jdk = pkgs.jdk21; + autoStart = true; + mem = "10G"; + cpu = 8; + } + { + name = "tl"; + jdk = pkgs.jdk21; + } + { + name = "botm"; + jdk = pkgs.jdk17; + } + { + name = "rogule-build"; + jdk = pkgs.jdk21; + } + { + name = "rogule-dev"; + jdk = pkgs.jdk21; + } + ]; +in +mkMcSet { + dataDir = "${data}/mc"; +} mcServers diff --git a/containers/server-n0/mtce.nix b/containers/server-n0/mtce.nix new file mode 100644 index 0000000..321d9bd --- /dev/null +++ b/containers/server-n0/mtce.nix @@ -0,0 +1,84 @@ +{ + name, + pkgs, + lib, + hostInfo, + dsl, + inputs, + ... +}: +let + inherit (dsl) mkContainer mkService; + inherit (hostInfo.paths) data secrets; + inherit (inputs) mtce; + dataDir = "/var/lib/mtce-bot"; + + threads = 64; + timeout = 1800; + + configJson = pkgs.writeText "mtce-bot-config.json" ( + builtins.toJSON { + inherit threads timeout; + } + ); + + mtce-engine = mtce.lib.mkMtce pkgs { + cpuFlags = [ + "-O3" + "-march=cascadelake" + ]; + vectorizedBitSize = 512; + }; + mtce-bot = mtce.lib.mkBot pkgs; +in +mkContainer { + inherit name; + svc.autoStart = true; + resources.fs = { + "${dataDir}/data.json" = { + hostPath = "${data}/misc/mtce-discord-bot/data.json"; + isReadOnly = false; + }; + "${dataDir}/audit" = { + hostPath = "${data}/misc/mtce-discord-bot/audit"; + isReadOnly = false; + }; + "${dataDir}/config.json" = { + hostPath = "${configJson}"; + isReadOnly = true; + }; + "${dataDir}/mtce" = { + hostPath = lib.getExe' mtce-engine "mtce"; + isReadOnly = true; + }; + "/run/secrets/mtce-discord-bot-env" = { + hostPath = "${secrets}/mtce-discord-bot-env"; + isReadOnly = true; + }; + }; + users = [ + { + name = "bot"; + uid = 1000; + } + ]; + service = + _: + mkService { + description = "Discord bot (monumenta-charm-eval)"; + execStart = lib.getExe' mtce-bot "mtce-bot"; + workingDir = dataDir; + networkOnline = true; + serviceConfig = { + User = "bot"; + EnvironmentFile = "/run/secrets/mtce-discord-bot-env"; + PrivateTmp = true; + CapabilityBoundingSet = ""; + ReadWritePaths = [ dataDir ]; + }; + }; + resources.mem = "2G"; + resources.cpu = 64; + net.access = [ { to = "@internet"; } ]; + includedInTests = false; +} diff --git a/containers/server-n0/nginx.nix b/containers/server-n0/nginx.nix new file mode 100644 index 0000000..52aa5c8 --- /dev/null +++ b/containers/server-n0/nginx.nix @@ -0,0 +1,215 @@ +{ + name, + hostInfo, + dsl, + lib, + containers, + ... +}: +let + inherit (dsl) mkContainer; + inherit (hostInfo.paths) data; + + mkNginxConfig = + { + email, + vhosts, + resolver, + }: + let + mkVhostEntries = + { + domain, + upstream ? null, + websockets ? true, + redirect ? true, + listenHttp ? false, + extraLocations ? { }, + }: + let + httpsListen = { + addr = "0.0.0.0"; + port = 443; + ssl = true; + }; + httpListen = { + addr = "0.0.0.0"; + port = 80; + ssl = false; + }; + defaultLoc = lib.optionalAttrs (upstream != null) { + "/" = { + proxyWebsockets = websockets; + # avoid startup error when containers aren't registered yet + # proxyPass is left unset for that reason, so recommendedProxySettings + # won't auto-include its headers here - set them manually to match + extraConfig = '' + set $backend "${upstream.container.name}:${toString upstream.port}"; + proxy_pass http://$backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $hostname; + ''; + }; + }; + acmePrimaryOnMain = !redirect && listenHttp; + mainVhost = lib.nameValuePair domain ( + { + listen = [ httpsListen ] ++ lib.optional listenHttp httpListen; + locations = defaultLoc // extraLocations; + } + // ( + if acmePrimaryOnMain then + { + addSSL = true; + enableACME = true; + } + else + { + onlySSL = true; + useACMEHost = domain; + } + ) + ); + redirectVhost = lib.nameValuePair "${domain}-redirect" { + serverName = domain; + enableACME = true; + listen = [ httpListen ]; + locations = { + "/" = { + return = "301 https://$host$request_uri"; + }; + }; + }; + in + [ mainVhost ] ++ lib.optional redirect redirectVhost; + in + { + access = map (v: { + to = v.upstream.container.name; + tcp = [ v.upstream.port ]; + }) (lib.filter (v: v.upstream or null != null) vhosts); + + containerConfig = + { + lib, + config, + ... + }: + { + services.nginx = { + enable = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + clientMaxBodySize = "100M"; + appendHttpConfig = '' + access_log syslog:server=unix:/dev/log,tag=nginx combined; + ''; + resolver = { + addresses = [ resolver ]; + ipv6 = false; + valid = "30s"; + }; + virtualHosts = lib.listToAttrs (lib.concatLists (map mkVhostEntries vhosts)) // { + "_" = { + listen = [ + { + addr = "0.0.0.0"; + port = 443; + extraParameters = [ + "ssl" + "default_server" + ]; + } + { + addr = "0.0.0.0"; + port = 80; + extraParameters = [ "default_server" ]; + } + ]; + extraConfig = "ssl_reject_handshake on;"; + locations."/".return = "404"; + }; + }; + }; + security.acme = { + acceptTerms = true; + defaults.email = email; + }; + systemd.services = + lib.mapAttrs' ( + cert: _: + lib.nameValuePair "acme-${cert}" { + after = [ "container-ready.target" ]; + wants = [ "container-ready.target" ]; + } + ) config.security.acme.certs + // { + nginx.serviceConfig.FailureAction = "exit-force"; + }; + }; + }; + + up = container: port: { inherit container port; }; + + nginxSetup = + hostAddr: + mkNginxConfig { + email = "floweydev@gmail.com"; + resolver = hostAddr; + vhosts = [ + { + domain = "devenv.floweytf.com"; + upstream = up containers.devenv 8080; + } + { + domain = "maven.floweytf.com"; + upstream = up containers.maven 8080; + } + { + domain = "git.floweytf.com"; + upstream = up containers.git 3000; + } + { + domain = "utils.floweytf.com"; + redirect = false; + listenHttp = true; + extraLocations = { + "/static/" = { + alias = "/usr/share/nginx/static/"; + extraConfig = "autoindex on;"; + }; + }; + } + { + # no upstream, vhost only exists for certs + domain = "mail.floweytf.com"; + } + ]; + }; +in +let + setup = nginxSetup hostAddr; + c = mkContainer { + inherit name; + net.access = setup.access ++ [ { to = "@internet"; } ]; + svc.autoStart = true; + svc.depends = lib.attrVals [ "devenv" "maven" "git" ] containers; + net.forward.tcp = [ + 80 + 443 + ]; + resources.fs."/var/lib/acme" = { + hostPath = "${data}/core/nginx/acme"; + isReadOnly = false; + }; + inherit (setup) containerConfig; + }; + hostAddr = c.hostAddr; +in +c diff --git a/containers/server-n0/pihole.nix b/containers/server-n0/pihole.nix new file mode 100644 index 0000000..be99e66 --- /dev/null +++ b/containers/server-n0/pihole.nix @@ -0,0 +1,64 @@ +{ + name, + hostInfo, + dsl, + ... +}: +let + inherit (dsl) mkContainer; + inherit (hostInfo.paths) data; +in +mkContainer { + inherit name; + net.access = [ { to = "@internet"; } ]; + svc.autoStart = false; + net.expose.tcp = [ 53 ]; + # TODO: figure out how to have this be the external dns resolver only + resources.fs."/var/lib/pihole" = { + hostPath = "${data}/misc/pihole"; + isReadOnly = false; + }; + containerConfig = _: { + services.pihole-ftl = { + enable = true; + stateDirectory = "/var/lib/pihole"; + openFirewallDNS = true; + settings.dns.upstreams = [ + "1.1.1.1" + "8.8.8.8" + ]; + lists = [ + { + url = "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"; + type = "block"; + description = "StevenBlack unified hosts"; + enabled = true; + } + { + url = "https://big.oisd.nl/domainswild"; + type = "block"; + description = "OISD Big"; + enabled = true; + } + { + url = "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@latest/domains/pro.txt"; + type = "block"; + description = "HaGeZi Multi Pro"; + enabled = true; + } + { + url = "https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@latest/domains/tif.txt"; + type = "block"; + description = "HaGeZi Threat Intelligence Feeds"; + enabled = true; + } + ]; + }; + services.pihole-web = { + enable = true; + ports = [ "443s" ]; + }; + systemd.services.pihole-ftl.serviceConfig.FailureAction = "exit-force"; + }; + resources.mem = "512M"; +} diff --git a/containers/server-n0/ssh.nix b/containers/server-n0/ssh.nix new file mode 100644 index 0000000..6b293a5 --- /dev/null +++ b/containers/server-n0/ssh.nix @@ -0,0 +1,186 @@ +{ + pkgs, + hostInfo, + dsl, + self, + lib, + customPkgs, + ... +}: +let + inherit (dsl) mkContainer; + inherit (hostInfo.paths) server data; + + sshTrait = { + resources.fs."/etc/ssh/authorized_keys" = { + hostPath = "${server}/keypair/id_ed25519.pub"; + isReadOnly = true; + }; + users = [ + { + name = "runner"; + uid = 1000; + } + ]; + containerConfig = + { + lib, + pkgs, + ... + }: + lib.mkMerge [ + { + services.openssh = { + enable = true; + authorizedKeysFiles = [ "/etc/ssh/authorized_keys" ]; + settings = { + PasswordAuthentication = false; + UseDns = false; + PrintMotd = false; + }; + }; + systemd.services.sshd.serviceConfig.FailureAction = "exit-force"; + } + { + environment.systemPackages = [ + customPkgs.kitty-terminfo + pkgs.wezterm.terminfo + ]; + } + ]; + }; + + mkSsh = + { + name, + packages ? [ ], + nixDaemon ? true, + routingInfo ? { + username = lib.strings.removePrefix "ssh-" name; + }, + net ? { }, + resources ? { }, + traits ? [ ], + }: + mkContainer { + inherit name; + inherit net resources; + svc.autoStart = true; + isolation.nixDaemon = nixDaemon; + traits = [ sshTrait ] ++ traits; + packages = with pkgs; [ tmux ] ++ packages; + miscProperties = self._sshpiperRegister routingInfo; + }; +in +{ + ssh-admin = mkSsh { + name = "ssh-admin"; + net.access = [ + { + to = "overseer"; + tcp = [ 80 ]; + } + ]; + resources.fs = { + "/home/runner/data" = { + hostPath = server; + isReadOnly = false; + }; + "/run/dbus/system_bus_socket" = { + hostPath = "/run/dbus/system_bus_socket"; + isReadOnly = false; + }; + }; + packages = with pkgs; [ + git + systemd + ]; + }; + + ssh-botm = mkSsh { + name = "ssh-botm"; + resources.fs."/home/runner/botm" = { + hostPath = "${data}/mc/botm"; + isReadOnly = false; + }; + packages = with pkgs; [ + python3 + jdk17 + tmux + ]; + }; + + ssh-generic = mkSsh { + name = "ssh-generic"; + net.access = [ { to = "@internet"; } ]; + resources = { + fs."/home/runner/data" = { + hostPath = "${data}/ssh/generic"; + isReadOnly = false; + }; + mem = "16G"; + cpu = 65; + }; + routingInfo = [ + { + username = "autumn"; + keysName = "generic"; + } + ]; + }; + + ssh-proxy = mkSsh { + name = "ssh-proxy"; + resources.fs."/home/runner/data" = { + hostPath = "${data}/ssh/nyaa"; + isReadOnly = true; + }; + packages = with pkgs; [ python3 ]; + routingInfo = [ + { + username = "nyaa"; + keysName = "nyaa"; + } + ]; + }; + + ssh-rogule = mkSsh { + name = "ssh-rogule"; + resources.fs = { + "/home/runner/rogule-dev" = { + hostPath = "${data}/mc/rogule-dev"; + isReadOnly = false; + }; + "/home/runner/rogule-build" = { + hostPath = "${data}/mc/rogule-build"; + isReadOnly = false; + }; + }; + packages = with pkgs; [ + python3 + jdk21 + ]; + }; + + ssh-starwalker = mkSsh { + name = "ssh-starwalker"; + resources.fs."/home/runner/starwalker" = { + hostPath = "${data}/mc/starwalker"; + isReadOnly = false; + }; + packages = with pkgs; [ + jdk21 + ]; + }; + + ssh-tl = mkSsh { + name = "ssh-tl"; + resources.fs."/home/runner/tl" = { + hostPath = "${data}/mc/tl"; + isReadOnly = false; + }; + packages = with pkgs; [ + jdk21 + ]; + }; +} diff --git a/containers/server-n0/sshpiper.nix b/containers/server-n0/sshpiper.nix new file mode 100644 index 0000000..876fc37 --- /dev/null +++ b/containers/server-n0/sshpiper.nix @@ -0,0 +1,122 @@ +{ + name, + pkgs, + hostInfo, + dsl, + lib, + containers, + customPkgs, + ... +}: +let + inherit (dsl) mkContainer mkService collectContainers; + inherit (hostInfo.paths) server secrets; + dataDir = hostInfo.paths.data; + + mkSshpiperConfig = + { + pipes, + privateKey ? "/common_id", + }: + let + header = '' + version: "1.0" + pipes: + ''; + mkPipe = + { + username, + host, + keysName ? username, + }: + '' + - from: + - username: "${username}" + authorized_keys: /etc/sshpiperd/keys/user_${keysName}/authorized_keys + to: + host: ${host}:22 + username: "runner" + private_key: "${privateKey}" + ignore_hostkey: true + ''; + in + header + lib.concatMapStrings mkPipe pipes; + + sshPipes = lib.concatMap ( + { container, data }: map (lib.mergeAttrs { host = container.name; }) data + ) (collectContainers "sshpiperRoutes" containers); + + sshpiperYaml = pkgs.writeText "sshpiper.yaml" (mkSshpiperConfig { + pipes = sshPipes; + }); +in +{ + ${name} = mkContainer { + inherit name; + net.access = map (p: { + to = p.host; + tcp = [ 22 ]; + }) sshPipes; + svc.autoStart = true; + net.forward.tcp = [ + { + host = 22; + container = 2222; + } + ]; + resources.fs = { + "/etc/sshpiperd/keys" = { + hostPath = "${dataDir}/core/sshpiper"; + isReadOnly = true; + }; + "/common_id" = { + hostPath = "${server}/keypair/id_ed25519"; + isReadOnly = true; + }; + "/etc/ssh/ssh_host_ed25519_key" = { + hostPath = "${secrets}/ssh_host_ed25519_key"; + isReadOnly = true; + }; + "/etc/ssh/ssh_host_rsa_key" = { + hostPath = "${secrets}/ssh_host_rsa_key"; + isReadOnly = true; + }; + "/etc/ssh/ssh_host_ecdsa_key" = { + hostPath = "${secrets}/ssh_host_ecdsa_key"; + isReadOnly = true; + }; + }; + service = + { pkgs, ... }: + mkService { + description = "sshpiper SSH multiplexer"; + execStart = "${lib.getExe' customPkgs.sshpiper "sshpiperd"} --log-level trace --server-key '/etc/ssh/ssh_host_*_key' ${lib.getExe' customPkgs.sshpiper "yaml"} --config /run/sshpiperd/config.yaml"; + networkOnline = true; + serviceConfig = { + # sshpiperd rejects world-readable configs; copy store (444) -> runtime (600) + ExecStartPre = "${lib.getExe' pkgs.coreutils "install"} -m 600 ${sshpiperYaml} /run/sshpiperd/config.yaml"; + RuntimeDirectory = "sshpiperd"; + PrivateTmp = true; + RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX"; + }; + }; + }; + + _sshpiperRegister = + let + doTranslate = + self: + { + username, + keysName ? username, + }: + { + inherit username keysName; + host = self.name; + }; + in + args: self: { + sshpiperRoutes = + if lib.isAttrs args then [ (doTranslate self args) ] else lib.map (doTranslate self) args; + }; +} diff --git a/dsl.nix b/dsl.nix new file mode 100644 index 0000000..ab4be0e --- /dev/null +++ b/dsl.nix @@ -0,0 +1,1318 @@ +{ lib, microvm }: +let + hexDigit = lib.listToAttrs ( + lib.imap0 (i: c: lib.nameValuePair c i) (lib.stringToCharacters "0123456789abcdef") + ); + hexToInt = s: lib.foldl' (acc: c: acc * 16 + hexDigit.${c}) 0 (lib.stringToCharacters s); + + deriveMac = + name: + "02:" + + lib.concatStringsSep ":" ( + lib.genList (i: lib.substring (i * 2) 2 (builtins.hashString "sha256" name)) 5 + ); + deriveIfName = prefix: name: lib.substring 0 15 "${prefix}-${builtins.hashString "sha256" name}"; # IFNAMSIZ - 1 + + deriveIp4AddrAt = + offset: name: + let + h = builtins.hashString "sha256" name; + octet = i: toString (hexToInt (lib.substring (offset + i * 2) 2 h)); + in + "10.${octet 0}.${octet 1}.${octet 2}"; + + ifPrefix = { + nspawn = "vh"; + vm = "vm"; + }; + + unitPrefix = { + nspawn = "container"; + vm = "microvm"; + }; + + deploymentIfGlobs = lib.mapAttrsToList (_: p: "${p}-*") ifPrefix; + + ofKind = k: lib.filterAttrs (_: c: c.kind == k); + + deriveIdentity = prefix: name: { + iface = deriveIfName prefix name; + mac = deriveMac name; + addr = deriveIp4AddrAt 12 name; + hostAddr = deriveIp4AddrAt 24 name; + }; + + specialTargets = [ + "@host" + "@internet" + ]; + + isSpecialTarget = to: lib.elem to specialTargets; + + hasInternetAccess = access: lib.any (a: a.to == "@internet") access; + + substrateNet = { + address = "10.0.0.0"; + prefixLength = 8; + }; + + needsGatewayFor = + access: hostPorts: hostUdpPorts: + hasInternetAccess access || hostPorts != [ ] || hostUdpPorts != [ ]; + + flattenMiscProp = + self: props: + if lib.isAttrs props then + props + else if lib.isList props then + lib.mergeAttrsList (map (flattenMiscProp self) props) + else + props self; + + mkUser = + { + name, + uid, + home ? "/home/${name}", + extraUserConfig ? { }, + }: + { + users.users.${name} = { + isNormalUser = true; + inherit uid home; + group = name; + } + // extraUserConfig; + users.groups.${name} = { }; + }; + + parseMemMiB = + s: + let + unit = lib.toUpper (lib.substring (lib.stringLength s - 1) 1 s); + n = lib.toInt (lib.substring 0 (lib.stringLength s - 1) s); + in + if unit == "G" then + n * 1024 + else if unit == "M" then + n + else + lib.toInt s; +in +rec { + /** + Expands a port spec. + + # Type + + ``` + normalizePort :: (Int | { host :: Int; container :: Int; }) -> { host :: Int; container :: Int; } + ``` + */ + normalizePort = + p: + if builtins.isInt p then + { + host = p; + container = p; + } + else + p; + + /** + Reads all `{name}.nix` files in `dir` and associates to { "${name}" = f name path } + + # Type + + ``` + listMap :: Path -> (String -> Path -> a) -> AttrSet + ``` + */ + listMap = + dir: f: + lib.mapAttrs' ( + fileName: _: + let + name = lib.removeSuffix ".nix" fileName; + in + lib.nameValuePair name (f name (dir + "/${fileName}")) + ) (lib.filterAttrs (n: t: t == "regular" && lib.hasSuffix ".nix" n) (builtins.readDir dir)); + + /** + Creates configuration for a hardened systemd service. + + # Type + + ``` + mkService :: AttrSet -> AttrSet + ``` + */ + mkService = + { + description, + execStart, + wantedBy ? [ "multi-user.target" ], + after ? [ ], + wants ? [ ], + networkOnline ? false, + workingDir ? null, + serviceConfig ? { }, + unitConfig ? { }, + path ? [ ], + }: + { + inherit + description + wantedBy + path + unitConfig + ; + after = after ++ lib.optional networkOnline "network-online.target"; + wants = wants ++ lib.optional networkOnline "network-online.target"; + serviceConfig = { + ExecStart = execStart; + FailureAction = "exit-force"; + NoNewPrivileges = true; + SystemCallFilter = "@system-service"; + SystemCallArchitectures = "native"; + ProtectSystem = "strict"; + } + // lib.optionalAttrs (workingDir != null) { WorkingDirectory = workingDir; } + // serviceConfig; + }; + + /** + Declares a systemd-nspawn container: networking, resource limits, bind + mounts, and the NixOS config that runs inside it. + + # Inputs + + `traits` + : Reusable partial configs (own `resources.fs`/`users`/`containerConfig`). + + `users` + : List of user (`{ name; uid; home?; extraUserConfig?; }`) specs + + `net.access` + : Outbound edges this container is allowed to open, as + `{ to; tcp ? [ ]; udp ? [ ]; }`. `to` is another deployment's name, or one + of the `@`-prefixed specials `"@host"` and `"@internet"` - the prefix + keeps them out of the deployment namespace. Everything not listed is + denied, and an edge is one-way: the peer needs its own entry to talk back. + Ports are mandatory except for `"@internet"`. DNS and ICMP to the host are + always allowed and are not declarable. + + `net.expose.tcp`, `net.expose.udp` + : Ports opened on the container's own firewall. + + `net.forward.tcp`, `net.forward.udp` + : Host ports forwarded into the container; see `normalizePort`. + + `svc.autoStart`, `svc.depends` + : Whether the container starts automatically, and other deployments it + depends on. + + `resources.cpu` + : CPU core count; converted to `CPUQuota` (`cpu * 100%`). + + `resources.mem` + : `MemoryMax`, e.g. `"512M"`. + + `resources.fs` + : Bind mounts, keyed by mount point. + + `isolation.nixDaemon` + : Whether the Nix daemon and CLI are available inside the container. + + `isolation.privateUsers` + : systemd-nspawn user namespacing; see `--private-users=` in + systemd-nspawn(1). Bind mounts always go through idmapped `--bind`. + + `miscProperties` + : Arbitrary data attached to the result. Used for cross-container discovery + via `collectContainers`. + + `includedInTests` + : Whether integration VM tests should includes this container. Ignored + for real deployment. Set `false` for containers the test VM shouldn't + run. + + # Type + + ``` + mkContainer :: AttrSet -> Deployment + ``` + */ + mkContainer = + { + name, + net ? { }, + svc ? { }, + resources ? { }, + isolation ? { }, + traits ? [ ], + users ? [ ], + packages ? [ ], + service ? null, + containerConfig ? (_: { }), + miscProperties ? { }, + includedInTests ? true, + }: + let + access = net.access or [ ]; + ports = net.expose.tcp or [ ]; + udpPorts = net.expose.udp or [ ]; + hostPorts = net.forward.tcp or [ ]; + hostUdpPorts = net.forward.udp or [ ]; + autoStart = svc.autoStart or false; + depends = svc.depends or [ ]; + cpu = resources.cpu or 1; + cpuQuota = "${toString (cpu * 100)}%"; + memoryMax = resources.mem or "512M"; + bindMounts = resources.fs or { }; + allowsNixDaemon = isolation.nixDaemon or false; + privateUsers = isolation.privateUsers or "pick"; + + allowInternet = hasInternetAccess access; + allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts; + userModules = map mkUser (lib.concatMap (t: t.users or [ ]) traits ++ users); + allPorts = ports ++ map (p: (normalizePort p).container) hostPorts; + useIdmap = privateUsers != "no"; + bindMountExtraFlags = lib.mapAttrsToList ( + mountPoint: bm: + (if bm.isReadOnly then "--bind-ro=" else "--bind=") + + "${bm.hostPath}:${mountPoint}" + + lib.optionalString useIdmap ":idmap" + ) allBindMounts; + bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts); + inherit (deriveIdentity ifPrefix.nspawn name) + iface + mac + addr + hostAddr + ; + needsGateway = needsGatewayFor access hostPorts hostUdpPorts; + + linkVeths.${iface} = { }; + + mergedConfig = + args@{ + pkgs, + lib, + ... + }: + let + traitConfigs = map (t: (t.containerConfig or (_: { })) args) traits; + pkgModule = lib.optional (packages != [ ]) { environment.systemPackages = packages; }; + nixModule = lib.optional allowsNixDaemon { + environment.systemPackages = [ pkgs.nix ]; + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + }; + svcModule = lib.optional (service != null) { + systemd.services.${name} = lib.mkMerge [ + (service args) + { + after = [ "container-ready.target" ]; + wants = [ "container-ready.target" ]; + } + ]; + }; + portModule = + lib.optional (allPorts != [ ]) { networking.firewall.allowedTCPPorts = allPorts; } + ++ lib.optional (udpPorts != [ ]) { networking.firewall.allowedUDPPorts = udpPorts; }; + readyTarget = { + systemd.targets.container-ready = { + description = "Container prerequisites satisfied"; + after = lib.optional allowInternet "network-online.target"; + wants = lib.optional allowInternet "network-online.target"; + wantedBy = [ "multi-user.target" ]; + }; + }; + in + lib.mkMerge ( + traitConfigs + ++ userModules + ++ pkgModule + ++ nixModule + ++ svcModule + ++ [ readyTarget ] + ++ [ (containerConfig args) ] + ++ portModule + ); + in + lib.fix (self: { + inherit + name + access + autoStart + depends + ports + hostPorts + hostUdpPorts + memoryMax + cpuQuota + privateUsers + allowInternet + needsGateway + iface + mac + addr + hostAddr + bindMountHostPaths + includedInTests + ; + bindMounts = allBindMounts; + kind = "nspawn"; + miscProperties = flattenMiscProp self miscProperties; + nixosContainer = { + inherit autoStart privateUsers; + bindMounts = { }; + extraFlags = bindMountExtraFlags; + privateNetwork = true; + extraVeths = linkVeths; + config = + { + pkgs, + config, + lib, + ... + }: + lib.mkMerge [ + { + networking.enableIPv6 = false; + networking.useHostResolvConf = lib.mkForce false; + networking.nameservers = [ hostAddr ]; + networking.interfaces.${iface} = { + ipv4.addresses = [ + { + address = addr; + prefixLength = 32; + } + ]; + ipv4.routes = [ + { + address = hostAddr; + prefixLength = 32; + } + ( + substrateNet + // { + via = hostAddr; + } + ) + ]; + }; + systemd.services."network-addresses-${iface}".before = [ "network-online.target" ]; + } + (lib.optionalAttrs needsGateway { + networking.defaultGateway = { + address = hostAddr; + interface = iface; + }; + }) + (mergedConfig { inherit pkgs config lib; }) + ]; + }; + }); + + /** + Declares a microvm: same networking interface as `mkContainer` (`access`, + `hostPorts` and friends all work the same way), virtiofs/9p shares, and the + NixOS config that runs inside it. The VM counterpart to `mkContainer` - + same shared interface, different backend. + + # Inputs + + `resources.fs` + : Passed straight through to `microvm.shares`. + + `guestConfig` + : NixOS module for the VM's own system; same role as `mkContainer`'s + `containerConfig`; gets full module args (`pkgs`, `utils`, ...) since + it's spliced in via `imports`, not called manually. + + `resources.cpu`, `resources.mem`, `hypervisor` + : Passed through to the corresponding `microvm.*` options. `resources.cpu` + is vcpu count. `resources.mem` is required, a string like `"8G"`, + converted to MiB via `parseMemMiB`. + + `net.access`, `net.forward.tcp`, `net.forward.udp`, `svc.autoStart`, + `svc.depends` + : See `mkContainer`. + + `miscProperties`, `includedInTests` + : See `mkContainer`. + + # Type + + ``` + mkVm :: AttrSet -> Deployment + ``` + */ + mkVm = + { + name, + net ? { }, + svc ? { }, + resources ? { }, + hypervisor ? "qemu", + guestConfig ? (_: { }), + miscProperties ? { }, + includedInTests ? true, + }: + let + access = net.access or [ ]; + hostPorts = net.forward.tcp or [ ]; + hostUdpPorts = net.forward.udp or [ ]; + autoStart = svc.autoStart or false; + depends = svc.depends or [ ]; + vcpu = resources.cpu or 1; + mem = parseMemMiB (resources.mem or (throw "mkVm ${name}: resources.mem is required")); + shares = resources.fs or [ ]; + + allowInternet = hasInternetAccess access; + needsGateway = needsGatewayFor access hostPorts hostUdpPorts; + inherit (deriveIdentity ifPrefix.vm name) + iface + mac + addr + hostAddr + ; + in + lib.fix (self: { + inherit + name + access + autoStart + depends + hostPorts + hostUdpPorts + allowInternet + iface + mac + addr + hostAddr + shares + vcpu + mem + hypervisor + includedInTests + ; + inherit needsGateway; + bindMountHostPaths = map (s: s.source) shares; + kind = "vm"; + miscProperties = flattenMiscProp self miscProperties; + microvmConfig = { + autostart = autoStart; + config = _: { + imports = [ + microvm.nixosModules.microvm + guestConfig + ]; + nixpkgs.overlays = [ microvm.overlay ]; + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + networking.hostName = name; + system.stateVersion = "26.05"; + microvm = { + inherit + vcpu + mem + hypervisor + shares + ; + interfaces = [ + { + type = "tap"; + id = iface; + inherit mac; + } + ]; + }; + networking.useNetworkd = true; + networking.useDHCP = false; + systemd.network.networks."10-main" = { + matchConfig.MACAddress = mac; + address = [ "${addr}/32" ]; + routes = [ + { + Destination = "${hostAddr}/32"; + Scope = "link"; + } + { + Destination = "${substrateNet.address}/${toString substrateNet.prefixLength}"; + Gateway = hostAddr; + } + ] + ++ lib.optional needsGateway { + Destination = "0.0.0.0/0"; + Gateway = hostAddr; + }; + linkConfig.RequiredForOnline = if allowInternet then "routable" else "degraded"; + }; + networking.enableIPv6 = false; + networking.nameservers = [ hostAddr ]; + }; + }; + }); + + /** + Subset of `svcs`, with `access` edges pointing at dropped deployments + removed. Filtering the attrset alone leaves those edges dangling, and + `mkNetworking` rejects an edge whose target it cannot resolve. + + # Type + + ``` + filterDeployments :: (Deployment -> Bool) -> AttrSet -> AttrSet + ``` + */ + filterDeployments = + pred: svcs: + let + kept = lib.filterAttrs (_: pred) svcs; + in + lib.mapAttrs ( + _: c: + c + // { + access = lib.filter (a: isSpecialTarget a.to || kept ? ${a.to}) c.access; + } + ) kept; + + /** + Finds every container that set `miscProperties.${key}`, paired with the + value. Used for cross-container discovery. + + # Type + + ``` + collectContainers :: String -> AttrSet -> [ { container :: Container; data :: Any; } ] + ``` + */ + collectContainers = + key: containers: + lib.mapAttrsToList (_: v: { + container = v; + data = v.miscProperties.${key}; + }) (lib.filterAttrs (_: v: v.miscProperties ? ${key}) containers); + + /** + Prepares networking for all deployments: dnsmasq for name resolution and + the nftables tables that carry isolation, NAT and port forwarding. Every + link is point-to-point, so `iifname` identifies the peer and nothing + reaches anything it has no `access` edge for. NixOS container builtin port + forwarding can't be used because we set up our own networking tables. + + # Type + + ``` + mkNetworking :: AttrSet -> AttrSet + ``` + */ + mkNetworking = + { + svcs, + }: + let + deploymentList = lib.attrValues svcs; + quote = lib.concatMapStringsSep ", " (v: "\"${v}\""); + + deploymentIfs = quote deploymentIfGlobs; + + netAccessEdges = lib.concatMap ( + deployment: + map ( + a: + let + portsOf = proto: a.${proto} or [ ]; + protos = lib.filter (p: portsOf p != [ ]) [ + "tcp" + "udp" + ]; + bad = msg: throw "${deployment.name}: access edge to \"${a.to}\" ${msg}"; + in + if a.to != "@internet" && protos == [ ] then + bad "${deployment.name} no tcp/udp ports" + else if isSpecialTarget a.to || svcs ? ${a.to} then + { + src = deployment; + inherit (a) to; + inherit protos portsOf; + } + else + bad "${deployment.name}: access target does not name a container, \"@host\" or \"@internet\"" + ) deployment.access + ) deploymentList; + + netEdgeRules = + pred: prefixOf: + lib.concatMapStrings ( + e: + lib.concatMapStrings ( + proto: + "${prefixOf e} ${proto} dport { ${ + lib.concatMapStringsSep ", " toString (e.portsOf proto) + } } accept\n" + ) e.protos + ) (lib.filter pred netAccessEdges); + + peerRules = netEdgeRules (e: !isSpecialTarget e.to) ( + e: "iifname \"${e.src.iface}\" oifname \"${svcs.${e.to}.iface}\"" + ); + + hostRules = netEdgeRules (e: e.to == "@host") ( + e: "iifname \"${e.src.iface}\" ip daddr ${e.src.hostAddr}" + ); + + srcGuard = lib.concatMapStrings (c: '' + iifname "${c.iface}" ip saddr != ${c.addr} counter name "spoofed" drop + '') deploymentList; + + internetIfs = map (c: c.iface) (lib.filter (c: c.allowInternet) deploymentList); + + uplinkIfs = quote [ + "en*" + "eth*" + "wl*" + ]; + + egressRules = lib.optionalString (internetIfs != [ ]) '' + iifname { ${quote internetIfs} } oifname { ${uplinkIfs} } accept + ''; + + masqRules = lib.optionalString (internetIfs != [ ]) '' + iifname { ${quote internetIfs} } oifname { ${uplinkIfs} } masquerade + ''; + + fwdElems = + sel: + lib.concatMapStringsSep ", " (f: "${toString f.host} : ${f.addr} . ${toString f.container}") ( + lib.concatMap (c: map (p: normalizePort p // { inherit (c) addr; }) (sel c)) deploymentList + ); + tcpElems = fwdElems (c: c.hostPorts); + udpElems = fwdElems (c: c.hostUdpPorts); + elemsOf = e: lib.optionalString (e != "") "elements = { ${e} }"; + in + { + boot.kernel.sysctl = { + "net.ipv4.ip_forward" = 1; + } + # nftables drops all container IPv6, but the host end of each link still + # autoconfs a fe80:: - a peerless link-local to aim at if a rule regresses. + # leading "-": the veth doesn't exist at the boot sysctl run, so ignore it + # there; systemd's net-device udev rule reapplies this per-iface at creation. + // lib.listToAttrs ( + map (c: lib.nameValuePair "-net.ipv6.conf.${c.iface}.disable_ipv6" 1) deploymentList + ); + networking.dhcpcd.denyInterfaces = deploymentIfGlobs; + + networking.firewall.trustedInterfaces = map (c: c.iface) deploymentList; + + services.dnsmasq = { + enable = true; + settings = { + interface = [ "lo" ] ++ map (c: c.iface) deploymentList; + local = "/containers/"; + no-resolv = true; + server = [ + "1.1.1.1" + "8.8.8.8" + ]; + host-record = map (c: "${c.name},${c.name}.containers,${c.addr}") deploymentList; + }; + }; + + networking.nftables.enable = true; + networking.nftables.tables = { + container-ip = { + family = "inet"; + content = '' + counter spoofed { } + chain prerouting { + type filter hook prerouting priority -300; policy accept; + ${srcGuard} + } + chain deployment-input { + meta nfproto ipv6 drop + ct state established,related accept + meta l4proto icmp accept + udp dport 53 accept + tcp dport 53 accept + ${hostRules} + drop + } + chain input { + type filter hook input priority filter - 10; policy accept; + iifname { ${deploymentIfs} } jump deployment-input + } + chain forward { + type filter hook forward priority 0; policy accept; + meta nfproto ipv6 drop + ct state established,related accept + ${peerRules} + ct status dnat accept + ${egressRules} + reject with icmp type host-unreachable + } + chain postrouting { + type nat hook postrouting priority srcnat; policy accept; + ${masqRules} + } + ''; + }; + container-portforward = { + family = "ip"; + content = '' + map tcp-fwd { + type inet_service : ipv4_addr . inet_service + ${elemsOf tcpElems} + } + map udp-fwd { + type inet_service : ipv4_addr . inet_service + ${elemsOf udpElems} + } + chain prerouting { + type nat hook prerouting priority dstnat; policy accept; + iifname { ${deploymentIfs} } return + fib daddr type local dnat to tcp dport map @tcp-fwd + fib daddr type local dnat to udp dport map @udp-fwd + } + chain output { + type nat hook output priority dstnat; policy accept; + fib daddr type local dnat to tcp dport map @tcp-fwd + fib daddr type local dnat to udp dport map @udp-fwd + } + ''; + }; + }; + }; + + /** + Throws if any two deployments derive the same interface name or address, or + if one lands on a reserved address. All three are hashes of the container + name, so a name collision is silent otherwise: two containers would share a + veth name, or an address two routes would claim. Set `address` on one of + them to resolve. + + # Type + + ``` + checkIdentities :: String -> AttrSet -> AttrSet + ``` + */ + checkIdentities = + mainGateway: svcs: + let + svcList = lib.attrValues svcs; + + reservedAddrs = [ + mainGateway + "10.0.0.0" + "10.255.255.255" + ]; + + # addr and hostAddr live in one routing namespace: a container addr equal to + # another's hostAddr makes the host claim it locally instead of routing to + # the container, so they must be checked against each other, not per-field. + ipEntries = + map (a: { + holder = "reserved"; + value = a; + }) reservedAddrs + ++ lib.concatMap (c: [ + { + holder = "${c.name}.addr"; + value = c.addr; + } + { + holder = "${c.name}.hostAddr"; + value = c.hostAddr; + } + ]) svcList; + + plainEntries = + field: + map (c: { + holder = c.name; + value = c.${field}; + }) svcList; + + dupsOf = + label: entries: + lib.mapAttrsToList (v: es: " ${label} ${v}: ${lib.concatMapStringsSep ", " (e: e.holder) es}") ( + lib.filterAttrs (_: es: lib.length es > 1) (lib.groupBy (e: e.value) entries) + ); + + problems = + dupsOf "ip" ipEntries ++ dupsOf "iface" (plainEntries "iface") ++ dupsOf "mac" (plainEntries "mac"); + in + if problems == [ ] then + svcs + else + throw "derived identity collision: \n${lib.concatStringsSep "\n" problems}"; + + checkForwardPorts = + svcs: + let + duplicateFor = + proto: selector: + let + entries = lib.concatMap ( + c: + map (p: { + holder = c.name; + port = (normalizePort p).host; + }) (selector c) + ) (lib.attrValues svcs); + duplicates = lib.filterAttrs (_: es: lib.length es > 1) (lib.groupBy (e: toString e.port) entries); + in + lib.mapAttrsToList ( + port: es: " ${proto}/${port}: ${lib.concatMapStringsSep ", " (e: e.holder) es}" + ) duplicates; + problems = duplicateFor "tcp" (c: c.hostPorts) ++ duplicateFor "udp" (c: c.hostUdpPorts); + in + if problems == [ ] then + svcs + else + throw "duplicate published ports:\n${lib.concatStringsSep "\n" problems}"; + + /** + Whether a value is a `mkContainer` or `mkVm` result. + + # Type + + ``` + isDeployment :: Any -> Bool + ``` + */ + isDeployment = c: lib.isAttrs c && c ? kind; + + /** + The systemd unit name a deployment runs as, without the `.service` suffix. + + # Type + + ``` + serviceUnitName :: Deployment -> String + ``` + */ + serviceUnitName = c: "${unitPrefix.${c.kind}}@${c.name}"; + + /** + Turns a set of containers into the actual NixOS configuration. + + # Type + + ``` + mkContainerSet :: { svcs :: AttrSet; } -> AttrSet + ``` + */ + mkContainerSet = + { svcs }: + let + nspawnSvcs = ofKind "nspawn" svcs; + vmSvcs = ofKind "vm" svcs; + + netUnitsCommon = [ "nftables.service" ]; + + mkDep = + c: + let + hasDeps = c.depends != [ ]; + depUnits = map (d: "${serviceUnitName d}.service") c.depends; + in + lib.nameValuePair (serviceUnitName c) { + wants = lib.optionals hasDeps depUnits; + after = lib.optionals hasDeps depUnits ++ netUnitsCommon; + requires = netUnitsCommon; + startLimitBurst = 5; + startLimitIntervalSec = 120; + unitConfig.RequiresMountsFor = c.bindMountHostPaths; + }; + + # the host end of a link, for both kinds. nixos-containers only configures + # an extraVeth's host end in the container unit's postStart, which by + # definition runs after the container is fully booted - so every service + # inside it sees a peerless, carrier-less link until then. Declaring the + # interface instead gets `network-addresses-`, which udev triggers off + # the device unit the moment the veth appears. + mkLink = c: { + ipv4.addresses = [ + { + address = c.hostAddr; + prefixLength = 32; + } + ]; + ipv4.routes = [ + { + address = c.addr; + prefixLength = 32; + } + ]; + }; + in + { + containers = lib.mapAttrs (_: c: c.nixosContainer) nspawnSvcs; + microvm.vms = lib.mapAttrs (_: c: c.microvmConfig) vmSvcs; + networking.interfaces = lib.mapAttrs' (_: c: lib.nameValuePair c.iface (mkLink c)) svcs; + systemd.services = lib.listToAttrs (map mkDep (lib.attrValues svcs)); + }; + + /** + Prometheus recording rules for each nspawn container's resource limits + (`container_limit_memory_bytes`, `container_limit_cpu_cores`). Used by + `overseer`. VM deployments are skipped - their `mem`/`vcpu` are an + allocation, not a cgroup ceiling, so they don't map onto the same + metric; not covered by these dashboards yet. + + # Type + + ``` + mkContainerLimitsRuleFile :: AttrSet -> AttrSet -> Derivation + ``` + */ + mkContainerLimitsRuleFile = + pkgs: deployments: + let + containers = ofKind "nspawn" deployments; + parseMemBytes = + s: + let + unit = lib.toUpper (lib.substring (lib.stringLength s - 1) 1 s); + n = lib.toInt (lib.substring 0 (lib.stringLength s - 1) s); + in + if unit == "G" then + n * 1024 * 1024 * 1024 + else if unit == "M" then + n * 1024 * 1024 + else if unit == "K" then + n * 1024 + else + lib.toInt s; + in + pkgs.writeText "container-limits.yml" ( + builtins.toJSON { + groups = [ + { + name = "container-limits"; + rules = lib.concatLists ( + lib.mapAttrsToList (cname: c: [ + { + record = "container_limit_memory_bytes"; + expr = toString (parseMemBytes c.memoryMax); + labels = { + container = cname; + }; + } + { + record = "container_limit_cpu_cores"; + expr = "${toString (lib.toInt (lib.removeSuffix "%" c.cpuQuota))}e-2"; + labels = { + container = cname; + }; + } + ]) containers + ); + } + ]; + } + ); + + /** + Loads a host's containers, as a attrset of name -> container. Does not + build NixOS config built yet. Use `mkHostFromContainers` for a real host. + + # Inputs + + `containerDirs` + : Directories to scan for `*.nix` container files (e.g. + `containers/common`, `containers/`). + + `ctx` + : Extra values passed into every container file's own top-level args + (`pkgs`, `hostInfo`, `inputs`, `customPkgs`, ...). + + # Type + + ``` + loadContainers :: { hostName :: String; containerDirs :: [ Path ]; ctx :: AttrSet; } -> AttrSet + ``` + */ + loadContainers = + { + hostName, + containerDirs, + ctx, + }: + let + dslCtx = { + inherit + listMap + mkService + mkContainer + mkVm + collectContainers + mkNetworking + isDeployment + mkContainerSet + mkContainerLimitsRuleFile + mkHostFromContainers + ; + }; + + fullCtx = ctx // { + inherit containers; + dsl = dslCtx; + }; + + loadWith = + extra: + let + perDir = map (dir: listMap dir (name: path: import path (fullCtx // extra // { inherit name; }))) ( + lib.filter builtins.pathExists containerDirs + ); + + nameCounts = lib.foldl' (acc: n: acc // { ${n} = (acc.${n} or 0) + 1; }) { } ( + lib.concatMap lib.attrNames perDir + ); + duplicateNames = lib.attrNames (lib.filterAttrs (_: count: count > 1) nameCounts); + in + if duplicateNames != [ ] then + throw "loadContainers ${hostName}: container name(s) defined in more than one of containerDirs: ${toString duplicateNames}" + else + lib.foldl' ( + acc: dirAttrs: + lib.foldl' ( + acc2: name: + let + m = dirAttrs.${name}; + in + acc2 // (if isDeployment m then { ${name} = m; } else m) + ) acc (lib.attrNames dirAttrs) + ) { } perDir; + + assertFn = + pred: x: m: + if pred x then x else throw m; + + containers = lib.mapAttrs (k: v: assertFn isDeployment v "${k} is not a deployment") ( + lib.filterAttrs (n: _: !lib.strings.hasPrefix "_" n) (lib.fix (self: loadWith { inherit self; })) + ); + in + containers; + + /** + Builds a host's NixOS config from container configuration. + + # Type + + ``` + mkHostFromContainers :: { containers :: AttrSet; ctx :: AttrSet; } -> Module + ``` + */ + mkHostFromContainers = + { containers, ctx }: + let + checked = checkForwardPorts (checkIdentities ctx.hostInfo.mainGateway containers); + hostConfigs = lib.concatMap (c: lib.optional (c ? hostConfig) c.hostConfig) ( + lib.attrValues checked + ); + nspawnContainers = ofKind "nspawn" checked; + in + { + config = lib.mkMerge ( + [ + (mkContainerSet { svcs = checked; }) + (mkNetworking { svcs = checked; }) + { + containers = lib.mapAttrs (_: _: { + config = { + nixpkgs.pkgs = lib.mkDefault ctx.pkgs; + system.stateVersion = lib.mkDefault "26.05"; + }; + }) nspawnContainers; + } + { + systemd.services = lib.mapAttrs' ( + _: c: + lib.nameValuePair (serviceUnitName c) { + serviceConfig = { + MemoryMax = c.memoryMax; + CPUQuota = c.cpuQuota; + }; + } + ) nspawnContainers; + } + ] + ++ hostConfigs + ); + }; + + /** + Host-independent boilerplate for a NixOS VM integration test. + # Type + + ``` + mkIntegrationTest :: { host :: AttrSet; testContainers :: AttrSet; pkgs :: AttrSet; } -> AttrSet + ``` + */ + mkIntegrationTest = + { + host, + testContainers, + pkgs, + }: + let + testHostInfo = host.specialArgs.hostInfo // { + isVm = true; + }; + + allTestContainerNames = lib.attrNames testContainers; + autoStartTestContainers = lib.filterAttrs (_: c: c.autoStart) testContainers; + includedContainerNames = lib.attrNames autoStartTestContainers; + nspawnNames = lib.attrNames (ofKind "nspawn" testContainers); + vmNames = lib.attrNames (ofKind "vm" testContainers); + allowInternetContainerNames = lib.attrNames ( + lib.filterAttrs (_: c: c.allowInternet) autoStartTestContainers + ); + gatewayContainerNames = lib.attrNames ( + lib.filterAttrs (_: c: c.needsGateway) autoStartTestContainers + ); + + pyList = names: "[" + lib.concatMapStringsSep ", " (n: "\"${n}\"") names + "]"; + pyDict = + render: attrs: + "{" + lib.concatStringsSep ", " (lib.mapAttrsToList (k: v: "\"${k}\": ${render v}") attrs) + "}"; + pyStr = v: "\"${v}\""; + pyInts = ps: "[" + lib.concatMapStringsSep ", " toString ps + "]"; + + pyEdges = lib.concatStringsSep ", " ( + lib.concatMap ( + c: + map (a: "(\"${c.name}\", \"${a.to}\", ${pyInts (a.tcp or [ ])}, ${pyInts (a.udp or [ ])})") c.access + ) (lib.attrValues autoStartTestContainers) + ); + + bin = lib.getExe'; + + testTools = [ + pkgs.curl + pkgs.iproute2 + pkgs.iputils + pkgs.util-linux + pkgs.openssh + pkgs.openssl + pkgs.coreutils + pkgs.dnsutils + pkgs.python3 + pkgs.strace + pkgs.tcpdump + ]; + in + { + inherit testHostInfo; + containerNames = { + all = allTestContainerNames; + included = includedContainerNames; + allowInternet = allowInternetContainerNames; + gateway = gatewayContainerNames; + }; + + internetNode = _: { + networking.firewall.allowedTCPPorts = [ 80 ]; + systemd.services.dummy-http = { + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${bin pkgs.python3 "python3"} -m http.server 80"; + DynamicUser = true; + AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; + }; + }; + }; + + serverModule = { + _module.args = host.specialArgs // { + hostInfo = testHostInfo; + }; + networking.hostName = host.hostName; + + # containers don't forward journal to console by default + containers = lib.genAttrs nspawnNames (_: { + timeoutStartSec = "10min"; + config.services.journald.extraConfig = "ForwardToConsole=yes"; + }); + + # a test VM has neither the memory to hand out nor the cores to divide. + # optimize.enable re-overrides qemu with nixosTestRunner; hydra caches + # that build of plain qemu but not of qemu_kvm, so naming it here is + # what keeps a nested guest from compiling qemu before it can boot + microvm.vms = lib.genAttrs vmNames (_: { + config.microvm = { + mem = lib.mkForce 512; + vcpu = lib.mkForce 1; + qemu.package = lib.mkForce pkgs.qemu_test; + }; + }); + + systemd.services = + lib.genAttrs (map serviceUnitName (lib.attrValues testContainers)) (_: { + serviceConfig = { + Restart = lib.mkForce "no"; + RestartForceExitStatus = lib.mkForce ""; + }; + }) + // { + # erofs /nix/store is read-only, breaks nix-store --load-db + register-nix-paths.enable = false; + }; + + virtualisation.fileSystems."/persist" = { + device = "none"; + fsType = "tmpfs"; + # needed before persist.mount runs (activation depends on it) + neededForBoot = true; + }; + services.dnsmasq.settings.server = lib.mkForce [ ]; + + virtualisation.memorySize = lib.mkDefault 4096; + virtualisation.cores = lib.mkDefault 4; + virtualisation.useNixStoreImage = true; + virtualisation.additionalPaths = testTools; + }; + + testScriptPreamble = '' + included = ${pyList includedContainerNames} + + ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.addr) autoStartTestContainers)} + HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)} + IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)} + KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)} + # only nspawn guests can be entered with nsenter; a vm has no host-side + # netns, so every probe that runs *inside* a guest is limited to these + enterable = [c for c in included if KIND[c] == "nspawn"] + + allow_internet = [c for c in enterable if c in ${pyList allowInternetContainerNames}] + has_gateway = [c for c in enterable if c in ${pyList gatewayContainerNames}] + not_allow_internet = [c for c in enterable if c not in allow_internet] + no_gateway = [c for c in enterable if c not in has_gateway] + + # (src, to, tcp ports, udp ports); `to` may be "@host" or "@internet" + EDGES = [${pyEdges}] + + NSENTER = "${bin pkgs.util-linux "nsenter"}" + CURL = "${bin pkgs.curl "curl"}" + IP = "${bin pkgs.iproute2 "ip"}" + PING = "${bin pkgs.iputils "ping"}" + DIG = "${bin pkgs.dnsutils "dig"}" + SSH = "${bin pkgs.openssh "ssh"}" + OPENSSL = "${bin pkgs.openssl "openssl"}" + TIMEOUT = "${bin pkgs.coreutils "timeout"}" + ''; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..6e14954 --- /dev/null +++ b/flake.lock @@ -0,0 +1,121 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1786300091, + "narHash": "sha256-4UFJOVGpaYtHW5yasSv80MaJxTBYdk2zyf3jhKtt0wA=", + "owner": "microvm-nix", + "repo": "microvm.nix", + "rev": "71beea0076cd46dafcee97a5a2e7d00cbba5bd2f", + "type": "github" + }, + "original": { + "owner": "microvm-nix", + "repo": "microvm.nix", + "type": "github" + } + }, + "mtce": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1782711974, + "narHash": "sha256-ju0/qU6ZGqRxc5yDhuljrprG0BSPgguc8Kz3txktSjI=", + "owner": "Floweynt", + "repo": "monumenta-charm-eval", + "rev": "1aa8b532975679787f7c5bed180fd7fc0da6b905", + "type": "github" + }, + "original": { + "owner": "Floweynt", + "ref": "master", + "repo": "monumenta-charm-eval", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1786384358, + "narHash": "sha256-RzPPiWeUtuvymnpuEWsdtzli5w4kjZs49FqEs3/1u+I=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2fcb964de67fcf60b43471c55d5d99e61a9ccb5a", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "microvm": "microvm", + "mtce": "mtce", + "nixpkgs": "nixpkgs" + } + }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1785761586, + "narHash": "sha256-MWOMVqJJERwjQGgRIv8d6rlEBqbLM3VZWxHkNKIIZNw=", + "ref": "refs/heads/main", + "rev": "a7762d6f54b40560dd5255ce902e6e6a5d980fe9", + "revCount": 1416, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..1e458ea --- /dev/null +++ b/flake.nix @@ -0,0 +1,154 @@ +{ + description = "Server infrastructure"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + mtce.url = "github:Floweynt/monumenta-charm-eval/master"; + mtce.inputs.nixpkgs.follows = "nixpkgs"; + microvm.url = "github:microvm-nix/microvm.nix"; + microvm.inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = + { + nixpkgs, + mtce, + microvm, + ... + }: + let + system = "x86_64-linux"; + inherit (nixpkgs) lib; + pkgs = import nixpkgs { inherit system; }; + + customPkgs = { + hopper-rs = pkgs.callPackage ./packages/hopper-rs.nix { }; + sshpiper = pkgs.callPackage ./packages/sshpiper.nix { }; + kitty-terminfo = pkgs.callPackage ./packages/kitty-terminfo.nix { }; + ct = pkgs.callPackage ./packages/ct.nix { }; + server-rebuild = pkgs.callPackage ./packages/server-rebuild.nix { }; + server-status = pkgs.callPackage ./packages/server-status.nix { }; + server-kexec = pkgs.callPackage ./packages/server-kexec.nix { }; + }; + + ci = import ./ci.nix { inherit lib pkgs; }; + + inputs = { inherit mtce microvm; }; + + dsl = import ./dsl.nix { inherit lib microvm; }; + introspect = import ./introspect.nix { inherit lib dsl; }; + + hostInfoFor = hostName: { + hostname = hostName; + mainGateway = "10.0.0.1"; + isVm = false; + paths = { + server = "/persist/@containers"; + data = "/persist/@containers/data"; + secrets = "/persist/@secrets/"; + }; + }; + + specialArgsFor = hostName: { + hostInfo = hostInfoFor hostName; + inherit inputs customPkgs; + }; + + hostDefFor = + hostName: + import (./hosts + "/${hostName}.nix") { + inherit lib; + hostInfo = hostInfoFor hostName; + }; + + commonModule = import ./configuration.nix; + + prepareHost = + hostName: + let + hostDef = hostDefFor hostName; + containerDirs = [ + ./containers/common + (./containers + "/${hostName}") + ]; + specialArgs = specialArgsFor hostName; + ctx = specialArgs // { + inherit pkgs lib; + }; + containers = dsl.loadContainers { + inherit + hostName + containerDirs + ctx + ; + }; + + self = { + inherit + hostName + hostDef + containerDirs + specialArgs + ctx + containers + ; + }; + + hostCfg = lib.nixosSystem { + inherit system specialArgs; + modules = [ + commonModule + microvm.nixosModules.host + hostDef.module + (dsl.mkHostFromContainers { inherit containers ctx; }) + ]; + }; + + testDir = ./tests + "/${hostName}"; + + tests = + if builtins.pathExists testDir then + lib.mapAttrs' (testName: testCfg: lib.nameValuePair "${testName}-${hostName}" testCfg) ( + dsl.listMap testDir ( + _name: path: + pkgs.testers.nixosTest ( + import path { + inherit lib dsl commonModule; + host = self; + } + ) + ) + ) + else + { }; + in + self // { inherit hostCfg tests; }; + + allHosts = lib.genAttrs (lib.attrNames (dsl.listMap ./hosts (hostName: _: hostName))) prepareHost; + in + { + nixosConfigurations = lib.mapAttrs (_: host: host.hostCfg) allHosts; + + inspect = lib.mapAttrs + (name: host: + pkgs.writeText "inspect-${name}.json" + (builtins.toJSON (introspect.hostReport { inherit host; }))) + allHosts; + + packages.${system} = customPkgs // { + inherit ci; + }; + + formatter.${system} = pkgs.nixfmt; + + checks.${system} = lib.foldl' (acc: host: acc // host.tests) { } (lib.attrValues allHosts) // { + inherit ci; + inherit (customPkgs) + ct + server-rebuild + server-status + server-kexec + ; + }; + }; +} diff --git a/hosts/example.nix b/hosts/example.nix new file mode 100644 index 0000000..194a503 --- /dev/null +++ b/hosts/example.nix @@ -0,0 +1,38 @@ +_: { + # New host: copy this file to hosts/.nix. Replace fileSystems/boot + # below with the real hardware (see hosts/server-n0.nix for a bigger + # example). Add a containers// directory for its containers. + + module = + { lib, hostInfo, ... }: + { + networking.hostName = hostInfo.hostname; + + fileSystems."/" = { + device = "/dev/disk/by-label/NIXOS_ROOT"; + fsType = "ext4"; + }; + + fileSystems."/persist" = { + device = "/dev/disk/by-label/NIXOS_PERSIST"; + fsType = "ext4"; + }; + + fileSystems."/nix" = { + device = "/dev/disk/by-label/NIXOS_NIX"; + fsType = "ext4"; + }; + + fileSystems."/var/log" = { + device = "/dev/disk/by-label/NIXOS_VARLOG"; + fsType = "ext4"; + neededForBoot = true; + }; + + boot.loader.grub.device = "nodev"; + boot.loader.grub.efiSupport = true; + boot.loader.efi.canTouchEfiVariables = true; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + }; +} diff --git a/hosts/server-n0-darling-erasure.sh b/hosts/server-n0-darling-erasure.sh new file mode 100644 index 0000000..2750d2d --- /dev/null +++ b/hosts/server-n0-darling-erasure.sh @@ -0,0 +1,23 @@ +mkdir -p /mnt + +echo "--- erasing system state begin ---" +# We first mount the btrfs root to /mnt +# so we can manipulate btrfs subvolumes. +mount -o subvol=/ /dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820 /mnt + +# back up the old root (important!!) +BOOTID=$(date +"@boot-%Y_%m_%d-%H_%M_%S-%s") +echo "backing up old root: /mnt/@root/ -> /mnt/@old-roots/$BOOTID..." +echo "note that the timestamp is of *this* boot, not of the boot that created the old root!" +mv /mnt/@root /mnt/@old-roots/$BOOTID +btrfs property set -f -ts /mnt/@old-roots/$BOOTID ro true + +echo "restoring blank /@root subvolume..." +btrfs subvolume snapshot /mnt/@root-blank /mnt/@root + +# Once we're done rolling back to a blank snapshot, +# we can unmount /mnt and continue on the boot process. +echo "--- erasing system state end ---" + +umount /mnt + diff --git a/hosts/server-n0.nix b/hosts/server-n0.nix new file mode 100644 index 0000000..b8d59d1 --- /dev/null +++ b/hosts/server-n0.nix @@ -0,0 +1,109 @@ +_: { + module = + { + config, + lib, + pkgs, + modulesPath, + hostInfo, + ... + }: + { + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + networking.hostName = hostInfo.hostname; + + fileSystems."/" = { + device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820"; + fsType = "btrfs"; + options = [ + "subvol=@root" + "noatime" + "compress=zstd" + ]; + }; + + fileSystems."/nix" = { + device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820"; + fsType = "btrfs"; + options = [ + "subvol=@nix" + "noatime" + "compress=zstd" + ]; + }; + + fileSystems."/persist" = { + device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820"; + fsType = "btrfs"; + options = [ + "subvol=@persist" + "noatime" + "compress=zstd" + ]; + }; + + fileSystems."/btrfs_root" = { + device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820"; + fsType = "btrfs"; + options = [ + "noatime" + "ro" + "compress=zstd" + ]; + }; + + fileSystems."/var/log" = { + device = "/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820"; + fsType = "btrfs"; + options = [ + "subvol=@logs" + "noatime" + "compress=zstd" + ]; + neededForBoot = true; + }; + + fileSystems."/boot" = { + device = "/dev/disk/by-uuid/7155-EF89"; + fsType = "vfat"; + options = [ + "fmask=0022" + "dmask=0022" + ]; + }; + + boot = { + loader.limine.enable = true; + + kernelPackages = pkgs.linuxPackages_latest; + kernelModules = [ "kvm-intel" ]; + extraModprobeConfig = "options kvm_intel nested=1"; + extraModulePackages = [ ]; + supportedFilesystems = [ "btrfs" ]; + initrd = { + compressorArgs = [ + "-22" + "-T0" + "--long" + "--ultra" + ]; + availableKernelModules = [ + "nvme" + "xhci_pci" + "ahci" + "usbhid" + ]; + kernelModules = [ ]; + systemd.enable = false; + postDeviceCommands = lib.mkBefore (builtins.readFile ./server-n0-darling-erasure.sh); + }; + }; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + hardware.enableRedistributableFirmware = true; + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; + }; +} diff --git a/introspect.nix b/introspect.nix new file mode 100644 index 0000000..f9d076a --- /dev/null +++ b/introspect.nix @@ -0,0 +1,399 @@ +{ lib, dsl }: +let + plain = + v: + if lib.isFunction v then + null + else if lib.isDerivation v then + "${v}" + else if lib.isList v then + map plain v + else if lib.isAttrs v then + lib.mapAttrs (_: plain) v + else if builtins.isPath v then + toString v + else + v; + + relations = [ + { + field = "after"; + kind = "after"; + reverse = false; + } + { + field = "before"; + kind = "after"; + reverse = true; + } + { + field = "wants"; + kind = "wants"; + reverse = false; + } + { + field = "wantedBy"; + kind = "wants"; + reverse = true; + } + { + field = "requires"; + kind = "requires"; + reverse = false; + } + { + field = "requiredBy"; + kind = "requires"; + reverse = true; + } + { + field = "upholds"; + kind = "upholds"; + reverse = false; + } + { + field = "upheldBy"; + kind = "upholds"; + reverse = true; + } + { + field = "bindsTo"; + kind = "bindsTo"; + reverse = false; + } + { + field = "partOf"; + kind = "partOf"; + reverse = false; + } + { + field = "requisite"; + kind = "requisite"; + reverse = false; + } + { + field = "conflicts"; + kind = "conflicts"; + reverse = false; + } + { + field = "onFailure"; + kind = "onFailure"; + reverse = false; + } + { + field = "onSuccess"; + kind = "onSuccess"; + reverse = false; + } + ]; + + relationFields = map (r: r.field) relations; + + # raw NixOS command options, mirrors every generated Exec* wrapper + commandFields = [ + "script" + "scriptArgs" + "preStart" + "postStart" + "reload" + "preStop" + "postStop" + ]; + + allUnits = + c: + lib.concatMap lib.attrValues [ + c.systemd.paths + c.systemd.services + c.systemd.slices + c.systemd.sockets + c.systemd.targets + c.systemd.timers + ] + ++ c.systemd.mounts + ++ c.systemd.automounts; + + unitSummary = + u: + { + inherit (u) name; + description = u.description or ""; + enable = u.enable or true; + unitConfig = u.unitConfig or { }; + } + // lib.filterAttrs (_: v: v != [ ]) (lib.genAttrs relationFields (f: u.${f} or [ ])) + // lib.optionalAttrs (u ? serviceConfig) ( + { + inherit (u) serviceConfig; + environment = u.environment or { }; + path = u.path or [ ]; + restartIfChanged = u.restartIfChanged or null; + stopIfChanged = u.stopIfChanged or null; + } + # raw command text: the real logic behind the store-path Exec* wrappers + // lib.filterAttrs (_: v: v != "") (lib.genAttrs commandFields (f: u.${f} or "")) + ) + // lib.optionalAttrs (u ? timerConfig) { inherit (u) timerConfig; } + // lib.optionalAttrs (u ? socketConfig) { + inherit (u) socketConfig; + listenStreams = u.listenStreams or [ ]; + listenDatagrams = u.listenDatagrams or [ ]; + } + // lib.optionalAttrs (u ? pathConfig) { inherit (u) pathConfig; } + // lib.optionalAttrs (u ? what) { + inherit (u) what; + where = u.where or ""; + type = u.type or ""; + options = u.options or ""; + }; + + unitEdges = + node: u: + let + ref = n: "${node}:${n}"; + in + lib.concatMap ( + r: + map ( + peer: + { + inherit (r) kind; + } + // ( + if r.reverse then + { + from = ref peer; + to = ref u.name; + } + else + { + from = ref u.name; + to = ref peer; + } + ) + ) (u.${r.field} or [ ]) + ) relations; + + fileSystemsOf = + c: + lib.mapAttrsToList (mountPoint: fs: { + inherit mountPoint; + inherit (fs) + device + fsType + options + neededForBoot + depends + ; + }) c.fileSystems; + + firewallOf = + c: + let + fw = c.networking.firewall; + portsOf = f: { + inherit (f) + allowedTCPPorts + allowedUDPPorts + allowedTCPPortRanges + allowedUDPPortRanges + ; + }; + in + { + inherit (fw) enable trustedInterfaces; + interfaces = lib.mapAttrs (_: portsOf) fw.interfaces; + } + // portsOf fw; + + isBuildUser = n: builtins.match "nixbld[0-9]+" n != null; + + usersOf = c: { + inherit (c.users) mutableUsers; + groups = lib.mapAttrs (_: g: { + inherit (g) gid; + members = lib.filter (m: !isBuildUser m) g.members; + }) c.users.groups; + users = lib.mapAttrs (_: u: { + inherit (u) + uid + group + home + shell + description + createHome + isNormalUser + isSystemUser + extraGroups + ; + authorizedKeys = u.openssh.authorizedKeys.keys; + passwordSet = lib.any (v: v != null) [ + u.hashedPassword + u.hashedPasswordFile + u.password + u.initialPassword + u.initialHashedPassword + ]; + }) (lib.filterAttrs (n: _: !isBuildUser n) c.users.users); + }; +in +{ + hostReport = + { host }: + let + cfg = host.hostCfg.config; + deployments = host.containers; + depList = lib.attrValues deployments; + + configOf = + c: + if c.kind == "nspawn" then + cfg.containers.${c.name}.config + else + cfg.microvm.vms.${c.name}.config.config; + unitOf = c: "${dsl.serviceUnitName c}.service"; + + nodes = { + "@host" = cfg; + } + // lib.mapAttrs (_: configOf) deployments; + + forward = + proto: c: p: + let + q = dsl.normalizePort p; + in + { + inherit proto; + hostPort = q.host; + target = c.name; + targetPort = q.container; + targetAddr = c.addr; + }; + + mountsOf = + c: + if c.kind == "nspawn" then + lib.mapAttrsToList (mountPoint: bm: { + inherit mountPoint; + inherit (bm) hostPath isReadOnly; + }) c.bindMounts + else + map (s: { + mountPoint = s.mountPoint or null; + hostPath = s.source; + isReadOnly = s.readOnly or false; + inherit (s) proto tag; + }) c.shares; + in + plain { + meta = { + host = host.hostName; + system = cfg.nixpkgs.hostPlatform.system; + inherit (cfg.system) stateVersion; + inherit (host.specialArgs.hostInfo) paths mainGateway; + toplevel = cfg.system.build.toplevel; + }; + + deployments = lib.mapAttrs (_: c: { + inherit (c) + name + kind + autoStart + access + allowInternet + needsGateway + includedInTests + ; + # names only; c.depends holds whole deployment objects + depends = map (d: d.name) c.depends; + unit = unitOf c; + limits = + if c.kind == "nspawn" then + { inherit (c) memoryMax cpuQuota privateUsers; } + else + { inherit (c) vcpu mem hypervisor; }; + inherit (c) miscProperties; + }) deployments; + + access = { + specials = [ + "@host" + "@internet" + ]; + edges = lib.concatMap ( + c: + map (a: { + from = c.name; + inherit (a) to; + tcp = a.tcp or [ ]; + udp = a.udp or [ ]; + }) c.access + ) depList; + }; + + kernel = { + sysctl = cfg.boot.kernel.sysctl; + params = cfg.boot.kernelParams; + modules = cfg.boot.kernelModules; + }; + + network = { + inherit (cfg.networking) nameservers; + links = lib.mapAttrs (_: c: { + inherit (c) + kind + iface + mac + addr + hostAddr + allowInternet + needsGateway + ; + }) deployments; + hostInterfaces = lib.mapAttrs (_: i: { inherit (i) ipv4; }) cfg.networking.interfaces; + dnsmasq = cfg.services.dnsmasq.settings; + }; + + ports = { + forwards = lib.concatMap ( + c: map (forward "tcp" c) c.hostPorts ++ map (forward "udp" c) c.hostUdpPorts + ) depList; + declared = lib.mapAttrs (_: c: c.ports or [ ]) deployments; + firewall = lib.mapAttrs (_: firewallOf) nodes; + }; + + mounts = { + fileSystems = lib.mapAttrs (_: fileSystemsOf) nodes; + declared = lib.mapAttrs (_: mountsOf) deployments; + }; + + users = lib.mapAttrs (_: usersOf) nodes; + + units = lib.mapAttrs ( + _: c: lib.listToAttrs (map (u: lib.nameValuePair u.name (unitSummary u)) (allUnits c)) + ) nodes; + + dependencies = { + edges = + lib.concatLists (lib.mapAttrsToList (node: c: lib.concatMap (unitEdges node) (allUnits c)) nodes) + ++ map (c: { + from = "@host:${unitOf c}"; + to = "${c.name}:multi-user.target"; + kind = "spawns"; + }) depList; + }; + + nftables = { + inherit (cfg.networking.nftables) enable ruleset; + tables = lib.mapAttrs (n: t: { + name = t.name or n; + inherit (t) family enable content; + }) cfg.networking.nftables.tables; + rendered = builtins.readFile ( + builtins.elemAt cfg.systemd.services.nftables.serviceConfig.ExecStart 1 + ); + }; + }; +} diff --git a/mount.sh b/mount.sh new file mode 100644 index 0000000..bc1ca46 --- /dev/null +++ b/mount.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +DEV=/dev/disk/by-uuid/5dfd3b74-2162-4167-825b-21259ca48820 +EFI=/dev/disk/by-uuid/7155-EF89 + +mount -o subvol=@root "$DEV" /mnt +mkdir -p /mnt/{nix,persist,var/log,btrfs_root,boot} +mount -o subvol=@nix "$DEV" /mnt/nix +mount -o subvol=@persist "$DEV" /mnt/persist +mount -o subvol=@logs "$DEV" /mnt/var/log +mount -o subvolid=5 "$DEV" /mnt/btrfs_root +mount "$EFI" /mnt/boot + +btrfs subvolume create /mnt/persist/@secrets diff --git a/packages/ct.nix b/packages/ct.nix new file mode 100644 index 0000000..3d877c8 --- /dev/null +++ b/packages/ct.nix @@ -0,0 +1,8 @@ +{ + writeShellApplication, + dtach, +}: +import ./mk-script.nix { inherit writeShellApplication; } { + name = "ct"; + runtimeInputs = [ dtach ]; +} diff --git a/packages/hopper-lazy-resolve.patch b/packages/hopper-lazy-resolve.patch new file mode 100644 index 0000000..220227e --- /dev/null +++ b/packages/hopper-lazy-resolve.patch @@ -0,0 +1,92 @@ +diff --git a/src/config/router.rs b/src/config/router.rs +index cfee4f8..9247cc3 100644 +--- a/src/config/router.rs ++++ b/src/config/router.rs +@@ -58,11 +58,10 @@ impl Router for RouterConfig { + .or(self.default.as_ref()) + .ok_or(RouterError::NoServer)?; + +- let address: SocketAddr = match route.ip { +- RouteType::Simple(address) => address.into(), +- RouteType::Balanced(ref list) => { +- let hash = client.hash(); +- list.get(hash as usize) ++ let address: SocketAddr = match &route.ip { ++ RouteType::Simple(addr) => addr.resolve().ok_or(RouterError::Unavailable)?, ++ RouteType::Balanced(list) => { ++ list.get(client.hash() as usize).ok_or(RouterError::Unavailable)? + } + }; + +diff --git a/src/config/router/balancer.rs b/src/config/router/balancer.rs +index 770a01b..b2e2491 100644 +--- a/src/config/router/balancer.rs ++++ b/src/config/router/balancer.rs +@@ -22,7 +22,7 @@ pub struct Balanced { + // } + + impl Balanced { +- pub(super) fn get(&self, n: usize) -> SocketAddr { +- self.servers[n & self.servers.len()].into() ++ pub(super) fn get(&self, n: usize) -> Option { ++ self.servers[n % self.servers.len()].resolve() + } + } +diff --git a/src/config/router/resolver.rs b/src/config/router/resolver.rs +index b66a16a..8fd3b0d 100644 +--- a/src/config/router/resolver.rs ++++ b/src/config/router/resolver.rs +@@ -2,26 +2,26 @@ use std::net::{SocketAddr, ToSocketAddrs}; + + use serde::{Deserialize, Deserializer}; + +-#[derive(Deserialize, Debug, Clone, Copy)] +-pub(super) struct ResolvableAddr(#[serde(deserialize_with = "resolve_hostname")] SocketAddr); ++#[derive(Deserialize, Debug, Clone)] ++pub(super) struct ResolvableAddr(#[serde(deserialize_with = "parse_addr_string")] String); + +-fn resolve_hostname<'de, D>(deserializer: D) -> Result ++fn parse_addr_string<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + +- let inner = ::deserialize(deserializer)?; +- +- let mut addr = inner +- .to_socket_addrs() +- .map_err(|err| Error::custom(format!("invalid hostname format: {err}")))?; +- +- addr.next().ok_or_else(|| Error::missing_field("address")) ++ let s = String::deserialize(deserializer)?; ++ if !s.contains(':') { ++ return Err(Error::custom(format!( ++ "invalid socket address syntax, expected host:port: {s}" ++ ))); ++ } ++ Ok(s) + } + +-impl From for SocketAddr { +- fn from(addr: ResolvableAddr) -> Self { +- addr.0 ++impl ResolvableAddr { ++ pub fn resolve(&self) -> Option { ++ self.0.to_socket_addrs().ok()?.next() + } + } +diff --git a/src/server/router.rs b/src/server/router.rs +index 4aae8e7..ad2849b 100644 +--- a/src/server/router.rs ++++ b/src/server/router.rs +@@ -7,6 +7,9 @@ use thiserror::Error; + pub enum RouterError { + #[error("no server with such hostname has been found")] + NoServer, ++ ++ #[error("backend server is currently unavailable")] ++ Unavailable, + } + + // #[async_trait::async_trait] diff --git a/packages/hopper-rs.nix b/packages/hopper-rs.nix new file mode 100644 index 0000000..fd3f4af --- /dev/null +++ b/packages/hopper-rs.nix @@ -0,0 +1,29 @@ +{ + lib, + fetchFromGitHub, + rustPlatform, +}: +rustPlatform.buildRustPackage { + pname = "hopper-rs"; + version = "v0.6.2-2-g73cb171"; + + src = fetchFromGitHub { + owner = "bra1l0r"; + repo = "hopper-rs"; + rev = "73cb171e203cef6c9c444b69c43ceefa2da213e4"; + hash = "sha256-QhDH03KHKtsh7Ch44TE02sr42i/CPD9BGK7QQ6PwHKc="; + }; + + patches = [ ./hopper-lazy-resolve.patch ]; + + cargoHash = "sha256-bC5BlP3opYYHkK/mauF9qdTczTdmGQD7yEOCOMylNOc="; + + doCheck = false; + + meta = with lib; { + description = "Lightweight Minecraft reverse proxy"; + homepage = "https://github.com/bra1l0r/hopper-rs"; + license = licenses.gpl3Only; + mainProgram = "hopper"; + }; +} diff --git a/packages/kitty-terminfo.nix b/packages/kitty-terminfo.nix new file mode 100644 index 0000000..b7ccb2e --- /dev/null +++ b/packages/kitty-terminfo.nix @@ -0,0 +1,9 @@ +{ + runCommand, + ncurses, + kitty, +}: +runCommand "kitty-terminfo" { nativeBuildInputs = [ ncurses ]; } '' + mkdir -p $out/share/terminfo + tic -x -o $out/share/terminfo ${kitty.src}/terminfo/kitty.terminfo +'' diff --git a/packages/mk-script.nix b/packages/mk-script.nix new file mode 100644 index 0000000..0acec74 --- /dev/null +++ b/packages/mk-script.nix @@ -0,0 +1,9 @@ +{ writeShellApplication }: +{ + name, + runtimeInputs ? [ ], +}: +writeShellApplication { + inherit name runtimeInputs; + text = builtins.readFile (../scripts + "/${name}.sh"); +} diff --git a/packages/server-kexec.nix b/packages/server-kexec.nix new file mode 100644 index 0000000..e2d56b8 --- /dev/null +++ b/packages/server-kexec.nix @@ -0,0 +1,8 @@ +{ + writeShellApplication, + kexec-tools, +}: +import ./mk-script.nix { inherit writeShellApplication; } { + name = "server-kexec"; + runtimeInputs = [ kexec-tools ]; +} diff --git a/packages/server-rebuild.nix b/packages/server-rebuild.nix new file mode 100644 index 0000000..886f525 --- /dev/null +++ b/packages/server-rebuild.nix @@ -0,0 +1,2 @@ +{ writeShellApplication }: +import ./mk-script.nix { inherit writeShellApplication; } { name = "server-rebuild"; } diff --git a/packages/server-status.nix b/packages/server-status.nix new file mode 100644 index 0000000..a2ece15 --- /dev/null +++ b/packages/server-status.nix @@ -0,0 +1,2 @@ +{ writeShellApplication }: +import ./mk-script.nix { inherit writeShellApplication; } { name = "server-status"; } diff --git a/packages/sshpiper.nix b/packages/sshpiper.nix new file mode 100644 index 0000000..d2f2c6b --- /dev/null +++ b/packages/sshpiper.nix @@ -0,0 +1,33 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, +}: +buildGoModule { + pname = "sshpiper"; + version = "unstable-2024"; + + src = fetchFromGitHub { + owner = "tg123"; + repo = "sshpiper"; + rev = "e5c5b904cfa57f7d60e37cb38cdb0bbcaadb022e"; + hash = "sha256-8Lqf7pCcAWAgz7tclKiV0DyHhbaaBule6/X30Fzm8DY="; + fetchSubmodules = true; + }; + + vendorHash = "sha256-6FU1r6vY6AorbP5z+C3zJpHQzpqzkzK9iQFnxDmYv8c="; + + subPackages = [ + "cmd/sshpiperd" + "plugin/yaml" + ]; + + tags = [ "full" ]; + + meta = with lib; { + description = "SSH reverse proxy / multiplexer"; + homepage = "https://github.com/tg123/sshpiper"; + license = licenses.mit; + mainProgram = "sshpiperd"; + }; +} diff --git a/scripts/ct.sh b/scripts/ct.sh new file mode 100644 index 0000000..f5958ad --- /dev/null +++ b/scripts/ct.sh @@ -0,0 +1,136 @@ +cmd=${1:-help} +ct_name=${2:-} + +require_name() { + if [ -z "$ct_name" ]; then + echo "usage: ct $cmd " >&2 + exit 1 + fi +} + +case "$cmd" in + list) + sep() { printf '%0.s-' {1..60}; printf '\n'; } + uptime_str() { + awk '{s=int($1);d=int(s/86400);h=int((s%86400)/3600);m=int((s%3600)/60); + if(d>0)printf "%d days %d hours %d minutes",d,h,m; + else if(h>0)printf "%d hours %d minutes",h,m; + else printf "%d minutes",m}' /proc/uptime + } + + printf 'HOST: %s UP: %s\n' "$(hostname)" "$(uptime_str)" + sep + + printf 'CONTAINERS\n' + mapfile -t all_units < <({ + systemctl list-unit-files --no-legend --no-pager --plain 2>/dev/null + systemctl list-units --all --no-legend --no-pager --plain 2>/dev/null + } | awk '$1 ~ /^container@.+\.service$/ {print $1}' | sort -u) + + if [ ${#all_units[@]} -gt 0 ]; then + systemctl show "${all_units[@]}" \ + --property=Id,UnitFileState,ActiveState --no-pager 2>/dev/null \ + | awk ' + /^Id=container@/ { n=$0; sub(/^Id=container@/,"",n); sub(/\.service$/,"",n) } + /^UnitFileState=/ { f=$0; sub(/^UnitFileState=/,"",f) } + /^ActiveState=/ { a=$0; sub(/^ActiveState=/,"",a) } + /^[[:space:]]*$/ { if (n!="") { _print(); n=f=a="" } } + END { if (n!="") _print() } + function _print() { + start = (f=="enabled") ? "auto" : "manual" + if (a=="active") st="running" + else if (a=="failed") st="FAILED" + else if (a=="inactive") st="stopped" + else st=(a!="") ? a : "not-loaded" + mid = "-" + if (a == "active") { + midcmd = "machinectl show " n " -p Id --value 2>/dev/null" + got = "" + midcmd | getline got + close(midcmd) + if (got != "") mid = substr(got,1,12) "..." + } + printf " %-35s %-8s %-10s %s\n", n, start, st, mid + } + ' + else + printf ' none\n' + fi + + sep + + printf 'FAILED UNITS (host)\n' + failed=$(systemctl list-units --state=failed --no-legend --no-pager --plain 2>/dev/null \ + | awk 'NF>0{print $1}') + if [ -z "$failed" ]; then printf ' none\n' + else printf '%s\n' "$failed" | sed 's/^/ /'; fi + + sep + + printf 'FAILED UNITS (containers)\n' + any_failed=0 + machines=$(machinectl list --no-legend --no-pager 2>/dev/null | awk 'NF>0{print $1}') + for m in $machines; do + ct_failed=$(sudo systemctl -M "$m" list-units --state=failed \ + --no-legend --no-pager --plain 2>/dev/null \ + | awk 'NF>0 && $1!~/^container@/{print $1}' || true) + if [ -n "$ct_failed" ]; then + printf ' [%s]\n' "$m" + printf '%s\n' "$ct_failed" | sed 's/^/ /' + any_failed=1 + fi + done + [ "$any_failed" -eq 0 ] && printf ' none\n' + + sep + + df -h /persist | awk 'NR==2{printf "DISK %s used / %s total (%s)\n",$3,$2,$5}' + free -h | awk '/^Mem:/{printf "MEM %s used / %s total\n",$3,$2}' + ;; + start) + require_name + sudo systemctl start "container@$ct_name" + ;; + stop) + require_name + sudo systemctl stop "container@$ct_name" + ;; + restart) + require_name + sudo systemctl restart "container@$ct_name" + ;; + status) + require_name + machinectl status "$ct_name" + ;; + logs) + require_name + shift 2 + sudo journalctl -M "$ct_name" --no-hostname "$@" + ;; + shell) + require_name + exec sudo machinectl shell "$ct_name" + ;; + attach) + require_name + leader=$(machinectl show "$ct_name" -p Leader --value 2>/dev/null) + if [ -z "$leader" ] || [ "$leader" = "0" ]; then + printf 'error: container %s is not running\n' "$ct_name" >&2 + exit 1 + fi + exec sudo nsenter -m -t "$leader" -- dtach -a -r none /run/ct-attach/attach.sock + ;; + *) + printf 'usage: ct [container]\n\n' + printf ' %-9s %-13s %s\n' \ + "list" "" "list all containers with state (running/stopped/FAILED)" \ + "start" "" "start a container" \ + "stop" "" "stop a container" \ + "restart" "" "restart a container" \ + "status" "" "show container status" \ + "logs" "" "view logs (extra args forwarded to journalctl)" \ + "shell" "" "open a shell" \ + "attach" "" "attach to Minecraft console (detach: Ctrl+\\)" + ;; +esac diff --git a/scripts/server-kexec.sh b/scripts/server-kexec.sh new file mode 100644 index 0000000..f1de874 --- /dev/null +++ b/scripts/server-kexec.sh @@ -0,0 +1,17 @@ +SYSTEM=${1:-/nix/var/nix/profiles/system} + +if [[ ! -e "$SYSTEM/kernel" ]]; then + echo "error: no kernel found at $SYSTEM" >&2 + exit 1 +fi + +echo "kexec: loading $SYSTEM" +echo "kernel-params: $(cat "$SYSTEM/kernel-params")" + +sudo kexec \ + --load "$SYSTEM/kernel" \ + --initrd="$SYSTEM/initrd" \ + --command-line="$(cat "$SYSTEM/kernel-params") init=$SYSTEM/init" + +echo "kexec: executing (systemd will stop services first)" +sudo systemctl kexec diff --git a/scripts/server-rebuild.sh b/scripts/server-rebuild.sh new file mode 100644 index 0000000..78fe94d --- /dev/null +++ b/scripts/server-rebuild.sh @@ -0,0 +1 @@ +exec sudo nixos-rebuild switch --flake "/persist/@configuration#$(hostname)" "$@" diff --git a/scripts/server-status.sh b/scripts/server-status.sh new file mode 100644 index 0000000..13778b0 --- /dev/null +++ b/scripts/server-status.sh @@ -0,0 +1,21 @@ +state=$(systemctl is-system-running 2>/dev/null || true) +uptime_str() { + awk '{s=int($1);d=int(s/86400);h=int((s%86400)/3600);m=int((s%3600)/60); + if(d>0)printf "%d days %d hours",d,h; + else if(h>0)printf "%d hours %d min",h,m; + else printf "%d min",m}' /proc/uptime +} +printf 'HOST: %s UP: %s STATE: %s\n' "$(hostname)" "$(uptime_str)" "$state" +printf '%0.s-' {1..60}; printf '\n' + +systemctl list-units --all --type=service \ + --state=active,failed --no-legend --no-pager --plain 2>/dev/null \ +| awk ' + NF >= 4 { + unit=$1; active=$3; sub=$4 + if (unit ~ /^(getty@|console-getty\.service)/) next + if (unit ~ /^systemd-(tmpfiles|binfmt|modules-load|sysctl|ask-password|remount|random-seed|update-done|hibernate|sleep|suspend|reboot|poweroff)/) next + state = (active == "failed") ? "FAILED" : sub + printf " %-50s %s\n", unit, state + } +' diff --git a/statix.toml b/statix.toml new file mode 100644 index 0000000..98e707c --- /dev/null +++ b/statix.toml @@ -0,0 +1,2 @@ +disabled = ["repeated_keys"] +ignore = [".direnv"] diff --git a/tests/server-n0/integration.nix b/tests/server-n0/integration.nix new file mode 100644 index 0000000..e985211 --- /dev/null +++ b/tests/server-n0/integration.nix @@ -0,0 +1,190 @@ +{ + lib, + host, + dsl, + commonModule, +}: +pkgs: +let + inherit (host.specialArgs.hostInfo) paths; + serverDataDir = paths.data; + serverSecretsDir = paths.secrets; + + testContainers = dsl.filterDeployments (c: c.includedInTests) host.containers; + + it = dsl.mkIntegrationTest { inherit host testContainers pkgs; }; + + nginxAcmeDomains = [ + "devenv.floweytf.com" + "maven.floweytf.com" + "git.floweytf.com" + "utils.floweytf.com" + "mail.floweytf.com" + ]; + + nginxAcmeDir = domain: "${serverDataDir}/core/nginx/acme/${domain}"; + + fakeCert = + domain: + pkgs.runCommand "fake-cert-${domain}" + { + nativeBuildInputs = [ pkgs.openssl ]; + } + '' + mkdir -p "$out" + openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj "/CN=${domain}" \ + -keyout "$out/key.pem" -out "$out/fullchain.pem" 2>/dev/null + ''; + + mkdirAcmeDirs = lib.concatMapStrings ( + domain: ''mkdir -p "${nginxAcmeDir domain}"'' + "\n" + ) nginxAcmeDomains; + + installFakeCerts = lib.concatMapStrings ( + domain: + let + dir = nginxAcmeDir domain; + cert = fakeCert domain; + in + '' + if [[ ! -f "${dir}/fullchain.pem" ]]; then + install -m 644 ${cert}/fullchain.pem "${dir}/fullchain.pem" + install -m 600 ${cert}/key.pem "${dir}/key.pem" + fi + '' + ) nginxAcmeDomains; + + # used in ssh-admin + testSshKey = + pkgs.runCommand "test-ssh-key" + { + nativeBuildInputs = [ pkgs.openssh ]; + } + '' + mkdir -p "$out" + ssh-keygen -q -t ed25519 -N "" -f "$out/key" -C test + ''; + sshpiperTestUser = "admin"; + + hostExporterPort = 9100; + hostCadvisorPort = 9200; + hostSshdPort = 5952; +in +{ + name = "integration-${host.hostName}"; + + nodes = { + internet = it.internetNode; + + server = + { pkgs, lib, ... }: + { + imports = [ + commonModule + host.specialArgs.inputs.microvm.nixosModules.host + (dsl.mkHostFromContainers { + containers = testContainers; + inherit (host) ctx; + }) + it.serverModule + ]; + + containers.nginx.config = { + systemd.services = lib.genAttrs (map (d: "acme-order-renew-${d}") nginxAcmeDomains) (_: { + enable = lib.mkForce false; + }); + }; + + system.activationScripts.test-secrets = + # bash + '' + mkdir -p /nix/var/nix/db /nix/var/nix/daemon-socket + + mkdir -p /var/log/journal + + ${mkdirAcmeDirs} + mkdir -p "${serverSecretsDir}" \ + "${serverDataDir}/core/sshpiper" \ + "${serverDataDir}/core/mail" \ + "${serverDataDir}/overseer/loki/data" \ + "${serverDataDir}/overseer/prometheus/data" \ + "${serverDataDir}/overseer/grafana" \ + "${serverDataDir}/web/git" \ + "${serverDataDir}/web/git-ci-runner/nix-rw-store" \ + "${serverDataDir}/web/git-ci-runner/docker" \ + "${serverDataDir}/web/git-ci-runner/runner-state" \ + "${serverSecretsDir}/ci-runner" \ + "${serverDataDir}/web/reposilite" \ + "${serverDataDir}/web/devenv/config" \ + "${serverDataDir}/web/devenv/workspace" \ + "${serverDataDir}/ssh/generic" \ + "${serverDataDir}/ssh/nyaa" \ + "${serverDataDir}/mc/botm" \ + "${serverDataDir}/mc/rogule-dev" \ + "${serverDataDir}/mc/rogule-build" \ + "${serverDataDir}/mc/starwalker" \ + "${serverDataDir}/mc/tl" + + chown -R 1000:1000 \ + "${serverDataDir}/web/git" \ + "${serverDataDir}/web/reposilite" \ + "${serverDataDir}/web/devenv/config" \ + "${serverDataDir}/web/devenv/workspace" \ + "${serverDataDir}/core/mail" \ + "${serverDataDir}/ssh/generic" \ + "${serverDataDir}/mc/botm" \ + "${serverDataDir}/mc/rogule-dev" \ + "${serverDataDir}/mc/rogule-build" \ + "${serverDataDir}/mc/starwalker" \ + "${serverDataDir}/mc/tl" + + for keytype in ed25519 rsa ecdsa; do + keyfile="${serverSecretsDir}/ssh_host_''${keytype}_key" + if [[ ! -f "$keyfile" ]]; then + ${lib.getExe' pkgs.openssh "ssh-keygen"} -q -t "$keytype" -f "$keyfile" -N "" + fi + done + + # stalwart rejects weak/dictionary passwords + for f in mail-admin-password mail-account-password; do + target="${serverSecretsDir}/$f" + if [[ ! -f "$target" ]]; then + head -c 32 /dev/urandom | base64 | tr -d '\n' > "$target" + chmod 600 "$target" + fi + done + + if [[ ! -f "${serverSecretsDir}/ci-runner/token" ]]; then + echo "TOKEN=test-registration-token" > "${serverSecretsDir}/ci-runner/token" + chmod 600 "${serverSecretsDir}/ci-runner/token" + fi + + if [[ ! -f "${serverSecretsDir}/code-server-env" ]]; then + echo "test-secret" > "${serverSecretsDir}/code-server-env" + chmod 600 "${serverSecretsDir}/code-server-env" + fi + + ${installFakeCerts} + mkdir -p "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}" + install -m 644 ${testSshKey}/key.pub \ + "${serverDataDir}/core/sshpiper/user_${sshpiperTestUser}/authorized_keys" + ''; + }; + }; + + testScript = + it.testScriptPreamble + + '' + # store paths are world-readable; ssh refuses a key with those permissions + SSH_KEY = "/root/test-ssh-key" + SSH_OPTS = f"-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -i {SSH_KEY}" + TEST_SSH_KEY_STORE = "${testSshKey}/key" + + SSHPIPER_TEST_USER = "${sshpiperTestUser}" + HOST_EXPORTER_PORT = ${toString hostExporterPort} + HOST_CADVISOR_PORT = ${toString hostCadvisorPort} + HOST_SSHD_PORT = ${toString hostSshdPort} + '' + + builtins.readFile ./integration.py; +} diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py new file mode 100644 index 0000000..265ccf9 --- /dev/null +++ b/tests/server-n0/integration.py @@ -0,0 +1,330 @@ +import json +import re +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + server: Any = None + internet: Any = None + start_all: Any = None + subtest: Any = None + retry: Any = None + + NSENTER: str = "" + CURL: str = "" + IP: str = "" + PING: str = "" + DIG: str = "" + SSH: str = "" + OPENSSL: str = "" + TIMEOUT: str = "" + included: list[str] = [] + enterable: list[str] = [] + allow_internet: list[str] = [] + not_allow_internet: list[str] = [] + has_gateway: list[str] = [] + no_gateway: list[str] = [] + ADDR: dict[str, str] = {} + HOST_ADDR: dict[str, str] = {} + IFACE: dict[str, str] = {} + KIND: dict[str, str] = {} + EDGES: list[tuple[str, str, list[int], list[int]]] = [] + SSH_KEY: str = "" + SSH_OPTS: str = "" + TEST_SSH_KEY_STORE: str = "" + SSHPIPER_TEST_USER: str = "" + HOST_EXPORTER_PORT: int = 0 + HOST_CADVISOR_PORT: int = 0 + HOST_SSHD_PORT: int = 0 + +SW = "/run/current-system/sw/bin" +SH = f"{SW}/sh" + +PEER_EDGES = {(src, to) for src, to, _, _ in EDGES if to not in ("@host", "@internet")} + + +def failed_units(node_arg=""): + out = server.succeed(f"systemctl {node_arg} list-units --state=failed --no-legend --plain || true") + return [ + line.split()[0] + for line in out.splitlines() + if line.strip() and not line.split()[0].startswith("acme-") + ] + + +def container_pid(name): + return server.succeed(f"machinectl show {name} -p Leader --value").strip() + + +def net_cmd(name, cmd): + pid = container_pid(name) + return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}" + + +def container_ip(name): + out = server.succeed(net_cmd(name, f"{IP} -4 -o addr show scope global")) + return out.split()[3].split("/")[0] + + +def has_ipv6_link_local(name): + out = server.succeed(net_cmd(name, f"{IP} -6 -o addr show scope link")) + return out.strip() != "" + + +def container_iface(name): + out = server.succeed(net_cmd(name, f"{IP} -j link show")) + for link in json.loads(out): + if link.get("link_type") == "ether" and "LOOPBACK" not in link.get("flags", []): + return link["ifname"] + raise Exception(f"no ethernet link found for {name}") + + +def connect(host, port, timeout=5): + """Open a TCP connection and close it again, using only tools every node has.""" + return f"{SW}/timeout {timeout} {SH} -c 'exec 3<>/dev/tcp/{host}/{port} && exec 3<&-'" + + +def ct_connect(name, host, port, timeout=5): + return net_cmd(name, connect(host, port, timeout)) + + +def ext_connect(port, timeout=5): + """A published port, probed the way a published port is actually used. + + Probing it from a container measures a different path - containers are + excluded from DNAT entirely. + """ + return connect(server_ip, port, timeout) + + +def spoof_counter(): + out = server.succeed("nft -j list counter inet container-ip spoofed") + return json.loads(out)["nftables"][1]["counter"]["packets"] + + +start_all() +server.succeed(f"install -m 600 {TEST_SSH_KEY_STORE} {SSH_KEY}") + +with subtest("(functional): containers autostart on boot with no errors"): + server.wait_for_unit("multi-user.target") + for name in included: + unit = "container" if KIND[name] == "nspawn" else "microvm" + server.wait_for_unit(f"{unit}@{name}.service") + server.succeed(f"systemctl is-active {unit}@{name}.service") + + assert failed_units() == [], f"host has failed units: {failed_units()}" + for name in enterable: + assert failed_units(f"-M {name}") == [], f"{name} has failed units: {failed_units(f'-M {name}')}" + +internet.wait_for_unit("dummy-http.service") +internet_ip = internet.succeed("ip -4 -o addr show dev eth1 scope global").split()[3].split("/")[0] +server_ip = server.succeed("ip -4 -o addr show dev eth1 scope global").split()[3].split("/")[0] + +with subtest("(functional): derived addressing matches what the ruleset assumes"): + for name in enterable: + assert container_ip(name) == ADDR[name], f"{name} has {container_ip(name)}, ruleset says {ADDR[name]}" + assert container_iface(name) == IFACE[name], f"{name} is on {container_iface(name)}, ruleset says {IFACE[name]}" + for name in [c for c in included if KIND[c] == "vm"]: + server.succeed(f"{IP} link show {IFACE[name]}") + server.wait_until_succeeds(f"{PING} -c 1 -W 2 {ADDR[name]}", timeout=180) + +with subtest("(isolation): internet egress follows the access list"): + for name in allow_internet: + server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) + for name in not_allow_internet: + server.fail(net_cmd(name, f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) + for name in has_gateway: + server.succeed(net_cmd(name, f"{IP} route show default | grep -q .")) + for name in no_gateway: + server.fail(net_cmd(name, f"{IP} route show default | grep -q .")) + +with subtest("(functional): container DNS still resolves"): + resolved = server.succeed( + net_cmd("git", f"{DIG} +short +time=3 @{HOST_ADDR['git']} nginx.containers") + ).strip() + assert resolved == ADDR["nginx"], f"nginx.containers resolved to {resolved!r}, expected {ADDR['nginx']}" + +with subtest("(functional): web services respond"): + for name, port in [("devenv", 8080), ("maven", 8080), ("git", 3000)]: + server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2")) + +with subtest("(functional): mail accepts connections"): + smtp_probe = net_cmd("mail", f"{SH} -c 'exec 3<>/dev/tcp/{ADDR['mail']}/25 && head -1 <&3'") + server.wait_until_succeeds(f"{smtp_probe} | grep -q '^220'", timeout=60) + + imap_probe = net_cmd( + "mail", + f"{TIMEOUT} 3 {OPENSSL} s_client -connect {ADDR['mail']}:993 -quiet -verify_quiet 2>/dev/null", + ) + + def imap_ready(_): + greeting = server.execute(f"echo | {imap_probe} | head -1")[1] + return greeting.strip().startswith("* OK") + + retry(imap_ready, timeout_seconds=60) + +with subtest("(functional): nginx proxies to backend containers"): + server.succeed( + net_cmd( + "nginx", + f"{CURL} -sfk --max-time 5 --resolve git.floweytf.com:443:{ADDR['nginx']} " + "https://git.floweytf.com/ >&2", + ) + ) + +with subtest("(functional): sshpiper authenticates/routes properly"): + whoami = server.succeed(f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} whoami").strip() + assert whoami == "runner", f"expected to land as runner, got {whoami}" + hostname = server.succeed( + f"{SSH} {SSH_OPTS} -p 2222 {SSHPIPER_TEST_USER}@{ADDR['sshpiper']} hostname" + ).strip() + assert hostname == "ssh-admin", f"expected to land on ssh-admin, got {hostname}" + +with subtest("(isolation): every declared container edge connects"): + for src, to, tcp_ports, _ in EDGES: + if to in ("@host", "@internet") or src not in enterable or to not in included: + continue + for port in tcp_ports: + server.succeed(ct_connect(src, ADDR[to], port)) + +with subtest("(isolation): undeclared container edges are refused"): + for src, dst, port in [ + ("devenv", "git", 3000), + ("git", "devenv", 8080), + ("maven", "git", 3000), + ("nginx", "mail", 25), + ("ssh-botm", "ssh-admin", 22), + ("overseer", "git", 3000), + ]: + assert (src, dst) not in PEER_EDGES, f"{src} -> {dst} is a declared edge, not a negative case" + server.fail(ct_connect(src, ADDR[dst], port, timeout=2)) + +with subtest("(isolation): edges are one-way"): + assert ("nginx", "git") in PEER_EDGES + assert ("git", "nginx") not in PEER_EDGES + server.succeed(ct_connect("nginx", ADDR["git"], 3000)) + server.fail(ct_connect("git", ADDR["nginx"], 443, timeout=2)) + +with subtest("(isolation): host access allowlist"): + server.succeed( + net_cmd( + "overseer", + f"{CURL} -sf --max-time 5 http://{HOST_ADDR['overseer']}:{HOST_EXPORTER_PORT}/metrics >&2", + ) + ) + for port in (HOST_EXPORTER_PORT, HOST_CADVISOR_PORT): + server.fail(ct_connect("git", HOST_ADDR["git"], port, timeout=2)) + server.fail(ct_connect("git", HOST_ADDR["overseer"], port, timeout=2)) + +with subtest("(isolation): host services outside the access list"): + internet.succeed(connect(server_ip, HOST_SSHD_PORT)) + for name in ("git", "overseer"): + server.fail(ct_connect(name, HOST_ADDR[name], HOST_SSHD_PORT, timeout=2)) + server.fail(ct_connect(name, server_ip, HOST_SSHD_PORT, timeout=2)) + +with subtest("(isolation): source address spoofing is dropped"): + before = spoof_counter() + server.succeed(net_cmd("git", f"{PING} -c 1 -W 2 {HOST_ADDR['git']} >&2")) + assert spoof_counter() == before, "an unspoofed packet hit the source guard" + + iface = IFACE["git"] + server.succeed(net_cmd("git", f"{IP} addr add {ADDR['nginx']}/32 dev {iface}")) + try: + server.execute( + net_cmd("git", f"{PING} -c 2 -W 2 -I {ADDR['nginx']} {HOST_ADDR['git']}") + ) + assert spoof_counter() > before, "a forged source address was not dropped" + finally: + server.succeed(net_cmd("git", f"{IP} addr del {ADDR['nginx']}/32 dev {iface}")) + +with subtest("(isolation): renaming the interface changes nothing"): + iface, host_addr = IFACE["nginx"], HOST_ADDR["nginx"] + + def rename(frm, to): + server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} down")) + server.succeed(net_cmd("nginx", f"{IP} link set dev {frm} name {to}")) + server.succeed(net_cmd("nginx", f"{IP} link set dev {to} up")) + for route in (host_addr, f"10.0.0.0/8 via {host_addr}", f"default via {host_addr}"): + server.succeed(net_cmd("nginx", f"{IP} route replace {route} dev {to}")) + + rename(iface, "evil0") + try: + server.succeed(ct_connect("nginx", ADDR["git"], 3000)) + server.fail(ct_connect("nginx", ADDR["mail"], 25, timeout=2)) + finally: + rename("evil0", iface) + +with subtest("(isolation): the veth cannot be moved out of the namespace"): + server.fail(net_cmd("git", f"{IP} link set dev {IFACE['git']} netns 1")) + +with subtest("(isolation): route bypass"): + iface = IFACE["hopper"] + server.succeed( + net_cmd("hopper", f"{IP} route replace default via {HOST_ADDR['hopper']} dev {iface} onlink") + ) + server.fail(net_cmd("hopper", f"{CURL} -sf --max-time 5 http://{internet_ip}/ >&2")) + server.fail(ct_connect("hopper", ADDR["git"], 3000, timeout=2)) + +with subtest("(isolation): no IPv6 link-local addresses"): + for name in ["git", "overseer"]: + assert not has_ipv6_link_local(name), f"{name} still has an IPv6 link-local address" + +with subtest("(isolation): published ports grant no reach between containers"): + for port in (80, 443, 25565, 22, 25): + server.fail(ct_connect("overseer", HOST_ADDR["overseer"], port, timeout=2)) + server.fail(ct_connect("git", server_ip, port, timeout=2)) + +with subtest("(isolation): privilege escalation / container escape"): + leader = container_pid("git") + + host_uid = server.succeed(f"awk '/^Uid:/{{print $2}}' /proc/{leader}/status").strip() + assert host_uid != "0", "container's PID 1 is not idmapped away from host uid 0" + + host_pidns = server.succeed("readlink /proc/1/ns/pid").strip() + container_pidns = server.succeed(f"readlink /proc/{leader}/ns/pid").strip() + assert host_pidns != container_pidns, "container shares the host's PID namespace" + + server.succeed( + f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " + f"{SH} -c 'echo x >> /var/lib/gitea/canary' >&2" + ) + + sshpiper_pid = container_pid("sshpiper") + server.fail( + f"{NSENTER} --mount=/proc/{sshpiper_pid}/ns/mnt --root=/proc/{sshpiper_pid}/root -- " + f"{SH} -c 'echo x >> /etc/ssh/ssh_host_ed25519_key' >&2" + ) + server.fail( + f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " + f"{SH} -c 'touch /nix/store/canary' >&2" + ) + server.fail( + f"{NSENTER} --mount=/proc/{leader}/ns/mnt --root=/proc/{leader}/root -- " + "/run/current-system/sw/bin/test -d /persist >&2" + ) + +with subtest("(functional): external clients reach published ports"): + for port in (80, 25565, 22): + internet.succeed(ext_connect(port)) + internet.fail(ext_connect(8080)) + +with subtest("(functional): no failing-reason restarts"): + assert failed_units() == [], f"host has failed units: {failed_units()}" + for name in included: + assert ( + failed_units(f"-M {name}") == [] + ), f"{name} has failed units: {failed_units(f'-M {name}')}" + restarts = server.succeed(f"systemctl show container@{name}.service -p NRestarts --value").strip() + assert restarts == "0", f"container@{name}.service restarted {restarts} times" + +with subtest("(functional): port forwards survive an nftables reload"): + server.succeed("systemctl reload nftables.service") + elements = server.succeed("nft list map ip container-portforward tcp-fwd") + assert re.search(r"\b80\s*:", elements), f"nftables reload emptied the map: {elements!r}" + internet.wait_until_succeeds(ext_connect(80), timeout=30) + +with subtest("(functional): published ports follow the container's lifetime"): + server.succeed("systemctl stop container@nginx") + internet.fail(ext_connect(80)) + server.succeed("systemctl start container@nginx") + server.wait_for_unit("container@nginx.service") + internet.wait_until_succeeds(ext_connect(80), timeout=30)