diff --git a/.github/workflows/diff-pr.yml b/.github/workflows/diff-pr.yml index 7ba4bd7..17b0abf 100644 --- a/.github/workflows/diff-pr.yml +++ b/.github/workflows/diff-pr.yml @@ -13,26 +13,21 @@ concurrency: jobs: dix: - runs-on: ubuntu-latest + runs-on: nix steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - name: worktree base for diff run: git worktree add "$RUNNER_TEMP/base" "${{ github.event.pull_request.base.sha }}" - name: build closures and render diff run: | - nix run .#ci -- diff "$RUNNER_TEMP/base" . \ + nix run -L .#ci -- diff "$RUNNER_TEMP/base" . \ "### System closure diff vs \`${{ github.event.pull_request.base.ref }}\`" \ > "$RUNNER_TEMP/comment.md" - nix run .#ci -- inspect-diff "$RUNNER_TEMP/base" . \ + nix run -L .#ci -- inspect-diff "$RUNNER_TEMP/base" . \ "### Inspection diff vs \`${{ github.event.pull_request.base.ref }}\`" \ >> "$RUNNER_TEMP/comment.md" @@ -42,4 +37,4 @@ jobs: API: ${{ github.api_url }} REPO: ${{ github.repository }} PR: ${{ github.event.pull_request.number }} - run: nix run .#ci -- post-comment "" "$RUNNER_TEMP/comment.md" + run: nix run -L .#ci -- post-comment "" "$RUNNER_TEMP/comment.md" diff --git a/.github/workflows/diff-tag.yml b/.github/workflows/diff-tag.yml index 8224366..381814d 100644 --- a/.github/workflows/diff-tag.yml +++ b/.github/workflows/diff-tag.yml @@ -9,17 +9,12 @@ permissions: jobs: dix: - runs-on: ubuntu-latest + runs-on: nix steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - name: diff latest tag against HEAD run: | tag=$(git tag --sort=-creatordate | head -n1) @@ -30,8 +25,8 @@ jobs: git worktree add "$RUNNER_TEMP/old" "$tag" head=$(git rev-parse --short HEAD) { - nix run .#ci -- diff "$RUNNER_TEMP/old" . \ + nix run -L .#ci -- diff "$RUNNER_TEMP/old" . \ "# System closure diff: \`$tag\` -> \`$head\`" - nix run .#ci -- inspect-diff "$RUNNER_TEMP/old" . \ + nix run -L .#ci -- inspect-diff "$RUNNER_TEMP/old" . \ "# Inspection diff: \`$tag\` -> \`$head\`" } | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/flake-check.yml b/.github/workflows/flake-check.yml index 452cfe6..9e6642d 100644 --- a/.github/workflows/flake-check.yml +++ b/.github/workflows/flake-check.yml @@ -14,13 +14,8 @@ concurrency: jobs: flake-check: - runs-on: ubuntu-latest + runs-on: nix steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - run: nix flake check -L diff --git a/.github/workflows/flake-update.yml b/.github/workflows/flake-update.yml index 2e57a9e..27ec96d 100644 --- a/.github/workflows/flake-update.yml +++ b/.github/workflows/flake-update.yml @@ -11,16 +11,11 @@ permissions: jobs: update: - runs-on: ubuntu-latest + runs-on: nix steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - - run: nix flake update + - run: nix flake update -L - name: open/update PR env: @@ -29,5 +24,5 @@ jobs: REPO: ${{ github.repository }} run: | printf 'Automated `nix flake update`.\n' > "$RUNNER_TEMP/body.md" - nix run .#ci -- open-pr \ + nix run -L .#ci -- open-pr \ flake-update "$GITHUB_REF_NAME" "flake.lock: update inputs" "$RUNNER_TEMP/body.md" automated diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 89a6d6f..feabeae 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,27 +15,21 @@ concurrency: jobs: lint: - runs-on: ubuntu-latest + runs-on: nix steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - - name: pick reporter - run: | - if [ "${{ github.event_name }}" = pull_request ]; then - echo "REPORTER=github-pr-review" >> "$GITHUB_ENV" - else - echo "REPORTER=local" >> "$GITHUB_ENV" - fi - - name: run linters - run: nix run .#ci -- lint . "$RUNNER_TEMP/findings" + run: nix run -L .#ci -- lint . "$RUNNER_TEMP/findings" - name: report findings + if: always() env: - REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: nix run .#ci -- lint-report "$REPORTER" "$RUNNER_TEMP/findings" + REVIEWDOG_GITEA_API_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + if [ "${{ github.event_name }}" = pull_request ]; then + nix run -L .#ci -- lint-report gitea-pr-review "$RUNNER_TEMP/findings" + else + nix run -L .#ci -- lint-summary "$RUNNER_TEMP/findings" \ + | tee -a "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/vulnix.yml b/.github/workflows/vulnix.yml index aa7e347..13a5440 100644 --- a/.github/workflows/vulnix.yml +++ b/.github/workflows/vulnix.yml @@ -11,18 +11,13 @@ permissions: jobs: scan: - runs-on: ubuntu-latest + runs-on: nix steps: - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v31 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - name: scan host closures id: scan - run: echo "count=$(nix run .#ci -- vulnix . "$RUNNER_TEMP/issue.md")" >> "$GITHUB_OUTPUT" + run: echo "count=$(nix run -L .#ci -- vulnix . "$RUNNER_TEMP/issue.md")" >> "$GITHUB_OUTPUT" - name: file/update issue if: steps.scan.outputs.count != '0' @@ -30,7 +25,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} API: ${{ github.api_url }} REPO: ${{ github.repository }} - run: nix run .#ci -- upsert-issue vulnix "vulnix: known vulnerabilities in system closure" "$RUNNER_TEMP/issue.md" + run: nix run -L .#ci -- upsert-issue vulnix "vulnix: known vulnerabilities in system closure" "$RUNNER_TEMP/issue.md" - name: close resolved issue if: steps.scan.outputs.count == '0' @@ -38,4 +33,4 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} API: ${{ github.api_url }} REPO: ${{ github.repository }} - run: nix run .#ci -- close-issue vulnix + run: nix run -L .#ci -- close-issue vulnix diff --git a/ci.nix b/ci.nix index 7bc375b..2a6d0d1 100644 --- a/ci.nix +++ b/ci.nix @@ -3,6 +3,7 @@ # runner's own nix, not a pinned one). pkgs.writers.writePython3Bin "ci" { flakeIgnore = [ "E501" ]; + libraries = [ pkgs.python3Packages.subprocess-tee ]; makeWrapperArgs = [ "--prefix" "PATH" diff --git a/ci.py b/ci.py index e4a3d57..b83f958 100644 --- a/ci.py +++ b/ci.py @@ -6,6 +6,7 @@ import json import os import re import subprocess +import subprocess_tee import sys import urllib.request from collections import Counter @@ -17,7 +18,7 @@ def run(*args, **kw): def nix(*args): - return run("nix", "--extra-experimental-features", "nix-command flakes", *args, check=True, capture_output=True).stdout + return subprocess_tee.run(("nix", "--extra-experimental-features", "nix-command flakes", "-L", *args), check=True, text=True).stdout def git(*args, check=True): @@ -362,10 +363,63 @@ def lint_report(args): codes = [reviewdog(f"{args.dir}/{name}", flags) for name, flags in LINTERS - if os.path.getsize(f"{args.dir}/{name}") > 0] + if os.path.isfile(f"{args.dir}/{name}") and os.path.getsize(f"{args.dir}/{name}") > 0] sys.exit(1 if any(codes) else 0) +def has_findings(path): + return os.path.isfile(path) and os.path.getsize(path) > 0 + + +def fmt_rdjsonl(path): + lines = [] + with open(path) as f: + for raw in f: + rec = json.loads(raw) + loc = rec["location"] + r = loc.get("range", {}).get("start") + pos = f"{loc['path']}:{r['line']}:{r['column']}" if r else loc["path"] + lines.append(f"- `{pos}`: {rec['message']}") + return "\n".join(lines) + + +def fmt_errfmt(path): + lines = [] + with open(path) as f: + for raw in f: + parts = raw.strip().split(">", 1) + if len(parts) == 2: + lines.append(f"- `{parts[0]}:{parts[1].split(':', 3)[-1].strip()}`" if ":" in parts[1] else f"- `{raw.strip()}`") + elif raw.strip(): + lines.append(f"- `{raw.strip()}`") + return "\n".join(lines) + + +def lint_summary(args): + sections = [] + d = args.dir + + if has_findings(f"{d}/nixfmt.diff"): + with open(f"{d}/nixfmt.diff") as f: + sections.append(f"## nixfmt\n\n```diff\n{f.read().rstrip()}\n```") + + if has_findings(f"{d}/statix.errfmt"): + sections.append(f"## statix\n\n{fmt_errfmt(f'{d}/statix.errfmt')}") + + if has_findings(f"{d}/deadnix.rdjsonl"): + sections.append(f"## deadnix\n\n{fmt_rdjsonl(f'{d}/deadnix.rdjsonl')}") + + if has_findings(f"{d}/cycles.rdjsonl"): + sections.append(f"## systemd ordering cycles\n\n{fmt_rdjsonl(f'{d}/cycles.rdjsonl')}") + + if sections: + print("\n\n".join(["# Lint findings"] + sections)) + sys.exit(1) + else: + print("No lint findings.") + sys.exit(0) + + def post_comment(args): body = args.marker + "\n" + Path(args.body_file).read_text() pr = os.environ["PR"] @@ -438,6 +492,7 @@ def main(): command("vulnix", vulnix, "ref", "body_out") command("lint", lint, "target", "out") command("lint-report", lint_report, "reporter", "dir") + command("lint-summary", lint_summary, "dir") command("post-comment", post_comment, "marker", "body_file") command("upsert-issue", upsert_issue, "label", "title", "body_file") command("close-issue", close_issue, "label") diff --git a/containers/common/overseer.nix b/containers/common/overseer.nix index 5ace706..426200b 100644 --- a/containers/common/overseer.nix +++ b/containers/common/overseer.nix @@ -381,6 +381,6 @@ let }; }; }; - hostAddr = c.hostAddr; + inherit (c) hostAddr; in c diff --git a/containers/server-n0/git.nix b/containers/server-n0/git.nix index f4510d7..07786df 100644 --- a/containers/server-n0/git.nix +++ b/containers/server-n0/git.nix @@ -8,6 +8,9 @@ let inherit (dsl) mkContainer mkVm; inherit (hostInfo.paths) data secrets; + pubSshPort = 23; + intSshPort = 23 + 8000; + ciRunnerData = "${data}/web/git-ci-runner"; ciRunnerSecrets = "${secrets}/ci-runner"; in @@ -17,8 +20,8 @@ in net.access = [ { to = "@internet"; } ]; net.forward.tcp = [ { - host = 23; - container = 8023; + host = pubSshPort; + container = intSshPort; } ]; svc.autoStart = true; @@ -38,7 +41,8 @@ in ROOT_URL = "https://git.floweytf.com/"; START_SSH_SERVER = true; BUILTIN_SSH_SERVER_USER = "git"; - SSH_PORT = 8023; + SSH_PORT = pubSshPort; + SSH_LISTEN_PORT = intSshPort; }; service.DISABLE_REGISTRATION = true; }; @@ -51,128 +55,180 @@ in net.expose.tcp = [ 3000 ]; }; - ci-runner = mkVm { - name = "ci-runner"; - net.access = [ - { - to = "git"; - tcp = [ 3000 ]; - } - { to = "@internet"; } - ]; - svc.autoStart = true; - resources.cpu = 4; - resources.mem = "8G"; - resources.fs = [ - { - proto = "virtiofs"; - tag = "nix-rw-store"; - source = "${ciRunnerData}/nix-rw-store"; - mountPoint = "/nix/.rw-store"; - } - { - proto = "virtiofs"; - tag = "docker-data"; - source = "${ciRunnerData}/docker"; - mountPoint = "/var/lib/docker"; - } - { - proto = "virtiofs"; - tag = "runner-state"; - source = "${ciRunnerData}/runner-state"; - mountPoint = "/var/lib/gitea-runner"; - } - { - proto = "virtiofs"; - tag = "runner-token"; - source = ciRunnerSecrets; - mountPoint = "/run/ci-runner-secrets"; - readOnly = true; - } - ]; - guestConfig = - { - pkgs, - utils, - lib, - ... - }: - let - runnerInstanceName = "ci-runner"; + ci-runner = + mkVm { + name = "ci-runner"; + includedInTests = false; + net.access = [ + { + to = "git"; + tcp = [ 3000 ]; + } + { to = "@internet"; } + ]; + svc.autoStart = true; + resources.cpu = 4; + resources.mem = "32G"; + resources.fs = [ + { + proto = "virtiofs"; + tag = "nix-rw-store"; + source = "${ciRunnerData}/nix-rw-store"; + mountPoint = "/nix/.rw-store"; + } + { + proto = "virtiofs"; + tag = "nix-db"; + source = "${ciRunnerData}/nix-db"; + mountPoint = "/nix/var/nix/db"; + } + { + proto = "virtiofs"; + tag = "docker-data"; + source = "${ciRunnerData}/docker"; + mountPoint = "/var/lib/docker"; + } + { + proto = "virtiofs"; + tag = "runner-state"; + source = "${ciRunnerData}/runner-state"; + mountPoint = "/var/lib/private/gitea-runner"; + } + { + proto = "virtiofs"; + tag = "runner-token"; + source = ciRunnerSecrets; + mountPoint = "/run/ci-runner-secrets"; + readOnly = true; + } + { + proto = "virtiofs"; + tag = "tmp"; + source = "/tmp"; + mountPoint = "/tmp"; + } + ]; + guestConfig = + { + pkgs, + utils, + lib, + ... + }: + let + runnerInstanceName = "ci-runner"; - ciJobImage = pkgs.dockerTools.buildLayeredImage { - name = "ci-job-image"; - tag = "latest"; - contents = with pkgs; [ + ciJobContents = with pkgs; [ bashInteractive coreutils gitMinimal nixVersions.latest nodejs cacert + (writeTextDir "etc/nix/nix.conf" "experimental-features = nix-command flakes\n") ]; - config = { - Cmd = [ "/bin/sh" ]; - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" - "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" - "NIX_REMOTE=daemon" + + ciJobImage = pkgs.dockerTools.buildLayeredImage { + name = "ci-job-image"; + tag = "latest"; + contents = ciJobContents; + config = { + Cmd = [ "/bin/sh" ]; + Env = [ + "PATH=${lib.makeBinPath ciJobContents}" + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "NIX_REMOTE=daemon" + ]; + }; + }; + in + { + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModprobeConfig = "options kvm_intel nested=1"; + + # volume-mounted into CI containers; must be in VM's store so + # image symlinks (/bin/node etc.) resolve through the mount + system.extraDependencies = ciJobContents; + + microvm.writableStoreOverlay = "/nix/.rw-store"; + + nix.settings = { + experimental-features = [ + "nix-command" + "flakes" ]; + max-jobs = "auto"; + substituters = [ "https://microvm.cachix.org" ]; + trusted-public-keys = [ "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys=" ]; }; - }; - in - { - boot.kernelModules = [ "kvm-intel" ]; - boot.extraModprobeConfig = "options kvm_intel nested=1"; - microvm.writableStoreOverlay = "/nix/.rw-store"; - - nix.settings = { - experimental-features = [ - "nix-command" - "flakes" - ]; - max-jobs = "auto"; - }; - - nix.gc = { - automatic = true; - dates = "daily"; - options = "--delete-older-than 14d"; - }; - # nix.optimise.automatic = true; - doesn't work with microvm.writableStoreOverlay - - virtualisation.docker.enable = true; - - systemd.services.load-ci-job-image = { - description = "Load the CI job Docker image"; - wantedBy = [ "multi-user.target" ]; - after = [ "docker.service" ]; - requires = [ "docker.service" ]; - before = [ "gitea-runner-${utils.escapeSystemdPath runnerInstanceName}.service" ]; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - ExecStart = "${lib.getExe' pkgs.docker "docker"} load -i ${ciJobImage}"; + nix.gc = { + automatic = true; + dates = "daily"; + options = "--delete-older-than 14d"; }; - }; + # nix.optimise.automatic = true; - doesn't work with microvm.writableStoreOverlay - services.gitea-actions-runner.instances.${runnerInstanceName} = { - enable = true; - name = runnerInstanceName; - url = "http://git:3000/"; - tokenFile = "/run/ci-runner-secrets/token"; - labels = [ - "nix:docker://ci-job-image:latest" - ]; - settings.container = { - options = "--device=/dev/kvm --volume=/nix/store:/nix/store:ro --volume=/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket"; - valid_volumes = [ - "/nix/store" - "/nix/var/nix/daemon-socket" + services.openssh = { + enable = true; + settings = { + PermitRootLogin = "yes"; + PermitEmptyPasswords = "yes"; + }; + }; + users.users.root.initialHashedPassword = ""; + services.journald.extraConfig = "ForwardToConsole=yes"; + + virtualisation.containerd.enable = true; + virtualisation.containerd.settings.plugins."io.containerd.snapshotter.v1.overlayfs".mount_options = + [ "userxattr" ]; + + virtualisation.docker.enable = true; + virtualisation.docker.extraOptions = "--containerd /run/containerd/containerd.sock"; + virtualisation.docker.daemon.settings.exec-opts = [ "native.cgroupdriver=cgroupfs" ]; + systemd.services.docker.wants = [ "containerd.service" ]; + systemd.services.docker.after = [ "containerd.service" ]; + + systemd.services.load-ci-job-image = { + description = "Load the CI job Docker image"; + wantedBy = [ "multi-user.target" ]; + after = [ "docker.service" ]; + requires = [ "docker.service" ]; + before = [ "gitea-runner-${utils.escapeSystemdPath runnerInstanceName}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${lib.getExe' pkgs.docker "docker"} load -i ${ciJobImage}"; + }; + }; + + services.gitea-actions-runner.instances.${runnerInstanceName} = { + enable = true; + name = runnerInstanceName; + url = "http://git:3000/"; + tokenFile = "/run/ci-runner-secrets/token"; + labels = [ + "nix:docker://ci-job-image:latest" ]; + settings.container = { + options = "--device=/dev/kvm --volume=/nix/store:/nix/store:ro --volume=/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket"; + valid_volumes = [ + "/nix/store" + "/nix/var/nix/daemon-socket" + ]; + }; }; }; + } + // { + hostConfig = { + systemd.tmpfiles.rules = [ + "d ${ciRunnerData}/nix-rw-store 0755 server server -" + "d ${ciRunnerData}/nix-db 0755 server server -" + "d ${ciRunnerData}/docker 0755 server server -" + "d ${ciRunnerData}/runner-state 0755 server server -" + ]; }; - }; + }; } diff --git a/containers/server-n0/nginx.nix b/containers/server-n0/nginx.nix index 52aa5c8..3c7b71d 100644 --- a/containers/server-n0/nginx.nix +++ b/containers/server-n0/nginx.nix @@ -210,6 +210,6 @@ let }; inherit (setup) containerConfig; }; - hostAddr = c.hostAddr; + inherit (c) hostAddr; in c diff --git a/dsl.nix b/dsl.nix index ab4be0e..1da09a6 100644 --- a/dsl.nix +++ b/dsl.nix @@ -553,6 +553,11 @@ rec { Destination = "0.0.0.0/0"; Gateway = hostAddr; }; + networkConfig.DNS = [ hostAddr ]; + networkConfig.Domains = [ + "~." + "containers" + ]; linkConfig.RequiredForOnline = if allowInternet then "routable" else "degraded"; }; networking.enableIPv6 = false; diff --git a/flake.nix b/flake.nix index 1e458ea..4eff366 100644 --- a/flake.nix +++ b/flake.nix @@ -129,11 +129,10 @@ { nixosConfigurations = lib.mapAttrs (_: host: host.hostCfg) allHosts; - inspect = lib.mapAttrs - (name: host: - pkgs.writeText "inspect-${name}.json" - (builtins.toJSON (introspect.hostReport { inherit host; }))) - allHosts; + inspect = lib.mapAttrs ( + name: host: + pkgs.writeText "inspect-${name}.json" (builtins.toJSON (introspect.hostReport { inherit host; })) + ) allHosts; packages.${system} = customPkgs // { inherit ci; diff --git a/hosts/example.nix b/hosts/example.nix index 194a503..609d641 100644 --- a/hosts/example.nix +++ b/hosts/example.nix @@ -1,7 +1,7 @@ _: { # New host: copy this file to hosts/.nix. Replace fileSystems/boot # below with the real hardware (see hosts/server-n0.nix for a bigger - # example). Add a containers// directory for its containers. + # example). Add a containers// directory for its containers. module = { lib, hostInfo, ... }: diff --git a/tests/server-n0/integration.py b/tests/server-n0/integration.py index 265ccf9..74e003f 100644 --- a/tests/server-n0/integration.py +++ b/tests/server-n0/integration.py @@ -205,12 +205,7 @@ with subtest("(isolation): edges are one-way"): server.fail(ct_connect("git", ADDR["nginx"], 443, timeout=2)) with subtest("(isolation): host access allowlist"): - server.succeed( - net_cmd( - "overseer", - f"{CURL} -sf --max-time 5 http://{HOST_ADDR['overseer']}:{HOST_EXPORTER_PORT}/metrics >&2", - ) - ) + server.succeed(ct_connect("overseer", HOST_ADDR["overseer"], HOST_EXPORTER_PORT)) for port in (HOST_EXPORTER_PORT, HOST_CADVISOR_PORT): server.fail(ct_connect("git", HOST_ADDR["git"], port, timeout=2)) server.fail(ct_connect("git", HOST_ADDR["overseer"], port, timeout=2))