6 Commits
Author SHA1 Message Date
gitea-actions[bot] 9877005d35 flake.lock: update inputs 2026-09-17 04:17:59 +00:00
Floweynt d347a2a139 bug: fix autodeploy
diff-tag / dix (push) Successful in 6m47s
flake-check / flake-check (push) Successful in 4m28s
lint / lint (push) Failing after 1m45s
2026-09-14 06:54:44 +00:00
floweyandFloweynt 678031aa0c feat: automated deployment (#7)
diff-tag / dix (push) Successful in 6m44s
flake-check / flake-check (push) Successful in 4m28s
lint / lint (push) Successful in 1m47s
Co-authored-by: Floweynt <50057682+Floweynt@users.noreply.github.com>
Reviewed-on: #7
2026-09-14 04:40:53 +00:00
autumnandAutumn Liu b083c03d5e bug: fix kvm device not being properly forwarded
diff-tag / dix (push) Successful in 6m7s
flake-check / flake-check (push) Successful in 52s
lint / lint (push) Successful in 1m48s
Co-authored-by: Autumn Liu <al63244@my.utexas.edu>
Reviewed-on: #9
Co-authored-by: autumn <aut.tea7@gmail.com>
2026-09-14 01:47:43 +00:00
Floweynt 060863f72f tests: require devices more strictly
diff-tag / dix (push) Successful in 6m2s
flake-check / flake-check (push) Successful in 4m5s
lint / lint (push) Successful in 1m46s
2026-09-12 12:28:25 -05:00
8190b0080d containerization (#3)
diff-tag / dix (push) Successful in 6m11s
flake-check / flake-check (push) Successful in 51s
lint / lint (push) Successful in 1m47s
Co-authored-by: Floweynt <50057682+Floweynt@users.noreply.github.com>
Co-authored-by: Autumn Liu <al63244@my.utexas.edu>
Reviewed-on: #3
Co-authored-by: autumn <aut.tea7@gmail.com>
2026-09-12 06:40:11 +00:00
12 changed files with 302 additions and 86 deletions
+1 -9
View File
@@ -17,18 +17,10 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: merge base branch
run: |
git config user.email "ci@localhost"
git config user.name "CI"
git fetch origin ${{ github.event.pull_request.base.ref }}
git merge --no-edit origin/${{ github.event.pull_request.base.ref }}
- name: worktree base for diff
run: git worktree add "$RUNNER_TEMP/base" "origin/${{ github.event.pull_request.base.ref }}"
run: git worktree add "$RUNNER_TEMP/base" "${{ github.event.pull_request.base.sha }}"
- name: build closures and render diff
run: |
+5 -3
View File
@@ -5,19 +5,21 @@ on:
- cron: "17 4 * * *"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
update:
runs-on: nix
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- run: nix flake update -L
- name: open/update PR
env:
GH_TOKEN: ${{ secrets.BOT_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
API: ${{ github.api_url }}
REPO: ${{ github.repository }}
run: |
+8 -25
View File
@@ -8,7 +8,6 @@ import re
import subprocess
import subprocess_tee
import sys
import urllib.error
import urllib.request
from collections import Counter
from pathlib import Path
@@ -27,10 +26,6 @@ def git(*args, check=True):
return run("git", *args, check=check, capture_output=True)
class GiteaError(Exception):
pass
def gitea(method, path, body=None):
url = f'{os.environ["API"]}/repos/{os.environ["REPO"]}{path}'
data = json.dumps(body).encode() if body is not None else None
@@ -42,11 +37,7 @@ def gitea(method, path, body=None):
with urllib.request.urlopen(req) as resp:
raw = resp.read()
except urllib.error.HTTPError as e:
try:
detail = json.loads(e.read()).get("message", e.reason)
except (json.JSONDecodeError, UnicodeDecodeError):
detail = e.reason
raise GiteaError(f"{method} {path}: {e.code} {detail}") from None
sys.exit(f"gitea {method} {path}: {e.code} {e.reason}\n{e.read().decode()}")
return json.loads(raw) if raw else None
@@ -475,24 +466,16 @@ def open_pr(args):
host = os.environ["API"].removesuffix("/api/v1")
auth = base64.b64encode(f'token:{os.environ["GH_TOKEN"]}'.encode()).decode()
push = git("-c", f"http.extraHeader=Authorization: Basic {auth}",
"push", "-f", f'{host}/{os.environ["REPO"]}.git', f"HEAD:{args.branch}",
check=False)
if push.returncode != 0:
raise RuntimeError(f"git push failed:\n{push.stderr}")
git("-c", f"http.extraHeader=Authorization: Basic {auth}",
"push", "-f", f'{host}/{os.environ["REPO"]}.git', f"HEAD:{args.branch}")
pulls = gitea("GET", f"/pulls?state=open&base_branch={args.base}")
existing = next((pr for pr in pulls if pr["head"]["ref"] == args.branch), None)
if existing:
print(f"PR #{existing['number']} already open, pushed update")
owner = os.environ["REPO"].split("/")[0]
if gitea("GET", f"/pulls?state=open&head={owner}:{args.branch}"):
return
labels = [label_id(args.label)] if args.label else []
pr = gitea("POST", "/pulls",
{"title": args.title, "head": args.branch, "base": args.base,
"body": Path(args.body_file).read_text(), "labels": labels})
if not pr or "number" not in pr:
raise GiteaError(f"POST /pulls: unexpected response: {json.dumps(pr)}")
print(f"opened PR #{pr['number']}")
gitea("POST", "/pulls",
{"title": args.title, "head": args.branch, "base": args.base,
"body": Path(args.body_file).read_text(), "labels": labels})
def main():
+23
View File
@@ -4,12 +4,27 @@
lib,
hostInfo,
customPkgs,
flakeRev,
...
}:
let
inherit (hostInfo) paths;
in
{
imports = [ ./modules/autodeploy.nix ];
services.server-autodeploy = {
enable = true;
giteaUrl = "https://git.floweytf.com/";
giteaRepo = "server/server-config";
tokenFile = "${hostInfo.paths.secrets}/gitea-autodeploy-token";
allowedSigners = [
"flowey@arch namespaces=\"git\" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPaGpTX3EzfezVc2nl3MgsKVamYYP2h2VJaRny8pZkvi"
];
stateDir = hostInfo.paths.config;
repoOwner = "server";
};
assertions =
let
isMounted =
@@ -47,6 +62,14 @@ in
time.timeZone = "UTC";
system.stateVersion = "26.05";
system.nixos.label =
let
labelFile = ./.nixos-label;
in
if builtins.pathExists labelFile then
lib.removeSuffix "\n" (builtins.readFile labelFile)
else
flakeRev;
nix = {
settings = {
+2
View File
@@ -146,6 +146,8 @@ let
in
mkContainer {
inherit name;
includedInTests = false; # TODO: fixme, clearly mail is flakey
net.access = [ { to = "@internet"; } ];
svc.autoStart = true;
net.forward.tcp = [
+11 -25
View File
@@ -75,15 +75,6 @@ let
packages = with pkgs; [ tmux ] ++ packages;
miscProperties = self._sshpiperRegister routingInfo;
};
kvmTrait = {
containerConfig =
{ pkgs, ... }:
{
users.users.runner.extraGroups = [ "kvm" ];
environment.systemPackages = [ pkgs.qemu_kvm ];
};
};
in
{
ssh-admin = mkSsh {
@@ -126,32 +117,27 @@ in
ssh-generic = mkSsh {
name = "ssh-generic";
net.access = [ { to = "@internet"; } ];
# /dev/kvm keeps its host ownership, so the kvm gid must not be shifted.
isolation.privateUsers = "identity";
resources = {
fs."/home/runner/data" = {
hostPath = "${data}/ssh/generic";
isReadOnly = false;
};
devices = [ "/dev/kvm" ];
mem = "16G";
cpu = 65;
};
traits = [ kvmTrait ];
isolation.allowedDevices = [
packages = with pkgs; [ qemu_kvm ];
traits = [
{
node = "/dev/kvm";
modifier = "rw";
containerConfig =
{ config, ... }:
{
users.groups.kvm.gid = config.ids.gids.kvm;
users.users.runner.extraGroups = [ "kvm" ];
};
}
{
node = "/dev/vhost-net";
modifier = "rw";
}
{
node = "/dev/net/tun";
modifier = "rwm";
}
];
isolation.additionalCapabilities = [
"CAP_NET_ADMIN"
"CAP_MKNOD"
];
routingInfo = [
{
+23 -21
View File
@@ -221,6 +221,17 @@ rec {
`resources.fs`
: Bind mounts, keyed by mount point.
`resources.devices`
: Host device nodes to expose, as paths. Bound into the container at the
same path and added to the unit's `DeviceAllow=` - the cgroup
permission alone doesn't get you a working node (nothing can `mknod` a
device inside a user namespace), and the bind alone leaves the cgroup
denying `open()`, so both are needed. The bind is unconditional, so
the device must actually exist on whatever host runs the container.
Nodes keep their host ownership, so a consumer inside needs
`isolation.privateUsers = "identity"` for the owning gid to mean
anything.
`isolation.nixDaemon`
: Whether the Nix daemon and CLI are available inside the container.
@@ -228,17 +239,6 @@ rec {
: systemd-nspawn user namespacing; see `--private-users=` in
systemd-nspawn(1). Bind mounts always go through idmapped `--bind`.
`isolation.allowedDevices`
: List of `{ node; modifier; }` host devices to grant the container's
cgroup access to; see `containers.<name>.allowedDevices` in the NixOS
manual. nspawn bind-mounts a handful of devices (`/dev/kvm`,
`/dev/net/tun`, ...) into the container itself once the cgroup allows it -
this doesn't need a matching `resources.fs` bind mount.
`isolation.additionalCapabilities`
: Extra Linux capabilities granted to the container; see
`containers.<name>.additionalCapabilities` in the NixOS manual.
`miscProperties`
: Arbitrary data attached to the result. Used for cross-container discovery
via `collectContainers`.
@@ -281,16 +281,15 @@ rec {
cpuQuota = "${toString (cpu * 100)}%";
memoryMax = resources.mem or "512M";
bindMounts = resources.fs or { };
devices = resources.devices or [ ];
allowsNixDaemon = isolation.nixDaemon or false;
privateUsers = isolation.privateUsers or "pick";
allowedDevices = isolation.allowedDevices or [ ];
additionalCapabilities = isolation.additionalCapabilities or [ ];
allowInternet = hasInternetAccess access;
allBindMounts = builtins.foldl' (acc: t: acc // (t.resources.fs or { })) { } traits // bindMounts;
userModules = map mkUser (lib.concatMap (t: t.users or [ ]) traits ++ users);
allPorts = ports ++ map (p: (normalizePort p).container) hostPorts;
useIdmap = privateUsers != "no";
useIdmap = privateUsers == "pick" || (builtins.isInt privateUsers && privateUsers > 0);
bindMountExtraFlags = lib.mapAttrsToList (
mountPoint: bm:
(if bm.isReadOnly then "--bind-ro=" else "--bind=")
@@ -298,6 +297,10 @@ rec {
+ lib.optionalString useIdmap ":idmap"
) allBindMounts;
bindMountHostPaths = map (bm: bm.hostPath) (lib.attrValues allBindMounts);
# DeviceAllow= alone only grants cgroup permission - nspawn can't
# mknod a device node inside a user namespace, so the node still
# needs an explicit bind from the host.
deviceFlags = map (d: "--bind=${d}") devices;
inherit (deriveIdentity ifPrefix.nspawn name)
iface
mac
@@ -376,19 +379,15 @@ rec {
hostAddr
bindMountHostPaths
includedInTests
devices
;
bindMounts = allBindMounts;
kind = "nspawn";
miscProperties = flattenMiscProp self miscProperties;
nixosContainer = {
inherit
autoStart
privateUsers
allowedDevices
additionalCapabilities
;
inherit autoStart privateUsers;
bindMounts = { };
extraFlags = bindMountExtraFlags;
extraFlags = bindMountExtraFlags ++ deviceFlags;
privateNetwork = true;
extraVeths = linkVeths;
config =
@@ -521,6 +520,7 @@ rec {
;
inherit needsGateway;
bindMountHostPaths = map (s: s.source) shares;
devices = [ ];
kind = "vm";
miscProperties = flattenMiscProp self miscProperties;
microvmConfig = {
@@ -957,6 +957,7 @@ rec {
startLimitBurst = 5;
startLimitIntervalSec = 120;
unitConfig.RequiresMountsFor = c.bindMountHostPaths;
serviceConfig.DeviceAllow = map (d: "${d} rw") c.devices;
};
# the host end of a link, for both kinds. nixos-containers only configures
@@ -1316,6 +1317,7 @@ rec {
HOST_ADDR = ${pyDict pyStr (lib.mapAttrs (_: c: c.hostAddr) autoStartTestContainers)}
IFACE = ${pyDict pyStr (lib.mapAttrs (_: c: c.iface) autoStartTestContainers)}
KIND = ${pyDict pyStr (lib.mapAttrs (_: c: c.kind) autoStartTestContainers)}
DEVICES = ${pyDict pyList (lib.mapAttrs (_: c: c.devices) autoStartTestContainers)}
# only nspawn guests can be entered with nsenter; a vm has no host-side
# netns, so every probe that runs *inside* a guest is limited to these
enterable = [c for c in included if KIND[c] == "nspawn"]
+7 -1
View File
@@ -11,6 +11,7 @@
outputs =
{
self,
nixpkgs,
mtce,
microvm,
@@ -21,6 +22,8 @@
inherit (nixpkgs) lib;
pkgs = import nixpkgs { inherit system; };
flakeRev = self.shortRev or self.dirtyShortRev or "unknown";
customPkgs = {
hopper-rs = pkgs.callPackage ./packages/hopper-rs.nix { };
sshpiper = pkgs.callPackage ./packages/sshpiper.nix { };
@@ -29,6 +32,7 @@
server-rebuild = pkgs.callPackage ./packages/server-rebuild.nix { };
server-status = pkgs.callPackage ./packages/server-status.nix { };
server-kexec = pkgs.callPackage ./packages/server-kexec.nix { };
server-autodeploy = pkgs.callPackage ./packages/server-autodeploy.nix { };
};
ci = import ./ci.nix { inherit lib pkgs; };
@@ -46,12 +50,13 @@
server = "/persist/@containers";
data = "/persist/@containers/data";
secrets = "/persist/@secrets/";
config = "/persist/@config/";
};
};
specialArgsFor = hostName: {
hostInfo = hostInfoFor hostName;
inherit inputs customPkgs;
inherit inputs customPkgs flakeRev;
};
hostDefFor =
@@ -147,6 +152,7 @@
server-rebuild
server-status
server-kexec
server-autodeploy
;
};
};
+96
View File
@@ -0,0 +1,96 @@
{
config,
lib,
pkgs,
customPkgs,
...
}:
let
cfg = config.services.server-autodeploy;
in
{
options.services.server-autodeploy = {
enable = lib.mkEnableOption "automatic deployment from signed git tags";
interval = lib.mkOption {
type = lib.types.str;
default = "15min";
};
branch = lib.mkOption {
type = lib.types.str;
default = "master";
};
giteaUrl = lib.mkOption {
type = lib.types.str;
};
giteaRepo = lib.mkOption {
type = lib.types.str;
};
tokenFile = lib.mkOption {
type = lib.types.str;
};
allowedSigners = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
};
repoOwner = lib.mkOption {
type = lib.types.str;
description = "User that owns the configuration checkout";
};
stateDir = lib.mkOption {
type = lib.types.str;
};
hostname = lib.mkOption {
type = lib.types.str;
default = config.networking.hostName;
};
};
config = lib.mkIf cfg.enable {
systemd.services.server-autodeploy = {
description = "Deploy latest signed tag";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
environment = {
GITEA_URL = cfg.giteaUrl;
GITEA_REPO = cfg.giteaRepo;
TOKEN_FILE = cfg.tokenFile;
ALLOWED_SIGNERS = builtins.toFile "signers" (lib.join "\n" cfg.allowedSigners);
DEPLOY_BRANCH = cfg.branch;
STATE_DIR = cfg.stateDir;
REPO_DIR = "/persist/@configuration";
REPO_OWNER = cfg.repoOwner;
HOSTNAME = cfg.hostname;
};
serviceConfig = {
Type = "oneshot";
ExecStart = lib.getExe customPkgs.server-autodeploy;
TimeoutStartSec = "30min";
};
path = [
config.nix.package
pkgs.nixos-rebuild
];
};
systemd.timers.server-autodeploy = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "2min";
OnUnitActiveSec = cfg.interval;
RandomizedDelaySec = "2min";
};
};
};
}
+18
View File
@@ -0,0 +1,18 @@
{
writeShellApplication,
git,
curl,
jq,
util-linux,
openssh,
}:
import ./mk-script.nix { inherit writeShellApplication; } {
name = "server-autodeploy";
runtimeInputs = [
git
curl
jq
util-linux
openssh
];
}
+60
View File
@@ -0,0 +1,60 @@
REPO_DIR="${REPO_DIR:-/persist/@configuration}"
STATE_DIR="${STATE_DIR:-/persist/@autodeploy}"
BRANCH="origin/${DEPLOY_BRANCH:-master}"
mkdir -p "$STATE_DIR"
exec 9>"$STATE_DIR/lock"
flock -n 9 || { echo "another deploy running"; exit 0; }
runas() { runuser -u "$REPO_OWNER" -- "$@"; }
report_status() {
echo "updating status to: $1 - $2"
[[ -f "$TOKEN_FILE" ]] || return 0
curl -sf -X POST \
-H "Authorization: token $(<"$TOKEN_FILE")" \
-H "Content-Type: application/json" \
-d "$(jq -n --arg s "$1" --arg d "$2" --arg c "deploy/$HOSTNAME" \
'{state:$s,description:$d,context:$c}')" \
"$GITEA_URL/api/v1/repos/$GITEA_REPO/statuses/$3" >/dev/null
}
cd "$REPO_DIR"
export GIT_CONFIG_COUNT=2
export GIT_CONFIG_KEY_0=gpg.format
export GIT_CONFIG_VALUE_0=ssh
export GIT_CONFIG_KEY_1=gpg.ssh.allowedSignersFile
export GIT_CONFIG_VALUE_1="$ALLOWED_SIGNERS"
runas git fetch origin --tags --prune-tags
latest=$(runas git describe --abbrev=0 "$BRANCH" 2>/dev/null) || {
echo "no tags on $BRANCH"
exit 0
}
current=$(cat "$STATE_DIR/tag" 2>/dev/null) || true
[[ "$latest" != "$current" ]] || { echo "already at $latest"; exit 0; }
echo "new tag: $latest (current: ${current:-none})"
runas git tag -v "$latest" || {
sha=$(runas git rev-list -1 "$latest")
report_status "failure" "tag signature verification failed" "$sha"
exit 1
}
sha=$(runas git rev-list -1 "$latest")
report_status "pending" "deploying $latest" "$sha"
flake_url="git+${GITEA_URL}/${GITEA_REPO}?ref=refs/tags/${latest}&rev=${sha}#$HOSTNAME"
if nixos-rebuild switch --flake "$flake_url"; then
echo "$latest" > "$STATE_DIR/tag"
report_status "success" "deployed $latest" "$sha"
echo "deployed $latest"
else
report_status "failure" "nixos-rebuild failed" "$sha"
exit 1
fi
+48 -2
View File
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
HOST_ADDR: dict[str, str] = {}
IFACE: dict[str, str] = {}
KIND: dict[str, str] = {}
DEVICES: dict[str, list[str]] = {}
EDGES: list[tuple[str, str, list[int], list[int]]] = []
SSH_KEY: str = ""
SSH_OPTS: str = ""
@@ -55,6 +56,11 @@ def container_pid(name):
return server.succeed(f"machinectl show {name} -p Leader --value").strip()
def ct_cmd(name, cmd):
pid = container_pid(name)
return f"{NSENTER} --mount=/proc/{pid}/ns/mnt --root=/proc/{pid}/root -- {SH} -c '{cmd}'"
def net_cmd(name, cmd):
pid = container_pid(name)
return f"{NSENTER} --net=/proc/{pid}/ns/net -- {cmd}"
@@ -147,6 +153,8 @@ with subtest("(functional): web services respond"):
for name, port in [("devenv", 8080), ("maven", 8080), ("git", 3000)]:
server.succeed(net_cmd(name, f"{CURL} -sf --max-time 5 -o /dev/null http://{ADDR[name]}:{port}/ >&2"))
# TODO: re-enable mail
"""
with subtest("(functional): mail accepts connections"):
smtp_probe = net_cmd("mail", f"{SH} -c 'exec 3<>/dev/tcp/{ADDR['mail']}/25 && head -1 <&3'")
server.wait_until_succeeds(f"{smtp_probe} | grep -q '^220'", timeout=60)
@@ -161,6 +169,7 @@ with subtest("(functional): mail accepts connections"):
return greeting.strip().startswith("* OK")
retry(imap_ready, timeout_seconds=60)
"""
with subtest("(functional): nginx proxies to backend containers"):
server.succeed(
@@ -191,7 +200,7 @@ with subtest("(isolation): undeclared container edges are refused"):
("devenv", "git", 3000),
("git", "devenv", 8080),
("maven", "git", 3000),
("nginx", "mail", 25),
# ("nginx", "mail", 25),
("ssh-botm", "ssh-admin", 22),
("overseer", "git", 3000),
]:
@@ -244,7 +253,7 @@ with subtest("(isolation): renaming the interface changes nothing"):
rename(iface, "evil0")
try:
server.succeed(ct_connect("nginx", ADDR["git"], 3000))
server.fail(ct_connect("nginx", ADDR["mail"], 25, timeout=2))
# server.fail(ct_connect("nginx", ADDR["mail"], 25, timeout=2))
finally:
rename("evil0", iface)
@@ -323,3 +332,40 @@ with subtest("(functional): published ports follow the container's lifetime"):
server.succeed("systemctl start container@nginx")
server.wait_for_unit("container@nginx.service")
internet.wait_until_succeeds(ext_connect(80), timeout=30)
with subtest("(functional): declared devices are usable in-container"):
for name, devices in DEVICES.items():
for dev in devices:
# opened, not stat'd: without DeviceAllow= the node is still visible
server.succeed(ct_cmd(name, f"exec 3<> {dev}"))
with subtest("(functional): /dev/kvm actually initializes, not just opens"):
# a wrong-gid or unmapped node still opens but fails the ioctls qemu
# needs, so boot with no disk (never completes) and kill it - the
# failure mode this catches is qemu refusing to start at all.
for name, devices in DEVICES.items():
if "/dev/kvm" not in devices:
continue
out = server.succeed(
ct_cmd(
name,
f"{TIMEOUT} 3 qemu-system-x86_64 -enable-kvm -m 256 -nographic -no-reboot "
">/tmp/qemu.out 2>&1; cat /tmp/qemu.out",
)
)
assert "Permission denied" not in out, f"{name}: qemu couldn't use /dev/kvm:\n{out}"
assert "Could not access KVM" not in out, f"{name}: qemu couldn't use /dev/kvm:\n{out}"
with subtest("(isolation): devices reach only the container that declared them"):
for name in enterable:
for dev in {d for ds in DEVICES.values() for d in ds} - set(DEVICES.get(name, [])):
server.fail(ct_cmd(name, f"test -e {dev}"))
with subtest("(isolation): identity uid mapping keeps the user namespace"):
for name in (n for n, ds in DEVICES.items() if ds):
host_ns = server.succeed("readlink /proc/1/ns/user").strip()
ct_ns = server.succeed(f"readlink /proc/{container_pid(name)}/ns/user").strip()
assert host_ns != ct_ns, f"{name} shares the host user namespace"