{ name, hostInfo, dsl, ... }: let inherit (dsl) mkContainer mkVm; inherit (hostInfo.paths) data secrets; pubSshPort = 23; intSshPort = 23 + 8000; ciRunnerData = "${data}/web/git-ci-runner"; ciRunnerSecrets = "${secrets}/ci-runner"; in { ${name} = mkContainer { inherit name; net.access = [ { to = "@internet"; } ]; net.forward.tcp = [ { host = pubSshPort; container = intSshPort; } ]; svc.autoStart = true; resources.fs."/var/lib/gitea" = { hostPath = "${data}/web/git"; isReadOnly = false; }; containerConfig = _: { services.gitea = { enable = true; stateDir = "/var/lib/gitea"; settings = { server = { HTTP_ADDR = "0.0.0.0"; HTTP_PORT = 3000; DOMAIN = "git.floweytf.com"; ROOT_URL = "https://git.floweytf.com/"; START_SSH_SERVER = true; BUILTIN_SSH_SERVER_USER = "git"; SSH_PORT = pubSshPort; SSH_LISTEN_PORT = intSshPort; }; service.DISABLE_REGISTRATION = true; }; }; users.users.gitea.uid = 1000; users.groups.gitea.gid = 1000; time.timeZone = "UTC"; systemd.services.gitea.serviceConfig.FailureAction = "exit-force"; }; net.expose.tcp = [ 3000 ]; miscProperties.nginx = { domain = "git.floweytf.com"; port = 3000; }; }; ci-runner = mkVm { name = "ci-runner"; includedInTests = false; net.access = [ { to = "git"; tcp = [ 3000 ]; } { to = "@internet"; } ]; svc.autoStart = true; resources.cpu = 4; resources.mem = "32G"; resources.fs = [ { proto = "virtiofs"; tag = "nix-rw-store"; source = "${ciRunnerData}/nix-rw-store"; mountPoint = "/nix/.rw-store"; } { proto = "virtiofs"; tag = "nix-db"; source = "${ciRunnerData}/nix-db"; mountPoint = "/nix/var/nix/db"; } { proto = "virtiofs"; tag = "docker-data"; source = "${ciRunnerData}/docker"; mountPoint = "/var/lib/docker"; } { proto = "virtiofs"; tag = "runner-state"; source = "${ciRunnerData}/runner-state"; mountPoint = "/var/lib/private/gitea-runner"; } { proto = "virtiofs"; tag = "runner-token"; source = ciRunnerSecrets; mountPoint = "/run/ci-runner-secrets"; readOnly = true; } { proto = "virtiofs"; tag = "tmp"; source = "/tmp"; mountPoint = "/tmp"; } ]; guestConfig = { pkgs, utils, lib, ... }: let runnerInstanceName = "ci-runner"; ciJobContents = with pkgs; [ bashInteractive coreutils gitMinimal nixVersions.latest nodejs cacert (writeTextDir "etc/nix/nix.conf" "experimental-features = nix-command flakes\n") ]; ciJobImage = pkgs.dockerTools.buildLayeredImage { name = "ci-job-image"; tag = "latest"; contents = ciJobContents; config = { Cmd = [ "/bin/sh" ]; Env = [ "PATH=${lib.makeBinPath ciJobContents}" "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" "NIX_SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" "NIX_REMOTE=daemon" ]; }; }; in { boot.kernelModules = [ "kvm-intel" ]; boot.extraModprobeConfig = "options kvm_intel nested=1"; # volume-mounted into CI containers; must be in VM's store so # image symlinks (/bin/node etc.) resolve through the mount system.extraDependencies = ciJobContents; microvm.writableStoreOverlay = "/nix/.rw-store"; nix.settings = { experimental-features = [ "nix-command" "flakes" ]; max-jobs = "auto"; substituters = [ "https://microvm.cachix.org" ]; trusted-public-keys = [ "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys=" ]; }; nix.settings = { min-free = 5 * 1024 * 1024 * 1024; max-free = 10 * 1024 * 1024 * 1024; }; # nix.optimise.automatic = true; - doesn't work with microvm.writableStoreOverlay services.openssh = { enable = true; settings = { PermitRootLogin = "yes"; PermitEmptyPasswords = "yes"; }; }; security.pam.services.sshd.allowNullPassword = true; users.users.root.initialHashedPassword = ""; services.journald.settings.Journal.ForwardToConsole = "yes"; virtualisation.containerd.enable = true; virtualisation.containerd.settings.plugins."io.containerd.snapshotter.v1.overlayfs".mount_options = [ "userxattr" ]; virtualisation.docker.enable = true; virtualisation.docker.extraOptions = "--containerd /run/containerd/containerd.sock"; virtualisation.docker.daemon.settings.exec-opts = [ "native.cgroupdriver=cgroupfs" ]; systemd.services.docker.wants = [ "containerd.service" ]; systemd.services.docker.after = [ "containerd.service" ]; systemd.services.load-ci-job-image = { description = "Load the CI job Docker image"; wantedBy = [ "multi-user.target" ]; after = [ "docker.service" ]; requires = [ "docker.service" ]; before = [ "gitea-runner-${utils.escapeSystemdPath runnerInstanceName}.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = "${lib.getExe' pkgs.docker "docker"} load -i ${ciJobImage}"; }; }; services.gitea-actions-runner.instances.${runnerInstanceName} = { enable = true; name = runnerInstanceName; url = "http://git:3000/"; tokenFile = "/run/ci-runner-secrets/token"; labels = [ "nix:docker://ci-job-image:latest" ]; settings.container = { options = "--device=/dev/kvm --volume=/nix/store:/nix/store:ro --volume=/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket"; valid_volumes = [ "/nix/store" "/nix/var/nix/daemon-socket" ]; }; }; }; } // { hostConfig = { systemd.tmpfiles.rules = [ "d ${ciRunnerData}/nix-rw-store 0755 server server -" "d ${ciRunnerData}/nix-db 0755 server server -" "d ${ciRunnerData}/docker 0755 server server -" "d ${ciRunnerData}/runner-state 0755 server server -" ]; }; }; }