server-config/containers/common/overseer.nix

{
  name,
  hostInfo,
  dsl,
  pkgs,
  lib,
  containers,
  ...
}:
let
  inherit (dsl) mkContainer;
  inherit (hostInfo.paths) data;

  nodePort = 9100;
  cadvPort = 9200;

  containerNetworkRulesFile =
    let
      mkRules =
        metric: record:
        lib.mapAttrsToList (cname: c: {
          inherit record;
          expr = "rate(${metric}{device=\"${c.iface}\"}[5m])";
          labels = {
            container = cname;
          };
        }) containers;
    in
    pkgs.writeText "container-network.yml" (
      builtins.toJSON {
        groups = [
          {
            name = "container-network";
            rules =
              mkRules "node_network_receive_bytes_total" "container_network_receive_rate"
              ++ mkRules "node_network_transmit_bytes_total" "container_network_transmit_rate";
          }
        ];
      }
    );

  containerDefinedRulesFile = pkgs.writeText "container-defined.yml" (
    builtins.toJSON {
      groups = [
        {
          name = "container-defined";
          rules = lib.mapAttrsToList (cname: _: {
            record = "container_defined";
            expr = "1";
            labels = {
              container = cname;
            };
          }) containers;
        }
      ];
    }
  );

  alloyConfig = pkgs.writeText "config.alloy" ''
    loki.source.journal "journal" {
      max_age    = "12h"
      path       = "/var/log/journal"
      labels     = { job = "systemd-journal" }
      relabel_rules = loki.relabel.journal.rules
      forward_to    = [loki.write.local.receiver]
    }

    loki.relabel "journal" {
      forward_to = []

      rule {
        source_labels = ["__journal__systemd_unit"]
        target_label  = "unit"
      }
      rule {
        source_labels = ["__journal__hostname"]
        target_label  = "host"
      }
      rule {
        source_labels = ["__journal_syslog_identifier"]
        target_label  = "syslog_id"
      }
      rule {
        source_labels = ["__journal_priority_keyword"]
        target_label  = "level"
      }
      rule {
        source_labels = ["__journal__systemd_machine_name"]
        target_label  = "container"
      }
    }

    loki.write "local" {
      endpoint {
        url = "http://127.0.0.1:3100/loki/api/v1/push"
      }
    }
  '';

  dashboards = lib.filterAttrs (name: _: lib.hasSuffix "-dashboard" name) (
    dsl.listMap ./overseer (_: path: import path { inherit pkgs lib; })
  );

  dashboardDir = pkgs.linkFarm "grafana-dashboards" (
    lib.mapAttrsToList (name: dashboard: {
      name = "${lib.removeSuffix "-dashboard" name}.json";
      path = dashboard;
    }) dashboards
  );

  landingRoot = pkgs.writeTextDir "index.html" ''
    <!DOCTYPE html>
    <html lang="en">
    <head>
      <meta charset="utf-8">
      <title>Overseer</title>
      <style>
        body { font-family: sans-serif; max-width: 480px; margin: 4rem auto; }
        ul   { padding: 0; list-style: none; }
        li   { margin: .5rem 0; }
      </style>
    </head>
    <body>
      <h1>Overseer</h1>
      <ul>
        <li><a data-host="stats.localhost">stats.localhost</a> — Grafana</li>
        <li><a data-host="prometheus.localhost">prometheus.localhost</a> — Prometheus</li>
      </ul>
      <script>
        const p = location.port && location.port !== "80" ? ":" + location.port : "";
        document.querySelectorAll("a[data-host]").forEach(a => {
          const h = a.dataset.host;
          a.href = "http://" + h + p;
          a.textContent = h + p;
        });
      </script>
    </body>
    </html>
  '';
in
let
  c =
    mkContainer {
      inherit name;
      net.access = [
        {
          to = "@host";
          tcp = [
            nodePort
            cadvPort
          ];
        }
      ];
      svc.autoStart = true;
      resources.fs = {
        "/var/lib/loki" = {
          hostPath = "${data}/overseer/loki/data";
          isReadOnly = false;
        };
        "/var/lib/prometheus2" = {
          hostPath = "${data}/overseer/prometheus/data";
          isReadOnly = false;
        };
        "/var/lib/grafana" = {
          hostPath = "${data}/overseer/grafana";
          isReadOnly = false;
        };
        "/var/log/journal" = {
          hostPath = "/var/log/journal";
          isReadOnly = true;
        };
      };
      containerConfig = _: {
        services.loki = {
          enable = true;
          configuration = {
            auth_enabled = false;
            server = {
              http_listen_port = 3100;
              grpc_listen_port = 9096;
            };
            common = {
              instance_addr = "127.0.0.1";
              path_prefix = "/var/lib/loki";
              storage.filesystem = {
                chunks_directory = "/var/lib/loki/chunks";
                rules_directory = "/var/lib/loki/rules";
              };
              replication_factor = 1;
              ring.kvstore.store = "inmemory";
            };
            schema_config.configs = [
              {
                from = "2020-10-24";
                store = "tsdb";
                object_store = "filesystem";
                schema = "v13";
                index = {
                  prefix = "index_";
                  period = "24h";
                };
              }
            ];
          };
        };

        services.alloy = {
          enable = true;
          configPath = alloyConfig;
        };

        services.prometheus = {
          enable = true;
          port = 9090;
          globalConfig.scrape_interval = "15s";
          scrapeConfigs = [
            {
              job_name = "node";
              static_configs = [ { targets = [ "${hostAddr}:${toString nodePort}" ]; } ];
            }
            {
              job_name = "cadvisor";
              static_configs = [ { targets = [ "${hostAddr}:${toString cadvPort}" ]; } ];
            }
            {
              job_name = "prometheus";
              static_configs = [ { targets = [ "localhost:9090" ]; } ];
            }
          ];
          ruleFiles = [
            containerNetworkRulesFile
            containerDefinedRulesFile
            (dsl.mkContainerLimitsRuleFile pkgs containers)
          ];
          rules = [
            ''
              groups:
                - name: containers
                  rules:
                    - alert: ContainerFailed
                      expr: node_systemd_unit_state{name=~"container@.*",state="failed"} == 1
                      for: 1m
                      labels:
                        severity: critical
                      annotations:
                        summary: "{{ $labels.name }} entered failed state"
                    - alert: ContainerOOMKill
                      expr: increase(container_oom_events_total{id=~"/machine.slice/container@.+\\.service.*"}[5m]) > 0
                      labels:
                        severity: warning
                      annotations:
                        summary: "OOM kill in {{ $labels.id }}"
            ''
          ];
        };

        services.grafana = {
          enable = true;
          dataDir = "/var/lib/grafana";
          settings = {
            server.http_port = 3000;
            server.http_addr = "127.0.0.1";
            # WORKAROUND: Grafana 10+ requires an explicit secret_key; omitting it
            # causes auth session tokens to be regenerated on every restart
            security.secret_key = "SW2YcwTIb9zpOOhoPsMm";
            "auth".disable_login_form = true;
            "auth.anonymous" = {
              enabled = true;
              org_role = "Admin";
            };
          };
          provision = {
            enable = true;
            datasources.settings = {
              apiVersion = 1;
              datasources = [
                {
                  name = "Prometheus";
                  type = "prometheus";
                  uid = "prometheus";
                  access = "proxy";
                  url = "http://localhost:9090";
                  isDefault = true;
                  jsonData.timeInterval = "15s";
                }
                {
                  name = "Loki";
                  type = "loki";
                  uid = "loki";
                  access = "proxy";
                  url = "http://127.0.0.1:3100";
                }
              ];
            };
            dashboards.settings = {
              apiVersion = 1;
              providers = [
                {
                  name = "default";
                  type = "file";
                  disableDeletion = true;
                  options.path = toString dashboardDir;
                }
              ];
            };
          };
        };

        services.nginx = {
          enable = true;
          virtualHosts."_" = {
            listen = [
              {
                addr = "0.0.0.0";
                port = 80;
                ssl = false;
                extraParameters = [ "default_server" ];
              }
            ];
            root = "${landingRoot}";
          };
          virtualHosts."stats.localhost" = {
            listen = [
              {
                addr = "0.0.0.0";
                port = 80;
                ssl = false;
              }
            ];
            locations."/" = {
              proxyPass = "http://127.0.0.1:3000";
              proxyWebsockets = true;
              extraConfig = "proxy_set_header Host $host;";
            };
          };
          virtualHosts."prometheus.localhost" = {
            listen = [
              {
                addr = "0.0.0.0";
                port = 80;
                ssl = false;
              }
            ];
            locations."/" = {
              proxyPass = "http://127.0.0.1:9090";
            };
          };
        };

        systemd.services = lib.genAttrs [ "loki" "alloy" "prometheus" "grafana" "nginx" ] (_: {
          serviceConfig.FailureAction = "exit-force";
        });
      };
      net.expose.tcp = [ 80 ];
      resources.mem = "2G";
    }
    // {
      hostConfig = {
        services.prometheus.exporters.node = {
          enable = true;
          listenAddress = "0.0.0.0";
          port = nodePort;
          enabledCollectors = [
            "systemd"
            "processes"
          ];
          disabledCollectors = [
            "netstat"
            "softnet"
          ];
          # radeon sometimes breaks on kexec
          extraFlags = [
            "--collector.systemd.enable-task-metrics"
            "--collector.hwmon.chip-exclude=radeon"
          ];
        };
        services.cadvisor = {
          enable = true;
          listenAddress = "0.0.0.0";
          port = cadvPort;
        };
      };
    };
  inherit (c) hostAddr;
in
c