{ name, hostInfo, dsl, pkgs, lib, containers, ... }: let inherit (dsl) mkContainer; inherit (hostInfo.paths) data; nodePort = 9100; cadvPort = 9200; containerNetworkRulesFile = let mkRules = metric: record: lib.mapAttrsToList (cname: c: { inherit record; expr = "rate(${metric}{device=\"${c.iface}\"}[5m])"; labels = { container = cname; }; }) containers; in pkgs.writeText "container-network.yml" ( builtins.toJSON { groups = [ { name = "container-network"; rules = mkRules "node_network_receive_bytes_total" "container_network_receive_rate" ++ mkRules "node_network_transmit_bytes_total" "container_network_transmit_rate"; } ]; } ); containerDefinedRulesFile = pkgs.writeText "container-defined.yml" ( builtins.toJSON { groups = [ { name = "container-defined"; rules = lib.mapAttrsToList (cname: _: { record = "container_defined"; expr = "1"; labels = { container = cname; }; }) containers; } ]; } ); alloyConfig = pkgs.writeText "config.alloy" '' loki.source.journal "journal" { max_age = "12h" path = "/var/log/journal" labels = { job = "systemd-journal" } relabel_rules = loki.relabel.journal.rules forward_to = [loki.write.local.receiver] } loki.relabel "journal" { forward_to = [] rule { source_labels = ["__journal__systemd_unit"] target_label = "unit" } rule { source_labels = ["__journal__hostname"] target_label = "host" } rule { source_labels = ["__journal_syslog_identifier"] target_label = "syslog_id" } rule { source_labels = ["__journal_priority_keyword"] target_label = "level" } rule { source_labels = ["__journal__systemd_machine_name"] target_label = "container" } } loki.write "local" { endpoint { url = "http://127.0.0.1:3100/loki/api/v1/push" } } ''; dashboards = lib.filterAttrs (name: _: lib.hasSuffix "-dashboard" name) ( dsl.listMap ./overseer (_: path: import path { inherit pkgs lib; }) ); dashboardDir = pkgs.linkFarm "grafana-dashboards" ( lib.mapAttrsToList (name: dashboard: { name = "${lib.removeSuffix "-dashboard" name}.json"; path = dashboard; }) dashboards ); landingRoot = pkgs.writeTextDir "index.html" '' Overseer

Overseer

''; in let c = mkContainer { inherit name; net.access = [ { to = "@host"; tcp = [ nodePort cadvPort ]; } ]; svc.autoStart = true; resources.fs = { "/var/lib/loki" = { hostPath = "${data}/overseer/loki/data"; isReadOnly = false; }; "/var/lib/prometheus2" = { hostPath = "${data}/overseer/prometheus/data"; isReadOnly = false; }; "/var/lib/grafana" = { hostPath = "${data}/overseer/grafana"; isReadOnly = false; }; "/var/log/journal" = { hostPath = "/var/log/journal"; isReadOnly = true; }; }; containerConfig = _: { services.loki = { enable = true; configuration = { auth_enabled = false; server = { http_listen_port = 3100; grpc_listen_port = 9096; }; common = { instance_addr = "127.0.0.1"; path_prefix = "/var/lib/loki"; storage.filesystem = { chunks_directory = "/var/lib/loki/chunks"; rules_directory = "/var/lib/loki/rules"; }; replication_factor = 1; ring.kvstore.store = "inmemory"; }; schema_config.configs = [ { from = "2020-10-24"; store = "tsdb"; object_store = "filesystem"; schema = "v13"; index = { prefix = "index_"; period = "24h"; }; } ]; }; }; services.alloy = { enable = true; configPath = alloyConfig; }; services.prometheus = { enable = true; port = 9090; globalConfig.scrape_interval = "15s"; scrapeConfigs = [ { job_name = "node"; static_configs = [ { targets = [ "${hostAddr}:${toString nodePort}" ]; } ]; } { job_name = "cadvisor"; static_configs = [ { targets = [ "${hostAddr}:${toString cadvPort}" ]; } ]; } { job_name = "prometheus"; static_configs = [ { targets = [ "localhost:9090" ]; } ]; } ]; ruleFiles = [ containerNetworkRulesFile containerDefinedRulesFile (dsl.mkContainerLimitsRuleFile pkgs containers) ]; rules = [ '' groups: - name: containers rules: - alert: ContainerFailed expr: node_systemd_unit_state{name=~"container@.*",state="failed"} == 1 for: 1m labels: severity: critical annotations: summary: "{{ $labels.name }} entered failed state" - alert: ContainerOOMKill expr: increase(container_oom_events_total{id=~"/machine.slice/container@.+\\.service.*"}[5m]) > 0 labels: severity: warning annotations: summary: "OOM kill in {{ $labels.id }}" '' ]; }; services.grafana = { enable = true; dataDir = "/var/lib/grafana"; settings = { server.http_port = 3000; server.http_addr = "127.0.0.1"; # WORKAROUND: Grafana 10+ requires an explicit secret_key; omitting it # causes auth session tokens to be regenerated on every restart security.secret_key = "SW2YcwTIb9zpOOhoPsMm"; "auth".disable_login_form = true; "auth.anonymous" = { enabled = true; org_role = "Admin"; }; }; provision = { enable = true; datasources.settings = { apiVersion = 1; datasources = [ { name = "Prometheus"; type = "prometheus"; uid = "prometheus"; access = "proxy"; url = "http://localhost:9090"; isDefault = true; jsonData.timeInterval = "15s"; } { name = "Loki"; type = "loki"; uid = "loki"; access = "proxy"; url = "http://127.0.0.1:3100"; } ]; }; dashboards.settings = { apiVersion = 1; providers = [ { name = "default"; type = "file"; disableDeletion = true; options.path = toString dashboardDir; } ]; }; }; }; services.nginx = { enable = true; virtualHosts."_" = { listen = [ { addr = "0.0.0.0"; port = 80; ssl = false; extraParameters = [ "default_server" ]; } ]; root = "${landingRoot}"; }; virtualHosts."stats.localhost" = { listen = [ { addr = "0.0.0.0"; port = 80; ssl = false; } ]; locations."/" = { proxyPass = "http://127.0.0.1:3000"; proxyWebsockets = true; extraConfig = "proxy_set_header Host $host;"; }; }; virtualHosts."prometheus.localhost" = { listen = [ { addr = "0.0.0.0"; port = 80; ssl = false; } ]; locations."/" = { proxyPass = "http://127.0.0.1:9090"; }; }; }; systemd.services = lib.genAttrs [ "loki" "alloy" "prometheus" "grafana" "nginx" ] (_: { serviceConfig.FailureAction = "exit-force"; }); }; net.expose.tcp = [ 80 ]; resources.mem = "2G"; } // { hostConfig = { services.prometheus.exporters.node = { enable = true; listenAddress = "0.0.0.0"; port = nodePort; enabledCollectors = [ "systemd" "processes" ]; disabledCollectors = [ "netstat" "softnet" ]; # radeon sometimes breaks on kexec extraFlags = [ "--collector.systemd.enable-task-metrics" "--collector.hwmon.chip-exclude=radeon" ]; }; services.cadvisor = { enable = true; listenAddress = "0.0.0.0"; port = cadvPort; }; }; }; inherit (c) hostAddr; in c