server-config/containers/common/overseer/resources-dashboard.nix

{ pkgs, lib }:
let
  inherit (import ./helpers.nix { inherit pkgs lib; })
    T
    mkRow
    mkStat
    mkTs
    prom
    mkDashboard
    ;

  ctid = "id=~\"/machine.slice/container@.+\\\\.service/payload/.+\"";
  ctsel = "id=~\"/machine.slice/container@$container\\\\.service/payload/.+\"";

  ctall = "id=~\"/machine.slice/container@.+\\\\.service.*\"";
  ctnet = "id=~\"/machine.slice/container@$container\\\\.service.*\"";

  labeled =
    m:
    "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service/payload.+\")";
  byContainer = m: "sum by (container) (${labeled m})";

  labeledAll =
    m:
    "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service.*\")";
  byContainerAll = m: "sum by (container) (${labeledAll m})";

  nmSvc = m: "label_replace(${m}, \"container\", \"$1\", \"name\", \"container@(.+)\\\\.service\")";

  orZero = m: fb: "(${m}) or on(container) (${fb} * 0)";

  cpuAll =
    rate:
    orZero (byContainer "rate(container_cpu_usage_seconds_total{${ctid}}[${rate}])") "container_limit_cpu_cores";
  memAll = orZero (byContainerAll "container_memory_rss{${ctall}}") "container_limit_memory_bytes";
  taskAll = nmSvc "node_systemd_unit_tasks_current{name=~\"container@.+\"}";
in
mkDashboard {
  name = "resources.json";
  title = "Container Resources";
  uid = "container-resources";

  templating = [
    {
      name = "container";
      type = "query";
      datasource = prom;
      query = "label_values(container_limit_cpu_cores, container)";
      refresh = 2;
      sort = 1;
      includeAll = true;
      multi = true;
      current = {
        selected = true;
        text = [ "All" ];
        value = [ "$__all" ];
      };
    }
  ];

  panels = [
    (mkRow "Overview")
    (mkStat {
      title = "Running";
      decimals = 0;
      expr = "count(node_systemd_unit_state{name=~\"container@.*\",state=\"active\"} == 1) or vector(0)";
    })
    (mkStat {
      title = "CPU used";
      decimals = 2;
      expr = "sum(${cpuAll "5m"}) or vector(0)";
    })
    (mkStat {
      title = "CPU limit";
      decimals = 2;
      expr = "sum(container_limit_cpu_cores) or vector(0)";
    })
    (mkStat {
      title = "Mem RSS";
      decimals = 1;
      unit = "bytes";
      expr = "sum(${memAll}) or vector(0)";
    })
    (mkStat {
      title = "Mem limit";
      decimals = 1;
      unit = "bytes";
      expr = "sum(container_limit_memory_bytes) or vector(0)";
    })
    (mkStat {
      title = "Tasks";
      decimals = 0;
      expr = "sum(${taskAll}) or vector(0)";
    })

    (mkRow "All Containers")
    (mkTs {
      title = "CPU (cores)";
      h = 24;
      targets = [ (T (cpuAll "5m") "{{container}}") ];
    })
    (mkTs {
      title = "Memory RSS";
      unit = "bytes";
      h = 24;
      targets = [ (T memAll "{{container}}") ];
    })
    (mkTs {
      title = "Tasks";
      h = 24;
      targets = [ (T taskAll "{{container}}") ];
    })
    (mkTs {
      title = "CPU throttled %";
      unit = "percent";
      h = 12;
      targets = [
        (T "(${byContainer "rate(container_cpu_cfs_throttled_periods_total{${ctid}}[5m])"} / ${byContainer "rate(container_cpu_cfs_periods_total{${ctid}}[5m])"}) * 100 or on(container) (container_limit_cpu_cores * 0)" "{{container}}")
      ];
    })
    (mkTs {
      title = "Network egress (bytes/s)";
      unit = "Bps";
      h = 24;
      targets = [ (T "sum by (container) (container_network_receive_rate)" "{{container}}") ];
    })
    (mkTs {
      title = "Network ingress (bytes/s)";
      unit = "Bps";
      h = 24;
      targets = [ (T "sum by (container) (container_network_transmit_rate)" "{{container}}") ];
    })
    (mkTs {
      title = "Disk read (bytes/s)";
      unit = "Bps";
      h = 24;
      targets = [ (T (byContainer "rate(container_fs_reads_bytes_total{${ctid}}[5m])") "{{container}}") ];
    })
    (mkTs {
      title = "Disk write (bytes/s)";
      unit = "Bps";
      h = 24;
      targets = [
        (T (byContainer "rate(container_fs_writes_bytes_total{${ctid}}[5m])") "{{container}}")
      ];
    })

    {
      type = "row";
      title = "$container";
      repeat = "container";
      collapsed = true;
      panels = [ ];
      gridPos = {
        w = 24;
        h = 1;
      };
    }
    (mkTs {
      title = "CPU (cores)";
      targets = [
        (T "sum(rate(container_cpu_usage_seconds_total{${ctsel}}[5m]))" "used")
        (T "container_limit_cpu_cores{container=\"$container\"}" "limit")
      ];
    })
    (mkTs {
      title = "CPU throttled %";
      unit = "percent";
      targets = [
        (T "(sum(rate(container_cpu_cfs_throttled_periods_total{${ctsel}}[5m])) / sum(rate(container_cpu_cfs_periods_total{${ctsel}}[5m]))) * 100 or vector(0)" "throttled %")
      ];
    })
    (mkTs {
      title = "Memory";
      unit = "bytes";
      targets = [
        (T "sum(container_memory_rss{${ctnet}})" "RSS")
        (T "sum(container_memory_working_set_bytes{${ctnet}})" "working set")
        (T "container_limit_memory_bytes{container=\"$container\"}" "limit")
      ];
    })
    (mkTs {
      title = "Network (bytes/s)";
      unit = "Bps";
      targets = [
        (T "container_network_receive_rate{container=\"$container\"}" "{{device}} ↑")
        (T "container_network_transmit_rate{container=\"$container\"}" "{{device}} ↓")
      ];
    })
    (mkTs {
      title = "Disk IO (bytes/s)";
      unit = "Bps";
      targets = [
        (T "sum(rate(container_fs_reads_bytes_total{${ctsel}}[5m]))" "read")
        (T "sum(rate(container_fs_writes_bytes_total{${ctsel}}[5m]))" "write")
      ];
    })
    (mkTs {
      title = "Tasks";
      targets = [ (T "node_systemd_unit_tasks_current{name=\"container@$container.service\"}" "tasks") ];
    })
  ];
}