initial commit

containers/common/overseer/resources-dashboard.nix

@@ -0,0 +1,203 @@
+{ pkgs, lib }:
+let
+  inherit (import ./helpers.nix { inherit pkgs lib; })
+    T
+    mkRow
+    mkStat
+    mkTs
+    prom
+    mkDashboard
+    ;
+
+  ctid = "id=~\"/machine.slice/container@.+\\\\.service/payload/.+\"";
+  ctsel = "id=~\"/machine.slice/container@$container\\\\.service/payload/.+\"";
+
+  ctall = "id=~\"/machine.slice/container@.+\\\\.service.*\"";
+  ctnet = "id=~\"/machine.slice/container@$container\\\\.service.*\"";
+
+  labeled =
+    m:
+    "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service/payload.+\")";
+  byContainer = m: "sum by (container) (${labeled m})";
+
+  labeledAll =
+    m:
+    "label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service.*\")";
+  byContainerAll = m: "sum by (container) (${labeledAll m})";
+
+  nmSvc = m: "label_replace(${m}, \"container\", \"$1\", \"name\", \"container@(.+)\\\\.service\")";
+
+  orZero = m: fb: "(${m}) or on(container) (${fb} * 0)";
+
+  cpuAll =
+    rate:
+    orZero (byContainer "rate(container_cpu_usage_seconds_total{${ctid}}[${rate}])") "container_limit_cpu_cores";
+  memAll = orZero (byContainerAll "container_memory_rss{${ctall}}") "container_limit_memory_bytes";
+  taskAll = nmSvc "node_systemd_unit_tasks_current{name=~\"container@.+\"}";
+in
+mkDashboard {
+  name = "resources.json";
+  title = "Container Resources";
+  uid = "container-resources";
+
+  templating = [
+    {
+      name = "container";
+      type = "query";
+      datasource = prom;
+      query = "label_values(container_limit_cpu_cores, container)";
+      refresh = 2;
+      sort = 1;
+      includeAll = true;
+      multi = true;
+      current = {
+        selected = true;
+        text = [ "All" ];
+        value = [ "$__all" ];
+      };
+    }
+  ];
+
+  panels = [
+    (mkRow "Overview")
+    (mkStat {
+      title = "Running";
+      decimals = 0;
+      expr = "count(node_systemd_unit_state{name=~\"container@.*\",state=\"active\"} == 1) or vector(0)";
+    })
+    (mkStat {
+      title = "CPU used";
+      decimals = 2;
+      expr = "sum(${cpuAll "5m"}) or vector(0)";
+    })
+    (mkStat {
+      title = "CPU limit";
+      decimals = 2;
+      expr = "sum(container_limit_cpu_cores) or vector(0)";
+    })
+    (mkStat {
+      title = "Mem RSS";
+      decimals = 1;
+      unit = "bytes";
+      expr = "sum(${memAll}) or vector(0)";
+    })
+    (mkStat {
+      title = "Mem limit";
+      decimals = 1;
+      unit = "bytes";
+      expr = "sum(container_limit_memory_bytes) or vector(0)";
+    })
+    (mkStat {
+      title = "Tasks";
+      decimals = 0;
+      expr = "sum(${taskAll}) or vector(0)";
+    })
+
+    (mkRow "All Containers")
+    (mkTs {
+      title = "CPU (cores)";
+      h = 24;
+      targets = [ (T (cpuAll "5m") "{{container}}") ];
+    })
+    (mkTs {
+      title = "Memory RSS";
+      unit = "bytes";
+      h = 24;
+      targets = [ (T memAll "{{container}}") ];
+    })
+    (mkTs {
+      title = "Tasks";
+      h = 24;
+      targets = [ (T taskAll "{{container}}") ];
+    })
+    (mkTs {
+      title = "CPU throttled %";
+      unit = "percent";
+      h = 12;
+      targets = [
+        (T "(${byContainer "rate(container_cpu_cfs_throttled_periods_total{${ctid}}[5m])"} / ${byContainer "rate(container_cpu_cfs_periods_total{${ctid}}[5m])"}) * 100 or on(container) (container_limit_cpu_cores * 0)" "{{container}}")
+      ];
+    })
+    (mkTs {
+      title = "Network egress (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [ (T "sum by (container) (container_network_receive_rate)" "{{container}}") ];
+    })
+    (mkTs {
+      title = "Network ingress (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [ (T "sum by (container) (container_network_transmit_rate)" "{{container}}") ];
+    })
+    (mkTs {
+      title = "Disk read (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [ (T (byContainer "rate(container_fs_reads_bytes_total{${ctid}}[5m])") "{{container}}") ];
+    })
+    (mkTs {
+      title = "Disk write (bytes/s)";
+      unit = "Bps";
+      h = 24;
+      targets = [
+        (T (byContainer "rate(container_fs_writes_bytes_total{${ctid}}[5m])") "{{container}}")
+      ];
+    })
+
+    {
+      type = "row";
+      title = "$container";
+      repeat = "container";
+      collapsed = true;
+      panels = [ ];
+      gridPos = {
+        w = 24;
+        h = 1;
+      };
+    }
+    (mkTs {
+      title = "CPU (cores)";
+      targets = [
+        (T "sum(rate(container_cpu_usage_seconds_total{${ctsel}}[5m]))" "used")
+        (T "container_limit_cpu_cores{container=\"$container\"}" "limit")
+      ];
+    })
+    (mkTs {
+      title = "CPU throttled %";
+      unit = "percent";
+      targets = [
+        (T "(sum(rate(container_cpu_cfs_throttled_periods_total{${ctsel}}[5m])) / sum(rate(container_cpu_cfs_periods_total{${ctsel}}[5m]))) * 100 or vector(0)" "throttled %")
+      ];
+    })
+    (mkTs {
+      title = "Memory";
+      unit = "bytes";
+      targets = [
+        (T "sum(container_memory_rss{${ctnet}})" "RSS")
+        (T "sum(container_memory_working_set_bytes{${ctnet}})" "working set")
+        (T "container_limit_memory_bytes{container=\"$container\"}" "limit")
+      ];
+    })
+    (mkTs {
+      title = "Network (bytes/s)";
+      unit = "Bps";
+      targets = [
+        (T "container_network_receive_rate{container=\"$container\"}" "{{device}} ↑")
+        (T "container_network_transmit_rate{container=\"$container\"}" "{{device}} ↓")
+      ];
+    })
+    (mkTs {
+      title = "Disk IO (bytes/s)";
+      unit = "Bps";
+      targets = [
+        (T "sum(rate(container_fs_reads_bytes_total{${ctsel}}[5m]))" "read")
+        (T "sum(rate(container_fs_writes_bytes_total{${ctsel}}[5m]))" "write")
+      ];
+    })
+    (mkTs {
+      title = "Tasks";
+      targets = [ (T "node_systemd_unit_tasks_current{name=\"container@$container.service\"}" "tasks") ];
+    })
+  ];
+}