initial commit
This commit is contained in:
Floweynt
@@ -0,0 +1,61 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; }) mkRow loki mkDashboard;
|
||||
|
||||
mkLogs =
|
||||
{
|
||||
title,
|
||||
expr,
|
||||
w ? 24,
|
||||
h ? 10,
|
||||
}:
|
||||
{
|
||||
inherit title;
|
||||
type = "logs";
|
||||
datasource = loki;
|
||||
gridPos = { inherit w h; };
|
||||
options = {
|
||||
showTime = true;
|
||||
showLabels = true;
|
||||
sortOrder = "Descending";
|
||||
wrapLogMessage = true;
|
||||
enableLogDetails = true;
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = loki;
|
||||
inherit expr;
|
||||
}
|
||||
];
|
||||
};
|
||||
in
|
||||
mkDashboard {
|
||||
name = "access-logs.json";
|
||||
title = "Access Logs";
|
||||
uid = "access-logs";
|
||||
from = "now-6h";
|
||||
panels = [
|
||||
(mkRow "HTTP")
|
||||
(mkLogs {
|
||||
title = "nginx (access + error)";
|
||||
expr = ''{syslog_id="nginx"}'';
|
||||
})
|
||||
(mkRow "SSH")
|
||||
(mkLogs {
|
||||
title = "sshpiper (all connections)";
|
||||
expr = ''{container="sshpiper"}'';
|
||||
})
|
||||
(mkLogs {
|
||||
title = "Failed / invalid auth attempts (all backends, incl. host)";
|
||||
expr = ''{syslog_id=~"sshd.*"} |~ "Failed|Invalid|error:"'';
|
||||
})
|
||||
(mkLogs {
|
||||
title = "sshd auth events (all backends, incl. host)";
|
||||
expr = ''{syslog_id=~"sshd.*"}'';
|
||||
})
|
||||
(mkLogs {
|
||||
title = "Host sshd (port 5952)";
|
||||
expr = ''{syslog_id=~"sshd.*", container=""}'';
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; }) prom mkDashboard;
|
||||
in
|
||||
mkDashboard {
|
||||
name = "containers.json";
|
||||
title = "Containers";
|
||||
uid = "containers";
|
||||
from = "now-3h";
|
||||
panels = [
|
||||
{
|
||||
title = "Container States";
|
||||
type = "piechart";
|
||||
gridPos = {
|
||||
h = 8;
|
||||
w = 8;
|
||||
};
|
||||
datasource = prom;
|
||||
options = {
|
||||
pieType = "donut";
|
||||
legend = {
|
||||
displayMode = "table";
|
||||
placement = "right";
|
||||
values = [
|
||||
"value"
|
||||
"percent"
|
||||
];
|
||||
};
|
||||
tooltip.mode = "single";
|
||||
};
|
||||
fieldConfig.overrides = [
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "active";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "green";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "failed";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "red";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "inactive";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "blue";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "activating";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "yellow";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "unknown";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "gray";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
expr = ''count by (state) ((label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service")) or on(container) label_replace(container_defined, "state", "unknown", "container", ".+"))'';
|
||||
instant = true;
|
||||
legendFormat = "{{state}}";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
title = "Container States";
|
||||
type = "table";
|
||||
gridPos = {
|
||||
h = 16;
|
||||
w = 24;
|
||||
};
|
||||
datasource = prom;
|
||||
options.footer.show = false;
|
||||
fieldConfig = {
|
||||
defaults = { };
|
||||
overrides = [
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "State";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "custom.displayMode";
|
||||
value = "color-text";
|
||||
}
|
||||
{
|
||||
id = "mappings";
|
||||
value = [
|
||||
{
|
||||
type = "value";
|
||||
options.active = {
|
||||
text = "active";
|
||||
color = "green";
|
||||
index = 0;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.failed = {
|
||||
text = "failed";
|
||||
color = "red";
|
||||
index = 1;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.activating = {
|
||||
text = "activating";
|
||||
color = "yellow";
|
||||
index = 2;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.inactive = {
|
||||
text = "inactive";
|
||||
color = "blue";
|
||||
index = 3;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.unknown = {
|
||||
text = "unknown";
|
||||
color = "gray";
|
||||
index = 4;
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
expr = ''
|
||||
(label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service"))
|
||||
or on(container)
|
||||
label_replace(container_defined, "state", "unknown", "container", ".+")
|
||||
'';
|
||||
instant = true;
|
||||
format = "table";
|
||||
legendFormat = "__auto";
|
||||
}
|
||||
];
|
||||
transformations = [
|
||||
{
|
||||
id = "organize";
|
||||
options = {
|
||||
excludeByName = {
|
||||
Time = true;
|
||||
Value = true;
|
||||
"__name__" = true;
|
||||
instance = true;
|
||||
job = true;
|
||||
name = true;
|
||||
};
|
||||
renameByName = {
|
||||
container = "Container";
|
||||
state = "State";
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
title = "Container State (1 = running, 0 = stopped/failed)";
|
||||
type = "timeseries";
|
||||
gridPos = {
|
||||
h = 8;
|
||||
w = 24;
|
||||
};
|
||||
datasource = prom;
|
||||
fieldConfig.defaults = {
|
||||
custom.lineWidth = 2;
|
||||
color.mode = "palette-classic";
|
||||
min = 0;
|
||||
max = 1;
|
||||
};
|
||||
options = {
|
||||
tooltip.mode = "multi";
|
||||
legend = {
|
||||
displayMode = "table";
|
||||
placement = "bottom";
|
||||
};
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
expr = ''label_replace(node_systemd_unit_state{name=~"container@.*",state="active"}, "container", "$1", "name", "container@(.+).service")'';
|
||||
legendFormat = "{{container}}";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
prom = {
|
||||
type = "prometheus";
|
||||
uid = "prometheus";
|
||||
};
|
||||
loki = {
|
||||
type = "loki";
|
||||
uid = "loki";
|
||||
};
|
||||
|
||||
autoLayout =
|
||||
panels:
|
||||
let
|
||||
step =
|
||||
acc: panel:
|
||||
let
|
||||
w = panel.gridPos.w;
|
||||
h = panel.gridPos.h;
|
||||
wrap = acc.x + w > 24;
|
||||
x = if wrap then 0 else acc.x;
|
||||
y = if wrap then acc.y + acc.rowH else acc.y;
|
||||
rowH = if wrap then h else lib.max acc.rowH h;
|
||||
in
|
||||
{
|
||||
x = x + w;
|
||||
inherit y rowH;
|
||||
result = acc.result ++ [
|
||||
(
|
||||
panel
|
||||
// {
|
||||
gridPos = panel.gridPos // {
|
||||
inherit x y;
|
||||
};
|
||||
}
|
||||
)
|
||||
];
|
||||
};
|
||||
in
|
||||
(lib.foldl' step {
|
||||
x = 0;
|
||||
y = 0;
|
||||
rowH = 0;
|
||||
result = [ ];
|
||||
} panels).result;
|
||||
|
||||
T = expr: legendFormat: {
|
||||
datasource = prom;
|
||||
inherit expr legendFormat;
|
||||
};
|
||||
|
||||
mkRow = title: {
|
||||
inherit title;
|
||||
type = "row";
|
||||
collapsed = false;
|
||||
panels = [ ];
|
||||
gridPos = {
|
||||
w = 24;
|
||||
h = 1;
|
||||
};
|
||||
};
|
||||
|
||||
mkStat =
|
||||
{
|
||||
title,
|
||||
w ? 4,
|
||||
h ? 4,
|
||||
expr,
|
||||
unit ? "short",
|
||||
decimals ? 2,
|
||||
}:
|
||||
{
|
||||
inherit title;
|
||||
type = "stat";
|
||||
datasource = prom;
|
||||
gridPos = { inherit w h; };
|
||||
fieldConfig.defaults = {
|
||||
inherit unit decimals;
|
||||
color.mode = "thresholds";
|
||||
thresholds = {
|
||||
mode = "absolute";
|
||||
steps = [
|
||||
{
|
||||
color = "blue";
|
||||
value = null;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
options = {
|
||||
reduceOptions.calcs = [ "lastNotNull" ];
|
||||
colorMode = "value";
|
||||
textMode = "value";
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
inherit expr;
|
||||
instant = true;
|
||||
legendFormat = "";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
mkGauge =
|
||||
{
|
||||
title,
|
||||
w ? 4,
|
||||
h ? 4,
|
||||
expr,
|
||||
unit ? "percent",
|
||||
min ? 0,
|
||||
max ? 100,
|
||||
}:
|
||||
{
|
||||
inherit title;
|
||||
type = "gauge";
|
||||
datasource = prom;
|
||||
gridPos = { inherit w h; };
|
||||
fieldConfig.defaults = {
|
||||
inherit unit min max;
|
||||
thresholds = {
|
||||
mode = "absolute";
|
||||
steps = [
|
||||
{
|
||||
color = "green";
|
||||
value = null;
|
||||
}
|
||||
{
|
||||
color = "yellow";
|
||||
value = 70;
|
||||
}
|
||||
{
|
||||
color = "red";
|
||||
value = 90;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
options = {
|
||||
reduceOptions.calcs = [ "lastNotNull" ];
|
||||
orientation = "auto";
|
||||
showThresholdLabels = false;
|
||||
showThresholdMarkers = true;
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
inherit expr;
|
||||
instant = true;
|
||||
legendFormat = "";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
mkTs =
|
||||
{
|
||||
title,
|
||||
w ? 24,
|
||||
h ? 8,
|
||||
unit ? "short",
|
||||
targets,
|
||||
}:
|
||||
{
|
||||
inherit title targets;
|
||||
type = "timeseries";
|
||||
datasource = prom;
|
||||
gridPos = { inherit w h; };
|
||||
fieldConfig.defaults = {
|
||||
inherit unit;
|
||||
custom = {
|
||||
lineWidth = 1;
|
||||
fillOpacity = 5;
|
||||
};
|
||||
color.mode = "palette-classic";
|
||||
};
|
||||
options = {
|
||||
tooltip.mode = "multi";
|
||||
legend = {
|
||||
displayMode = "table";
|
||||
placement = "bottom";
|
||||
calcs = [
|
||||
"mean"
|
||||
"max"
|
||||
"lastNotNull"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
mkDashboard =
|
||||
{
|
||||
name,
|
||||
title,
|
||||
uid,
|
||||
from ? "now-1h",
|
||||
panels,
|
||||
templating ? [ ],
|
||||
}:
|
||||
pkgs.writeText name (
|
||||
builtins.toJSON (
|
||||
{
|
||||
inherit title uid;
|
||||
schemaVersion = 37;
|
||||
refresh = "30s";
|
||||
time = {
|
||||
inherit from;
|
||||
to = "now";
|
||||
};
|
||||
panels = lib.imap1 (id: p: p // { inherit id; }) (autoLayout panels);
|
||||
}
|
||||
// lib.optionalAttrs (templating != [ ]) { templating.list = templating; }
|
||||
)
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit
|
||||
prom
|
||||
loki
|
||||
autoLayout
|
||||
T
|
||||
mkRow
|
||||
mkStat
|
||||
mkGauge
|
||||
mkTs
|
||||
mkDashboard
|
||||
;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; })
|
||||
T
|
||||
mkRow
|
||||
mkStat
|
||||
mkGauge
|
||||
mkTs
|
||||
mkDashboard
|
||||
;
|
||||
in
|
||||
mkDashboard {
|
||||
name = "host.json";
|
||||
title = "Host";
|
||||
uid = "host";
|
||||
panels = [
|
||||
|
||||
(mkRow "Overview")
|
||||
(mkGauge {
|
||||
title = "CPU";
|
||||
expr = ''100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])))'';
|
||||
})
|
||||
(mkGauge {
|
||||
title = "Memory used";
|
||||
expr = "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Load (1m)";
|
||||
unit = "short";
|
||||
decimals = 2;
|
||||
expr = "node_load1";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Load (5m)";
|
||||
unit = "short";
|
||||
decimals = 2;
|
||||
expr = "node_load5";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Uptime";
|
||||
unit = "dtdurations";
|
||||
decimals = 0;
|
||||
expr = "node_time_seconds - node_boot_time_seconds";
|
||||
})
|
||||
|
||||
(mkRow "CPU")
|
||||
(mkTs {
|
||||
title = "CPU usage (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="user"}[5m])) * 100'' "user")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="system"}[5m])) * 100'' "system")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100'' "iowait")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100'' "steal")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="softirq"}[5m])) * 100'' "softirq")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Per-core usage (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''(1 - rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100'' "cpu{{cpu}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "CPU temperatures";
|
||||
unit = "celsius";
|
||||
targets = [
|
||||
(T "node_hwmon_temp_celsius" "{{chip}} {{sensor}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Load average";
|
||||
targets = [
|
||||
(T "node_load1" "1m")
|
||||
(T "node_load5" "5m")
|
||||
(T "node_load15" "15m")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "Memory")
|
||||
(mkTs {
|
||||
title = "Memory";
|
||||
unit = "bytes";
|
||||
targets = [
|
||||
(T "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes" "used")
|
||||
(T "node_memory_Cached_bytes + node_memory_Buffers_bytes" "cache + buffers")
|
||||
(T "node_memory_MemAvailable_bytes" "available")
|
||||
(T "node_memory_MemTotal_bytes" "total")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Swap";
|
||||
unit = "bytes";
|
||||
targets = [
|
||||
(T "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes" "used")
|
||||
(T "node_memory_SwapTotal_bytes" "total")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "Disk")
|
||||
(mkTs {
|
||||
title = "Filesystem usage (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''100 * (1 - node_filesystem_avail_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"} / node_filesystem_size_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"})'' "{{mountpoint}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk IO (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T ''rate(node_disk_read_bytes_total{device!~"loop.*"}[5m])'' "{{device}} read")
|
||||
(T ''rate(node_disk_written_bytes_total{device!~"loop.*"}[5m])'' "{{device}} write")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk IO utilisation (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''rate(node_disk_io_time_seconds_total{device!~"loop.*"}[5m]) * 100'' "{{device}}")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "Network")
|
||||
(mkTs {
|
||||
title = "Network (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T ''rate(node_network_receive_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx")
|
||||
(T ''rate(node_network_transmit_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network errors + drops (pps)";
|
||||
targets = [
|
||||
(T ''rate(node_network_receive_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx err")
|
||||
(T ''rate(node_network_transmit_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx err")
|
||||
(T ''rate(node_network_receive_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx drop")
|
||||
(T ''rate(node_network_transmit_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx drop")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "System")
|
||||
(mkTs {
|
||||
title = "Context switches + forks (ops/s)";
|
||||
targets = [
|
||||
(T "rate(node_context_switches_total[5m])" "context switches")
|
||||
(T "rate(node_forks_total[5m])" "forks")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "File descriptors";
|
||||
targets = [
|
||||
(T "node_filefd_allocated" "allocated")
|
||||
(T "node_filefd_maximum" "limit")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Processes";
|
||||
targets = [
|
||||
(T "node_procs_running" "running")
|
||||
(T "node_procs_blocked" "blocked on IO")
|
||||
];
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; })
|
||||
T
|
||||
mkRow
|
||||
mkStat
|
||||
mkTs
|
||||
prom
|
||||
mkDashboard
|
||||
;
|
||||
|
||||
ctid = "id=~\"/machine.slice/container@.+\\\\.service/payload/.+\"";
|
||||
ctsel = "id=~\"/machine.slice/container@$container\\\\.service/payload/.+\"";
|
||||
|
||||
ctall = "id=~\"/machine.slice/container@.+\\\\.service.*\"";
|
||||
ctnet = "id=~\"/machine.slice/container@$container\\\\.service.*\"";
|
||||
|
||||
labeled =
|
||||
m:
|
||||
"label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service/payload.+\")";
|
||||
byContainer = m: "sum by (container) (${labeled m})";
|
||||
|
||||
labeledAll =
|
||||
m:
|
||||
"label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service.*\")";
|
||||
byContainerAll = m: "sum by (container) (${labeledAll m})";
|
||||
|
||||
nmSvc = m: "label_replace(${m}, \"container\", \"$1\", \"name\", \"container@(.+)\\\\.service\")";
|
||||
|
||||
orZero = m: fb: "(${m}) or on(container) (${fb} * 0)";
|
||||
|
||||
cpuAll =
|
||||
rate:
|
||||
orZero (byContainer "rate(container_cpu_usage_seconds_total{${ctid}}[${rate}])") "container_limit_cpu_cores";
|
||||
memAll = orZero (byContainerAll "container_memory_rss{${ctall}}") "container_limit_memory_bytes";
|
||||
taskAll = nmSvc "node_systemd_unit_tasks_current{name=~\"container@.+\"}";
|
||||
in
|
||||
mkDashboard {
|
||||
name = "resources.json";
|
||||
title = "Container Resources";
|
||||
uid = "container-resources";
|
||||
|
||||
templating = [
|
||||
{
|
||||
name = "container";
|
||||
type = "query";
|
||||
datasource = prom;
|
||||
query = "label_values(container_limit_cpu_cores, container)";
|
||||
refresh = 2;
|
||||
sort = 1;
|
||||
includeAll = true;
|
||||
multi = true;
|
||||
current = {
|
||||
selected = true;
|
||||
text = [ "All" ];
|
||||
value = [ "$__all" ];
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
panels = [
|
||||
(mkRow "Overview")
|
||||
(mkStat {
|
||||
title = "Running";
|
||||
decimals = 0;
|
||||
expr = "count(node_systemd_unit_state{name=~\"container@.*\",state=\"active\"} == 1) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "CPU used";
|
||||
decimals = 2;
|
||||
expr = "sum(${cpuAll "5m"}) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "CPU limit";
|
||||
decimals = 2;
|
||||
expr = "sum(container_limit_cpu_cores) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Mem RSS";
|
||||
decimals = 1;
|
||||
unit = "bytes";
|
||||
expr = "sum(${memAll}) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Mem limit";
|
||||
decimals = 1;
|
||||
unit = "bytes";
|
||||
expr = "sum(container_limit_memory_bytes) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Tasks";
|
||||
decimals = 0;
|
||||
expr = "sum(${taskAll}) or vector(0)";
|
||||
})
|
||||
|
||||
(mkRow "All Containers")
|
||||
(mkTs {
|
||||
title = "CPU (cores)";
|
||||
h = 24;
|
||||
targets = [ (T (cpuAll "5m") "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Memory RSS";
|
||||
unit = "bytes";
|
||||
h = 24;
|
||||
targets = [ (T memAll "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Tasks";
|
||||
h = 24;
|
||||
targets = [ (T taskAll "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "CPU throttled %";
|
||||
unit = "percent";
|
||||
h = 12;
|
||||
targets = [
|
||||
(T "(${byContainer "rate(container_cpu_cfs_throttled_periods_total{${ctid}}[5m])"} / ${byContainer "rate(container_cpu_cfs_periods_total{${ctid}}[5m])"}) * 100 or on(container) (container_limit_cpu_cores * 0)" "{{container}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network egress (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [ (T "sum by (container) (container_network_receive_rate)" "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network ingress (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [ (T "sum by (container) (container_network_transmit_rate)" "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk read (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [ (T (byContainer "rate(container_fs_reads_bytes_total{${ctid}}[5m])") "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk write (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [
|
||||
(T (byContainer "rate(container_fs_writes_bytes_total{${ctid}}[5m])") "{{container}}")
|
||||
];
|
||||
})
|
||||
|
||||
{
|
||||
type = "row";
|
||||
title = "$container";
|
||||
repeat = "container";
|
||||
collapsed = true;
|
||||
panels = [ ];
|
||||
gridPos = {
|
||||
w = 24;
|
||||
h = 1;
|
||||
};
|
||||
}
|
||||
(mkTs {
|
||||
title = "CPU (cores)";
|
||||
targets = [
|
||||
(T "sum(rate(container_cpu_usage_seconds_total{${ctsel}}[5m]))" "used")
|
||||
(T "container_limit_cpu_cores{container=\"$container\"}" "limit")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "CPU throttled %";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T "(sum(rate(container_cpu_cfs_throttled_periods_total{${ctsel}}[5m])) / sum(rate(container_cpu_cfs_periods_total{${ctsel}}[5m]))) * 100 or vector(0)" "throttled %")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Memory";
|
||||
unit = "bytes";
|
||||
targets = [
|
||||
(T "sum(container_memory_rss{${ctnet}})" "RSS")
|
||||
(T "sum(container_memory_working_set_bytes{${ctnet}})" "working set")
|
||||
(T "container_limit_memory_bytes{container=\"$container\"}" "limit")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T "container_network_receive_rate{container=\"$container\"}" "{{device}} ↑")
|
||||
(T "container_network_transmit_rate{container=\"$container\"}" "{{device}} ↓")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk IO (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T "sum(rate(container_fs_reads_bytes_total{${ctsel}}[5m]))" "read")
|
||||
(T "sum(rate(container_fs_writes_bytes_total{${ctsel}}[5m]))" "write")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Tasks";
|
||||
targets = [ (T "node_systemd_unit_tasks_current{name=\"container@$container.service\"}" "tasks") ];
|
||||
})
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user