initial commit
This commit is contained in:
Floweynt
@@ -0,0 +1,386 @@
|
||||
{
|
||||
name,
|
||||
hostInfo,
|
||||
dsl,
|
||||
pkgs,
|
||||
lib,
|
||||
containers,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (dsl) mkContainer;
|
||||
inherit (hostInfo.paths) data;
|
||||
|
||||
nodePort = 9100;
|
||||
cadvPort = 9200;
|
||||
|
||||
containerNetworkRulesFile =
|
||||
let
|
||||
mkRules =
|
||||
metric: record:
|
||||
lib.mapAttrsToList (cname: c: {
|
||||
inherit record;
|
||||
expr = "rate(${metric}{device=\"${c.iface}\"}[5m])";
|
||||
labels = {
|
||||
container = cname;
|
||||
};
|
||||
}) containers;
|
||||
in
|
||||
pkgs.writeText "container-network.yml" (
|
||||
builtins.toJSON {
|
||||
groups = [
|
||||
{
|
||||
name = "container-network";
|
||||
rules =
|
||||
mkRules "node_network_receive_bytes_total" "container_network_receive_rate"
|
||||
++ mkRules "node_network_transmit_bytes_total" "container_network_transmit_rate";
|
||||
}
|
||||
];
|
||||
}
|
||||
);
|
||||
|
||||
containerDefinedRulesFile = pkgs.writeText "container-defined.yml" (
|
||||
builtins.toJSON {
|
||||
groups = [
|
||||
{
|
||||
name = "container-defined";
|
||||
rules = lib.mapAttrsToList (cname: _: {
|
||||
record = "container_defined";
|
||||
expr = "1";
|
||||
labels = {
|
||||
container = cname;
|
||||
};
|
||||
}) containers;
|
||||
}
|
||||
];
|
||||
}
|
||||
);
|
||||
|
||||
alloyConfig = pkgs.writeText "config.alloy" ''
|
||||
loki.source.journal "journal" {
|
||||
max_age = "12h"
|
||||
path = "/var/log/journal"
|
||||
labels = { job = "systemd-journal" }
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
forward_to = [loki.write.local.receiver]
|
||||
}
|
||||
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal__hostname"]
|
||||
target_label = "host"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_syslog_identifier"]
|
||||
target_label = "syslog_id"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_machine_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
}
|
||||
|
||||
loki.write "local" {
|
||||
endpoint {
|
||||
url = "http://127.0.0.1:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
'';
|
||||
|
||||
dashboards = lib.filterAttrs (name: _: lib.hasSuffix "-dashboard" name) (
|
||||
dsl.listMap ./overseer (_: path: import path { inherit pkgs lib; })
|
||||
);
|
||||
|
||||
dashboardDir = pkgs.linkFarm "grafana-dashboards" (
|
||||
lib.mapAttrsToList (name: dashboard: {
|
||||
name = "${lib.removeSuffix "-dashboard" name}.json";
|
||||
path = dashboard;
|
||||
}) dashboards
|
||||
);
|
||||
|
||||
landingRoot = pkgs.writeTextDir "index.html" ''
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Overseer</title>
|
||||
<style>
|
||||
body { font-family: sans-serif; max-width: 480px; margin: 4rem auto; }
|
||||
ul { padding: 0; list-style: none; }
|
||||
li { margin: .5rem 0; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Overseer</h1>
|
||||
<ul>
|
||||
<li><a data-host="stats.localhost">stats.localhost</a> — Grafana</li>
|
||||
<li><a data-host="prometheus.localhost">prometheus.localhost</a> — Prometheus</li>
|
||||
</ul>
|
||||
<script>
|
||||
const p = location.port && location.port !== "80" ? ":" + location.port : "";
|
||||
document.querySelectorAll("a[data-host]").forEach(a => {
|
||||
const h = a.dataset.host;
|
||||
a.href = "http://" + h + p;
|
||||
a.textContent = h + p;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'';
|
||||
in
|
||||
let
|
||||
c =
|
||||
mkContainer {
|
||||
inherit name;
|
||||
net.access = [
|
||||
{
|
||||
to = "@host";
|
||||
tcp = [
|
||||
nodePort
|
||||
cadvPort
|
||||
];
|
||||
}
|
||||
];
|
||||
svc.autoStart = true;
|
||||
resources.fs = {
|
||||
"/var/lib/loki" = {
|
||||
hostPath = "${data}/overseer/loki/data";
|
||||
isReadOnly = false;
|
||||
};
|
||||
"/var/lib/prometheus2" = {
|
||||
hostPath = "${data}/overseer/prometheus/data";
|
||||
isReadOnly = false;
|
||||
};
|
||||
"/var/lib/grafana" = {
|
||||
hostPath = "${data}/overseer/grafana";
|
||||
isReadOnly = false;
|
||||
};
|
||||
"/var/log/journal" = {
|
||||
hostPath = "/var/log/journal";
|
||||
isReadOnly = true;
|
||||
};
|
||||
};
|
||||
containerConfig = _: {
|
||||
services.loki = {
|
||||
enable = true;
|
||||
configuration = {
|
||||
auth_enabled = false;
|
||||
server = {
|
||||
http_listen_port = 3100;
|
||||
grpc_listen_port = 9096;
|
||||
};
|
||||
common = {
|
||||
instance_addr = "127.0.0.1";
|
||||
path_prefix = "/var/lib/loki";
|
||||
storage.filesystem = {
|
||||
chunks_directory = "/var/lib/loki/chunks";
|
||||
rules_directory = "/var/lib/loki/rules";
|
||||
};
|
||||
replication_factor = 1;
|
||||
ring.kvstore.store = "inmemory";
|
||||
};
|
||||
schema_config.configs = [
|
||||
{
|
||||
from = "2020-10-24";
|
||||
store = "tsdb";
|
||||
object_store = "filesystem";
|
||||
schema = "v13";
|
||||
index = {
|
||||
prefix = "index_";
|
||||
period = "24h";
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.alloy = {
|
||||
enable = true;
|
||||
configPath = alloyConfig;
|
||||
};
|
||||
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
port = 9090;
|
||||
globalConfig.scrape_interval = "15s";
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [ { targets = [ "${hostAddr}:${toString nodePort}" ]; } ];
|
||||
}
|
||||
{
|
||||
job_name = "cadvisor";
|
||||
static_configs = [ { targets = [ "${hostAddr}:${toString cadvPort}" ]; } ];
|
||||
}
|
||||
{
|
||||
job_name = "prometheus";
|
||||
static_configs = [ { targets = [ "localhost:9090" ]; } ];
|
||||
}
|
||||
];
|
||||
ruleFiles = [
|
||||
containerNetworkRulesFile
|
||||
containerDefinedRulesFile
|
||||
(dsl.mkContainerLimitsRuleFile pkgs containers)
|
||||
];
|
||||
rules = [
|
||||
''
|
||||
groups:
|
||||
- name: containers
|
||||
rules:
|
||||
- alert: ContainerFailed
|
||||
expr: node_systemd_unit_state{name=~"container@.*",state="failed"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.name }} entered failed state"
|
||||
- alert: ContainerOOMKill
|
||||
expr: increase(container_oom_events_total{id=~"/machine.slice/container@.+\\.service.*"}[5m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "OOM kill in {{ $labels.id }}"
|
||||
''
|
||||
];
|
||||
};
|
||||
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
dataDir = "/var/lib/grafana";
|
||||
settings = {
|
||||
server.http_port = 3000;
|
||||
server.http_addr = "127.0.0.1";
|
||||
# WORKAROUND: Grafana 10+ requires an explicit secret_key; omitting it
|
||||
# causes auth session tokens to be regenerated on every restart
|
||||
security.secret_key = "SW2YcwTIb9zpOOhoPsMm";
|
||||
"auth".disable_login_form = true;
|
||||
"auth.anonymous" = {
|
||||
enabled = true;
|
||||
org_role = "Admin";
|
||||
};
|
||||
};
|
||||
provision = {
|
||||
enable = true;
|
||||
datasources.settings = {
|
||||
apiVersion = 1;
|
||||
datasources = [
|
||||
{
|
||||
name = "Prometheus";
|
||||
type = "prometheus";
|
||||
uid = "prometheus";
|
||||
access = "proxy";
|
||||
url = "http://localhost:9090";
|
||||
isDefault = true;
|
||||
jsonData.timeInterval = "15s";
|
||||
}
|
||||
{
|
||||
name = "Loki";
|
||||
type = "loki";
|
||||
uid = "loki";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:3100";
|
||||
}
|
||||
];
|
||||
};
|
||||
dashboards.settings = {
|
||||
apiVersion = 1;
|
||||
providers = [
|
||||
{
|
||||
name = "default";
|
||||
type = "file";
|
||||
disableDeletion = true;
|
||||
options.path = toString dashboardDir;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
virtualHosts."_" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = 80;
|
||||
ssl = false;
|
||||
extraParameters = [ "default_server" ];
|
||||
}
|
||||
];
|
||||
root = "${landingRoot}";
|
||||
};
|
||||
virtualHosts."stats.localhost" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = 80;
|
||||
ssl = false;
|
||||
}
|
||||
];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:3000";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = "proxy_set_header Host $host;";
|
||||
};
|
||||
};
|
||||
virtualHosts."prometheus.localhost" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = 80;
|
||||
ssl = false;
|
||||
}
|
||||
];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:9090";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = lib.genAttrs [ "loki" "alloy" "prometheus" "grafana" "nginx" ] (_: {
|
||||
serviceConfig.FailureAction = "exit-force";
|
||||
});
|
||||
};
|
||||
net.expose.tcp = [ 80 ];
|
||||
resources.mem = "2G";
|
||||
}
|
||||
// {
|
||||
hostConfig = {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
port = nodePort;
|
||||
enabledCollectors = [
|
||||
"systemd"
|
||||
"processes"
|
||||
];
|
||||
disabledCollectors = [
|
||||
"netstat"
|
||||
"softnet"
|
||||
];
|
||||
# radeon sometimes breaks on kexec
|
||||
extraFlags = [
|
||||
"--collector.systemd.enable-task-metrics"
|
||||
"--collector.hwmon.chip-exclude=radeon"
|
||||
];
|
||||
};
|
||||
services.cadvisor = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
port = cadvPort;
|
||||
};
|
||||
};
|
||||
};
|
||||
hostAddr = c.hostAddr;
|
||||
in
|
||||
c
|
||||
@@ -0,0 +1,61 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; }) mkRow loki mkDashboard;
|
||||
|
||||
mkLogs =
|
||||
{
|
||||
title,
|
||||
expr,
|
||||
w ? 24,
|
||||
h ? 10,
|
||||
}:
|
||||
{
|
||||
inherit title;
|
||||
type = "logs";
|
||||
datasource = loki;
|
||||
gridPos = { inherit w h; };
|
||||
options = {
|
||||
showTime = true;
|
||||
showLabels = true;
|
||||
sortOrder = "Descending";
|
||||
wrapLogMessage = true;
|
||||
enableLogDetails = true;
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = loki;
|
||||
inherit expr;
|
||||
}
|
||||
];
|
||||
};
|
||||
in
|
||||
mkDashboard {
|
||||
name = "access-logs.json";
|
||||
title = "Access Logs";
|
||||
uid = "access-logs";
|
||||
from = "now-6h";
|
||||
panels = [
|
||||
(mkRow "HTTP")
|
||||
(mkLogs {
|
||||
title = "nginx (access + error)";
|
||||
expr = ''{syslog_id="nginx"}'';
|
||||
})
|
||||
(mkRow "SSH")
|
||||
(mkLogs {
|
||||
title = "sshpiper (all connections)";
|
||||
expr = ''{container="sshpiper"}'';
|
||||
})
|
||||
(mkLogs {
|
||||
title = "Failed / invalid auth attempts (all backends, incl. host)";
|
||||
expr = ''{syslog_id=~"sshd.*"} |~ "Failed|Invalid|error:"'';
|
||||
})
|
||||
(mkLogs {
|
||||
title = "sshd auth events (all backends, incl. host)";
|
||||
expr = ''{syslog_id=~"sshd.*"}'';
|
||||
})
|
||||
(mkLogs {
|
||||
title = "Host sshd (port 5952)";
|
||||
expr = ''{syslog_id=~"sshd.*", container=""}'';
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; }) prom mkDashboard;
|
||||
in
|
||||
mkDashboard {
|
||||
name = "containers.json";
|
||||
title = "Containers";
|
||||
uid = "containers";
|
||||
from = "now-3h";
|
||||
panels = [
|
||||
{
|
||||
title = "Container States";
|
||||
type = "piechart";
|
||||
gridPos = {
|
||||
h = 8;
|
||||
w = 8;
|
||||
};
|
||||
datasource = prom;
|
||||
options = {
|
||||
pieType = "donut";
|
||||
legend = {
|
||||
displayMode = "table";
|
||||
placement = "right";
|
||||
values = [
|
||||
"value"
|
||||
"percent"
|
||||
];
|
||||
};
|
||||
tooltip.mode = "single";
|
||||
};
|
||||
fieldConfig.overrides = [
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "active";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "green";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "failed";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "red";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "inactive";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "blue";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "activating";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "yellow";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "unknown";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "color";
|
||||
value = {
|
||||
mode = "fixed";
|
||||
fixedColor = "gray";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
expr = ''count by (state) ((label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service")) or on(container) label_replace(container_defined, "state", "unknown", "container", ".+"))'';
|
||||
instant = true;
|
||||
legendFormat = "{{state}}";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
title = "Container States";
|
||||
type = "table";
|
||||
gridPos = {
|
||||
h = 16;
|
||||
w = 24;
|
||||
};
|
||||
datasource = prom;
|
||||
options.footer.show = false;
|
||||
fieldConfig = {
|
||||
defaults = { };
|
||||
overrides = [
|
||||
{
|
||||
matcher = {
|
||||
id = "byName";
|
||||
options = "State";
|
||||
};
|
||||
properties = [
|
||||
{
|
||||
id = "custom.displayMode";
|
||||
value = "color-text";
|
||||
}
|
||||
{
|
||||
id = "mappings";
|
||||
value = [
|
||||
{
|
||||
type = "value";
|
||||
options.active = {
|
||||
text = "active";
|
||||
color = "green";
|
||||
index = 0;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.failed = {
|
||||
text = "failed";
|
||||
color = "red";
|
||||
index = 1;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.activating = {
|
||||
text = "activating";
|
||||
color = "yellow";
|
||||
index = 2;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.inactive = {
|
||||
text = "inactive";
|
||||
color = "blue";
|
||||
index = 3;
|
||||
};
|
||||
}
|
||||
{
|
||||
type = "value";
|
||||
options.unknown = {
|
||||
text = "unknown";
|
||||
color = "gray";
|
||||
index = 4;
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
expr = ''
|
||||
(label_replace(node_systemd_unit_state{name=~"container@.*"} == 1, "container", "$1", "name", "container@(.+)\\.service"))
|
||||
or on(container)
|
||||
label_replace(container_defined, "state", "unknown", "container", ".+")
|
||||
'';
|
||||
instant = true;
|
||||
format = "table";
|
||||
legendFormat = "__auto";
|
||||
}
|
||||
];
|
||||
transformations = [
|
||||
{
|
||||
id = "organize";
|
||||
options = {
|
||||
excludeByName = {
|
||||
Time = true;
|
||||
Value = true;
|
||||
"__name__" = true;
|
||||
instance = true;
|
||||
job = true;
|
||||
name = true;
|
||||
};
|
||||
renameByName = {
|
||||
container = "Container";
|
||||
state = "State";
|
||||
};
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
title = "Container State (1 = running, 0 = stopped/failed)";
|
||||
type = "timeseries";
|
||||
gridPos = {
|
||||
h = 8;
|
||||
w = 24;
|
||||
};
|
||||
datasource = prom;
|
||||
fieldConfig.defaults = {
|
||||
custom.lineWidth = 2;
|
||||
color.mode = "palette-classic";
|
||||
min = 0;
|
||||
max = 1;
|
||||
};
|
||||
options = {
|
||||
tooltip.mode = "multi";
|
||||
legend = {
|
||||
displayMode = "table";
|
||||
placement = "bottom";
|
||||
};
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
expr = ''label_replace(node_systemd_unit_state{name=~"container@.*",state="active"}, "container", "$1", "name", "container@(.+).service")'';
|
||||
legendFormat = "{{container}}";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
prom = {
|
||||
type = "prometheus";
|
||||
uid = "prometheus";
|
||||
};
|
||||
loki = {
|
||||
type = "loki";
|
||||
uid = "loki";
|
||||
};
|
||||
|
||||
autoLayout =
|
||||
panels:
|
||||
let
|
||||
step =
|
||||
acc: panel:
|
||||
let
|
||||
w = panel.gridPos.w;
|
||||
h = panel.gridPos.h;
|
||||
wrap = acc.x + w > 24;
|
||||
x = if wrap then 0 else acc.x;
|
||||
y = if wrap then acc.y + acc.rowH else acc.y;
|
||||
rowH = if wrap then h else lib.max acc.rowH h;
|
||||
in
|
||||
{
|
||||
x = x + w;
|
||||
inherit y rowH;
|
||||
result = acc.result ++ [
|
||||
(
|
||||
panel
|
||||
// {
|
||||
gridPos = panel.gridPos // {
|
||||
inherit x y;
|
||||
};
|
||||
}
|
||||
)
|
||||
];
|
||||
};
|
||||
in
|
||||
(lib.foldl' step {
|
||||
x = 0;
|
||||
y = 0;
|
||||
rowH = 0;
|
||||
result = [ ];
|
||||
} panels).result;
|
||||
|
||||
T = expr: legendFormat: {
|
||||
datasource = prom;
|
||||
inherit expr legendFormat;
|
||||
};
|
||||
|
||||
mkRow = title: {
|
||||
inherit title;
|
||||
type = "row";
|
||||
collapsed = false;
|
||||
panels = [ ];
|
||||
gridPos = {
|
||||
w = 24;
|
||||
h = 1;
|
||||
};
|
||||
};
|
||||
|
||||
mkStat =
|
||||
{
|
||||
title,
|
||||
w ? 4,
|
||||
h ? 4,
|
||||
expr,
|
||||
unit ? "short",
|
||||
decimals ? 2,
|
||||
}:
|
||||
{
|
||||
inherit title;
|
||||
type = "stat";
|
||||
datasource = prom;
|
||||
gridPos = { inherit w h; };
|
||||
fieldConfig.defaults = {
|
||||
inherit unit decimals;
|
||||
color.mode = "thresholds";
|
||||
thresholds = {
|
||||
mode = "absolute";
|
||||
steps = [
|
||||
{
|
||||
color = "blue";
|
||||
value = null;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
options = {
|
||||
reduceOptions.calcs = [ "lastNotNull" ];
|
||||
colorMode = "value";
|
||||
textMode = "value";
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
inherit expr;
|
||||
instant = true;
|
||||
legendFormat = "";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
mkGauge =
|
||||
{
|
||||
title,
|
||||
w ? 4,
|
||||
h ? 4,
|
||||
expr,
|
||||
unit ? "percent",
|
||||
min ? 0,
|
||||
max ? 100,
|
||||
}:
|
||||
{
|
||||
inherit title;
|
||||
type = "gauge";
|
||||
datasource = prom;
|
||||
gridPos = { inherit w h; };
|
||||
fieldConfig.defaults = {
|
||||
inherit unit min max;
|
||||
thresholds = {
|
||||
mode = "absolute";
|
||||
steps = [
|
||||
{
|
||||
color = "green";
|
||||
value = null;
|
||||
}
|
||||
{
|
||||
color = "yellow";
|
||||
value = 70;
|
||||
}
|
||||
{
|
||||
color = "red";
|
||||
value = 90;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
options = {
|
||||
reduceOptions.calcs = [ "lastNotNull" ];
|
||||
orientation = "auto";
|
||||
showThresholdLabels = false;
|
||||
showThresholdMarkers = true;
|
||||
};
|
||||
targets = [
|
||||
{
|
||||
datasource = prom;
|
||||
inherit expr;
|
||||
instant = true;
|
||||
legendFormat = "";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
mkTs =
|
||||
{
|
||||
title,
|
||||
w ? 24,
|
||||
h ? 8,
|
||||
unit ? "short",
|
||||
targets,
|
||||
}:
|
||||
{
|
||||
inherit title targets;
|
||||
type = "timeseries";
|
||||
datasource = prom;
|
||||
gridPos = { inherit w h; };
|
||||
fieldConfig.defaults = {
|
||||
inherit unit;
|
||||
custom = {
|
||||
lineWidth = 1;
|
||||
fillOpacity = 5;
|
||||
};
|
||||
color.mode = "palette-classic";
|
||||
};
|
||||
options = {
|
||||
tooltip.mode = "multi";
|
||||
legend = {
|
||||
displayMode = "table";
|
||||
placement = "bottom";
|
||||
calcs = [
|
||||
"mean"
|
||||
"max"
|
||||
"lastNotNull"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
mkDashboard =
|
||||
{
|
||||
name,
|
||||
title,
|
||||
uid,
|
||||
from ? "now-1h",
|
||||
panels,
|
||||
templating ? [ ],
|
||||
}:
|
||||
pkgs.writeText name (
|
||||
builtins.toJSON (
|
||||
{
|
||||
inherit title uid;
|
||||
schemaVersion = 37;
|
||||
refresh = "30s";
|
||||
time = {
|
||||
inherit from;
|
||||
to = "now";
|
||||
};
|
||||
panels = lib.imap1 (id: p: p // { inherit id; }) (autoLayout panels);
|
||||
}
|
||||
// lib.optionalAttrs (templating != [ ]) { templating.list = templating; }
|
||||
)
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit
|
||||
prom
|
||||
loki
|
||||
autoLayout
|
||||
T
|
||||
mkRow
|
||||
mkStat
|
||||
mkGauge
|
||||
mkTs
|
||||
mkDashboard
|
||||
;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; })
|
||||
T
|
||||
mkRow
|
||||
mkStat
|
||||
mkGauge
|
||||
mkTs
|
||||
mkDashboard
|
||||
;
|
||||
in
|
||||
mkDashboard {
|
||||
name = "host.json";
|
||||
title = "Host";
|
||||
uid = "host";
|
||||
panels = [
|
||||
|
||||
(mkRow "Overview")
|
||||
(mkGauge {
|
||||
title = "CPU";
|
||||
expr = ''100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])))'';
|
||||
})
|
||||
(mkGauge {
|
||||
title = "Memory used";
|
||||
expr = "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Load (1m)";
|
||||
unit = "short";
|
||||
decimals = 2;
|
||||
expr = "node_load1";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Load (5m)";
|
||||
unit = "short";
|
||||
decimals = 2;
|
||||
expr = "node_load5";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Uptime";
|
||||
unit = "dtdurations";
|
||||
decimals = 0;
|
||||
expr = "node_time_seconds - node_boot_time_seconds";
|
||||
})
|
||||
|
||||
(mkRow "CPU")
|
||||
(mkTs {
|
||||
title = "CPU usage (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="user"}[5m])) * 100'' "user")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="system"}[5m])) * 100'' "system")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100'' "iowait")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100'' "steal")
|
||||
(T ''avg(rate(node_cpu_seconds_total{mode="softirq"}[5m])) * 100'' "softirq")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Per-core usage (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''(1 - rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100'' "cpu{{cpu}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "CPU temperatures";
|
||||
unit = "celsius";
|
||||
targets = [
|
||||
(T "node_hwmon_temp_celsius" "{{chip}} {{sensor}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Load average";
|
||||
targets = [
|
||||
(T "node_load1" "1m")
|
||||
(T "node_load5" "5m")
|
||||
(T "node_load15" "15m")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "Memory")
|
||||
(mkTs {
|
||||
title = "Memory";
|
||||
unit = "bytes";
|
||||
targets = [
|
||||
(T "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes" "used")
|
||||
(T "node_memory_Cached_bytes + node_memory_Buffers_bytes" "cache + buffers")
|
||||
(T "node_memory_MemAvailable_bytes" "available")
|
||||
(T "node_memory_MemTotal_bytes" "total")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Swap";
|
||||
unit = "bytes";
|
||||
targets = [
|
||||
(T "node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes" "used")
|
||||
(T "node_memory_SwapTotal_bytes" "total")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "Disk")
|
||||
(mkTs {
|
||||
title = "Filesystem usage (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''100 * (1 - node_filesystem_avail_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"} / node_filesystem_size_bytes{fstype=~"btrfs|ext4|xfs|vfat|f2fs"})'' "{{mountpoint}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk IO (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T ''rate(node_disk_read_bytes_total{device!~"loop.*"}[5m])'' "{{device}} read")
|
||||
(T ''rate(node_disk_written_bytes_total{device!~"loop.*"}[5m])'' "{{device}} write")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk IO utilisation (%)";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T ''rate(node_disk_io_time_seconds_total{device!~"loop.*"}[5m]) * 100'' "{{device}}")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "Network")
|
||||
(mkTs {
|
||||
title = "Network (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T ''rate(node_network_receive_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx")
|
||||
(T ''rate(node_network_transmit_bytes_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network errors + drops (pps)";
|
||||
targets = [
|
||||
(T ''rate(node_network_receive_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx err")
|
||||
(T ''rate(node_network_transmit_errs_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx err")
|
||||
(T ''rate(node_network_receive_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} rx drop")
|
||||
(T ''rate(node_network_transmit_drop_total{device!~"lo|br-.*|vh-.*|vc-.*"}[5m])'' "{{device}} tx drop")
|
||||
];
|
||||
})
|
||||
|
||||
(mkRow "System")
|
||||
(mkTs {
|
||||
title = "Context switches + forks (ops/s)";
|
||||
targets = [
|
||||
(T "rate(node_context_switches_total[5m])" "context switches")
|
||||
(T "rate(node_forks_total[5m])" "forks")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "File descriptors";
|
||||
targets = [
|
||||
(T "node_filefd_allocated" "allocated")
|
||||
(T "node_filefd_maximum" "limit")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Processes";
|
||||
targets = [
|
||||
(T "node_procs_running" "running")
|
||||
(T "node_procs_blocked" "blocked on IO")
|
||||
];
|
||||
})
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
{ pkgs, lib }:
|
||||
let
|
||||
inherit (import ./helpers.nix { inherit pkgs lib; })
|
||||
T
|
||||
mkRow
|
||||
mkStat
|
||||
mkTs
|
||||
prom
|
||||
mkDashboard
|
||||
;
|
||||
|
||||
ctid = "id=~\"/machine.slice/container@.+\\\\.service/payload/.+\"";
|
||||
ctsel = "id=~\"/machine.slice/container@$container\\\\.service/payload/.+\"";
|
||||
|
||||
ctall = "id=~\"/machine.slice/container@.+\\\\.service.*\"";
|
||||
ctnet = "id=~\"/machine.slice/container@$container\\\\.service.*\"";
|
||||
|
||||
labeled =
|
||||
m:
|
||||
"label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service/payload.+\")";
|
||||
byContainer = m: "sum by (container) (${labeled m})";
|
||||
|
||||
labeledAll =
|
||||
m:
|
||||
"label_replace(${m}, \"container\", \"$1\", \"id\", \"/machine.slice/container@(.+)\\\\.service.*\")";
|
||||
byContainerAll = m: "sum by (container) (${labeledAll m})";
|
||||
|
||||
nmSvc = m: "label_replace(${m}, \"container\", \"$1\", \"name\", \"container@(.+)\\\\.service\")";
|
||||
|
||||
orZero = m: fb: "(${m}) or on(container) (${fb} * 0)";
|
||||
|
||||
cpuAll =
|
||||
rate:
|
||||
orZero (byContainer "rate(container_cpu_usage_seconds_total{${ctid}}[${rate}])") "container_limit_cpu_cores";
|
||||
memAll = orZero (byContainerAll "container_memory_rss{${ctall}}") "container_limit_memory_bytes";
|
||||
taskAll = nmSvc "node_systemd_unit_tasks_current{name=~\"container@.+\"}";
|
||||
in
|
||||
mkDashboard {
|
||||
name = "resources.json";
|
||||
title = "Container Resources";
|
||||
uid = "container-resources";
|
||||
|
||||
templating = [
|
||||
{
|
||||
name = "container";
|
||||
type = "query";
|
||||
datasource = prom;
|
||||
query = "label_values(container_limit_cpu_cores, container)";
|
||||
refresh = 2;
|
||||
sort = 1;
|
||||
includeAll = true;
|
||||
multi = true;
|
||||
current = {
|
||||
selected = true;
|
||||
text = [ "All" ];
|
||||
value = [ "$__all" ];
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
panels = [
|
||||
(mkRow "Overview")
|
||||
(mkStat {
|
||||
title = "Running";
|
||||
decimals = 0;
|
||||
expr = "count(node_systemd_unit_state{name=~\"container@.*\",state=\"active\"} == 1) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "CPU used";
|
||||
decimals = 2;
|
||||
expr = "sum(${cpuAll "5m"}) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "CPU limit";
|
||||
decimals = 2;
|
||||
expr = "sum(container_limit_cpu_cores) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Mem RSS";
|
||||
decimals = 1;
|
||||
unit = "bytes";
|
||||
expr = "sum(${memAll}) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Mem limit";
|
||||
decimals = 1;
|
||||
unit = "bytes";
|
||||
expr = "sum(container_limit_memory_bytes) or vector(0)";
|
||||
})
|
||||
(mkStat {
|
||||
title = "Tasks";
|
||||
decimals = 0;
|
||||
expr = "sum(${taskAll}) or vector(0)";
|
||||
})
|
||||
|
||||
(mkRow "All Containers")
|
||||
(mkTs {
|
||||
title = "CPU (cores)";
|
||||
h = 24;
|
||||
targets = [ (T (cpuAll "5m") "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Memory RSS";
|
||||
unit = "bytes";
|
||||
h = 24;
|
||||
targets = [ (T memAll "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Tasks";
|
||||
h = 24;
|
||||
targets = [ (T taskAll "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "CPU throttled %";
|
||||
unit = "percent";
|
||||
h = 12;
|
||||
targets = [
|
||||
(T "(${byContainer "rate(container_cpu_cfs_throttled_periods_total{${ctid}}[5m])"} / ${byContainer "rate(container_cpu_cfs_periods_total{${ctid}}[5m])"}) * 100 or on(container) (container_limit_cpu_cores * 0)" "{{container}}")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network egress (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [ (T "sum by (container) (container_network_receive_rate)" "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network ingress (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [ (T "sum by (container) (container_network_transmit_rate)" "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk read (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [ (T (byContainer "rate(container_fs_reads_bytes_total{${ctid}}[5m])") "{{container}}") ];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk write (bytes/s)";
|
||||
unit = "Bps";
|
||||
h = 24;
|
||||
targets = [
|
||||
(T (byContainer "rate(container_fs_writes_bytes_total{${ctid}}[5m])") "{{container}}")
|
||||
];
|
||||
})
|
||||
|
||||
{
|
||||
type = "row";
|
||||
title = "$container";
|
||||
repeat = "container";
|
||||
collapsed = true;
|
||||
panels = [ ];
|
||||
gridPos = {
|
||||
w = 24;
|
||||
h = 1;
|
||||
};
|
||||
}
|
||||
(mkTs {
|
||||
title = "CPU (cores)";
|
||||
targets = [
|
||||
(T "sum(rate(container_cpu_usage_seconds_total{${ctsel}}[5m]))" "used")
|
||||
(T "container_limit_cpu_cores{container=\"$container\"}" "limit")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "CPU throttled %";
|
||||
unit = "percent";
|
||||
targets = [
|
||||
(T "(sum(rate(container_cpu_cfs_throttled_periods_total{${ctsel}}[5m])) / sum(rate(container_cpu_cfs_periods_total{${ctsel}}[5m]))) * 100 or vector(0)" "throttled %")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Memory";
|
||||
unit = "bytes";
|
||||
targets = [
|
||||
(T "sum(container_memory_rss{${ctnet}})" "RSS")
|
||||
(T "sum(container_memory_working_set_bytes{${ctnet}})" "working set")
|
||||
(T "container_limit_memory_bytes{container=\"$container\"}" "limit")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Network (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T "container_network_receive_rate{container=\"$container\"}" "{{device}} ↑")
|
||||
(T "container_network_transmit_rate{container=\"$container\"}" "{{device}} ↓")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Disk IO (bytes/s)";
|
||||
unit = "Bps";
|
||||
targets = [
|
||||
(T "sum(rate(container_fs_reads_bytes_total{${ctsel}}[5m]))" "read")
|
||||
(T "sum(rate(container_fs_writes_bytes_total{${ctsel}}[5m]))" "write")
|
||||
];
|
||||
})
|
||||
(mkTs {
|
||||
title = "Tasks";
|
||||
targets = [ (T "node_systemd_unit_tasks_current{name=\"container@$container.service\"}" "tasks") ];
|
||||
})
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user