initial commit
This commit is contained in:
Floweynt
@@ -0,0 +1,386 @@
|
||||
{
|
||||
name,
|
||||
hostInfo,
|
||||
dsl,
|
||||
pkgs,
|
||||
lib,
|
||||
containers,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (dsl) mkContainer;
|
||||
inherit (hostInfo.paths) data;
|
||||
|
||||
nodePort = 9100;
|
||||
cadvPort = 9200;
|
||||
|
||||
containerNetworkRulesFile =
|
||||
let
|
||||
mkRules =
|
||||
metric: record:
|
||||
lib.mapAttrsToList (cname: c: {
|
||||
inherit record;
|
||||
expr = "rate(${metric}{device=\"${c.iface}\"}[5m])";
|
||||
labels = {
|
||||
container = cname;
|
||||
};
|
||||
}) containers;
|
||||
in
|
||||
pkgs.writeText "container-network.yml" (
|
||||
builtins.toJSON {
|
||||
groups = [
|
||||
{
|
||||
name = "container-network";
|
||||
rules =
|
||||
mkRules "node_network_receive_bytes_total" "container_network_receive_rate"
|
||||
++ mkRules "node_network_transmit_bytes_total" "container_network_transmit_rate";
|
||||
}
|
||||
];
|
||||
}
|
||||
);
|
||||
|
||||
containerDefinedRulesFile = pkgs.writeText "container-defined.yml" (
|
||||
builtins.toJSON {
|
||||
groups = [
|
||||
{
|
||||
name = "container-defined";
|
||||
rules = lib.mapAttrsToList (cname: _: {
|
||||
record = "container_defined";
|
||||
expr = "1";
|
||||
labels = {
|
||||
container = cname;
|
||||
};
|
||||
}) containers;
|
||||
}
|
||||
];
|
||||
}
|
||||
);
|
||||
|
||||
alloyConfig = pkgs.writeText "config.alloy" ''
|
||||
loki.source.journal "journal" {
|
||||
max_age = "12h"
|
||||
path = "/var/log/journal"
|
||||
labels = { job = "systemd-journal" }
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
forward_to = [loki.write.local.receiver]
|
||||
}
|
||||
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal__hostname"]
|
||||
target_label = "host"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_syslog_identifier"]
|
||||
target_label = "syslog_id"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_machine_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
}
|
||||
|
||||
loki.write "local" {
|
||||
endpoint {
|
||||
url = "http://127.0.0.1:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
'';
|
||||
|
||||
dashboards = lib.filterAttrs (name: _: lib.hasSuffix "-dashboard" name) (
|
||||
dsl.listMap ./overseer (_: path: import path { inherit pkgs lib; })
|
||||
);
|
||||
|
||||
dashboardDir = pkgs.linkFarm "grafana-dashboards" (
|
||||
lib.mapAttrsToList (name: dashboard: {
|
||||
name = "${lib.removeSuffix "-dashboard" name}.json";
|
||||
path = dashboard;
|
||||
}) dashboards
|
||||
);
|
||||
|
||||
landingRoot = pkgs.writeTextDir "index.html" ''
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Overseer</title>
|
||||
<style>
|
||||
body { font-family: sans-serif; max-width: 480px; margin: 4rem auto; }
|
||||
ul { padding: 0; list-style: none; }
|
||||
li { margin: .5rem 0; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Overseer</h1>
|
||||
<ul>
|
||||
<li><a data-host="stats.localhost">stats.localhost</a> — Grafana</li>
|
||||
<li><a data-host="prometheus.localhost">prometheus.localhost</a> — Prometheus</li>
|
||||
</ul>
|
||||
<script>
|
||||
const p = location.port && location.port !== "80" ? ":" + location.port : "";
|
||||
document.querySelectorAll("a[data-host]").forEach(a => {
|
||||
const h = a.dataset.host;
|
||||
a.href = "http://" + h + p;
|
||||
a.textContent = h + p;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'';
|
||||
in
|
||||
let
|
||||
c =
|
||||
mkContainer {
|
||||
inherit name;
|
||||
net.access = [
|
||||
{
|
||||
to = "@host";
|
||||
tcp = [
|
||||
nodePort
|
||||
cadvPort
|
||||
];
|
||||
}
|
||||
];
|
||||
svc.autoStart = true;
|
||||
resources.fs = {
|
||||
"/var/lib/loki" = {
|
||||
hostPath = "${data}/overseer/loki/data";
|
||||
isReadOnly = false;
|
||||
};
|
||||
"/var/lib/prometheus2" = {
|
||||
hostPath = "${data}/overseer/prometheus/data";
|
||||
isReadOnly = false;
|
||||
};
|
||||
"/var/lib/grafana" = {
|
||||
hostPath = "${data}/overseer/grafana";
|
||||
isReadOnly = false;
|
||||
};
|
||||
"/var/log/journal" = {
|
||||
hostPath = "/var/log/journal";
|
||||
isReadOnly = true;
|
||||
};
|
||||
};
|
||||
containerConfig = _: {
|
||||
services.loki = {
|
||||
enable = true;
|
||||
configuration = {
|
||||
auth_enabled = false;
|
||||
server = {
|
||||
http_listen_port = 3100;
|
||||
grpc_listen_port = 9096;
|
||||
};
|
||||
common = {
|
||||
instance_addr = "127.0.0.1";
|
||||
path_prefix = "/var/lib/loki";
|
||||
storage.filesystem = {
|
||||
chunks_directory = "/var/lib/loki/chunks";
|
||||
rules_directory = "/var/lib/loki/rules";
|
||||
};
|
||||
replication_factor = 1;
|
||||
ring.kvstore.store = "inmemory";
|
||||
};
|
||||
schema_config.configs = [
|
||||
{
|
||||
from = "2020-10-24";
|
||||
store = "tsdb";
|
||||
object_store = "filesystem";
|
||||
schema = "v13";
|
||||
index = {
|
||||
prefix = "index_";
|
||||
period = "24h";
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.alloy = {
|
||||
enable = true;
|
||||
configPath = alloyConfig;
|
||||
};
|
||||
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
port = 9090;
|
||||
globalConfig.scrape_interval = "15s";
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [ { targets = [ "${hostAddr}:${toString nodePort}" ]; } ];
|
||||
}
|
||||
{
|
||||
job_name = "cadvisor";
|
||||
static_configs = [ { targets = [ "${hostAddr}:${toString cadvPort}" ]; } ];
|
||||
}
|
||||
{
|
||||
job_name = "prometheus";
|
||||
static_configs = [ { targets = [ "localhost:9090" ]; } ];
|
||||
}
|
||||
];
|
||||
ruleFiles = [
|
||||
containerNetworkRulesFile
|
||||
containerDefinedRulesFile
|
||||
(dsl.mkContainerLimitsRuleFile pkgs containers)
|
||||
];
|
||||
rules = [
|
||||
''
|
||||
groups:
|
||||
- name: containers
|
||||
rules:
|
||||
- alert: ContainerFailed
|
||||
expr: node_systemd_unit_state{name=~"container@.*",state="failed"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.name }} entered failed state"
|
||||
- alert: ContainerOOMKill
|
||||
expr: increase(container_oom_events_total{id=~"/machine.slice/container@.+\\.service.*"}[5m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "OOM kill in {{ $labels.id }}"
|
||||
''
|
||||
];
|
||||
};
|
||||
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
dataDir = "/var/lib/grafana";
|
||||
settings = {
|
||||
server.http_port = 3000;
|
||||
server.http_addr = "127.0.0.1";
|
||||
# WORKAROUND: Grafana 10+ requires an explicit secret_key; omitting it
|
||||
# causes auth session tokens to be regenerated on every restart
|
||||
security.secret_key = "SW2YcwTIb9zpOOhoPsMm";
|
||||
"auth".disable_login_form = true;
|
||||
"auth.anonymous" = {
|
||||
enabled = true;
|
||||
org_role = "Admin";
|
||||
};
|
||||
};
|
||||
provision = {
|
||||
enable = true;
|
||||
datasources.settings = {
|
||||
apiVersion = 1;
|
||||
datasources = [
|
||||
{
|
||||
name = "Prometheus";
|
||||
type = "prometheus";
|
||||
uid = "prometheus";
|
||||
access = "proxy";
|
||||
url = "http://localhost:9090";
|
||||
isDefault = true;
|
||||
jsonData.timeInterval = "15s";
|
||||
}
|
||||
{
|
||||
name = "Loki";
|
||||
type = "loki";
|
||||
uid = "loki";
|
||||
access = "proxy";
|
||||
url = "http://127.0.0.1:3100";
|
||||
}
|
||||
];
|
||||
};
|
||||
dashboards.settings = {
|
||||
apiVersion = 1;
|
||||
providers = [
|
||||
{
|
||||
name = "default";
|
||||
type = "file";
|
||||
disableDeletion = true;
|
||||
options.path = toString dashboardDir;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
virtualHosts."_" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = 80;
|
||||
ssl = false;
|
||||
extraParameters = [ "default_server" ];
|
||||
}
|
||||
];
|
||||
root = "${landingRoot}";
|
||||
};
|
||||
virtualHosts."stats.localhost" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = 80;
|
||||
ssl = false;
|
||||
}
|
||||
];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:3000";
|
||||
proxyWebsockets = true;
|
||||
extraConfig = "proxy_set_header Host $host;";
|
||||
};
|
||||
};
|
||||
virtualHosts."prometheus.localhost" = {
|
||||
listen = [
|
||||
{
|
||||
addr = "0.0.0.0";
|
||||
port = 80;
|
||||
ssl = false;
|
||||
}
|
||||
];
|
||||
locations."/" = {
|
||||
proxyPass = "http://127.0.0.1:9090";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services = lib.genAttrs [ "loki" "alloy" "prometheus" "grafana" "nginx" ] (_: {
|
||||
serviceConfig.FailureAction = "exit-force";
|
||||
});
|
||||
};
|
||||
net.expose.tcp = [ 80 ];
|
||||
resources.mem = "2G";
|
||||
}
|
||||
// {
|
||||
hostConfig = {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
port = nodePort;
|
||||
enabledCollectors = [
|
||||
"systemd"
|
||||
"processes"
|
||||
];
|
||||
disabledCollectors = [
|
||||
"netstat"
|
||||
"softnet"
|
||||
];
|
||||
# radeon sometimes breaks on kexec
|
||||
extraFlags = [
|
||||
"--collector.systemd.enable-task-metrics"
|
||||
"--collector.hwmon.chip-exclude=radeon"
|
||||
];
|
||||
};
|
||||
services.cadvisor = {
|
||||
enable = true;
|
||||
listenAddress = "0.0.0.0";
|
||||
port = cadvPort;
|
||||
};
|
||||
};
|
||||
};
|
||||
hostAddr = c.hostAddr;
|
||||
in
|
||||
c
|
||||
Reference in New Issue
Block a user