infra/config/services/monitoring.nix

132 lines
3.5 KiB
Nix

{
config,
lib,
pkgs,
...
}: {
services.grafana = {
enable = true;
settings = {
server = {
http_addr = "127.0.0.1";
http_port = 21983;
domain = "grafana.winston.sh";
serve_from_sub_path = true;
};
};
provision = {
enable = true;
datasources.settings.datasources = [
(with config.services.prometheus; {
name = "Prometheus";
type = "prometheus";
url = "http://${listenAddress}:${toString port}";
})
];
};
};
services.nginx.statusPage = true;
services.prometheus = {
enable = true;
extraFlags = ["--web.enable-admin-api"];
globalConfig.scrape_interval = "10s";
scrapeConfigs =
builtins.map (config: {
inherit (config) job_name;
static_configs = [{targets = ["localhost:${toString config.port}"];}];
}) [
{
job_name = "fail2ban";
port = 9191;
}
{
job_name = "nginx";
port = config.services.prometheus.exporters.nginx.port;
}
{
job_name = "nginxlog";
port = config.services.prometheus.exporters.nginxlog.port;
}
{
job_name = "node";
port = config.services.prometheus.exporters.node.port;
}
{
job_name = "postgres";
port = config.services.prometheus.exporters.postgres.port;
}
];
exporters = {
nginx.enable = true;
nginxlog = {
enable = true;
group = "nginx";
settings.namespaces =
builtins.map (app: {
name = app;
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\"";
metrics_override.prefix = "nginxlog";
namespace_label = "vhost";
relabel = {
city.from = "geoip2_data_city_name";
country.from = "geoip2_data_country_name";
};
source.files = ["/var/log/nginx/${app}.access.log"];
}) [
"attic"
"atuin"
"forgejo"
"freshrss"
"minio"
"nextcloud"
"wakapi"
];
};
node = {
enable = true;
enabledCollectors = ["processes" "systemd"];
disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
};
postgres = {
enable = true;
# FIXME: this is not ideal...
runAsLocalSuperUser = true;
};
};
};
systemd.services.prometheus-fail2ban-exporter = {
wantedBy = ["multi-user.target"];
after = ["network.target"];
requires = ["network-online.target"];
serviceConfig = {
ExecStart = [(lib.getExe pkgs.prometheus-fail2ban-exporter)];
Restart = "on-failure";
NoNewPrivileges = true;
User = "root";
Group = "root";
};
};
services.nginx.virtualHosts = with config.services.grafana.settings.server; {
${domain} = {
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
locations."/" = {
proxyPass = "http://${http_addr}:${toString http_port}";
proxyWebsockets = true;
recommendedProxySettings = true;
};
};
};
}