2024-09-13 02:28:10 +02:00
|
|
|
{
|
|
|
|
config,
|
|
|
|
lib,
|
|
|
|
pkgs,
|
|
|
|
...
|
2024-09-13 18:01:30 +02:00
|
|
|
}: let
|
|
|
|
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
|
|
|
|
|
|
|
|
log_params = [
|
|
|
|
"remote_addr"
|
|
|
|
"time_local"
|
|
|
|
"scheme"
|
|
|
|
"server_name"
|
|
|
|
"status"
|
|
|
|
"body_bytes_sent"
|
|
|
|
"http_user_agent"
|
|
|
|
"request_length"
|
|
|
|
"request_time"
|
|
|
|
"request_uri"
|
|
|
|
"upstream_response_time"
|
|
|
|
"geoip2_data_country_code"
|
|
|
|
"geoip2_data_city_name"
|
|
|
|
"geoip2_data_latitude"
|
|
|
|
"geoip2_data_longitude"
|
|
|
|
];
|
|
|
|
log_format = lib.concatStringsSep " " (map (p: "${p}=\"\$${p}\"") log_params);
|
|
|
|
log_file = "/var/log/nginx/access.log";
|
|
|
|
in {
|
2024-09-11 09:05:52 +02:00
|
|
|
services.grafana = {
|
|
|
|
enable = true;
|
|
|
|
settings = {
|
|
|
|
server = {
|
|
|
|
http_addr = "127.0.0.1";
|
|
|
|
http_port = 21983;
|
|
|
|
domain = "grafana.winston.sh";
|
|
|
|
serve_from_sub_path = true;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
provision = {
|
|
|
|
enable = true;
|
|
|
|
datasources.settings.datasources = [
|
2024-09-13 02:28:10 +02:00
|
|
|
(with config.services.prometheus; {
|
2024-09-11 09:05:52 +02:00
|
|
|
name = "Prometheus";
|
|
|
|
type = "prometheus";
|
2024-09-13 02:28:10 +02:00
|
|
|
url = "http://${listenAddress}:${toString port}";
|
|
|
|
})
|
2024-09-11 09:05:52 +02:00
|
|
|
];
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
2024-09-13 18:01:30 +02:00
|
|
|
services.nginx = {
|
|
|
|
additionalModules = [pkgs.nginxModules.geoip2];
|
|
|
|
commonHttpConfig =
|
|
|
|
# nginx
|
|
|
|
''
|
|
|
|
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
|
|
|
|
auto_reload 5m;
|
|
|
|
$geoip2_metadata_country_build metadata build_epoch;
|
|
|
|
$geoip2_data_country_code default=Unknown country iso_code;
|
|
|
|
}
|
|
|
|
|
|
|
|
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
|
|
|
|
auto_reload 5m;
|
|
|
|
$geoip2_data_city_name city names en;
|
|
|
|
$geoip2_data_latitude default=0.0 location latitude;
|
|
|
|
$geoip2_data_longitude default=0.0 location longitude;
|
|
|
|
}
|
|
|
|
|
|
|
|
log_format combined_geoip '${log_format}';
|
|
|
|
access_log ${log_file} combined_geoip;
|
|
|
|
'';
|
|
|
|
statusPage = true;
|
|
|
|
};
|
2024-09-11 09:05:52 +02:00
|
|
|
|
2024-09-13 18:01:30 +02:00
|
|
|
age.secrets."services/prometheus/minio-bearer-token".owner = "prometheus";
|
2024-09-11 09:05:52 +02:00
|
|
|
services.prometheus = {
|
|
|
|
enable = true;
|
2024-09-13 18:01:30 +02:00
|
|
|
|
|
|
|
# using agenix means secrets are mounted in /run/agenix which is inaccessible for the `promtool check` derivation
|
|
|
|
checkConfig = "syntax-only";
|
|
|
|
|
2024-09-11 09:05:52 +02:00
|
|
|
globalConfig.scrape_interval = "10s";
|
2024-09-13 02:28:10 +02:00
|
|
|
scrapeConfigs =
|
2024-09-13 18:01:30 +02:00
|
|
|
(builtins.map (config: {
|
|
|
|
inherit (config) job_name;
|
|
|
|
static_configs = [{targets = ["localhost:${toString config.port}"];}];
|
|
|
|
}) [
|
|
|
|
{
|
|
|
|
job_name = "fail2ban";
|
|
|
|
port = 9191;
|
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "nginx";
|
|
|
|
port = config.services.prometheus.exporters.nginx.port;
|
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "nginxlog";
|
|
|
|
port = config.services.prometheus.exporters.nginxlog.port;
|
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "node";
|
|
|
|
port = config.services.prometheus.exporters.node.port;
|
|
|
|
}
|
|
|
|
{
|
|
|
|
job_name = "postgres";
|
|
|
|
port = config.services.prometheus.exporters.postgres.port;
|
|
|
|
}
|
|
|
|
])
|
|
|
|
++ [
|
2024-09-13 02:28:10 +02:00
|
|
|
{
|
2024-09-13 18:01:30 +02:00
|
|
|
job_name = "minio";
|
|
|
|
bearer_token_file = config.age.secrets."services/prometheus/minio-bearer-token".path;
|
|
|
|
metrics_path = "/minio/v2/metrics/cluster";
|
|
|
|
static_configs = [{targets = [config.services.minio.listenAddress];}];
|
2024-09-13 02:28:10 +02:00
|
|
|
}
|
|
|
|
{
|
2024-09-13 18:01:30 +02:00
|
|
|
job_name = "forgejo";
|
|
|
|
static_configs = with config.services.forgejo.settings.server; [{targets = ["${HTTP_ADDR}:${toString HTTP_PORT}"];}];
|
2024-09-13 02:28:10 +02:00
|
|
|
}
|
|
|
|
];
|
2024-09-11 09:05:52 +02:00
|
|
|
|
|
|
|
exporters = {
|
|
|
|
nginx.enable = true;
|
|
|
|
nginxlog = {
|
|
|
|
enable = true;
|
|
|
|
group = "nginx";
|
2024-09-13 18:01:30 +02:00
|
|
|
settings.namespaces = [
|
|
|
|
{
|
|
|
|
format = log_format;
|
2024-09-13 02:28:10 +02:00
|
|
|
|
|
|
|
metrics_override.prefix = "nginxlog";
|
|
|
|
|
2024-09-13 18:01:30 +02:00
|
|
|
relabel_configs = [
|
|
|
|
{
|
|
|
|
target_label = "vhost";
|
|
|
|
from = "server_name";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
target_label = "status";
|
|
|
|
from = "status";
|
|
|
|
only_counter = true;
|
|
|
|
}
|
|
|
|
{
|
|
|
|
target_label = "geo_city";
|
|
|
|
from = "geoip2_data_city_name";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
target_label = "geo_country";
|
|
|
|
from = "geoip2_data_country_code";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
target_label = "latitude";
|
|
|
|
from = "geoip2_data_latitude";
|
|
|
|
}
|
|
|
|
{
|
|
|
|
target_label = "longitude";
|
|
|
|
from = "geoip2_data_longitude";
|
|
|
|
}
|
|
|
|
];
|
2024-09-13 02:28:10 +02:00
|
|
|
|
2024-09-13 18:01:30 +02:00
|
|
|
source.files = [log_file];
|
|
|
|
}
|
|
|
|
];
|
2024-09-13 02:28:10 +02:00
|
|
|
};
|
|
|
|
node = {
|
|
|
|
enable = true;
|
2024-09-13 18:01:30 +02:00
|
|
|
enabledCollectors = ["logind" "processes" "systemd"];
|
2024-09-13 02:28:10 +02:00
|
|
|
disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
|
|
|
|
};
|
|
|
|
postgres = {
|
|
|
|
enable = true;
|
|
|
|
# FIXME: this is not ideal...
|
|
|
|
runAsLocalSuperUser = true;
|
2024-09-11 09:05:52 +02:00
|
|
|
};
|
|
|
|
};
|
2024-09-13 02:28:10 +02:00
|
|
|
};
|
2024-09-11 09:05:52 +02:00
|
|
|
|
2024-09-13 02:28:10 +02:00
|
|
|
systemd.services.prometheus-fail2ban-exporter = {
|
|
|
|
wantedBy = ["multi-user.target"];
|
|
|
|
after = ["network.target"];
|
|
|
|
requires = ["network-online.target"];
|
|
|
|
serviceConfig = {
|
|
|
|
ExecStart = [(lib.getExe pkgs.prometheus-fail2ban-exporter)];
|
|
|
|
Restart = "on-failure";
|
|
|
|
NoNewPrivileges = true;
|
|
|
|
User = "root";
|
|
|
|
Group = "root";
|
2024-09-11 09:05:52 +02:00
|
|
|
};
|
|
|
|
};
|
|
|
|
|
2024-09-13 02:28:10 +02:00
|
|
|
services.nginx.virtualHosts = with config.services.grafana.settings.server; {
|
|
|
|
${domain} = {
|
|
|
|
forceSSL = true;
|
|
|
|
enableACME = false;
|
|
|
|
useACMEHost = "winston.sh";
|
2024-09-11 09:05:52 +02:00
|
|
|
|
2024-09-13 02:28:10 +02:00
|
|
|
locations."/" = {
|
|
|
|
proxyPass = "http://${http_addr}:${toString http_port}";
|
|
|
|
proxyWebsockets = true;
|
|
|
|
recommendedProxySettings = true;
|
|
|
|
};
|
2024-09-11 09:05:52 +02:00
|
|
|
};
|
|
|
|
};
|
|
|
|
}
|