feat: simplify nginx monitoring setup
This commit is contained in:
parent
ddcf3f836b
commit
86722826de
12 changed files with 140 additions and 104 deletions
|
@ -34,6 +34,8 @@ in {
|
|||
"services/nextcloud/admin-password.age".publicKeys = default;
|
||||
"services/nextcloud/s3-secret.age".publicKeys = default;
|
||||
|
||||
"services/prometheus/minio-bearer-token.age".publicKeys = default;
|
||||
|
||||
"services/wakapi/password-salt.env.age".publicKeys = default;
|
||||
|
||||
"system/password-root.age".publicKeys = default;
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
age-encryption.org/v1
|
||||
-> piv-p256 ML6NcA AqqvLAKvyJwwZYSpx3wINpgLbt6jnPNaz3FsdLrgc6AI
|
||||
akR0ix3DNUB4oZXEficyp4rbkOZF0y/ms7r8wr/d9Cg
|
||||
-> ssh-ed25519 zj2A2A Ul6EUnAnh4IOurvpr0i6StlrtRDMDM8HdZWBgvcQuxg
|
||||
ECDnJYCV5aS2t7o+4ixo37nMemJdANvGlsLdQO2vPCc
|
||||
--- acN/h+fasItr1Yp7ZJAUCz9K9nOo//zKMDPGI3rvgsM
|
||||
2TH’L<>È.b<>ú::.N¤Ÿ0î·ÓZ-q-šöß΃ê²6œI<C593>ÖÆ"K;V{ËÚrØRg¦Î]ªsfHVøkƒoö@ûÑè!eÎbãSy‰Þµ=‘ô öNë„î ¹d¶NŒÚLÑû7'„/’@¤¿'H¬’ÌŒ1:$ʈñ&;nZþrøÁÔ%—˸9U«‰úé¸îñrÃû{äåHÍÄð¢
%¿ý<C2BF>៲ã©ÔΆ¹¤¹¬—eeUïØ85FÜReïTÎgéy!³r˜c×D7‘k¤
|
|
@ -28,7 +28,6 @@
|
|||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/attic.access.log combined_geoip;
|
||||
client_max_body_size 512M;
|
||||
'';
|
||||
proxyPass = "http://${config.services.atticd.settings.listen}";
|
||||
|
|
|
@ -12,11 +12,6 @@
|
|||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = with config.services.atuin; {
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/atuin.access.log combined_geoip;
|
||||
'';
|
||||
proxyPass = "http://${host}:${toString port}";
|
||||
};
|
||||
};
|
||||
|
|
|
@ -55,6 +55,12 @@ in {
|
|||
ISSUE_INDEXER_TYPE = indexer;
|
||||
};
|
||||
|
||||
metrics = {
|
||||
ENABLED = true;
|
||||
ENABLED_ISSUE_BY_REPOSITORY = true;
|
||||
ENABLED_ISSUE_BY_LABEL = true;
|
||||
};
|
||||
|
||||
repository.ENABLE_PUSH_CREATE_USER = true;
|
||||
|
||||
server = rec {
|
||||
|
@ -118,19 +124,21 @@ in {
|
|||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
locations = {
|
||||
"/" = with config.services.forgejo.settings.server; {
|
||||
locations = with config.services.forgejo.settings.server; {
|
||||
"/" = {
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/forgejo.access.log combined_geoip;
|
||||
client_max_body_size 512M;
|
||||
'';
|
||||
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
|
||||
};
|
||||
|
||||
# don't spam the log with runner polls
|
||||
"/api/actions/runner.v1.RunnerService/FetchTask".extraConfig = "access_log off;";
|
||||
"/api/actions/runner.v1.RunnerService/FetchTask" = {
|
||||
extraConfig = "access_log off;";
|
||||
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -15,10 +15,5 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/freshrss.access.log combined_geoip;
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
@ -22,11 +22,6 @@
|
|||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = {
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/minio.access.log combined_geoip;
|
||||
'';
|
||||
proxyPass = "http://${config.services.minio.consoleAddress}";
|
||||
};
|
||||
};
|
||||
|
@ -39,7 +34,6 @@
|
|||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/minio.access.log combined_geoip;
|
||||
client_max_body_size 512M;
|
||||
'';
|
||||
proxyPass = "http://${config.services.minio.listenAddress}";
|
||||
|
|
|
@ -3,7 +3,29 @@
|
|||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: {
|
||||
}: let
|
||||
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
|
||||
|
||||
log_params = [
|
||||
"remote_addr"
|
||||
"time_local"
|
||||
"scheme"
|
||||
"server_name"
|
||||
"status"
|
||||
"body_bytes_sent"
|
||||
"http_user_agent"
|
||||
"request_length"
|
||||
"request_time"
|
||||
"request_uri"
|
||||
"upstream_response_time"
|
||||
"geoip2_data_country_code"
|
||||
"geoip2_data_city_name"
|
||||
"geoip2_data_latitude"
|
||||
"geoip2_data_longitude"
|
||||
];
|
||||
log_format = lib.concatStringsSep " " (map (p: "${p}=\"\$${p}\"") log_params);
|
||||
log_file = "/var/log/nginx/access.log";
|
||||
in {
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
settings = {
|
||||
|
@ -27,36 +49,74 @@
|
|||
};
|
||||
};
|
||||
|
||||
services.nginx.statusPage = true;
|
||||
services.nginx = {
|
||||
additionalModules = [pkgs.nginxModules.geoip2];
|
||||
commonHttpConfig =
|
||||
# nginx
|
||||
''
|
||||
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
|
||||
auto_reload 5m;
|
||||
$geoip2_metadata_country_build metadata build_epoch;
|
||||
$geoip2_data_country_code default=Unknown country iso_code;
|
||||
}
|
||||
|
||||
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
|
||||
auto_reload 5m;
|
||||
$geoip2_data_city_name city names en;
|
||||
$geoip2_data_latitude default=0.0 location latitude;
|
||||
$geoip2_data_longitude default=0.0 location longitude;
|
||||
}
|
||||
|
||||
log_format combined_geoip '${log_format}';
|
||||
access_log ${log_file} combined_geoip;
|
||||
'';
|
||||
statusPage = true;
|
||||
};
|
||||
|
||||
age.secrets."services/prometheus/minio-bearer-token".owner = "prometheus";
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
extraFlags = ["--web.enable-admin-api"];
|
||||
|
||||
# using agenix means secrets are mounted in /run/agenix which is inaccessible for the `promtool check` derivation
|
||||
checkConfig = "syntax-only";
|
||||
|
||||
globalConfig.scrape_interval = "10s";
|
||||
scrapeConfigs =
|
||||
builtins.map (config: {
|
||||
inherit (config) job_name;
|
||||
static_configs = [{targets = ["localhost:${toString config.port}"];}];
|
||||
}) [
|
||||
(builtins.map (config: {
|
||||
inherit (config) job_name;
|
||||
static_configs = [{targets = ["localhost:${toString config.port}"];}];
|
||||
}) [
|
||||
{
|
||||
job_name = "fail2ban";
|
||||
port = 9191;
|
||||
}
|
||||
{
|
||||
job_name = "nginx";
|
||||
port = config.services.prometheus.exporters.nginx.port;
|
||||
}
|
||||
{
|
||||
job_name = "nginxlog";
|
||||
port = config.services.prometheus.exporters.nginxlog.port;
|
||||
}
|
||||
{
|
||||
job_name = "node";
|
||||
port = config.services.prometheus.exporters.node.port;
|
||||
}
|
||||
{
|
||||
job_name = "postgres";
|
||||
port = config.services.prometheus.exporters.postgres.port;
|
||||
}
|
||||
])
|
||||
++ [
|
||||
{
|
||||
job_name = "fail2ban";
|
||||
port = 9191;
|
||||
job_name = "minio";
|
||||
bearer_token_file = config.age.secrets."services/prometheus/minio-bearer-token".path;
|
||||
metrics_path = "/minio/v2/metrics/cluster";
|
||||
static_configs = [{targets = [config.services.minio.listenAddress];}];
|
||||
}
|
||||
{
|
||||
job_name = "nginx";
|
||||
port = config.services.prometheus.exporters.nginx.port;
|
||||
}
|
||||
{
|
||||
job_name = "nginxlog";
|
||||
port = config.services.prometheus.exporters.nginxlog.port;
|
||||
}
|
||||
{
|
||||
job_name = "node";
|
||||
port = config.services.prometheus.exporters.node.port;
|
||||
}
|
||||
{
|
||||
job_name = "postgres";
|
||||
port = config.services.prometheus.exporters.postgres.port;
|
||||
job_name = "forgejo";
|
||||
static_configs = with config.services.forgejo.settings.server; [{targets = ["${HTTP_ADDR}:${toString HTTP_PORT}"];}];
|
||||
}
|
||||
];
|
||||
|
||||
|
@ -65,34 +125,47 @@
|
|||
nginxlog = {
|
||||
enable = true;
|
||||
group = "nginx";
|
||||
settings.namespaces =
|
||||
builtins.map (app: {
|
||||
name = app;
|
||||
|
||||
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\"";
|
||||
settings.namespaces = [
|
||||
{
|
||||
format = log_format;
|
||||
|
||||
metrics_override.prefix = "nginxlog";
|
||||
namespace_label = "vhost";
|
||||
|
||||
relabel = {
|
||||
city.from = "geoip2_data_city_name";
|
||||
country.from = "geoip2_data_country_name";
|
||||
};
|
||||
relabel_configs = [
|
||||
{
|
||||
target_label = "vhost";
|
||||
from = "server_name";
|
||||
}
|
||||
{
|
||||
target_label = "status";
|
||||
from = "status";
|
||||
only_counter = true;
|
||||
}
|
||||
{
|
||||
target_label = "geo_city";
|
||||
from = "geoip2_data_city_name";
|
||||
}
|
||||
{
|
||||
target_label = "geo_country";
|
||||
from = "geoip2_data_country_code";
|
||||
}
|
||||
{
|
||||
target_label = "latitude";
|
||||
from = "geoip2_data_latitude";
|
||||
}
|
||||
{
|
||||
target_label = "longitude";
|
||||
from = "geoip2_data_longitude";
|
||||
}
|
||||
];
|
||||
|
||||
source.files = ["/var/log/nginx/${app}.access.log"];
|
||||
}) [
|
||||
"attic"
|
||||
"atuin"
|
||||
"forgejo"
|
||||
"freshrss"
|
||||
"minio"
|
||||
"nextcloud"
|
||||
"wakapi"
|
||||
];
|
||||
source.files = [log_file];
|
||||
}
|
||||
];
|
||||
};
|
||||
node = {
|
||||
enable = true;
|
||||
enabledCollectors = ["processes" "systemd"];
|
||||
enabledCollectors = ["logind" "processes" "systemd"];
|
||||
disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
|
||||
};
|
||||
postgres = {
|
||||
|
|
|
@ -53,11 +53,5 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/nextcloud.access.log combined_geoip;
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,8 +1,4 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
{pkgs, ...}: let
|
||||
snakeoilCert = pkgs.runCommand "nginx-snakeoil-cert" {buildInputs = [pkgs.openssl];} ''
|
||||
mkdir "$out"
|
||||
openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 -subj '/CN=Snakeoil CA' -nodes -out "$out/cert.pem" -keyout "$out/cert.key"
|
||||
|
@ -11,34 +7,12 @@ in {
|
|||
services.nginx = {
|
||||
enable = true;
|
||||
package = pkgs.nginxMainline;
|
||||
additionalModules = [pkgs.nginxModules.geoip2];
|
||||
|
||||
recommendedGzipSettings = true;
|
||||
recommendedOptimisation = true;
|
||||
recommendedProxySettings = true;
|
||||
recommendedTlsSettings = true;
|
||||
|
||||
commonHttpConfig = let
|
||||
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
|
||||
in
|
||||
# nginx
|
||||
''
|
||||
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
|
||||
auto_reload 5m;
|
||||
$geoip2_metadata_country_build metadata build_epoch;
|
||||
$geoip2_data_country_code country iso_code;
|
||||
$geoip2_data_country_name country names en;
|
||||
}
|
||||
|
||||
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
|
||||
auto_reload 5m;
|
||||
$geoip2_data_city_name city names en;
|
||||
}
|
||||
|
||||
log_format combined_geoip '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time" "$geoip2_data_country_name" "$geoip2_data_city_name"';
|
||||
access_log /var/log/nginx/access.log combined_geoip;
|
||||
'';
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/issues/180980#issuecomment-1179723811
|
||||
virtualHosts = {
|
||||
"defaultDummy404" = {
|
||||
|
|
|
@ -25,12 +25,6 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/wakapi.access.log combined_geoip;
|
||||
'';
|
||||
};
|
||||
|
||||
# for agenix owner permissions
|
||||
|
|
|
@ -68,6 +68,7 @@
|
|||
inherit (config.pre-commit.devShell) shellHook;
|
||||
buildInputs = [
|
||||
inputs'.agenix.packages.agenix
|
||||
pkgs.age-plugin-yubikey
|
||||
pkgs.unstable.nh
|
||||
pkgs.unstable.nil
|
||||
self'.formatter
|
||||
|
|
Loading…
Reference in a new issue