winston/infra
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

feat: simplify nginx monitoring setup

Browse source
This commit is contained in:
winston 2024-09-13 18:01:30 +02:00
parent ddcf3f836b
commit 86722826de
12 changed files with 140 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/secrets/secrets.nix
View file

@ -34,6 +34,8 @@ in {
"services/nextcloud/admin-password.age".publicKeys = default;
"services/nextcloud/s3-secret.age".publicKeys = default;
"services/prometheus/minio-bearer-token.age".publicKeys = default;
"services/wakapi/password-salt.env.age".publicKeys = default;
"system/password-root.age".publicKeys = default;

7
config/secrets/services/prometheus/minio-bearer-token.age Normal file
View file

@ -0,0 +1,7 @@
age-encryption.org/v1
-> piv-p256 ML6NcA AqqvLAKvyJwwZYSpx3wINpgLbt6jnPNaz3FsdLrgc6AI
akR0ix3DNUB4oZXEficyp4rbkOZF0y/ms7r8wr/d9Cg
-> ssh-ed25519 zj2A2A Ul6EUnAnh4IOurvpr0i6StlrtRDMDM8HdZWBgvcQuxg
ECDnJYCV5aS2t7o+4ixo37nMemJdANvGlsLdQO2vPCc
--- acN/h+fasItr1Yp7ZJAUCz9K9nOo//zKMDPGI3rvgsM
2TH’ L<>È.b<>ú::.N¤Ÿ0î·ÓZ-q-šöß΃ê²6œI<C593>ÖÆ"K;V{ËÚrØRg¦Î]ªsfHVøkƒoö@ûÑè!e­ÎbãSy‰Þµ=‘ô öNë„î ¹d¶NŒÚLÑû7' „/’@¤¿'H¬’ÌŒ1:$ʈñ&;nZþrøÁ­Ô%—˸9U«‰úé¸îñrÃû{äåHÍÄð¢ %¿ý<C2BF>៲ã©ÔΆ¹­¤¹¬—eeUïØ85FÜReïTÎgéy!³r˜c×D7‘k¤

1
config/services/attic.nix
View file

@ -28,7 +28,6 @@
extraConfig =
# nginx
''
access_log /var/log/nginx/attic.access.log combined_geoip;
client_max_body_size 512M;
'';
proxyPass = "http://${config.services.atticd.settings.listen}";

5
config/services/atuin.nix
View file

@ -12,11 +12,6 @@
useACMEHost = "winston.sh";
locations."/" = with config.services.atuin; {
extraConfig =
# nginx
''
access_log /var/log/nginx/atuin.access.log combined_geoip;
'';
proxyPass = "http://${host}:${toString port}";
};
};

16
config/services/forgejo.nix
View file

@ -55,6 +55,12 @@ in {
ISSUE_INDEXER_TYPE = indexer;
};
metrics = {
ENABLED = true;
ENABLED_ISSUE_BY_REPOSITORY = true;
ENABLED_ISSUE_BY_LABEL = true;
};
repository.ENABLE_PUSH_CREATE_USER = true;
server = rec {
@ -118,19 +124,21 @@ in {
enableACME = false;
useACMEHost = "winston.sh";
locations = {
"/" = with config.services.forgejo.settings.server; {
locations = with config.services.forgejo.settings.server; {
"/" = {
extraConfig =
# nginx
''
access_log /var/log/nginx/forgejo.access.log combined_geoip;
client_max_body_size 512M;
'';
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
};
# don't spam the log with runner polls
"/api/actions/runner.v1.RunnerService/FetchTask".extraConfig = "access_log off;";
"/api/actions/runner.v1.RunnerService/FetchTask" = {
extraConfig = "access_log off;";
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
};
};
};
}

5
config/services/freshrss.nix
View file

@ -15,10 +15,5 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
extraConfig =
# nginx
''
access_log /var/log/nginx/freshrss.access.log combined_geoip;
'';
};
}

6
config/services/minio.nix
View file

@ -22,11 +22,6 @@
useACMEHost = "winston.sh";
locations."/" = {
extraConfig =
# nginx
''
access_log /var/log/nginx/minio.access.log combined_geoip;
'';
proxyPass = "http://${config.services.minio.consoleAddress}";
};
};
@ -39,7 +34,6 @@
extraConfig =
# nginx
''
access_log /var/log/nginx/minio.access.log combined_geoip;
client_max_body_size 512M;
'';
proxyPass = "http://${config.services.minio.listenAddress}";

161
config/services/monitoring.nix
View file

@ -3,7 +3,29 @@
lib,
pkgs,
...
}: {
}: let
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
log_params = [
"remote_addr"
"time_local"
"scheme"
"server_name"
"status"
"body_bytes_sent"
"http_user_agent"
"request_length"
"request_time"
"request_uri"
"upstream_response_time"
"geoip2_data_country_code"
"geoip2_data_city_name"
"geoip2_data_latitude"
"geoip2_data_longitude"
];
log_format = lib.concatStringsSep " " (map (p: "${p}=\"\$${p}\"") log_params);
log_file = "/var/log/nginx/access.log";
in {
services.grafana = {
enable = true;
settings = {
@ -27,36 +49,74 @@
};
};
services.nginx.statusPage = true;
services.nginx = {
additionalModules = [pkgs.nginxModules.geoip2];
commonHttpConfig =
# nginx
''
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
auto_reload 5m;
$geoip2_metadata_country_build metadata build_epoch;
$geoip2_data_country_code default=Unknown country iso_code;
}
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
auto_reload 5m;
$geoip2_data_city_name city names en;
$geoip2_data_latitude default=0.0 location latitude;
$geoip2_data_longitude default=0.0 location longitude;
}
log_format combined_geoip '${log_format}';
access_log ${log_file} combined_geoip;
'';
statusPage = true;
};
age.secrets."services/prometheus/minio-bearer-token".owner = "prometheus";
services.prometheus = {
enable = true;
extraFlags = ["--web.enable-admin-api"];
# using agenix means secrets are mounted in /run/agenix which is inaccessible for the `promtool check` derivation
checkConfig = "syntax-only";
globalConfig.scrape_interval = "10s";
scrapeConfigs =
builtins.map (config: {
inherit (config) job_name;
static_configs = [{targets = ["localhost:${toString config.port}"];}];
}) [
(builtins.map (config: {
inherit (config) job_name;
static_configs = [{targets = ["localhost:${toString config.port}"];}];
}) [
{
job_name = "fail2ban";
port = 9191;
}
{
job_name = "nginx";
port = config.services.prometheus.exporters.nginx.port;
}
{
job_name = "nginxlog";
port = config.services.prometheus.exporters.nginxlog.port;
}
{
job_name = "node";
port = config.services.prometheus.exporters.node.port;
}
{
job_name = "postgres";
port = config.services.prometheus.exporters.postgres.port;
}
])
++ [
{
job_name = "fail2ban";
port = 9191;
job_name = "minio";
bearer_token_file = config.age.secrets."services/prometheus/minio-bearer-token".path;
metrics_path = "/minio/v2/metrics/cluster";
static_configs = [{targets = [config.services.minio.listenAddress];}];
}
{
job_name = "nginx";
port = config.services.prometheus.exporters.nginx.port;
}
{
job_name = "nginxlog";
port = config.services.prometheus.exporters.nginxlog.port;
}
{
job_name = "node";
port = config.services.prometheus.exporters.node.port;
}
{
job_name = "postgres";
port = config.services.prometheus.exporters.postgres.port;
job_name = "forgejo";
static_configs = with config.services.forgejo.settings.server; [{targets = ["${HTTP_ADDR}:${toString HTTP_PORT}"];}];
}
];
@ -65,34 +125,47 @@
nginxlog = {
enable = true;
group = "nginx";
settings.namespaces =
builtins.map (app: {
name = app;
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\"";
settings.namespaces = [
{
format = log_format;
metrics_override.prefix = "nginxlog";
namespace_label = "vhost";
relabel = {
city.from = "geoip2_data_city_name";
country.from = "geoip2_data_country_name";
};
relabel_configs = [
{
target_label = "vhost";
from = "server_name";
}
{
target_label = "status";
from = "status";
only_counter = true;
}
{
target_label = "geo_city";
from = "geoip2_data_city_name";
}
{
target_label = "geo_country";
from = "geoip2_data_country_code";
}
{
target_label = "latitude";
from = "geoip2_data_latitude";
}
{
target_label = "longitude";
from = "geoip2_data_longitude";
}
];
source.files = ["/var/log/nginx/${app}.access.log"];
}) [
"attic"
"atuin"
"forgejo"
"freshrss"
"minio"
"nextcloud"
"wakapi"
];
source.files = [log_file];
}
];
};
node = {
enable = true;
enabledCollectors = ["processes" "systemd"];
enabledCollectors = ["logind" "processes" "systemd"];
disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
};
postgres = {

6
config/services/nextcloud.nix
View file

@ -53,11 +53,5 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
extraConfig =
# nginx
''
access_log /var/log/nginx/nextcloud.access.log combined_geoip;
'';
};
}

28
config/services/nginx.nix
View file

@ -1,8 +1,4 @@
{
config,
pkgs,
...
}: let
{pkgs, ...}: let
snakeoilCert = pkgs.runCommand "nginx-snakeoil-cert" {buildInputs = [pkgs.openssl];} ''
mkdir "$out"
openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 -subj '/CN=Snakeoil CA' -nodes -out "$out/cert.pem" -keyout "$out/cert.key"
@ -11,34 +7,12 @@ in {
services.nginx = {
enable = true;
package = pkgs.nginxMainline;
additionalModules = [pkgs.nginxModules.geoip2];
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedProxySettings = true;
recommendedTlsSettings = true;
commonHttpConfig = let
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
in
# nginx
''
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
auto_reload 5m;
$geoip2_metadata_country_build metadata build_epoch;
$geoip2_data_country_code country iso_code;
$geoip2_data_country_name country names en;
}
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
auto_reload 5m;
$geoip2_data_city_name city names en;
}
log_format combined_geoip '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time" "$geoip2_data_country_name" "$geoip2_data_city_name"';
access_log /var/log/nginx/access.log combined_geoip;
'';
# https://github.com/NixOS/nixpkgs/issues/180980#issuecomment-1179723811
virtualHosts = {
"defaultDummy404" = {

6
config/services/wakapi.nix
View file

@ -25,12 +25,6 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
extraConfig =
# nginx
''
access_log /var/log/nginx/wakapi.access.log combined_geoip;
'';
};
# for agenix owner permissions

1
flake.nix
View file

@ -68,6 +68,7 @@
inherit (config.pre-commit.devShell) shellHook;
buildInputs = [
inputs'.agenix.packages.agenix
pkgs.age-plugin-yubikey
pkgs.unstable.nh
pkgs.unstable.nil
self'.formatter