winston/infra
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

feat: rework nginx, add per-vhost logging & monitoring

Browse source
This commit is contained in:
winston 2024-09-13 02:28:10 +02:00
parent 66aa57dec1
commit d9e616b135
17 changed files with 242 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
config/network.nix
View file

@ -1,10 +1,11 @@
{
networking.firewall.enable = true;
services = {
fail2ban.enable = true;
openssh = {
enable = true;
ports = [1322];
ports = [22];
settings = {
PasswordAuthentication = false;
PermitRootLogin = "no";

2
config/secrets/secrets.nix
View file

@ -17,6 +17,8 @@ in {
"services/forgejo/minio-secretkey.age".publicKeys = default;
"services/forgejo/runner-token.age".publicKeys = default;
"services/geoipupdate/license.age".publicKeys = default;
"services/gitlab/dbFile.age".publicKeys = default;
"services/gitlab/jwsFile.age".publicKeys = default;
"services/gitlab/otpFile.age".publicKeys = default;

BIN
config/secrets/services/geoipupdate/license.age Normal file
View file

Binary file not shown.

7
config/services/attic.nix
View file

@ -25,7 +25,12 @@
useACMEHost = "winston.sh";
locations."/" = {
extraConfig = "client_max_body_size 512M;";
extraConfig =
# nginx
''
access_log /var/log/nginx/attic.access.log combined_geoip;
client_max_body_size 512M;
'';
proxyPass = "http://${config.services.atticd.settings.listen}";
};
};

5
config/services/atuin.nix
View file

@ -12,6 +12,11 @@
useACMEHost = "winston.sh";
locations."/" = with config.services.atuin; {
extraConfig =
# nginx
''
access_log /var/log/nginx/atuin.access.log combined_geoip;
'';
proxyPass = "http://${host}:${toString port}";
};
};

1
config/services/default.nix
View file

@ -5,6 +5,7 @@
./containers.nix
./forgejo.nix
./freshrss.nix
./geoipupdate.nix
./minio.nix
./monitoring.nix
./nextcloud.nix

26
config/services/forgejo.nix
View file

@ -5,7 +5,6 @@
...
}: let
modules = ["services/misc/forgejo.nix" "services/continuous-integration/gitea-actions-runner.nix"];
pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.system};
in {
# swap out stable for unstable modules
disabledModules = modules;
@ -27,7 +26,7 @@ in {
services.forgejo = {
enable = true;
package = pkgsUnstable.forgejo;
package = pkgs.unstable.forgejo;
database = {
type = "postgres";
@ -60,9 +59,12 @@ in {
server = rec {
DOMAIN = "code.winston.sh";
ROOT_URL = "https://${DOMAIN}/";
HTTP_ADDR = "127.0.0.1";
HTTP_PORT = 12492;
ROOT_URL = "https://${DOMAIN}/";
# allow fetch from gravatar etc.
OFFLINE_MODE = false;
};
@ -97,7 +99,7 @@ in {
virtualisation.podman.enable = true;
services.gitea-actions-runner = {
package = pkgsUnstable.forgejo-runner;
package = pkgs.unstable.forgejo-runner;
instances.main = {
enable = true;
name = "main";
@ -116,9 +118,19 @@ in {
enableACME = false;
useACMEHost = "winston.sh";
locations."/" = with config.services.forgejo.settings.server; {
extraConfig = "client_max_body_size 512M;";
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
locations = {
"/" = with config.services.forgejo.settings.server; {
extraConfig =
# nginx
''
access_log /var/log/nginx/forgejo.access.log combined_geoip;
client_max_body_size 512M;
'';
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
};
# don't spam the log with runner polls
"/api/actions/runner.v1.RunnerService/FetchTask".extraConfig = "access_log off;";
};
};
}

5
config/services/freshrss.nix
View file

@ -15,5 +15,10 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
extraConfig =
# nginx
''
access_log /var/log/nginx/freshrss.access.log combined_geoip;
'';
};
}

10
config/services/geoipupdate.nix Normal file
View file

@ -0,0 +1,10 @@
{config, ...}: {
services.geoipupdate = {
enable = true;
settings = {
AccountID = 1062126;
LicenseKey = config.age.secrets."services/geoipupdate/license".path;
EditionIDs = ["GeoLite2-ASN" "GeoLite2-City" "GeoLite2-Country"];
};
};
}

17
config/services/minio.nix
View file

@ -20,7 +20,15 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
locations."/".proxyPass = "http://${config.services.minio.consoleAddress}";
locations."/" = {
extraConfig =
# nginx
''
access_log /var/log/nginx/minio.access.log combined_geoip;
'';
proxyPass = "http://${config.services.minio.consoleAddress}";
};
};
"s3.winston.sh" = {
forceSSL = true;
@ -28,7 +36,12 @@
useACMEHost = "winston.sh";
locations."/" = {
extraConfig = "client_max_body_size 512M;";
extraConfig =
# nginx
''
access_log /var/log/nginx/minio.access.log combined_geoip;
client_max_body_size 512M;
'';
proxyPass = "http://${config.services.minio.listenAddress}";
};
};

168
config/services/monitoring.nix
View file

@ -1,4 +1,9 @@
{config, ...}: {
{
config,
lib,
pkgs,
...
}: {
services.grafana = {
enable = true;
settings = {
@ -12,14 +17,12 @@
provision = {
enable = true;
datasources.settings.datasources = [
# "Built-in" datasources can be provisioned - c.f. https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources
{
(with config.services.prometheus; {
name = "Prometheus";
type = "prometheus";
url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
}
url = "http://${listenAddress}:${toString port}";
})
];
};
};
@ -28,87 +31,102 @@
services.prometheus = {
enable = true;
extraFlags = ["--web.enable-admin-api"];
globalConfig.scrape_interval = "10s";
scrapeConfigs = [
{
job_name = "nginx";
static_configs = [
{
targets = ["localhost:${toString config.services.prometheus.exporters.nginx.port}"];
}
];
}
{
job_name = "nginxlog";
static_configs = [
{
targets = ["localhost:${toString config.services.prometheus.exporters.nginxlog.port}"];
}
];
}
{
job_name = "node";
static_configs = [
{
targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
}
];
}
# {
# job_name = "minio";
# static_configs = [
# {
# targets = ["localhost:${toString config.services.prometheus.exporters.minio.port}"];
# }
# ];
# }
# {
# job_name = "postgres";
# static_configs = [
# {
# targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
# }
# ];
# }
];
scrapeConfigs =
builtins.map (config: {
inherit (config) job_name;
static_configs = [{targets = ["localhost:${toString config.port}"];}];
}) [
{
job_name = "fail2ban";
port = 9191;
}
{
job_name = "nginx";
port = config.services.prometheus.exporters.nginx.port;
}
{
job_name = "nginxlog";
port = config.services.prometheus.exporters.nginxlog.port;
}
{
job_name = "node";
port = config.services.prometheus.exporters.node.port;
}
{
job_name = "postgres";
port = config.services.prometheus.exporters.postgres.port;
}
];
exporters = {
nginx.enable = true;
nginxlog = {
enable = true;
group = "nginx";
settings.namespaces = [
{
name = "filelogger";
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"";
source.files = ["/var/log/nginx/access.log"];
}
];
};
# FIXME: set up auth!
# minio.enable = true;
# postgres = {
# enable = true;
# dataSourceName = "postgresql://localhost:5432/postgres?sslmode=disable";
# };
};
settings.namespaces =
builtins.map (app: {
name = app;
exporters.node = {
enable = true;
port = 9000;
enabledCollectors = ["processes" "systemd"];
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\"";
metrics_override.prefix = "nginxlog";
namespace_label = "vhost";
relabel = {
city.from = "geoip2_data_city_name";
country.from = "geoip2_data_country_name";
};
source.files = ["/var/log/nginx/${app}.access.log"];
}) [
"attic"
"atuin"
"forgejo"
"freshrss"
"minio"
"nextcloud"
"wakapi"
];
};
node = {
enable = true;
enabledCollectors = ["processes" "systemd"];
disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
};
postgres = {
enable = true;
# FIXME: this is not ideal...
runAsLocalSuperUser = true;
};
};
};
services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
systemd.services.prometheus-fail2ban-exporter = {
wantedBy = ["multi-user.target"];
after = ["network.target"];
requires = ["network-online.target"];
serviceConfig = {
ExecStart = [(lib.getExe pkgs.prometheus-fail2ban-exporter)];
Restart = "on-failure";
NoNewPrivileges = true;
User = "root";
Group = "root";
};
};
locations."/" = {
proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
proxyWebsockets = true;
recommendedProxySettings = true;
services.nginx.virtualHosts = with config.services.grafana.settings.server; {
${domain} = {
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
locations."/" = {
proxyPass = "http://${http_addr}:${toString http_port}";
proxyWebsockets = true;
recommendedProxySettings = true;
};
};
};
}

6
config/services/nextcloud.nix
View file

@ -53,5 +53,11 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
extraConfig =
# nginx
''
access_log /var/log/nginx/nextcloud.access.log combined_geoip;
'';
};
}

28
config/services/nginx.nix
View file

@ -1,4 +1,8 @@
{pkgs, ...}: let
{
config,
pkgs,
...
}: let
snakeoilCert = pkgs.runCommand "nginx-snakeoil-cert" {buildInputs = [pkgs.openssl];} ''
mkdir "$out"
openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 -subj '/CN=Snakeoil CA' -nodes -out "$out/cert.pem" -keyout "$out/cert.key"
@ -7,12 +11,34 @@ in {
services.nginx = {
enable = true;
package = pkgs.nginxMainline;
additionalModules = [pkgs.nginxModules.geoip2];
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedProxySettings = true;
recommendedTlsSettings = true;
commonHttpConfig = let
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
in
# nginx
''
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
auto_reload 5m;
$geoip2_metadata_country_build metadata build_epoch;
$geoip2_data_country_code country iso_code;
$geoip2_data_country_name country names en;
}
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
auto_reload 5m;
$geoip2_data_city_name city names en;
}
log_format combined_geoip '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time" "$geoip2_data_country_name" "$geoip2_data_city_name"';
access_log /var/log/nginx/access.log combined_geoip;
'';
# https://github.com/NixOS/nixpkgs/issues/180980#issuecomment-1179723811
virtualHosts = {
"defaultDummy404" = {

16
config/services/wakapi.nix
View file

@ -25,11 +25,21 @@
forceSSL = true;
enableACME = false;
useACMEHost = "winston.sh";
extraConfig =
# nginx
''
access_log /var/log/nginx/wakapi.access.log combined_geoip;
'';
};
# for agenix owner permissions
users.users.wakapi.isSystemUser = true;
users.users.wakapi.group = "wakapi";
users.groups.wakapi = {};
age.secrets."services/wakapi/password-salt.env".owner = "wakapi";
users = {
groups.wakapi = {};
users.wakapi = {
isSystemUser = true;
group = "wakapi";
};
};
}

14
config/services/website/default.nix
View file

@ -14,12 +14,14 @@
locations."/" = {
root = "/var/lib/winston.sh/experiments";
extraConfig = ''
autoindex on;
autoindex_exact_size off;
autoindex_format html;
autoindex_localtime on;
'';
extraConfig =
# nginx
''
autoindex on;
autoindex_exact_size off;
autoindex_format html;
autoindex_localtime on;
'';
};
};
}

5
flake.nix
View file

@ -5,9 +5,10 @@
...
} @ inputs: let
overlays = [
(_: prev: rec {
(final: _: rec {
atuin = unstable.atuin;
unstable = inputs.nixpkgs-unstable.legacyPackages.${prev.system};
prometheus-fail2ban-exporter = final.callPackage ./pkgs/prometheus-fail2ban-exporter {};
unstable = inputs.nixpkgs-unstable.legacyPackages.${final.system};
})
];
in

27
pkgs/prometheus-fail2ban-exporter/default.nix Normal file
View file

@ -0,0 +1,27 @@
{
buildGoModule,
fetchFromGitLab,
lib,
}: let
version = "0.10.1";
in
buildGoModule {
pname = "prometheus-fail2ban-exporter";
inherit version;
src = fetchFromGitLab {
owner = "hectorjsmith";
repo = "fail2ban-prometheus-exporter";
rev = "v${version}";
sha256 = "sha256-zGEhDy3uXIbvx4agSA8Mx7bRtiZZtoDZGbNbHc9L+yI=";
};
vendorHash = "sha256-5o8p5p0U/c0WAIV5dACnWA3ThzSh2tt5LIFMb59i9GY=";
meta = with lib; {
mainProgram = "fail2ban-prometheus-exporter";
description = "Collect and export metrics on Fail2Ban";
homepage = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
license = licenses.mit;
};
}