feat: rework nginx, add per-vhost logging & monitoring
This commit is contained in:
parent
66aa57dec1
commit
d9e616b135
17 changed files with 242 additions and 98 deletions
|
@ -1,10 +1,11 @@
|
|||
{
|
||||
networking.firewall.enable = true;
|
||||
|
||||
services = {
|
||||
fail2ban.enable = true;
|
||||
openssh = {
|
||||
enable = true;
|
||||
ports = [1322];
|
||||
ports = [22];
|
||||
settings = {
|
||||
PasswordAuthentication = false;
|
||||
PermitRootLogin = "no";
|
||||
|
|
|
@ -17,6 +17,8 @@ in {
|
|||
"services/forgejo/minio-secretkey.age".publicKeys = default;
|
||||
"services/forgejo/runner-token.age".publicKeys = default;
|
||||
|
||||
"services/geoipupdate/license.age".publicKeys = default;
|
||||
|
||||
"services/gitlab/dbFile.age".publicKeys = default;
|
||||
"services/gitlab/jwsFile.age".publicKeys = default;
|
||||
"services/gitlab/otpFile.age".publicKeys = default;
|
||||
|
|
BIN
config/secrets/services/geoipupdate/license.age
Normal file
BIN
config/secrets/services/geoipupdate/license.age
Normal file
Binary file not shown.
|
@ -25,7 +25,12 @@
|
|||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = {
|
||||
extraConfig = "client_max_body_size 512M;";
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/attic.access.log combined_geoip;
|
||||
client_max_body_size 512M;
|
||||
'';
|
||||
proxyPass = "http://${config.services.atticd.settings.listen}";
|
||||
};
|
||||
};
|
||||
|
|
|
@ -12,6 +12,11 @@
|
|||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = with config.services.atuin; {
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/atuin.access.log combined_geoip;
|
||||
'';
|
||||
proxyPass = "http://${host}:${toString port}";
|
||||
};
|
||||
};
|
||||
|
|
|
@ -5,6 +5,7 @@
|
|||
./containers.nix
|
||||
./forgejo.nix
|
||||
./freshrss.nix
|
||||
./geoipupdate.nix
|
||||
./minio.nix
|
||||
./monitoring.nix
|
||||
./nextcloud.nix
|
||||
|
|
|
@ -5,7 +5,6 @@
|
|||
...
|
||||
}: let
|
||||
modules = ["services/misc/forgejo.nix" "services/continuous-integration/gitea-actions-runner.nix"];
|
||||
pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.system};
|
||||
in {
|
||||
# swap out stable for unstable modules
|
||||
disabledModules = modules;
|
||||
|
@ -27,7 +26,7 @@ in {
|
|||
services.forgejo = {
|
||||
enable = true;
|
||||
|
||||
package = pkgsUnstable.forgejo;
|
||||
package = pkgs.unstable.forgejo;
|
||||
|
||||
database = {
|
||||
type = "postgres";
|
||||
|
@ -60,9 +59,12 @@ in {
|
|||
|
||||
server = rec {
|
||||
DOMAIN = "code.winston.sh";
|
||||
ROOT_URL = "https://${DOMAIN}/";
|
||||
|
||||
HTTP_ADDR = "127.0.0.1";
|
||||
HTTP_PORT = 12492;
|
||||
ROOT_URL = "https://${DOMAIN}/";
|
||||
|
||||
# allow fetch from gravatar etc.
|
||||
OFFLINE_MODE = false;
|
||||
};
|
||||
|
||||
|
@ -97,7 +99,7 @@ in {
|
|||
|
||||
virtualisation.podman.enable = true;
|
||||
services.gitea-actions-runner = {
|
||||
package = pkgsUnstable.forgejo-runner;
|
||||
package = pkgs.unstable.forgejo-runner;
|
||||
instances.main = {
|
||||
enable = true;
|
||||
name = "main";
|
||||
|
@ -116,9 +118,19 @@ in {
|
|||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = with config.services.forgejo.settings.server; {
|
||||
extraConfig = "client_max_body_size 512M;";
|
||||
locations = {
|
||||
"/" = with config.services.forgejo.settings.server; {
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/forgejo.access.log combined_geoip;
|
||||
client_max_body_size 512M;
|
||||
'';
|
||||
proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
|
||||
};
|
||||
|
||||
# don't spam the log with runner polls
|
||||
"/api/actions/runner.v1.RunnerService/FetchTask".extraConfig = "access_log off;";
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -15,5 +15,10 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/freshrss.access.log combined_geoip;
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
10
config/services/geoipupdate.nix
Normal file
10
config/services/geoipupdate.nix
Normal file
|
@ -0,0 +1,10 @@
|
|||
{config, ...}: {
|
||||
services.geoipupdate = {
|
||||
enable = true;
|
||||
settings = {
|
||||
AccountID = 1062126;
|
||||
LicenseKey = config.age.secrets."services/geoipupdate/license".path;
|
||||
EditionIDs = ["GeoLite2-ASN" "GeoLite2-City" "GeoLite2-Country"];
|
||||
};
|
||||
};
|
||||
}
|
|
@ -20,7 +20,15 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
locations."/".proxyPass = "http://${config.services.minio.consoleAddress}";
|
||||
|
||||
locations."/" = {
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/minio.access.log combined_geoip;
|
||||
'';
|
||||
proxyPass = "http://${config.services.minio.consoleAddress}";
|
||||
};
|
||||
};
|
||||
"s3.winston.sh" = {
|
||||
forceSSL = true;
|
||||
|
@ -28,7 +36,12 @@
|
|||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = {
|
||||
extraConfig = "client_max_body_size 512M;";
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/minio.access.log combined_geoip;
|
||||
client_max_body_size 512M;
|
||||
'';
|
||||
proxyPass = "http://${config.services.minio.listenAddress}";
|
||||
};
|
||||
};
|
||||
|
|
|
@ -1,4 +1,9 @@
|
|||
{config, ...}: {
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}: {
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
settings = {
|
||||
|
@ -12,14 +17,12 @@
|
|||
|
||||
provision = {
|
||||
enable = true;
|
||||
|
||||
datasources.settings.datasources = [
|
||||
# "Built-in" datasources can be provisioned - c.f. https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources
|
||||
{
|
||||
(with config.services.prometheus; {
|
||||
name = "Prometheus";
|
||||
type = "prometheus";
|
||||
url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
|
||||
}
|
||||
url = "http://${listenAddress}:${toString port}";
|
||||
})
|
||||
];
|
||||
};
|
||||
};
|
||||
|
@ -28,87 +31,102 @@
|
|||
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
extraFlags = ["--web.enable-admin-api"];
|
||||
globalConfig.scrape_interval = "10s";
|
||||
scrapeConfigs = [
|
||||
scrapeConfigs =
|
||||
builtins.map (config: {
|
||||
inherit (config) job_name;
|
||||
static_configs = [{targets = ["localhost:${toString config.port}"];}];
|
||||
}) [
|
||||
{
|
||||
job_name = "fail2ban";
|
||||
port = 9191;
|
||||
}
|
||||
{
|
||||
job_name = "nginx";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["localhost:${toString config.services.prometheus.exporters.nginx.port}"];
|
||||
}
|
||||
];
|
||||
port = config.services.prometheus.exporters.nginx.port;
|
||||
}
|
||||
{
|
||||
job_name = "nginxlog";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["localhost:${toString config.services.prometheus.exporters.nginxlog.port}"];
|
||||
}
|
||||
];
|
||||
port = config.services.prometheus.exporters.nginxlog.port;
|
||||
}
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [
|
||||
port = config.services.prometheus.exporters.node.port;
|
||||
}
|
||||
{
|
||||
targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
|
||||
job_name = "postgres";
|
||||
port = config.services.prometheus.exporters.postgres.port;
|
||||
}
|
||||
];
|
||||
}
|
||||
# {
|
||||
# job_name = "minio";
|
||||
# static_configs = [
|
||||
# {
|
||||
# targets = ["localhost:${toString config.services.prometheus.exporters.minio.port}"];
|
||||
# }
|
||||
# ];
|
||||
# }
|
||||
# {
|
||||
# job_name = "postgres";
|
||||
# static_configs = [
|
||||
# {
|
||||
# targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
|
||||
# }
|
||||
# ];
|
||||
# }
|
||||
];
|
||||
|
||||
exporters = {
|
||||
nginx.enable = true;
|
||||
nginxlog = {
|
||||
enable = true;
|
||||
group = "nginx";
|
||||
settings.namespaces = [
|
||||
{
|
||||
name = "filelogger";
|
||||
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"";
|
||||
source.files = ["/var/log/nginx/access.log"];
|
||||
}
|
||||
settings.namespaces =
|
||||
builtins.map (app: {
|
||||
name = app;
|
||||
|
||||
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\"";
|
||||
|
||||
metrics_override.prefix = "nginxlog";
|
||||
namespace_label = "vhost";
|
||||
|
||||
relabel = {
|
||||
city.from = "geoip2_data_city_name";
|
||||
country.from = "geoip2_data_country_name";
|
||||
};
|
||||
|
||||
source.files = ["/var/log/nginx/${app}.access.log"];
|
||||
}) [
|
||||
"attic"
|
||||
"atuin"
|
||||
"forgejo"
|
||||
"freshrss"
|
||||
"minio"
|
||||
"nextcloud"
|
||||
"wakapi"
|
||||
];
|
||||
};
|
||||
# FIXME: set up auth!
|
||||
# minio.enable = true;
|
||||
# postgres = {
|
||||
# enable = true;
|
||||
# dataSourceName = "postgresql://localhost:5432/postgres?sslmode=disable";
|
||||
# };
|
||||
};
|
||||
|
||||
exporters.node = {
|
||||
node = {
|
||||
enable = true;
|
||||
port = 9000;
|
||||
enabledCollectors = ["processes" "systemd"];
|
||||
disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
|
||||
};
|
||||
postgres = {
|
||||
enable = true;
|
||||
# FIXME: this is not ideal...
|
||||
runAsLocalSuperUser = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
|
||||
systemd.services.prometheus-fail2ban-exporter = {
|
||||
wantedBy = ["multi-user.target"];
|
||||
after = ["network.target"];
|
||||
requires = ["network-online.target"];
|
||||
serviceConfig = {
|
||||
ExecStart = [(lib.getExe pkgs.prometheus-fail2ban-exporter)];
|
||||
Restart = "on-failure";
|
||||
NoNewPrivileges = true;
|
||||
User = "root";
|
||||
Group = "root";
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx.virtualHosts = with config.services.grafana.settings.server; {
|
||||
${domain} = {
|
||||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
locations."/" = {
|
||||
proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
|
||||
proxyPass = "http://${http_addr}:${toString http_port}";
|
||||
proxyWebsockets = true;
|
||||
recommendedProxySettings = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -53,5 +53,11 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/nextcloud.access.log combined_geoip;
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
{pkgs, ...}: let
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
snakeoilCert = pkgs.runCommand "nginx-snakeoil-cert" {buildInputs = [pkgs.openssl];} ''
|
||||
mkdir "$out"
|
||||
openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 -subj '/CN=Snakeoil CA' -nodes -out "$out/cert.pem" -keyout "$out/cert.key"
|
||||
|
@ -7,12 +11,34 @@ in {
|
|||
services.nginx = {
|
||||
enable = true;
|
||||
package = pkgs.nginxMainline;
|
||||
additionalModules = [pkgs.nginxModules.geoip2];
|
||||
|
||||
recommendedGzipSettings = true;
|
||||
recommendedOptimisation = true;
|
||||
recommendedProxySettings = true;
|
||||
recommendedTlsSettings = true;
|
||||
|
||||
commonHttpConfig = let
|
||||
geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
|
||||
in
|
||||
# nginx
|
||||
''
|
||||
geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
|
||||
auto_reload 5m;
|
||||
$geoip2_metadata_country_build metadata build_epoch;
|
||||
$geoip2_data_country_code country iso_code;
|
||||
$geoip2_data_country_name country names en;
|
||||
}
|
||||
|
||||
geoip2 ${geoipDir}/GeoLite2-City.mmdb {
|
||||
auto_reload 5m;
|
||||
$geoip2_data_city_name city names en;
|
||||
}
|
||||
|
||||
log_format combined_geoip '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time" "$geoip2_data_country_name" "$geoip2_data_city_name"';
|
||||
access_log /var/log/nginx/access.log combined_geoip;
|
||||
'';
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/issues/180980#issuecomment-1179723811
|
||||
virtualHosts = {
|
||||
"defaultDummy404" = {
|
||||
|
|
|
@ -25,11 +25,21 @@
|
|||
forceSSL = true;
|
||||
enableACME = false;
|
||||
useACMEHost = "winston.sh";
|
||||
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
access_log /var/log/nginx/wakapi.access.log combined_geoip;
|
||||
'';
|
||||
};
|
||||
|
||||
# for agenix owner permissions
|
||||
users.users.wakapi.isSystemUser = true;
|
||||
users.users.wakapi.group = "wakapi";
|
||||
users.groups.wakapi = {};
|
||||
age.secrets."services/wakapi/password-salt.env".owner = "wakapi";
|
||||
users = {
|
||||
groups.wakapi = {};
|
||||
users.wakapi = {
|
||||
isSystemUser = true;
|
||||
group = "wakapi";
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -14,7 +14,9 @@
|
|||
|
||||
locations."/" = {
|
||||
root = "/var/lib/winston.sh/experiments";
|
||||
extraConfig = ''
|
||||
extraConfig =
|
||||
# nginx
|
||||
''
|
||||
autoindex on;
|
||||
autoindex_exact_size off;
|
||||
autoindex_format html;
|
||||
|
|
|
@ -5,9 +5,10 @@
|
|||
...
|
||||
} @ inputs: let
|
||||
overlays = [
|
||||
(_: prev: rec {
|
||||
(final: _: rec {
|
||||
atuin = unstable.atuin;
|
||||
unstable = inputs.nixpkgs-unstable.legacyPackages.${prev.system};
|
||||
prometheus-fail2ban-exporter = final.callPackage ./pkgs/prometheus-fail2ban-exporter {};
|
||||
unstable = inputs.nixpkgs-unstable.legacyPackages.${final.system};
|
||||
})
|
||||
];
|
||||
in
|
||||
|
|
27
pkgs/prometheus-fail2ban-exporter/default.nix
Normal file
27
pkgs/prometheus-fail2ban-exporter/default.nix
Normal file
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
buildGoModule,
|
||||
fetchFromGitLab,
|
||||
lib,
|
||||
}: let
|
||||
version = "0.10.1";
|
||||
in
|
||||
buildGoModule {
|
||||
pname = "prometheus-fail2ban-exporter";
|
||||
inherit version;
|
||||
|
||||
src = fetchFromGitLab {
|
||||
owner = "hectorjsmith";
|
||||
repo = "fail2ban-prometheus-exporter";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-zGEhDy3uXIbvx4agSA8Mx7bRtiZZtoDZGbNbHc9L+yI=";
|
||||
};
|
||||
|
||||
vendorHash = "sha256-5o8p5p0U/c0WAIV5dACnWA3ThzSh2tt5LIFMb59i9GY=";
|
||||
|
||||
meta = with lib; {
|
||||
mainProgram = "fail2ban-prometheus-exporter";
|
||||
description = "Collect and export metrics on Fail2Ban";
|
||||
homepage = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
|
||||
license = licenses.mit;
|
||||
};
|
||||
}
|
Loading…
Reference in a new issue