feat: rework nginx, add per-vhost logging & monitoring

config/network.nix

@@ -1,10 +1,11 @@
 {
   networking.firewall.enable = true;
+
   services = {
     fail2ban.enable = true;
     openssh = {
       enable = true;
-      ports = [1322];
+      ports = [22];
       settings = {
         PasswordAuthentication = false;
         PermitRootLogin = "no";

config/secrets/secrets.nix

@@ -17,6 +17,8 @@ in {
   "services/forgejo/minio-secretkey.age".publicKeys = default;
   "services/forgejo/runner-token.age".publicKeys = default;
 
+  "services/geoipupdate/license.age".publicKeys = default;
+
   "services/gitlab/dbFile.age".publicKeys = default;
   "services/gitlab/jwsFile.age".publicKeys = default;
   "services/gitlab/otpFile.age".publicKeys = default;

config/services/attic.nix

@@ -25,7 +25,12 @@
     useACMEHost = "winston.sh";
 
     locations."/" = {
-      extraConfig = "client_max_body_size 512M;";
+      extraConfig =
+        # nginx
+        ''
+          access_log /var/log/nginx/attic.access.log combined_geoip;
+          client_max_body_size 512M;
+        '';
       proxyPass = "http://${config.services.atticd.settings.listen}";
     };
   };

config/services/atuin.nix

@@ -12,6 +12,11 @@
     useACMEHost = "winston.sh";
 
     locations."/" = with config.services.atuin; {
+      extraConfig =
+        # nginx
+        ''
+          access_log /var/log/nginx/atuin.access.log combined_geoip;
+        '';
       proxyPass = "http://${host}:${toString port}";
     };
   };

config/services/default.nix

@@ -5,6 +5,7 @@
     ./containers.nix
     ./forgejo.nix
     ./freshrss.nix
+    ./geoipupdate.nix
     ./minio.nix
     ./monitoring.nix
     ./nextcloud.nix

config/services/forgejo.nix

@@ -5,7 +5,6 @@
   ...
 }: let
   modules = ["services/misc/forgejo.nix" "services/continuous-integration/gitea-actions-runner.nix"];
-  pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.system};
 in {
   # swap out stable for unstable modules
   disabledModules = modules;
@@ -27,7 +26,7 @@ in {
   services.forgejo = {
     enable = true;
 
-    package = pkgsUnstable.forgejo;
+    package = pkgs.unstable.forgejo;
 
     database = {
       type = "postgres";
@@ -60,9 +59,12 @@ in {
 
       server = rec {
         DOMAIN = "code.winston.sh";
+        ROOT_URL = "https://${DOMAIN}/";
+
         HTTP_ADDR = "127.0.0.1";
         HTTP_PORT = 12492;
-        ROOT_URL = "https://${DOMAIN}/";
+
+        # allow fetch from gravatar etc.
         OFFLINE_MODE = false;
       };
 
@@ -97,7 +99,7 @@ in {
 
   virtualisation.podman.enable = true;
   services.gitea-actions-runner = {
-    package = pkgsUnstable.forgejo-runner;
+    package = pkgs.unstable.forgejo-runner;
     instances.main = {
       enable = true;
       name = "main";
@@ -116,9 +118,19 @@ in {
     enableACME = false;
     useACMEHost = "winston.sh";
 
-    locations."/" = with config.services.forgejo.settings.server; {
+    locations = {
-      extraConfig = "client_max_body_size 512M;";
+      "/" = with config.services.forgejo.settings.server; {
+        extraConfig =
+          # nginx
+          ''
+            access_log /var/log/nginx/forgejo.access.log combined_geoip;
+            client_max_body_size 512M;
+          '';
         proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}";
       };
+
+      # don't spam the log with runner polls
+      "/api/actions/runner.v1.RunnerService/FetchTask".extraConfig = "access_log off;";
+    };
   };
 }

config/services/freshrss.nix

@@ -15,5 +15,10 @@
     forceSSL = true;
     enableACME = false;
     useACMEHost = "winston.sh";
+    extraConfig =
+      # nginx
+      ''
+        access_log /var/log/nginx/freshrss.access.log combined_geoip;
+      '';
   };
 }

config/services/geoipupdate.nix

@@ -0,0 +1,10 @@
+{config, ...}: {
+  services.geoipupdate = {
+    enable = true;
+    settings = {
+      AccountID = 1062126;
+      LicenseKey = config.age.secrets."services/geoipupdate/license".path;
+      EditionIDs = ["GeoLite2-ASN" "GeoLite2-City" "GeoLite2-Country"];
+    };
+  };
+}

config/services/minio.nix

@@ -20,7 +20,15 @@
       forceSSL = true;
       enableACME = false;
       useACMEHost = "winston.sh";
-      locations."/".proxyPass = "http://${config.services.minio.consoleAddress}";
+
+      locations."/" = {
+        extraConfig =
+          # nginx
+          ''
+            access_log /var/log/nginx/minio.access.log combined_geoip;
+          '';
+        proxyPass = "http://${config.services.minio.consoleAddress}";
+      };
     };
     "s3.winston.sh" = {
       forceSSL = true;
@@ -28,7 +36,12 @@
       useACMEHost = "winston.sh";
 
       locations."/" = {
-        extraConfig = "client_max_body_size 512M;";
+        extraConfig =
+          # nginx
+          ''
+            access_log /var/log/nginx/minio.access.log combined_geoip;
+            client_max_body_size 512M;
+          '';
         proxyPass = "http://${config.services.minio.listenAddress}";
       };
     };

config/services/monitoring.nix

@@ -1,4 +1,9 @@
-{config, ...}: {
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: {
   services.grafana = {
     enable = true;
     settings = {
@@ -12,14 +17,12 @@
 
     provision = {
       enable = true;
-
       datasources.settings.datasources = [
-        # "Built-in" datasources can be provisioned - c.f. https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources
+        (with config.services.prometheus; {
-        {
           name = "Prometheus";
           type = "prometheus";
-          url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+          url = "http://${listenAddress}:${toString port}";
-        }
+        })
       ];
     };
   };
@@ -28,87 +31,102 @@
 
   services.prometheus = {
     enable = true;
+    extraFlags = ["--web.enable-admin-api"];
     globalConfig.scrape_interval = "10s";
-    scrapeConfigs = [
+    scrapeConfigs =
+      builtins.map (config: {
+        inherit (config) job_name;
+        static_configs = [{targets = ["localhost:${toString config.port}"];}];
+      }) [
+        {
+          job_name = "fail2ban";
+          port = 9191;
+        }
         {
           job_name = "nginx";
-        static_configs = [
+          port = config.services.prometheus.exporters.nginx.port;
-          {
-            targets = ["localhost:${toString config.services.prometheus.exporters.nginx.port}"];
-          }
-        ];
         }
         {
           job_name = "nginxlog";
-        static_configs = [
+          port = config.services.prometheus.exporters.nginxlog.port;
-          {
-            targets = ["localhost:${toString config.services.prometheus.exporters.nginxlog.port}"];
-          }
-        ];
         }
         {
           job_name = "node";
-        static_configs = [
+          port = config.services.prometheus.exporters.node.port;
+        }
         {
-            targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+          job_name = "postgres";
+          port = config.services.prometheus.exporters.postgres.port;
         }
       ];
-      }
-      # {
-      #   job_name = "minio";
-      #   static_configs = [
-      #     {
-      #       targets = ["localhost:${toString config.services.prometheus.exporters.minio.port}"];
-      #     }
-      #   ];
-      # }
-      # {
-      #   job_name = "postgres";
-      #   static_configs = [
-      #     {
-      #       targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
-      #     }
-      #   ];
-      # }
-    ];
 
     exporters = {
       nginx.enable = true;
       nginxlog = {
         enable = true;
         group = "nginx";
-        settings.namespaces = [
+        settings.namespaces =
-          {
+          builtins.map (app: {
-            name = "filelogger";
+            name = app;
-            format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"";
+
-            source.files = ["/var/log/nginx/access.log"];
+            format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\"";
-          }
+
+            metrics_override.prefix = "nginxlog";
+            namespace_label = "vhost";
+
+            relabel = {
+              city.from = "geoip2_data_city_name";
+              country.from = "geoip2_data_country_name";
+            };
+
+            source.files = ["/var/log/nginx/${app}.access.log"];
+          }) [
+            "attic"
+            "atuin"
+            "forgejo"
+            "freshrss"
+            "minio"
+            "nextcloud"
+            "wakapi"
           ];
       };
-      # FIXME: set up auth!
+      node = {
-      # minio.enable = true;
-      # postgres = {
-      #   enable = true;
-      #   dataSourceName = "postgresql://localhost:5432/postgres?sslmode=disable";
-      # };
-    };
-
-    exporters.node = {
         enable = true;
-      port = 9000;
         enabledCollectors = ["processes" "systemd"];
+        disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"];
+      };
+      postgres = {
+        enable = true;
+        # FIXME: this is not ideal...
+        runAsLocalSuperUser = true;
+      };
     };
   };
 
-  services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
+  systemd.services.prometheus-fail2ban-exporter = {
+    wantedBy = ["multi-user.target"];
+    after = ["network.target"];
+    requires = ["network-online.target"];
+    serviceConfig = {
+      ExecStart = [(lib.getExe pkgs.prometheus-fail2ban-exporter)];
+      Restart = "on-failure";
+      NoNewPrivileges = true;
+      User = "root";
+      Group = "root";
+    };
+  };
+
+  services.nginx.virtualHosts = with config.services.grafana.settings.server; {
+    ${domain} = {
       forceSSL = true;
       enableACME = false;
       useACMEHost = "winston.sh";
 
       locations."/" = {
-      proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
+        proxyPass = "http://${http_addr}:${toString http_port}";
         proxyWebsockets = true;
         recommendedProxySettings = true;
       };
     };
+  };
 }

config/services/nextcloud.nix

@@ -53,5 +53,11 @@
     forceSSL = true;
     enableACME = false;
     useACMEHost = "winston.sh";
+
+    extraConfig =
+      # nginx
+      ''
+        access_log /var/log/nginx/nextcloud.access.log combined_geoip;
+      '';
   };
 }

config/services/nginx.nix

@@ -1,4 +1,8 @@
-{pkgs, ...}: let
+{
+  config,
+  pkgs,
+  ...
+}: let
   snakeoilCert = pkgs.runCommand "nginx-snakeoil-cert" {buildInputs = [pkgs.openssl];} ''
     mkdir "$out"
     openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 -subj '/CN=Snakeoil CA' -nodes -out "$out/cert.pem" -keyout "$out/cert.key"
@@ -7,12 +11,34 @@ in {
   services.nginx = {
     enable = true;
     package = pkgs.nginxMainline;
+    additionalModules = [pkgs.nginxModules.geoip2];
 
     recommendedGzipSettings = true;
     recommendedOptimisation = true;
     recommendedProxySettings = true;
     recommendedTlsSettings = true;
 
+    commonHttpConfig = let
+      geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;
+    in
+      # nginx
+      ''
+        geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
+          auto_reload 5m;
+          $geoip2_metadata_country_build metadata build_epoch;
+          $geoip2_data_country_code country iso_code;
+          $geoip2_data_country_name country names en;
+        }
+
+        geoip2 ${geoipDir}/GeoLite2-City.mmdb {
+          auto_reload 5m;
+          $geoip2_data_city_name city names en;
+        }
+
+        log_format combined_geoip '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time" "$geoip2_data_country_name" "$geoip2_data_city_name"';
+        access_log /var/log/nginx/access.log combined_geoip;
+      '';
+
     # https://github.com/NixOS/nixpkgs/issues/180980#issuecomment-1179723811
     virtualHosts = {
       "defaultDummy404" = {

config/services/wakapi.nix

@@ -25,11 +25,21 @@
     forceSSL = true;
     enableACME = false;
     useACMEHost = "winston.sh";
+
+    extraConfig =
+      # nginx
+      ''
+        access_log /var/log/nginx/wakapi.access.log combined_geoip;
+      '';
   };
 
   # for agenix owner permissions
-  users.users.wakapi.isSystemUser = true;
-  users.users.wakapi.group = "wakapi";
-  users.groups.wakapi = {};
   age.secrets."services/wakapi/password-salt.env".owner = "wakapi";
+  users = {
+    groups.wakapi = {};
+    users.wakapi = {
+      isSystemUser = true;
+      group = "wakapi";
+    };
+  };
 }

config/services/website/default.nix

@@ -14,7 +14,9 @@
 
     locations."/" = {
       root = "/var/lib/winston.sh/experiments";
-      extraConfig = ''
+      extraConfig =
+        # nginx
+        ''
           autoindex on;
           autoindex_exact_size off;
           autoindex_format html;

flake.nix

@@ -5,9 +5,10 @@
     ...
   } @ inputs: let
     overlays = [
-      (_: prev: rec {
+      (final: _: rec {
         atuin = unstable.atuin;
-        unstable = inputs.nixpkgs-unstable.legacyPackages.${prev.system};
+        prometheus-fail2ban-exporter = final.callPackage ./pkgs/prometheus-fail2ban-exporter {};
+        unstable = inputs.nixpkgs-unstable.legacyPackages.${final.system};
       })
     ];
   in

pkgs/prometheus-fail2ban-exporter/default.nix

@@ -0,0 +1,27 @@
+{
+  buildGoModule,
+  fetchFromGitLab,
+  lib,
+}: let
+  version = "0.10.1";
+in
+  buildGoModule {
+    pname = "prometheus-fail2ban-exporter";
+    inherit version;
+
+    src = fetchFromGitLab {
+      owner = "hectorjsmith";
+      repo = "fail2ban-prometheus-exporter";
+      rev = "v${version}";
+      sha256 = "sha256-zGEhDy3uXIbvx4agSA8Mx7bRtiZZtoDZGbNbHc9L+yI=";
+    };
+
+    vendorHash = "sha256-5o8p5p0U/c0WAIV5dACnWA3ThzSh2tt5LIFMb59i9GY=";
+
+    meta = with lib; {
+      mainProgram = "fail2ban-prometheus-exporter";
+      description = "Collect and export metrics on Fail2Ban";
+      homepage = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter";
+      license = licenses.mit;
+    };
+  }