infra/config/services/monitoring.nix

{
  config,
  lib,
  pkgs,
  ...
}:
let
  geoipDir = config.services.geoipupdate.settings.DatabaseDirectory;

  log_params = [
    "remote_addr"
    "time_local"
    "scheme"
    "server_name"
    "status"
    "body_bytes_sent"
    "http_user_agent"
    "request_length"
    "request_time"
    "request_uri"
    "upstream_response_time"
    "geoip2_data_country_code"
    "geoip2_data_city_name"
    "geoip2_data_latitude"
    "geoip2_data_longitude"
  ];
  log_format = lib.concatStringsSep " " (map (p: "${p}=\"\$${p}\"") log_params);
  log_file = "/var/log/nginx/access.log";
in
{
  services.grafana = {
    enable = true;
    settings = {
      server = {
        http_addr = "127.0.0.1";
        http_port = 21983;
        domain = "grafana.winston.sh";
        serve_from_sub_path = true;
      };
    };

    provision = {
      enable = true;
      datasources.settings.datasources = [
        (with config.services.prometheus; {
          name = "Prometheus";
          type = "prometheus";
          url = "http://${listenAddress}:${toString port}";
        })
      ];
    };
  };

  services.nginx = {
    additionalModules = [ pkgs.nginxModules.geoip2 ];
    commonHttpConfig =
      # nginx
      ''
        geoip2 ${geoipDir}/GeoLite2-Country.mmdb {
          auto_reload 5m;
          $geoip2_metadata_country_build metadata build_epoch;
          $geoip2_data_country_code default=Unknown country iso_code;
        }

        geoip2 ${geoipDir}/GeoLite2-City.mmdb {
          auto_reload 5m;
          $geoip2_data_city_name city names en;
          $geoip2_data_latitude default=0.0 location latitude;
          $geoip2_data_longitude default=0.0 location longitude;
        }

        log_format combined_geoip '${log_format}';
        access_log ${log_file} combined_geoip;
      '';
    statusPage = true;
  };

  age.secrets."services/prometheus/minio-bearer-token".owner = "prometheus";
  services.prometheus = {
    enable = true;

    # using agenix means secrets are mounted in /run/agenix which is inaccessible for the `promtool check` derivation
    checkConfig = "syntax-only";

    globalConfig.scrape_interval = "10s";
    scrapeConfigs =
      (builtins.map
        (config: {
          inherit (config) job_name;
          static_configs = [ { targets = [ "localhost:${toString config.port}" ]; } ];
        })
        [
          {
            job_name = "fail2ban";
            port = 9191;
          }
          {
            job_name = "nginx";
            port = config.services.prometheus.exporters.nginx.port;
          }
          {
            job_name = "nginxlog";
            port = config.services.prometheus.exporters.nginxlog.port;
          }
          {
            job_name = "node";
            port = config.services.prometheus.exporters.node.port;
          }
          {
            job_name = "postgres";
            port = config.services.prometheus.exporters.postgres.port;
          }
        ]
      )
      ++ [
        {
          job_name = "minio";
          bearer_token_file = config.age.secrets."services/prometheus/minio-bearer-token".path;
          metrics_path = "/minio/v2/metrics/cluster";
          static_configs = [ { targets = [ config.services.minio.listenAddress ]; } ];
        }
        {
          job_name = "forgejo";
          static_configs = with config.services.forgejo.settings.server; [
            { targets = [ "${HTTP_ADDR}:${toString HTTP_PORT}" ]; }
          ];
        }
      ];

    exporters = {
      nginx.enable = true;
      nginxlog = {
        enable = true;
        group = "nginx";
        settings.namespaces = [
          {
            format = log_format;

            metrics_override.prefix = "nginxlog";

            relabel_configs = [
              {
                target_label = "vhost";
                from = "server_name";
              }
              {
                target_label = "status";
                from = "status";
                only_counter = true;
              }
              {
                target_label = "geo_city";
                from = "geoip2_data_city_name";
              }
              {
                target_label = "geo_country";
                from = "geoip2_data_country_code";
              }
              {
                target_label = "latitude";
                from = "geoip2_data_latitude";
              }
              {
                target_label = "longitude";
                from = "geoip2_data_longitude";
              }
            ];

            source.files = [ log_file ];
          }
        ];
      };
      node = {
        enable = true;
        enabledCollectors = [
          "logind"
          "processes"
          "systemd"
        ];
        disabledCollectors = [
          "bonding"
          "fibrechannel"
          "infiniband"
          "ipvs"
          "mdadm"
          "nfs"
          "nfsd"
          "nvme"
          "tapestats"
          "watchdog"
          "zfs"
        ];
      };
      postgres = {
        enable = true;
        # FIXME: this is not ideal...
        runAsLocalSuperUser = true;
      };
    };
  };

  systemd.services.prometheus-fail2ban-exporter = {
    wantedBy = [ "multi-user.target" ];
    after = [ "network.target" ];
    requires = [ "network-online.target" ];
    serviceConfig = {
      ExecStart = [ (lib.getExe pkgs.prometheus-fail2ban-exporter) ];
      Restart = "on-failure";
      NoNewPrivileges = true;
      User = "root";
      Group = "root";
    };
  };

  services.nginx.virtualHosts = with config.services.grafana.settings.server; {
    ${domain} = {
      forceSSL = true;
      enableACME = false;
      useACMEHost = "winston.sh";

      locations."/" = {
        proxyPass = "http://${http_addr}:${toString http_port}";
        proxyWebsockets = true;
        recommendedProxySettings = true;
      };
    };
  };
}