diff --git a/config/network.nix b/config/network.nix index 55eb1e8..64642f2 100644 --- a/config/network.nix +++ b/config/network.nix @@ -1,10 +1,11 @@ { networking.firewall.enable = true; + services = { fail2ban.enable = true; openssh = { enable = true; - ports = [1322]; + ports = [22]; settings = { PasswordAuthentication = false; PermitRootLogin = "no"; diff --git a/config/secrets/secrets.nix b/config/secrets/secrets.nix index 2316363..37c427e 100644 --- a/config/secrets/secrets.nix +++ b/config/secrets/secrets.nix @@ -17,6 +17,8 @@ in { "services/forgejo/minio-secretkey.age".publicKeys = default; "services/forgejo/runner-token.age".publicKeys = default; + "services/geoipupdate/license.age".publicKeys = default; + "services/gitlab/dbFile.age".publicKeys = default; "services/gitlab/jwsFile.age".publicKeys = default; "services/gitlab/otpFile.age".publicKeys = default; diff --git a/config/secrets/services/geoipupdate/license.age b/config/secrets/services/geoipupdate/license.age new file mode 100644 index 0000000..c1bb036 Binary files /dev/null and b/config/secrets/services/geoipupdate/license.age differ diff --git a/config/services/attic.nix b/config/services/attic.nix index 3b1eb2f..81c8c9b 100644 --- a/config/services/attic.nix +++ b/config/services/attic.nix @@ -25,7 +25,12 @@ useACMEHost = "winston.sh"; locations."/" = { - extraConfig = "client_max_body_size 512M;"; + extraConfig = + # nginx + '' + access_log /var/log/nginx/attic.access.log combined_geoip; + client_max_body_size 512M; + ''; proxyPass = "http://${config.services.atticd.settings.listen}"; }; }; diff --git a/config/services/atuin.nix b/config/services/atuin.nix index d9ada7f..1b206d9 100644 --- a/config/services/atuin.nix +++ b/config/services/atuin.nix @@ -12,6 +12,11 @@ useACMEHost = "winston.sh"; locations."/" = with config.services.atuin; { + extraConfig = + # nginx + '' + access_log /var/log/nginx/atuin.access.log combined_geoip; + ''; proxyPass = "http://${host}:${toString port}"; }; }; diff --git a/config/services/default.nix b/config/services/default.nix index eedb06a..0883c99 100644 --- a/config/services/default.nix +++ b/config/services/default.nix @@ -5,6 +5,7 @@ ./containers.nix ./forgejo.nix ./freshrss.nix + ./geoipupdate.nix ./minio.nix ./monitoring.nix ./nextcloud.nix diff --git a/config/services/forgejo.nix b/config/services/forgejo.nix index 296be90..603093c 100644 --- a/config/services/forgejo.nix +++ b/config/services/forgejo.nix @@ -5,7 +5,6 @@ ... }: let modules = ["services/misc/forgejo.nix" "services/continuous-integration/gitea-actions-runner.nix"]; - pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.system}; in { # swap out stable for unstable modules disabledModules = modules; @@ -27,7 +26,7 @@ in { services.forgejo = { enable = true; - package = pkgsUnstable.forgejo; + package = pkgs.unstable.forgejo; database = { type = "postgres"; @@ -60,9 +59,12 @@ in { server = rec { DOMAIN = "code.winston.sh"; + ROOT_URL = "https://${DOMAIN}/"; + HTTP_ADDR = "127.0.0.1"; HTTP_PORT = 12492; - ROOT_URL = "https://${DOMAIN}/"; + + # allow fetch from gravatar etc. OFFLINE_MODE = false; }; @@ -97,7 +99,7 @@ in { virtualisation.podman.enable = true; services.gitea-actions-runner = { - package = pkgsUnstable.forgejo-runner; + package = pkgs.unstable.forgejo-runner; instances.main = { enable = true; name = "main"; @@ -116,9 +118,19 @@ in { enableACME = false; useACMEHost = "winston.sh"; - locations."/" = with config.services.forgejo.settings.server; { - extraConfig = "client_max_body_size 512M;"; - proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}"; + locations = { + "/" = with config.services.forgejo.settings.server; { + extraConfig = + # nginx + '' + access_log /var/log/nginx/forgejo.access.log combined_geoip; + client_max_body_size 512M; + ''; + proxyPass = "http://${HTTP_ADDR}:${toString HTTP_PORT}"; + }; + + # don't spam the log with runner polls + "/api/actions/runner.v1.RunnerService/FetchTask".extraConfig = "access_log off;"; }; }; } diff --git a/config/services/freshrss.nix b/config/services/freshrss.nix index d73ed4e..87cabb9 100644 --- a/config/services/freshrss.nix +++ b/config/services/freshrss.nix @@ -15,5 +15,10 @@ forceSSL = true; enableACME = false; useACMEHost = "winston.sh"; + extraConfig = + # nginx + '' + access_log /var/log/nginx/freshrss.access.log combined_geoip; + ''; }; } diff --git a/config/services/geoipupdate.nix b/config/services/geoipupdate.nix new file mode 100644 index 0000000..9e76a5b --- /dev/null +++ b/config/services/geoipupdate.nix @@ -0,0 +1,10 @@ +{config, ...}: { + services.geoipupdate = { + enable = true; + settings = { + AccountID = 1062126; + LicenseKey = config.age.secrets."services/geoipupdate/license".path; + EditionIDs = ["GeoLite2-ASN" "GeoLite2-City" "GeoLite2-Country"]; + }; + }; +} diff --git a/config/services/minio.nix b/config/services/minio.nix index 91771e3..c79ae7c 100644 --- a/config/services/minio.nix +++ b/config/services/minio.nix @@ -20,7 +20,15 @@ forceSSL = true; enableACME = false; useACMEHost = "winston.sh"; - locations."/".proxyPass = "http://${config.services.minio.consoleAddress}"; + + locations."/" = { + extraConfig = + # nginx + '' + access_log /var/log/nginx/minio.access.log combined_geoip; + ''; + proxyPass = "http://${config.services.minio.consoleAddress}"; + }; }; "s3.winston.sh" = { forceSSL = true; @@ -28,7 +36,12 @@ useACMEHost = "winston.sh"; locations."/" = { - extraConfig = "client_max_body_size 512M;"; + extraConfig = + # nginx + '' + access_log /var/log/nginx/minio.access.log combined_geoip; + client_max_body_size 512M; + ''; proxyPass = "http://${config.services.minio.listenAddress}"; }; }; diff --git a/config/services/monitoring.nix b/config/services/monitoring.nix index e7dc1f2..025084b 100644 --- a/config/services/monitoring.nix +++ b/config/services/monitoring.nix @@ -1,4 +1,9 @@ -{config, ...}: { +{ + config, + lib, + pkgs, + ... +}: { services.grafana = { enable = true; settings = { @@ -12,14 +17,12 @@ provision = { enable = true; - datasources.settings.datasources = [ - # "Built-in" datasources can be provisioned - c.f. https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources - { + (with config.services.prometheus; { name = "Prometheus"; type = "prometheus"; - url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}"; - } + url = "http://${listenAddress}:${toString port}"; + }) ]; }; }; @@ -28,87 +31,102 @@ services.prometheus = { enable = true; + extraFlags = ["--web.enable-admin-api"]; globalConfig.scrape_interval = "10s"; - scrapeConfigs = [ - { - job_name = "nginx"; - static_configs = [ - { - targets = ["localhost:${toString config.services.prometheus.exporters.nginx.port}"]; - } - ]; - } - { - job_name = "nginxlog"; - static_configs = [ - { - targets = ["localhost:${toString config.services.prometheus.exporters.nginxlog.port}"]; - } - ]; - } - { - job_name = "node"; - static_configs = [ - { - targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; - } - ]; - } - # { - # job_name = "minio"; - # static_configs = [ - # { - # targets = ["localhost:${toString config.services.prometheus.exporters.minio.port}"]; - # } - # ]; - # } - # { - # job_name = "postgres"; - # static_configs = [ - # { - # targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"]; - # } - # ]; - # } - ]; + scrapeConfigs = + builtins.map (config: { + inherit (config) job_name; + static_configs = [{targets = ["localhost:${toString config.port}"];}]; + }) [ + { + job_name = "fail2ban"; + port = 9191; + } + { + job_name = "nginx"; + port = config.services.prometheus.exporters.nginx.port; + } + { + job_name = "nginxlog"; + port = config.services.prometheus.exporters.nginxlog.port; + } + { + job_name = "node"; + port = config.services.prometheus.exporters.node.port; + } + { + job_name = "postgres"; + port = config.services.prometheus.exporters.postgres.port; + } + ]; exporters = { nginx.enable = true; nginxlog = { enable = true; group = "nginx"; - settings.namespaces = [ - { - name = "filelogger"; - format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\""; - source.files = ["/var/log/nginx/access.log"]; - } - ]; - }; - # FIXME: set up auth! - # minio.enable = true; - # postgres = { - # enable = true; - # dataSourceName = "postgresql://localhost:5432/postgres?sslmode=disable"; - # }; - }; + settings.namespaces = + builtins.map (app: { + name = app; - exporters.node = { - enable = true; - port = 9000; - enabledCollectors = ["processes" "systemd"]; + format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" rt=$request_time uct=\"$upstream_connect_time\" uht=\"$upstream_header_time\" urt=\"$upstream_response_time\" \"$geoip2_data_country_name\" \"$geoip2_data_city_name\""; + + metrics_override.prefix = "nginxlog"; + namespace_label = "vhost"; + + relabel = { + city.from = "geoip2_data_city_name"; + country.from = "geoip2_data_country_name"; + }; + + source.files = ["/var/log/nginx/${app}.access.log"]; + }) [ + "attic" + "atuin" + "forgejo" + "freshrss" + "minio" + "nextcloud" + "wakapi" + ]; + }; + node = { + enable = true; + enabledCollectors = ["processes" "systemd"]; + disabledCollectors = ["bonding" "fibrechannel" "infiniband" "ipvs" "mdadm" "nfs" "nfsd" "nvme" "tapestats" "watchdog" "zfs"]; + }; + postgres = { + enable = true; + # FIXME: this is not ideal... + runAsLocalSuperUser = true; + }; }; }; - services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = { - forceSSL = true; - enableACME = false; - useACMEHost = "winston.sh"; + systemd.services.prometheus-fail2ban-exporter = { + wantedBy = ["multi-user.target"]; + after = ["network.target"]; + requires = ["network-online.target"]; + serviceConfig = { + ExecStart = [(lib.getExe pkgs.prometheus-fail2ban-exporter)]; + Restart = "on-failure"; + NoNewPrivileges = true; + User = "root"; + Group = "root"; + }; + }; - locations."/" = { - proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}"; - proxyWebsockets = true; - recommendedProxySettings = true; + services.nginx.virtualHosts = with config.services.grafana.settings.server; { + ${domain} = { + forceSSL = true; + enableACME = false; + useACMEHost = "winston.sh"; + + locations."/" = { + proxyPass = "http://${http_addr}:${toString http_port}"; + proxyWebsockets = true; + recommendedProxySettings = true; + }; }; }; } diff --git a/config/services/nextcloud.nix b/config/services/nextcloud.nix index 6ae9519..c12072e 100644 --- a/config/services/nextcloud.nix +++ b/config/services/nextcloud.nix @@ -53,5 +53,11 @@ forceSSL = true; enableACME = false; useACMEHost = "winston.sh"; + + extraConfig = + # nginx + '' + access_log /var/log/nginx/nextcloud.access.log combined_geoip; + ''; }; } diff --git a/config/services/nginx.nix b/config/services/nginx.nix index fe47181..8dd6c51 100644 --- a/config/services/nginx.nix +++ b/config/services/nginx.nix @@ -1,4 +1,8 @@ -{pkgs, ...}: let +{ + config, + pkgs, + ... +}: let snakeoilCert = pkgs.runCommand "nginx-snakeoil-cert" {buildInputs = [pkgs.openssl];} '' mkdir "$out" openssl req -newkey rsa:4096 -x509 -sha256 -days 36500 -subj '/CN=Snakeoil CA' -nodes -out "$out/cert.pem" -keyout "$out/cert.key" @@ -7,12 +11,34 @@ in { services.nginx = { enable = true; package = pkgs.nginxMainline; + additionalModules = [pkgs.nginxModules.geoip2]; recommendedGzipSettings = true; recommendedOptimisation = true; recommendedProxySettings = true; recommendedTlsSettings = true; + commonHttpConfig = let + geoipDir = config.services.geoipupdate.settings.DatabaseDirectory; + in + # nginx + '' + geoip2 ${geoipDir}/GeoLite2-Country.mmdb { + auto_reload 5m; + $geoip2_metadata_country_build metadata build_epoch; + $geoip2_data_country_code country iso_code; + $geoip2_data_country_name country names en; + } + + geoip2 ${geoipDir}/GeoLite2-City.mmdb { + auto_reload 5m; + $geoip2_data_city_name city names en; + } + + log_format combined_geoip '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time" "$geoip2_data_country_name" "$geoip2_data_city_name"'; + access_log /var/log/nginx/access.log combined_geoip; + ''; + # https://github.com/NixOS/nixpkgs/issues/180980#issuecomment-1179723811 virtualHosts = { "defaultDummy404" = { diff --git a/config/services/wakapi.nix b/config/services/wakapi.nix index fcbcf2b..870bbc3 100644 --- a/config/services/wakapi.nix +++ b/config/services/wakapi.nix @@ -25,11 +25,21 @@ forceSSL = true; enableACME = false; useACMEHost = "winston.sh"; + + extraConfig = + # nginx + '' + access_log /var/log/nginx/wakapi.access.log combined_geoip; + ''; }; # for agenix owner permissions - users.users.wakapi.isSystemUser = true; - users.users.wakapi.group = "wakapi"; - users.groups.wakapi = {}; age.secrets."services/wakapi/password-salt.env".owner = "wakapi"; + users = { + groups.wakapi = {}; + users.wakapi = { + isSystemUser = true; + group = "wakapi"; + }; + }; } diff --git a/config/services/website/default.nix b/config/services/website/default.nix index 44b1b71..4aaa63a 100644 --- a/config/services/website/default.nix +++ b/config/services/website/default.nix @@ -14,12 +14,14 @@ locations."/" = { root = "/var/lib/winston.sh/experiments"; - extraConfig = '' - autoindex on; - autoindex_exact_size off; - autoindex_format html; - autoindex_localtime on; - ''; + extraConfig = + # nginx + '' + autoindex on; + autoindex_exact_size off; + autoindex_format html; + autoindex_localtime on; + ''; }; }; } diff --git a/flake.nix b/flake.nix index 6fec85e..b09fc3b 100644 --- a/flake.nix +++ b/flake.nix @@ -5,9 +5,10 @@ ... } @ inputs: let overlays = [ - (_: prev: rec { + (final: _: rec { atuin = unstable.atuin; - unstable = inputs.nixpkgs-unstable.legacyPackages.${prev.system}; + prometheus-fail2ban-exporter = final.callPackage ./pkgs/prometheus-fail2ban-exporter {}; + unstable = inputs.nixpkgs-unstable.legacyPackages.${final.system}; }) ]; in diff --git a/pkgs/prometheus-fail2ban-exporter/default.nix b/pkgs/prometheus-fail2ban-exporter/default.nix new file mode 100644 index 0000000..9e97ff4 --- /dev/null +++ b/pkgs/prometheus-fail2ban-exporter/default.nix @@ -0,0 +1,27 @@ +{ + buildGoModule, + fetchFromGitLab, + lib, +}: let + version = "0.10.1"; +in + buildGoModule { + pname = "prometheus-fail2ban-exporter"; + inherit version; + + src = fetchFromGitLab { + owner = "hectorjsmith"; + repo = "fail2ban-prometheus-exporter"; + rev = "v${version}"; + sha256 = "sha256-zGEhDy3uXIbvx4agSA8Mx7bRtiZZtoDZGbNbHc9L+yI="; + }; + + vendorHash = "sha256-5o8p5p0U/c0WAIV5dACnWA3ThzSh2tt5LIFMb59i9GY="; + + meta = with lib; { + mainProgram = "fail2ban-prometheus-exporter"; + description = "Collect and export metrics on Fail2Ban"; + homepage = "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter"; + license = licenses.mit; + }; + }