lux/modules/capabilities/services/vps-insights.nix

{ ... }:
{
  flake.modules.nixos.vps-insights =
    { config, pkgs, ... }:
    let
      localAddress = "127.0.0.1";
      grafanaDataDir = config.services.grafana.dataDir;
      lokiDataDir = config.services.loki.dataDir;
      lokiUrl = "http://${localAddress}:3100";
      prometheusUrl = "http://${localAddress}:9090";
    in
    {
      environment = {
        etc."alloy/config.alloy".text = ''
          loki.relabel "journal" {
            forward_to = []

            rule {
              source_labels = ["__journal__systemd_unit"]
              target_label  = "unit"
            }

            rule {
              source_labels = ["__journal_syslog_identifier"]
              target_label  = "syslog_identifier"
            }

            rule {
              source_labels = ["__journal_priority_keyword"]
              target_label  = "level"
            }
          }

          loki.source.journal "system" {
            forward_to    = [loki.write.local.receiver]
            relabel_rules = loki.relabel.journal.rules
            max_age       = "24h"
            labels        = {
              host = "${config.networking.hostName}",
            }
          }

          loki.write "local" {
            endpoint {
              url = "${lokiUrl}/loki/api/v1/push"
            }
          }
        '';

        systemPackages = with pkgs; [
          goaccess
          lynis
        ];
      };

      services = {
        # Keep local system logs available for Loki and manual inspection.
        journald.extraConfig = ''
          Storage=persistent
          SystemMaxUse=1G
          MaxRetentionSec=30day
        '';

        # Detect and block common attacks against SSH and Caddy.
        crowdsec = {
          enable = true;
          hub.collections = [
            "crowdsecurity/linux"
            "crowdsecurity/caddy"
          ];
          localConfig.acquisitions = [
            {
              source = "journalctl";
              journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
              labels.type = "syslog";
            }
            {
              filenames = [ "/var/log/caddy/*.log" ];
              labels.type = "caddy";
            }
          ];
        };

        crowdsec-firewall-bouncer = {
          enable = true;
          registerBouncer.bouncerName = "${config.networking.hostName}-firewall-bouncer";
        };

        # Grafana defaults to 127.0.0.1:3000; add secrets and datasources only.
        grafana = {
          enable = true;
          settings = {
            analytics.reporting_enabled = false;
            security = {
              admin_password = "$__file{${grafanaDataDir}/admin-password}";
              secret_key = "$__file{${grafanaDataDir}/secret-key}";
            };
          };
          provision.datasources.settings = {
            prune = true;
            datasources = [
              {
                name = "Prometheus";
                type = "prometheus";
                uid = "prometheus";
                url = prometheusUrl;
                isDefault = true;
              }
              {
                name = "Loki";
                type = "loki";
                uid = "loki";
                url = lokiUrl;
              }
            ];
          };
        };

        # Store local logs in Loki and feed them from journald through Alloy.
        loki = {
          enable = true;
          configuration = {
            analytics.reporting_enabled = false;
            auth_enabled = false;

            server = {
              http_listen_address = localAddress;
              http_listen_port = 3100;
              grpc_listen_address = localAddress;
              grpc_listen_port = 9096;
            };

            common = {
              path_prefix = lokiDataDir;
              replication_factor = 1;
              instance_interface_names = [ "lo" ];
              ring = {
                instance_addr = localAddress;
                kvstore.store = "inmemory";
              };
            };

            schema_config.configs = [
              {
                from = "2025-01-01";
                store = "tsdb";
                object_store = "filesystem";
                schema = "v13";
                index = {
                  prefix = "index_";
                  period = "24h";
                };
              }
            ];

            storage_config.filesystem.directory = "${lokiDataDir}/chunks";

            compactor = {
              working_directory = "${lokiDataDir}/compactor";
              retention_enabled = true;
              delete_request_store = "filesystem";
            };

            limits_config.retention_period = "720h";
          };
        };

        alloy = {
          enable = true;
          extraFlags = [ "--server.http.listen-addr=${localAddress}:12345" ];
        };

        # Collect basic VPS health metrics for Grafana.
        prometheus = {
          enable = true;
          listenAddress = localAddress;
          retentionTime = "30d";
          scrapeConfigs = [
            {
              job_name = "prometheus";
              static_configs = [
                {
                  targets = [ "${localAddress}:9090" ];
                }
              ];
            }
            {
              job_name = "node";
              static_configs = [
                {
                  targets = [ "${localAddress}:9100" ];
                }
              ];
            }
          ];
          exporters.node = {
            enable = true;
            listenAddress = localAddress;
          };
        };
      };

      systemd.services.grafana.preStart = ''
        umask 077

        if [ ! -s ${grafanaDataDir}/admin-password ]; then
          ${pkgs.openssl}/bin/openssl rand -base64 32 > ${grafanaDataDir}/admin-password
        fi

        if [ ! -s ${grafanaDataDir}/secret-key ]; then
          ${pkgs.openssl}/bin/openssl rand -hex 32 > ${grafanaDataDir}/secret-key
        fi
      '';
    };
}