diff --git a/modules/capabilities/services/vps-insights.nix b/modules/capabilities/services/vps-insights.nix new file mode 100644 index 0000000..2724b85 --- /dev/null +++ b/modules/capabilities/services/vps-insights.nix @@ -0,0 +1,215 @@ +{ ... }: +{ + flake.modules.nixos.vps-insights = + { config, pkgs, ... }: + let + localAddress = "127.0.0.1"; + grafanaDataDir = config.services.grafana.dataDir; + lokiDataDir = config.services.loki.dataDir; + lokiUrl = "http://${localAddress}:3100"; + prometheusUrl = "http://${localAddress}:9090"; + in + { + environment = { + etc."alloy/config.alloy".text = '' + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal_syslog_identifier"] + target_label = "syslog_identifier" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + loki.source.journal "system" { + forward_to = [loki.write.local.receiver] + relabel_rules = loki.relabel.journal.rules + max_age = "24h" + labels = { + host = "${config.networking.hostName}", + } + } + + loki.write "local" { + endpoint { + url = "${lokiUrl}/loki/api/v1/push" + } + } + ''; + + systemPackages = with pkgs; [ + goaccess + lynis + ]; + }; + + services = { + # Keep local system logs available for Loki and manual inspection. + journald.extraConfig = '' + Storage=persistent + SystemMaxUse=1G + MaxRetentionSec=30day + ''; + + # Detect and block common attacks against SSH and Caddy. + crowdsec = { + enable = true; + hub.collections = [ + "crowdsecurity/linux" + "crowdsecurity/caddy" + ]; + localConfig.acquisitions = [ + { + source = "journalctl"; + journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ]; + labels.type = "syslog"; + } + { + filenames = [ "/var/log/caddy/*.log" ]; + labels.type = "caddy"; + } + ]; + }; + + crowdsec-firewall-bouncer = { + enable = true; + registerBouncer.bouncerName = "${config.networking.hostName}-firewall-bouncer"; + }; + + # Grafana defaults to 127.0.0.1:3000; add secrets and datasources only. + grafana = { + enable = true; + settings = { + analytics.reporting_enabled = false; + security = { + admin_password = "$__file{${grafanaDataDir}/admin-password}"; + secret_key = "$__file{${grafanaDataDir}/secret-key}"; + }; + }; + provision.datasources.settings = { + prune = true; + datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + uid = "prometheus"; + url = prometheusUrl; + isDefault = true; + } + { + name = "Loki"; + type = "loki"; + uid = "loki"; + url = lokiUrl; + } + ]; + }; + }; + + # Store local logs in Loki and feed them from journald through Alloy. + loki = { + enable = true; + configuration = { + analytics.reporting_enabled = false; + auth_enabled = false; + + server = { + http_listen_address = localAddress; + http_listen_port = 3100; + grpc_listen_address = localAddress; + grpc_listen_port = 9096; + }; + + common = { + path_prefix = lokiDataDir; + replication_factor = 1; + instance_interface_names = [ "lo" ]; + ring = { + instance_addr = localAddress; + kvstore.store = "inmemory"; + }; + }; + + schema_config.configs = [ + { + from = "2025-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + + storage_config.filesystem.directory = "${lokiDataDir}/chunks"; + + compactor = { + working_directory = "${lokiDataDir}/compactor"; + retention_enabled = true; + delete_request_store = "filesystem"; + }; + + limits_config.retention_period = "720h"; + }; + }; + + alloy = { + enable = true; + extraFlags = [ "--server.http.listen-addr=${localAddress}:12345" ]; + }; + + # Collect basic VPS health metrics for Grafana. + prometheus = { + enable = true; + listenAddress = localAddress; + retentionTime = "30d"; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ "${localAddress}:9090" ]; + } + ]; + } + { + job_name = "node"; + static_configs = [ + { + targets = [ "${localAddress}:9100" ]; + } + ]; + } + ]; + exporters.node = { + enable = true; + listenAddress = localAddress; + }; + }; + }; + + systemd.services.grafana.preStart = '' + umask 077 + + if [ ! -s ${grafanaDataDir}/admin-password ]; then + ${pkgs.openssl}/bin/openssl rand -base64 32 > ${grafanaDataDir}/admin-password + fi + + if [ ! -s ${grafanaDataDir}/secret-key ]; then + ${pkgs.openssl}/bin/openssl rand -hex 32 > ${grafanaDataDir}/secret-key + fi + ''; + }; +} diff --git a/modules/hosts/orion/default.nix b/modules/hosts/orion/default.nix index 5bf25f3..10174f8 100644 --- a/modules/hosts/orion/default.nix +++ b/modules/hosts/orion/default.nix @@ -25,6 +25,7 @@ in nixosModules.caddy nixosModules.server-firewall nixosModules.sudo-ssh-agent-auth + nixosModules.vps-insights nixosModules.vaultwarden nixosModules.radicale nixosModules.actual