From d00b184d91c01f4e2d0e4c753234dbcb11249c3b Mon Sep 17 00:00:00 2001 From: ctsk <9384305+ctsk@users.noreply.github.com> Date: Sun, 1 Oct 2023 12:26:11 +0200 Subject: [PATCH] [observatory] Set up monitoring (grafana, prometheus, loki) --- lib/modules/observatory/default.nix | 207 ++++++++++++++++++++++++++++ lib/systems/fugitive/default.nix | 21 ++- secrets/grafana.age | 8 ++ 3 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 lib/modules/observatory/default.nix create mode 100644 secrets/grafana.age diff --git a/lib/modules/observatory/default.nix b/lib/modules/observatory/default.nix new file mode 100644 index 0000000..d6ae08c --- /dev/null +++ b/lib/modules/observatory/default.nix @@ -0,0 +1,207 @@ +{ config, pkgs, lib, ...}: + +with lib; + +let + + cfg = config.services.observatory; + +in { + + options.services.observatory = { + enable = mkEnableOption "observatory"; + domain = mkOption { + type = types.str; + default = "observe.enclave.ctsk.dev"; + }; + certDomain = mkOption { + type = types.str; + default = "enclave.ctsk.dev"; + }; + + ports = { + grafana = mkOption { + type = types.port; + default = 11001; + }; + prometheus = mkOption { + type = types.port; + default = 11002; + }; + }; + }; + + config = mkIf cfg.enable { + services.grafana.enable = true; + services.grafana.settings = { + server = { + domain = cfg.domain; + http_port = cfg.ports.grafana; + http_addr = "127.0.0.1"; + }; + + security = mkIf (builtins.hasAttr "grafana" config.age.secrets) { + admin_user = "c"; + admin_password = "$__file{${config.age.secrets.grafana.path}}"; + }; + }; + + services.grafana.provision.enable = true; + services.grafana.provision.datasources.settings.datasources = + [ (mkIf config.services.prometheus.enable + { + name = "Prometheus"; + type = "prometheus"; + access = "proxy"; + url = "http://127.0.0.1:${toString config.services.prometheus.port}"; + } + ) + (mkIf config.services.loki.enable + { + name = "Loki"; + type = "loki"; + access = "proxy"; + url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}"; + } + ) + ]; + + services.prometheus.enable = true; + services.prometheus = { + port = cfg.ports.prometheus; + exporters = { + node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 11003; + }; + }; + scrapeConfigs = [ + { + job_name = "local_systemd"; + static_configs = [{ + targets = [ "127.0.0.1:11003" ]; + }]; + } + ]; + }; + + services.loki.enable = true; + services.loki.configuration = { + auth_enabled = false; + + server = { + http_listen_port = 11004; + }; + + ingester = { + lifecycler = { + address = "127.0.0.1"; + ring = { + kvstore.store = "inmemory"; + replication_factor = 1; + }; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 1048576; + chunk_retain_period = "30s"; + max_transfer_retries = 0; + }; + + schema_config = { + configs = [{ + from = "2023-01-01"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v12"; + index = { + prefix = "index_"; + period = "24h"; + }; + }]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + shared_store = "filesystem"; + }; + + filesystem.directory = "/var/lib/loki/chunks"; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + }; + + chunk_store_config = { + max_look_back_period = "0s"; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + + compactor = { + working_directory = "/var/lib/loki"; + shared_store = "filesystem"; + compactor_ring.kvstore.store = "inmemory"; + }; + }; + + services.promtail.enable = true; + services.promtail.configuration = { + server = { + http_listen_port = 11005; + grpc_listen_port = 0; + }; + + positions = { + filename = "/tmp/positions.yaml"; + }; + + clients = [{ + url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push"; + }]; + + scrape_configs = [{ + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = "observatory"; + }; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + }; + + + # services.nginx.upstreams = { + # "grafana".servers."127.0.0.1:${toString config.services.grafana.port}" = {}; + # "prometheus".servers."127.0.0.1:${toString config.services.prometheus.port}" = {}; + # "loki".servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {}; + # "promtail".servers."127.0.0.1:${toString config.services.promtail.configuration.server.http_listen_port}" = {}; + # }; + + services.nginx.virtualHosts."${cfg.domain}" = { + forceSSL = true; + useACMEHost = cfg.certDomain; + locations."/".proxyPass = "http://127.0.0.1:${toString cfg.ports.grafana}"; + locations."/".proxyWebsockets = true; + extraConfig = '' + allow 10.11.0.0/16; + deny all; + ''; + }; + }; +} \ No newline at end of file diff --git a/lib/systems/fugitive/default.nix b/lib/systems/fugitive/default.nix index 7499051..895454c 100644 --- a/lib/systems/fugitive/default.nix +++ b/lib/systems/fugitive/default.nix @@ -19,17 +19,25 @@ let in { + imports = [ ./hardware.nix + ../../modules/gitea ../../modules/headscale + ../../modules/observatory + ../../users/christian ]; services = { gitea.enable = true; - nginx.enable = true; headscale.enable = true; + observatory.enable = true; + nginx = { + enable = true; + recommendedProxySettings = true; + }; openssh = { enable = true; ports = ports.ssh; @@ -46,6 +54,9 @@ in }; }; + + networking.hostName = "fugitive"; + networking.firewall = { enable = true; allowedTCPPorts = [ 80 443 ]; @@ -67,8 +78,12 @@ in }; }; - age = { - secrets.namecheap.file = ../../../secrets/namecheap.age; + age.secrets = { + namecheap.file = ../../../secrets/namecheap.age; + grafana = { + file = ../../../secrets/grafana.age; + owner = "grafana"; + }; }; security = { diff --git a/secrets/grafana.age b/secrets/grafana.age new file mode 100644 index 0000000..1a6b7c2 --- /dev/null +++ b/secrets/grafana.age @@ -0,0 +1,8 @@ +age-encryption.org/v1 +-> ssh-ed25519 jfi4TQ a9RF/wMLY4K4QdRerb/884GDUbxCbAYEHDtGDGUEbH0 +XW7JAh516Z0Ld02/NK5FaZ1XSpCJbW7mwTZQaLmC8Nk +-> =U.,!-=_-grease +CjxZKY66lnUzRVMYNXFNEJh2nNdlQ/RVQHq2fZrWptelVXqe3bYXuI2d47Sm4E+7 +jD9++lM +--- /toZRWJ5mLoptTqJyyFaw1NiBG1JNLxcDqfifyMjjLo +FG=*9Ng0'-ZCARA|-FeXcd6.Y(-J \ No newline at end of file