[observatory] Set up monitoring (grafana, prometheus, loki)

lib/modules/observatory/default.nix

@@ -0,0 +1,207 @@
+{ config, pkgs, lib, ...}:
+
+with lib;
+
+let
+
+  cfg = config.services.observatory;
+
+in {
+
+  options.services.observatory = {
+    enable = mkEnableOption "observatory";
+    domain = mkOption {
+      type = types.str;
+      default = "observe.enclave.ctsk.dev";
+    };
+    certDomain = mkOption {
+      type = types.str;
+      default = "enclave.ctsk.dev";
+    };
+
+    ports = {
+      grafana = mkOption {
+        type = types.port;
+        default = 11001;
+      };
+      prometheus = mkOption {
+        type = types.port;
+        default = 11002;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.grafana.enable = true;
+    services.grafana.settings = {
+      server = {
+        domain = cfg.domain;
+        http_port = cfg.ports.grafana;
+        http_addr = "127.0.0.1";  
+      };
+
+      security = mkIf (builtins.hasAttr "grafana" config.age.secrets) {
+        admin_user = "c";
+        admin_password = "$__file{${config.age.secrets.grafana.path}}";
+      };
+    };
+
+    services.grafana.provision.enable = true;
+    services.grafana.provision.datasources.settings.datasources =
+      [ (mkIf config.services.prometheus.enable 
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+          }
+        )
+        (mkIf config.services.loki.enable
+          {
+            name = "Loki";
+            type = "loki";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+          }
+        )
+      ];
+
+    services.prometheus.enable = true;
+    services.prometheus = {
+      port = cfg.ports.prometheus;
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" ];
+          port = 11003;
+        };
+      };
+      scrapeConfigs = [
+        {
+          job_name = "local_systemd";
+          static_configs = [{
+            targets = [ "127.0.0.1:11003" ];
+          }];
+        }
+      ];
+    };
+
+    services.loki.enable = true;
+    services.loki.configuration = {
+      auth_enabled = false;
+
+      server = {
+        http_listen_port = 11004;
+      };
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore.store = "inmemory";
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 1048576;
+        chunk_retain_period = "30s";
+        max_transfer_retries = 0;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2023-01-01";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v12";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+
+        filesystem.directory = "/var/lib/loki/chunks";
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      chunk_store_config = {
+        max_look_back_period = "0s";
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki";
+        shared_store = "filesystem";
+        compactor_ring.kvstore.store = "inmemory";
+      };
+    };
+
+    services.promtail.enable = true;
+    services.promtail.configuration = {
+      server = {
+        http_listen_port = 11005;
+        grpc_listen_port = 0;
+      };
+
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+
+      clients = [{
+        url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+      }];
+
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "12h";
+          labels = {
+            job = "systemd-journal";
+            host = "observatory";
+          };
+        };
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+
+
+    # services.nginx.upstreams = {
+    #   "grafana".servers."127.0.0.1:${toString config.services.grafana.port}" = {};
+    #   "prometheus".servers."127.0.0.1:${toString config.services.prometheus.port}" = {};
+    #   "loki".servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {};
+    #   "promtail".servers."127.0.0.1:${toString config.services.promtail.configuration.server.http_listen_port}" = {};
+    # };
+
+    services.nginx.virtualHosts."${cfg.domain}" = {
+      forceSSL = true;
+      useACMEHost = cfg.certDomain;
+      locations."/".proxyPass = "http://127.0.0.1:${toString cfg.ports.grafana}";
+      locations."/".proxyWebsockets = true;
+      extraConfig = ''
+      allow 10.11.0.0/16;
+      deny all;
+      '';
+    };
+  };
+}

lib/systems/fugitive/default.nix

@@ -19,17 +19,25 @@ let
 in
 {
 
+
   imports = [
     ./hardware.nix
+    
     ../../modules/gitea
     ../../modules/headscale
+    ../../modules/observatory
+
     ../../users/christian
   ];
 
   services = {
     gitea.enable = true;
-    nginx.enable = true;
     headscale.enable = true;
+    observatory.enable = true;
+    nginx = {
+      enable = true;
+      recommendedProxySettings = true;
+    };
     openssh = {
       enable = true;
       ports = ports.ssh;
@@ -46,6 +54,9 @@ in
     };
   };
 
+
+  networking.hostName = "fugitive";
+
   networking.firewall = {
     enable = true;
     allowedTCPPorts = [ 80 443 ];
@@ -67,8 +78,12 @@ in
     };
   };
 
-  age = {
-    secrets.namecheap.file = ../../../secrets/namecheap.age;
+  age.secrets = {
+    namecheap.file = ../../../secrets/namecheap.age;
+    grafana = {
+      file = ../../../secrets/grafana.age;
+      owner = "grafana";
+    };
   };
 
   security = {

secrets/grafana.age

@@ -0,0 +1,8 @@
+age-encryption.org/v1
+-> ssh-ed25519 jfi4TQ a9RF/wMLY4K4QdRerb/884GDUbxCbAYEHDtGDGUEbH0
+XW7JAh516Z0Ld02/NK5FaZ1XSpCJbW7mwTZQaLmC8Nk
+-> =U.,!-=_-grease
+CjxZKY66lnUzRVMYNXFNEJh2nNdlQ/RVQHq2fZrWptelVXqe3bYXuI2d47Sm4E+7
+jD9++lM
+--- /toZRWJ5mLoptTqJyyFaw1NiBG1JNLxcDqfifyMjjLo
+<EFBFBD><EFBFBD>F<><46><EFBFBD>G=*9<><39>N<EFBFBD><4E>g0'<27><><EFBFBD><EFBFBD>-<14>Z<EFBFBD>CA<43>R<EFBFBD>A<EFBFBD><41>|<7C><><1F>-Fe<46><65>Xc<58>d<EFBFBD>6.<2E>Y<EFBFBD>(-J<><4A><EFBFBD><EFBFBD><EFBFBD>