infra/lib/modules/observatory/default.nix

{ config, pkgs, lib, ...}:

with lib;

let

  cfg = config.services.observatory;

in {

  options.services.observatory = {
    enable = mkEnableOption "observatory";
    domain = mkOption {
      type = types.str;
      default = "observe.enclave.ctsk.dev";
    };
    certDomain = mkOption {
      type = types.str;
      default = "enclave.ctsk.dev";
    };

    ports = {
      grafana = mkOption {
        type = types.port;
        default = 11001;
      };
      prometheus = mkOption {
        type = types.port;
        default = 11002;
      };
    };
  };

  config = mkIf cfg.enable {
    services.grafana.enable = true;
    services.grafana.settings = {
      server = {
        domain = cfg.domain;
        http_port = cfg.ports.grafana;
        http_addr = "127.0.0.1";  
      };

      security = mkIf (builtins.hasAttr "grafana" config.age.secrets) {
        admin_user = "c";
        admin_password = "$__file{${config.age.secrets.grafana.path}}";
      };
    };

    services.grafana.provision.enable = true;
    services.grafana.provision.datasources.settings.datasources =
      [ (mkIf config.services.prometheus.enable 
          {
            name = "Prometheus";
            type = "prometheus";
            access = "proxy";
            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
          }
        )
        (mkIf config.services.loki.enable
          {
            name = "Loki";
            type = "loki";
            access = "proxy";
            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
          }
        )
      ];

    services.prometheus.enable = true;
    services.prometheus = {
      port = cfg.ports.prometheus;
      exporters = {
        node = {
          enable = true;
          enabledCollectors = [ "systemd" ];
          port = 11003;
        };
      };
      scrapeConfigs = [
        {
          job_name = "local_systemd";
          static_configs = [{
            targets = [ "127.0.0.1:11003" ];
          }];
        }
      ];
    };

    services.loki.enable = true;
    services.loki.configuration = {
      auth_enabled = false;

      server = {
        http_listen_port = 11004;
      };

      ingester = {
        lifecycler = {
          address = "127.0.0.1";
          ring = {
            kvstore.store = "inmemory";
            replication_factor = 1;
          };
        };
        chunk_idle_period = "1h";
        max_chunk_age = "1h";
        chunk_target_size = 1048576;
        chunk_retain_period = "30s";
      };

      schema_config = {
        configs = [
          {
            from = "2023-01-01";
            store = "boltdb-shipper";
            object_store = "filesystem";
            schema = "v12";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
          {
            from = "2024-10-21";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };

      storage_config = {
        boltdb_shipper = {
          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
          cache_location = "/var/lib/loki/boltdb-shipper-cache";
          cache_ttl = "24h";
        };
        tsdb_shipper = {
          active_index_directory = "/var/lib/loki/tsdb-shipper-active";
          cache_location = "/var/lib/loki/tsdb-shipper-cache";
          cache_ttl = "24h";
        };
        
        filesystem.directory = "/var/lib/loki/chunks";
      };

      limits_config = {
        reject_old_samples = true;
        reject_old_samples_max_age = "168h";
      };
      
      table_manager = {
        retention_deletes_enabled = false;
        retention_period = "0s";
      };

      compactor = {
        working_directory = "/var/lib/loki";
        compactor_ring.kvstore.store = "inmemory";
      };
    };

    services.promtail.enable = true;
    services.promtail.configuration = {
      server = {
        http_listen_port = 11005;
        grpc_listen_port = 0;
      };

      positions = {
        filename = "/tmp/positions.yaml";
      };

      clients = [{
        url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
      }];

      scrape_configs = [{
        job_name = "journal";
        journal = {
          max_age = "12h";
          labels = {
            job = "systemd-journal";
            host = "observatory";
          };
        };
        relabel_configs = [{
          source_labels = [ "__journal__systemd_unit" ];
          target_label = "unit";
        }];
      }];
    };


    # services.nginx.upstreams = {
    #   "grafana".servers."127.0.0.1:${toString config.services.grafana.port}" = {};
    #   "prometheus".servers."127.0.0.1:${toString config.services.prometheus.port}" = {};
    #   "loki".servers."127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}" = {};
    #   "promtail".servers."127.0.0.1:${toString config.services.promtail.configuration.server.http_listen_port}" = {};
    # };

    services.nginx.virtualHosts."${cfg.domain}" = {
      forceSSL = true;
      useACMEHost = cfg.certDomain;
      locations."/".proxyPass = "http://127.0.0.1:${toString cfg.ports.grafana}";
      locations."/".proxyWebsockets = true;
      extraConfig = ''
      allow 10.11.0.0/16;
      deny all;
      '';
    };
  };
}