prole/supabase/helm/knoe-supabase/templates/vector/config.yaml

{{- if .Values.deployment.vector.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ include "supabase.vector.fullname" . }}-config
  labels:
    {{- include "supabase.labels" . | nindent 4 }}
data:
  vector.yml: |
    api:
      enabled: true
      address: 0.0.0.0:{{ .Values.service.vector.port }}

    sources:
      kubernetes_host:
        type: kubernetes_logs
        extra_label_selector: app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/name!={{ include "supabase.vector.name" . }}

    transforms:
      project_logs:
        type: remap
        inputs:
          - kubernetes_host
        source: |-
          .project = "default"
          .event_message = del(.message)
          .appname = del(.kubernetes.container_name)
          del(.file)
          del(.kubernetes)
          del(.source_type)
          del(.stream)
      router:
        type: route
        inputs:
          - project_logs
        route:
          kong: '.appname == {{ include "supabase.kong.name" . | quote }}'
          auth: '.appname == {{ include "supabase.auth.name" . | quote }}'
          rest: '.appname == {{ include "supabase.rest.name" . | quote }}'
          realtime: '.appname == {{ include "supabase.realtime.name" . | quote }}'
          storage: '.appname == {{ include "supabase.storage.name" . | quote }}'
          functions: '.appname == {{ include "supabase.functions.name" . | quote }}'
          db: '.appname == {{ include "supabase.db.name" . | quote }}'
      # Ignores non nginx errors since they are related with kong booting up
      kong_logs:
        type: remap
        inputs:
          - router.kong
        source: |-
          req, err = parse_nginx_log(.event_message, "combined")
          if err == null {
              .timestamp = req.timestamp
              .metadata.request.headers.referer = req.referer
              .metadata.request.headers.user_agent = req.agent
              .metadata.request.headers.cf_connecting_ip = req.client
              .metadata.response.status_code = req.status
              url, split_err = split(req.request, " ")
              if split_err == null {
                  .metadata.request.method = url[0]
                  .metadata.request.path = url[1]
                  .metadata.request.protocol = url[2]
              }
          }
          if err != null {
            abort
          }
      # Ignores non nginx errors since they are related with kong booting up
      kong_err:
        type: remap
        inputs:
          - router.kong
        source: |-
          .metadata.request.method = "GET"
          .metadata.response.status_code = 200
          parsed, err = parse_nginx_log(.event_message, "error")
          if err == null {
              .timestamp = parsed.timestamp
              .severity = parsed.severity
              .metadata.request.host = parsed.host
              .metadata.request.headers.cf_connecting_ip = parsed.client
              url, err = split(parsed.request, " ")
              if err == null {
                  .metadata.request.method = url[0]
                  .metadata.request.path = url[1]
                  .metadata.request.protocol = url[2]
              }
          }
          if err != null {
            abort
          }
      # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.
      auth_logs:
        type: remap
        inputs:
          - router.auth
        source: |-
          parsed, err = parse_json(.event_message)
          if err == null {
              .metadata.timestamp = parsed.time
              .metadata = merge!(.metadata, parsed)
          }
      # PostgREST logs are structured so we separate timestamp from message using regex
      rest_logs:
        type: remap
        inputs:
          - router.rest
        source: |-
          parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')
          if err == null {
              .event_message = parsed.msg
              .timestamp = parse_timestamp!(value: parsed.time,format: "%d/%b/%Y:%H:%M:%S %z")
              .metadata.host = .project
          }
      # Filter out healthcheck logs from Realtime
      realtime_logs_filtered:
        type: filter
        inputs:
          - router.realtime
        condition: '!contains(string!(.event_message), "/health")' 
      # Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)
      realtime_logs:
        type: remap
        inputs:
          - realtime_logs_filtered
        source: |-
          .metadata.project = del(.project)
          .metadata.external_id = .metadata.project
          parsed, err = parse_regex(.event_message, r'^(?P<time>\d+:\d+:\d+\.\d+) \[(?P<level>\w+)\] (?P<msg>.*)$')
          if err == null {
              .event_message = parsed.msg
              .metadata.level = parsed.level
          }
      # Function logs are unstructured messages on stderr
      functions_logs:
        type: remap
        inputs:
          - router.functions
        source: |-
          .metadata.project_ref = del(.project)
      # Storage logs may contain json objects so we parse them for completeness
      storage_logs:
        type: remap
        inputs:
          - router.storage
        source: |-
          .metadata.project = del(.project)
          .metadata.tenantId = .metadata.project
          parsed, err = parse_json(.event_message)
          if err == null {
              .event_message = parsed.msg
              .metadata.level = parsed.level
              .metadata.timestamp = parsed.time
              .metadata.context[0].host = parsed.hostname
              .metadata.context[0].pid = parsed.pid
          }
      # Postgres logs some messages to stderr which we map to warning severity level
      db_logs:
        type: remap
        inputs:
          - router.db
        source: |-
          .metadata.host = "db-default"
          .metadata.parsed.timestamp = .timestamp
          
          parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)

          if err != null || parsed == null {
            .metadata.parsed.error_severity = "info"
          }
          if parsed.level != null {
          .metadata.parsed.error_severity = parsed.level
          }
          if .metadata.parsed.error_severity == "info" {
              .metadata.parsed.error_severity = "log"
          }
          .metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)

    {{- if .Values.deployment.analytics.enabled }}
    sinks:
      logflare_auth:
        type: 'http'
        inputs:
          - auth_logs
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=gotrue.logs.prod'
      logflare_realtime:
        type: 'http'
        inputs:
          - realtime_logs
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=realtime.logs.prod'
      logflare_rest:
        type: 'http'
        inputs:
          - rest_logs
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=postgREST.logs.prod'
      logflare_db:
        type: 'http'
        inputs:
          - db_logs
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        # We must route the sink through kong because ingesting logs before logflare is fully initialised will
        # lead to broken queries from studio. This works by the assumption that containers are started in the
        # following order: vector > db > logflare > kong
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=postgres.logs'
      logflare_functions:
        type: 'http'
        inputs:
          - functions_logs
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=deno-relay-logs'
      logflare_storage:
        type: 'http'
        inputs:
          - storage_logs
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=storage.logs.prod.2'
      logflare_kong:
        type: 'http'
        inputs:
          - kong_logs
          - kong_err
        encoding:
          codec: 'json'
        method: 'post'
        request:
          retry_max_duration_secs: 30
          headers:
            x-api-key: ${LOGFLARE_PUBLIC_ACCESS_TOKEN?LOGFLARE_PUBLIC_ACCESS_TOKEN is required}
        uri: 'http://{{ include "supabase.analytics.fullname" . }}:{{ .Values.service.analytics.port }}/api/logs?source_name=cloudflare.logs.prod'
    {{- end }}    
{{- end }}