values-full.yaml

# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

################################################################################
# NVSentinel Complete Configuration Reference
#
# This file documents every possible configuration option for NVSentinel.
# All options are set to their default values unless otherwise noted.
#
# NVSentinel is a Kubernetes-native system for detecting, quarantining,
# and remediating node failures in GPU clusters. It consists of:
#
# - Health Monitors: Detect hardware and software failures
# - Fault Quarantine: Cordon and taint nodes with detected faults  
# - Node Drainer: Gracefully evict workloads from failing nodes
# - Fault Remediation: Create maintenance resources to trigger node recovery
# - Janitor: Execute node reboots and terminations via cloud provider APIs
#
# For production use, enable the core modules and configure them for your
# specific environment.
################################################################################

################################################################################
# GLOBAL CONFIGURATION
#
# Settings that apply to all NVSentinel components.
# These can be overridden per-module in their individual sections.
################################################################################
global:
  # Docker image configuration
  image:
    # Image tag to use for all NVSentinel containers
    # Examples: "main" (latest development), "v1.0.0" (specific release)
    # Leave empty to use the appVersion from Chart.yaml
    tag: "main"

  # Metrics port for Prometheus scraping
  # All components expose health and metrics endpoints on this port unless using ctrl-runtime mgmt
  # Prometheus should scrape these endpoints to monitor NVSentinel
  metricsPort: 2112

  # Health Check port for healthz and readyz scraping
  # This port is only used when using ctrl-runtime
  # components expose health and metrics endpoints on this port
  # Prometheus should scrape these endpoints to monitor NVSentinel
  healthPort: 9440

  # Dry-run mode - when enabled, all actions are logged but not executed
  # Useful for:
  # - Testing configuration changes safely
  # - Validating rule behavior before production use
  # - Debugging without affecting the cluster
  # Components will log what they WOULD do but won't actually:
  # - Cordon or taint nodes (fault-quarantine)
  # - Evict or delete pods (node-drainer)
  # - Create maintenance resources (fault-remediation)
  # - Reboot or terminate nodes (janitor)
  dryRun: false

  # Image pull secrets for private container registries
  # Create these secrets in Kubernetes before installing NVSentinel:
  # kubectl create secret docker-registry my-registry-secret \
  #   --docker-server=myregistry.com \
  #   --docker-username=myuser \
  #   --docker-password=mypassword
  # Then reference them here:
  # imagePullSecrets:
  #   - name: my-registry-secret
  imagePullSecrets: []
  
  # Node selector for all NVSentinel workloads
  # Used to constrain pods to nodes with specific labels
  # Example - run only on nodes with label "nvsentinel=enabled":
  # nodeSelector:
  #   nvsentinel: "enabled"
  # Leave empty to allow scheduling on any node
  nodeSelector: {}
  
  # Tolerations for all NVSentinel workloads
  # Allows pods to schedule on nodes with matching taints
  # Example - tolerate GPU nodes:
  # tolerations:
  #   - key: "nvidia.com/gpu"
  #     operator: "Exists"
  #     effect: "NoSchedule"
  # Leave empty for no tolerations
  tolerations: []
  
  # Affinity rules for all NVSentinel workloads
  # Controls pod placement based on node properties or other pods
  # Example - spread pods across availability zones:
  # affinity:
  #   podAntiAffinity:
  #     preferredDuringSchedulingIgnoredDuringExecution:
  #       - weight: 100
  #         podAffinityTerm:
  #           topologyKey: topology.kubernetes.io/zone
  # Leave empty for no affinity rules
  affinity: {}
  
  # Node selector for system components
  # System components include: platform-connectors, fault-quarantine,
  # node-drainer, fault-remediation (control plane workloads)
  # Example - run system components on control plane nodes:
  # systemNodeSelector:
  #   node-role.kubernetes.io/control-plane: ""
  # Leave empty to use same as global nodeSelector
  systemNodeSelector: {}
  
  # Tolerations for system components
  # Allows system pods to run on tainted nodes (like control plane)
  # Example - tolerate control plane taint:
  # systemNodeTolerations:
  #   - key: "node-role.kubernetes.io/control-plane"
  #     operator: "Exists"
  #     effect: "NoSchedule"
  # Leave empty to use same as global tolerations
  systemNodeTolerations: []

  ################################################################################
  # MODULE ENABLE/DISABLE FLAGS
  #
  # Control which NVSentinel components are deployed.
  # For production, typically enable all modules except healthEventsAnalyzer.
  ################################################################################
  
  # GPU Health Monitor - monitors GPU hardware health via DCGM
  # Detects: XID errors, thermal issues, ECC errors, memory errors, NVLink failures
  # Requires: NVIDIA GPU Operator with DCGM installed
  gpuHealthMonitor:
    enabled: true

  # Health Events Analyzer - analyzes patterns in health events
  # Provides insights and recommendations based on historical data
  # Requires: MongoDB store enabled
  healthEventsAnalyzer:
    enabled: false

  # Fault Quarantine Module - cordons/taints nodes with detected faults
  # Prevents new workloads from scheduling on failing nodes
  # Uses rule-based evaluation to determine when to quarantine
  # Requires: MongoDB store enabled
  faultQuarantine:
    enabled: false

  # Node Drainer Module - evicts workloads from quarantined nodes
  # Gracefully terminates pods before node maintenance
  # Supports multiple eviction strategies per namespace
  # Requires: MongoDB store enabled, fault-quarantine enabled
  nodeDrainer:
    enabled: false

  # Fault Remediation Module - creates maintenance resources for node recovery
  # Triggers janitor to reboot or terminate failing nodes
  # Uses configurable templates for different remediation actions
  # Requires: MongoDB store enabled, janitor enabled
  faultRemediation:
    enabled: false

  # Janitor Module - executes node reboots and terminations
  # Integrates with cloud provider APIs to perform actual maintenance
  # Supports: AWS, GCP, Azure, OCI, kind (for testing)
  # Requires: Cloud provider credentials configured
  janitor:
    enabled: false

  # CSP Health Monitor - monitors cloud provider maintenance events
  # Detects scheduled maintenance windows from cloud provider APIs
  # Supports: GCP (Compute Engine), AWS (EC2)
  # Requires: Cloud provider credentials configured
  cspHealthMonitor:
    enabled: false

  # Syslog Health Monitor - analyzes system logs for fault patterns
  # Reads journalctl logs to detect kernel errors, driver issues
  # Runs as DaemonSet on all nodes
  # Requires: Access to host journalctl
  syslogHealthMonitor:
    enabled: true
    # XID sidecar - parses NVIDIA XID errors from dmesg
    # Extracts detailed XID error information
    xidSideCar:
      enabled: false

  # Labeler Module - applies labels to nodes based on capabilities
  # Detects: Kata Containers support, node features
  # Runs as Deployment to label all nodes in the cluster
  labeler:
    enabled: true

  # In-cluster File Server - HTTP server for log uploads
  # Used by log-collector jobs to upload diagnostic bundles
  # Only needed if fault-remediation log collection is enabled
  inclusterFileServer:
    enabled: false
    # Metrics port for file server
    metricsPort: 9001
    # Metrics port for cleanup job
    cleanupMetricsPort: 9002

  # MongoDB Store - database for health events
  # Stores all health events from monitors
  # Required by: fault-quarantine, node-drainer, fault-remediation
  # Uses Bitnami MongoDB chart with TLS enabled
  mongodbStore:
    enabled: false

  # Event Exporter - exports health events to a remote endpoint
  # Requires: MongoDB store enabled
  eventExporter:
    enabled: false

  # Preflight - mutating admission webhook for GPU diagnostic init containers
  # Does not use MongoDB. Requires cert-manager (or OpenShift service CA), DCGM,
  # and labeled namespaces. Subchart values: top-level preflight: key and
  # charts/preflight/values.yaml. See docs/configuration/preflight.md
  preflight:
    enabled: false

################################################################################
# PLATFORM CONNECTOR CONFIGURATION
#
# Central communication hub that receives health events from all monitors.
# Persists events to MongoDB and updates Kubernetes node status.
# This is the core component that connects health monitors to the remediation
# pipeline.
################################################################################
platformConnector:
  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/platform-connectors
    # Image pull policy - when to pull the image
    # IfNotPresent: Only pull if not already present (recommended)
    # Always: Always pull the latest image
    # Never: Never pull, use local image only
    pullPolicy: IfNotPresent
    # Image tag override - leave empty to use global.image.tag
    tag: ""

  # Resource limits and requests
  # Limits: Maximum resources the pod can use
  # Requests: Resources guaranteed to be available
  resources:
    limits:
      cpu: 200m
      memory: 512Mi
    requests:
      cpu: 200m
      memory: 512Mi

  # Additional pod annotations
  # Example - for Istio service mesh:
  # podAnnotations:
  #   sidecar.istio.io/inject: "false"
  podAnnotations: {}

  # Tolerations for platform-connector pods
  # Used when global.tolerations is not specified
  tolerations: []

  # Kubernetes connector configuration
  # Handles updating Kubernetes node status and conditions
  k8sConnector:
    # Maximum length of the node condition message
    # This is the maximum length of the node condition message that will be stored in the Kubernetes node status
    # If the node condition message is longer than this value, it will be truncated
    # This is to prevent the node condition message from being too long and causing the Kubernetes node status to be too large
    maxNodeConditionMessageLength: 1024
    # Maximum length of the compacted message prefix (before Recommended Action) per health event.
    # Controls how much of the ErrorCode and entity identifiers (GPU, PCI, etc.) are preserved
    # when messages are compacted to fit within maxNodeConditionMessageLength.
    compactedHealthEventMsgLen: 72
    # Enable Kubernetes status updates
    enabled: true
    # Rate limiting for Kubernetes API calls
    # QPS: Maximum queries per second to API server
    qps: 5.0
    # Burst: Maximum burst size above QPS
    burst: 10

  # Health event transformers pipeline
  # Transformers process health events before storage and propagation
  # They execute in the order specified in the pipeline
  pipeline:
    # Metadata augmentor - enriches events with node labels and provider info
    - name: MetadataAugmentor
      enabled: false  # Disabled by default
      config: /etc/config/metadata.toml
    
    # Property overrides - uses CEL expressions to override event properties
    - name: OverrideTransformer
      enabled: false  # Disabled by default
      config: /etc/config/overrides.toml
  
  # Transformer-specific configurations
  transformers:
    # Metadata augmentor configuration
    # Adds node labels and provider information to health events
    MetadataAugmentor:
      # LRU cache size for node metadata
      # Higher values use more memory but reduce API calls
      cacheSize: 50
      # Cache TTL in seconds (supports duration strings like "1h", "30m")
      # How long to cache node metadata before refreshing
      cacheTTLSeconds: 3600
      # Node labels to include in health event metadata
      # Only these labels will be copied from nodes to events
      # This allows CEL rules to match on topology, instance type, etc.
      allowedLabels:
        - "topology.kubernetes.io/zone"
        - "topology.kubernetes.io/region"
        - "node.kubernetes.io/instance-type"
        - "nvidia.com/cuda.driver-version.major"
        - "nvidia.com/cuda.driver-version.minor"
        - "nvidia.com/cuda.driver-version.revision"
        - "nvidia.com/cuda.driver-version.full"
        - "nvidia.com/cuda.runtime-version.major"
        - "nvidia.com/cuda.runtime-version.minor"
        - "nvidia.com/cuda.runtime-version.full"
        - "topology.k8s.aws/capacity-block-id"
        - "topology.k8s.aws/network-node-layer-1"
        - "topology.k8s.aws/network-node-layer-2"
        - "topology.k8s.aws/network-node-layer-3"
        - "oci.oraclecloud.com/host.id"
        - "oci.oraclecloud.com/host.network_block_id"
        - "oci.oraclecloud.com/host.rack_id"
        - "oci.oraclecloud.com/host.serial_number"
        - "cloud.google.com/gce-topology-block"
        - "cloud.google.com/gce-topology-host"
        - "cloud.google.com/gce-topology-subblock"
        # Topograph-applied topology labels (present when Topograph is deployed in the cluster;
        # see https://github.com/NVIDIA/topograph/blob/main/docs/reference/node-labels.md)
        - "network.topology.nvidia.com/accelerator"
        - "network.topology.nvidia.com/leaf"
        - "network.topology.nvidia.com/spine"
        - "network.topology.nvidia.com/core"

    # Property overrides configuration
    # Uses CEL expressions to modify health event properties
    # Allows operators to suppress errors or change recommended actions
    OverrideTransformer:
      # List of override rules (evaluated in order, first match wins)
      rules: []
      # Example rule - suppress XID 109 errors:
      # - name: "suppress-xid-109"
      #   when: 'event.agent == "syslog-health-monitor" && "109" in event.errorCode'
      #   override:
      #     isFatal: false
      #     recommendedAction: "NONE"
      #
      # Example rule - zone-specific override:
      # - name: "zone-specific-override"
      #   when: 'event.metadata["topology.kubernetes.io/zone"] == "us-west1-a" && event.componentClass == "GPU"'
      #   override:
      #     isFatal: false
      #     recommendedAction: "CONTACT_SUPPORT"

# Unix socket path for inter-process communication
# Health monitors connect to platform-connectors via this socket
# Must be accessible by both monitors and platform-connectors
socketPath: "/var/run/nvsentinel.sock"

################################################################################
# FAULT-QUARANTINE MODULE CONFIGURATION
#
# Watches MongoDB for health events and applies quarantine actions to nodes.
# Quarantine actions include: cordoning (preventing new pods) and tainting
# (repelling pods without tolerations). Also adds labels and annotations
# for tracking quarantine state.
################################################################################
fault-quarantine:
  # Number of pod replicas (recommend 1 for consistency)
  replicaCount: 1

  # Log level - controls verbosity of logging
  # Options: debug, info, warn, error
  # debug: Very verbose, includes all details
  # info: Normal operations and important events
  # warn: Warning conditions
  # error: Error conditions only
  logLevel: info

  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/fault-quarantine
    pullPolicy: IfNotPresent
    tag: ""

  # Pod resource limits and requests
  resources: 
    limits:
      cpu: "1"
      memory: "1Gi"
    requests:
      cpu: "1"
      memory: "1Gi"

  # Scheduling configuration
  # These override global values if set
  nodeSelector: {}
  affinity: {}
  tolerations: []
  podAnnotations: {}

  # Label prefix for node labels and annotations
  # Used to create labels like: <labelPrefix>cordon-by, <labelPrefix>cordon-reason
  # These labels track when and why nodes were quarantined
  # Also used for uncordon tracking
  labelPrefix: "k8saas.nvidia.com/"

  # Circuit breaker prevents quarantining too many nodes at once
  # If the threshold is exceeded, new quarantines are blocked
  # This prevents cascading failures from taking down the entire cluster
  # State is persisted in a ConfigMap and survives restarts
  circuitBreaker:
    # Enable circuit breaker protection
    enabled: true
    # Maximum percentage of cluster nodes that can be cordoned
    # Example: 50 means if 50% or more nodes are already cordoned,
    # no new quarantines will be allowed
    # Range: 1-100
    percentage: 50
    # Cooldown period after circuit breaker trips
    # Circuit breaker waits this long before attempting to close again
    # Even if node count drops below threshold during cooldown, it stays open
    # Examples: "5m", "10m", "1h"
    duration: "5m"

  # Quarantine rules define when and how to quarantine nodes
  # Each rule has:
  # - Match conditions: When the rule should trigger
  # - Actions: What to do when conditions match (cordon, taint, label)
  # Rules are evaluated in order for each health event
  ruleSets:
    # Example rule 1: Quarantine nodes with fatal GPU errors
    # Enable or disable this ruleset; disabled rulesets are skipped entirely
    - enabled: true
      version: "1"
      name: "GPU fatal error ruleset"
      
      # Match conditions - all must be true
      match:
        # Use 'all' for AND logic (all conditions must match)
        # Use 'any' for OR logic (at least one condition must match)
        all:
          # Check health event properties
          - kind: "HealthEvent"
            # CEL expression with access to 'event' object
            # Available fields: agent, componentClass, isFatal, nodeName, severity, etc.
            expression: "event.agent == 'gpu-health-monitor' && event.componentClass == 'GPU' && event.isFatal == true"
          
          # Check node properties
          - kind: "Node"
            # CEL expression with access to 'node' object
            # Available fields: node.metadata.labels, node.metadata.annotations, node.spec, etc.
            # This checks if node is explicitly excluded from management
            expression: |
              !('k8saas.nvidia.com/ManagedByNVSentinel' in node.metadata.labels && node.metadata.labels['k8saas.nvidia.com/ManagedByNVSentinel'] == "false")
      
      # Actions to take when rule matches
      # Cordon action - mark node as unschedulable
      cordon:
        # When true, sets node.spec.unschedulable=true
        # Prevents new pods from being scheduled on the node
        # Existing pods continue running
        shouldCordon: true
      
      # Optional: Taint action - repel pods without matching tolerations
      # Uncomment to enable tainting
      # taint:
      #   # Taint key identifier
      #   key: "nvidia.com/gpu-error"
      #   # Taint value (additional context)
      #   value: "fatal"
      #   # Effect determines pod behavior:
      #   # NoSchedule: New pods without toleration won't schedule
      #   # PreferNoSchedule: Scheduler tries to avoid (soft version)
      #   # NoExecute: Existing pods without toleration are evicted
      #   effect: "NoSchedule"

    # Example rule 2: Quarantine nodes with CSP maintenance events
    - enabled: true
      version: "1"
      name: "CSP health monitor fatal error ruleset"
      match:
        all:
          - kind: "HealthEvent"
            expression: "event.agent == 'csp-health-monitor' && event.checkName == 'CSPMaintenance' && event.isFatal == true"
          - kind: "Node"
            expression: |
              !('k8saas.nvidia.com/ManagedByNVSentinel' in node.metadata.labels && node.metadata.labels['k8saas.nvidia.com/ManagedByNVSentinel'] == "false")
      cordon:
        shouldCordon: true

    # Example rule 3: Quarantine nodes with syslog errors
    - enabled: true
      version: "1"
      name: "Syslog fatal error ruleset"
      match:
        all:
          - kind: "HealthEvent"
            expression: "event.agent == 'syslog-health-monitor' && event.componentClass == 'GPU' && event.isFatal == true"
          - kind: "Node"
            expression: |
              !('k8saas.nvidia.com/ManagedByNVSentinel' in node.metadata.labels && node.metadata.labels['k8saas.nvidia.com/ManagedByNVSentinel'] == "false")
      cordon:
        shouldCordon: true

################################################################################
# NODE-DRAINER MODULE CONFIGURATION
#
# Evicts workloads from quarantined nodes to prepare for maintenance.
# Supports different eviction strategies per namespace:
# - Immediate: Evict right away
# - AllowCompletion: Wait for graceful termination
# - DeleteAfterTimeout: Wait for timeout, then force delete
################################################################################
node-drainer:
  # Number of pod replicas (recommend 1 for consistency)
  replicaCount: 1

  # Log level for debugging and monitoring
  logLevel: info

  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/node-drainer
    pullPolicy: IfNotPresent
    tag: ""

  # Pod resource limits and requests
  resources:
    limits:
      cpu: "200m"
      memory: "300Mi"
    requests:
      cpu: "200m"
      memory: "300Mi"

  podAnnotations: {}

  # Eviction timeout in seconds
  # Maximum time to wait for a pod to terminate gracefully
  # After this timeout, pod may be force deleted (depends on eviction mode)
  # Must be a positive integer
  evictionTimeoutInSeconds: "60"

  # System namespaces regex pattern
  # Pods in these namespaces are skipped during drain
  # This protects critical infrastructure pods from eviction
  # The pattern matches namespace names
  systemNamespaces: "^(nvsentinel|kube-system|gpu-operator|gmp-system|network-operator|skyhook)$"

  # Delete timeout in minutes for DeleteAfterTimeout mode
  # Calculated from health event creation time
  # When timeout is reached, remaining pods are force deleted
  # Used to prevent drains from waiting indefinitely
  # Default: 60 minutes
  deleteAfterTimeoutMinutes: 60

  # NotReady timeout in minutes
  # Time after which a pod in NotReady state is considered stuck
  # Stuck pods are candidates for force deletion
  # Helps identify and handle unhealthy pods that won't complete gracefully
  # Default: 5 minutes
  notReadyTimeoutMinutes: 5

  # Namespace-specific eviction strategies
  # Define how pods in different namespaces should be evicted
  # Multiple rules can be defined with namespace patterns
  userNamespaces:
    # Match all user namespaces (not matched by systemNamespaces)
    - name: "*"
      # Eviction mode options:
      # "Immediate": Evict pod immediately, don't wait for graceful termination
      #              Use for: Stateless apps, batch jobs
      # "AllowCompletion": Wait for pod to complete gracefully
      #                   Respects pod's terminationGracePeriodSeconds
      #                   Use for: Long-running jobs, stateful apps
      # "DeleteAfterTimeout": Wait for deleteAfterTimeoutMinutes
      #                       Then force delete if still running
      #                       Use for: Jobs that should complete but might hang
      mode: "AllowCompletion"
    
    # Example: Immediate eviction for a specific namespace
    # - name: "batch-jobs"
    #   mode: "Immediate"
    
    # Example: Delete after timeout for another namespace
    # - name: "training-jobs"
    #   mode: "DeleteAfterTimeout"

  # If enabled, the node-drainer will only drain pods which are leveraging the GPU_UUID impacted entity in
  # COMPONENT_RESET HealthEvents. If disabled, the node-drainer will drain all eligible pods
  # on the impacted node for the configured namespaces regardless of the remediation action.
  # HealthEvents with the COMPONENT_RESET remediation action must include an impacted entity for the
  # unhealthy GPU_UUID or else the drain will fail. IMPORTANT: If this setting is enabled, the COMPONENT_RESET
  # action in fault-remediation must map to a custom resource which takes action only against the GPU_UUID.
  # If partial drain was enabled in node-drainer but fault-remediation mapped COMPONENT_RESET to a reboot
  # action, pods which weren't drained would be restarted as part of the reboot.
  partialDrainEnabled: false

################################################################################
# FAULT-REMEDIATION MODULE CONFIGURATION
#
# Creates maintenance CRDs (Custom Resource Definitions) to trigger node recovery.
# The janitor module watches these CRDs and performs actual cloud provider API
# calls to reboot or terminate nodes.
################################################################################
fault-remediation:
  # Number of pod replicas (recommend 1 for consistency)
  replicaCount: 1

  # Log level for debugging and monitoring
  logLevel: info

  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/fault-remediation
    pullPolicy: IfNotPresent
    tag: ""

  # Pod resource limits and requests
  resources: 
    limits:
      cpu: "200m"
      memory: "300Mi"
    requests:
      cpu: "200m"
      memory: "300Mi"

  # Scheduling configuration
  nodeSelector: {}
  affinity: {}
  # Special tolerations - allow running on all nodes for log collection
  tolerations:
    - operator: "Exists"
  podAnnotations: {}

  # Multi-template remediation configuration
  # Allows different remediation actions to use different CRDs and operators
  maintenance:
    # Per-action remediation definitions
    # Key is the RecommendedAction string (e.g., "COMPONENT_RESET", "RESTART_BM")
    actions:
      COMPONENT_RESET:
        apiGroup: "janitor.dgxc.nvidia.com"
        version: "v1alpha1"
        kind: "RebootNode"
        scope: "Cluster"
        completeConditionType: "NodeReady"
        templateFileName: "nvidia-reboot.yaml"
        equivalenceGroup: "restart"

      # RESTART_VM is used as a fallback when GPU UUID is not available from metadata-collector.
      RESTART_VM:
        apiGroup: "janitor.dgxc.nvidia.com"
        version: "v1alpha1"
        kind: "RebootNode"
        scope: "Cluster"
        completeConditionType: "NodeReady"
        templateFileName: "nvidia-reboot.yaml"
        equivalenceGroup: "restart"

      # RESTART_BM is used for bare-metal node restarts.
      RESTART_BM:
        apiGroup: "janitor.dgxc.nvidia.com"
        version: "v1alpha1"
        kind: "RebootNode"
        scope: "Cluster"
        completeConditionType: "NodeReady"
        templateFileName: "nvidia-reboot.yaml"
        equivalenceGroup: "restart"

      # REPLACE_VM is used when the VM needs to be terminated and replaced.
      REPLACE_VM:
        apiGroup: "janitor.dgxc.nvidia.com"
        version: "v1alpha1"
        kind: "TerminateNode"
        scope: "Cluster"
        completeConditionType: "NodeTerminated"
        templateFileName: "terminate-node.yaml"
        equivalenceGroup: "terminate"

      # NOTE: Resource names for RBAC are generated by appending 's' to lowercase kind.
      # This works for regular nouns but may fail for irregular plurals:
      #   RebootNode → rebootnodes (✓ correct)
      #   Policy → policys (✗ should be policies)
      # Use CRD kinds that follow regular pluralization rules.

    # Template content for each remediation action
    # Key matches the templateFileName name from actions above
    templates:
      "nvidia-reboot.yaml": |
        apiVersion: janitor.dgxc.nvidia.com/v1alpha1
        kind: RebootNode
        metadata:
          name: maintenance-{{ .HealthEvent.NodeName }}-{{ .HealthEventID }}
          labels:
            app.kubernetes.io/managed-by: nvsentinel
          annotations:
            nvsentinel.nvidia.com/ttl: "336h"
        spec:
          nodeName: {{ .HealthEvent.NodeName }}
          force: false
      "terminate-node.yaml": |
        apiVersion: janitor.dgxc.nvidia.com/v1alpha1
        kind: TerminateNode
        metadata:
          name: maintenance-{{ .HealthEvent.NodeName }}-{{ .HealthEventID }}
          labels:
            app.kubernetes.io/managed-by: nvsentinel
          annotations:
            nvsentinel.nvidia.com/ttl: "336h"
        spec:
          nodeName: {{ .HealthEvent.NodeName }}
          force: false
      # Additional template examples:
      # "namespaced-restart.yaml": |
      #   apiVersion: remediation.example.com/v1alpha1
      #   kind: RestartNode
      #   metadata:
      #     name: maintenance-{{ .NodeName }}-{{ .HealthEventID }}
      #     namespace: remediation
      #     labels:
      #       app.kubernetes.io/managed-by: nvsentinel
      #   spec:
      #     nodeName: {{ .NodeName }}
      #     ttlSecondsAfterFinished: 60

  # Retry configuration for node annotation updates
  # After creating maintenance resource, module updates node annotations
  # These settings control retry behavior on failures (conflicts, network issues)
  updateRetry:
    # Maximum retry attempts
    # After this many failures, the update is abandoned
    maxRetries: 5
    
    # Delay between retries in seconds
    # Uses exponential backoff (delay increases each retry)
    # First retry: retryDelaySeconds
    # Second retry: retryDelaySeconds * 2
    # Third retry: retryDelaySeconds * 4, etc.
    retryDelaySeconds: 10

  # Log collector configuration
  # When enabled, creates Kubernetes Jobs to collect diagnostic logs
  # from failing nodes before they are rebooted or terminated
  logCollector:
    # Enable log collection feature
    # When true, a log collector job is created for each node failure
    # Logs are uploaded to the configured upload URL
    enabled: false
    
    # Container image for log collector
    image:
      repository: ghcr.io/nvidia/nvsentinel/log-collector
      pullPolicy: IfNotPresent
    
    # HTTP endpoint where logs will be uploaded
    # Should be the incluster-file-server or external storage
    # Logs are uploaded as tar.gz files
    uploadURL: "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local/upload"
    
    # Namespaces where GPU operator components run
    # Log collector will gather logs from these namespaces
    # Can be multiple namespaces separated by commas
    gpuOperatorNamespaces: "gpu-operator"
    
    # Enable GPU Operator must-gather collection (disabled by default)
    # WARNING: must-gather collects logs from ALL nodes in the cluster, which can be very
    # time-consuming for large clusters
    # If enabling this, you MUST increase the timeout accordingly.
    # Recommended timeout: ~2-3 minutes per node 
    enableGpuOperatorMustGather: false
    
    # Timeout for log collection job (default: 10m)
    # If enableGpuOperatorMustGather is true, increase to: ~2-3 min per node in cluster
    timeout: "10m"
    
    # Enable GCP-specific SOS report collection
    # Requires running on GCP and appropriate permissions
    enableGcpSosCollection: false
    
    # Enable AWS-specific SOS report collection  
    # Requires running on AWS and appropriate permissions
    enableAwsSosCollection: false

################################################################################
# GPU-HEALTH-MONITOR CONFIGURATION
#
# Monitors GPU hardware health using NVIDIA DCGM (Data Center GPU Manager).
# Detects: XID errors, thermal throttling, ECC errors, memory errors, NVLink
# failures, and other GPU hardware faults.
#
# Requires: NVIDIA GPU Operator with DCGM deployed in the cluster
################################################################################
gpu-health-monitor:
  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/gpu-health-monitor
    pullPolicy: IfNotPresent
    tag: ""

  # DCGM integration configuration
  # DCGM must be running in the cluster (typically via GPU Operator)
  dcgm:
    # Enable DCGM Kubernetes service discovery
    # When true, monitor connects to DCGM via Kubernetes service
    # When false, must use host networking to connect to local DCGM
    dcgmK8sServiceEnabled: true
    
    # DCGM service endpoint
    service:
      # Service endpoint in format: servicename.namespace.svc
      # Default GPU Operator deployment uses: nvidia-dcgm.gpu-operator.svc
      endpoint: "nvidia-dcgm.gpu-operator.svc"
      
      # DCGM service port
      # Default DCGM port is 5555
      port: 5555

  # Use host networking for GPU health monitor pods
  # Required when:
  # - DCGM is running as hostProcess (not as service)
  # - Need direct access to host-level GPU metrics
  # - dcgmK8sServiceEnabled is false
  # Most deployments should use service-based access (false)
  useHostNetworking: false

  # Additional volume mounts for GPU health monitor
  # Use when monitor needs access to host files
  # Example:
  # additionalVolumeMounts:
  #   - name: custom-config
  #     mountPath: /etc/custom
  #     readOnly: true
  additionalVolumeMounts: []

  # Additional host volumes for GPU health monitor
  # Use with additionalVolumeMounts to mount host paths
  # Example:
  # additionalHostVolumes:
  #   - name: custom-config
  #     hostPath:
  #       path: /etc/custom
  #       type: Directory
  additionalHostVolumes: []

  # Pod resource limits and requests
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 100m
      memory: 128Mi

  # Scheduling configuration
  nodeSelector: {}
  affinity: {}
  tolerations: []
  podAnnotations: {}

################################################################################
# LABELER CONFIGURATION
#
# Labels nodes based on their capabilities and features.
# Currently detects: Kata Containers support
# Runs as Deployment to label all nodes in the cluster.
################################################################################
labeler:
  # Number of replicas for the labeler deployment
  replicaCount: 1

  # Log level for debugging
  logLevel: info

  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/labeler
    pullPolicy: IfNotPresent
    tag: ""

  podAnnotations: {}

  # Kata Containers detection configuration
  # Labeler checks node labels to determine Kata support
  # Sets output label: nvsentinel.dgxc.nvidia.com/kata.enabled
  #
  # Detection logic:
  # 1. Check default label: katacontainers.io/kata-runtime
  # 2. If kataLabelOverride is set, also check that label
  # 3. Label value must be truthy: "true", "enabled", "1", or "yes" (case-insensitive)
  # 4. Set output label to "true" if detected, "false" otherwise
  #
  # Use kataLabelOverride when:
  # - Your cluster uses custom labels for Kata detection
  # - You have additional Kata runtime indicators
  #
  # Example: 'io.katacontainers.config.runtime.oci_runtime'
  # Leave empty to only check default label
  kataLabelOverride: ""

  # Pod resource limits and requests
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: 500m
      memory: 256Mi

################################################################################
# JANITOR CONFIGURATION
#
# Executes node reboots and terminations by calling cloud provider APIs.
# Watches maintenance CRDs created by fault-remediation and performs the
# actual infrastructure operations.
#
# Supports: AWS, GCP, Azure, OCI, kind (for testing), kwok (for testing)
################################################################################
janitor:
  # Number of pod replicas (recommend 1 for consistency)
  replicaCount: 1

  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/janitor
    pullPolicy: IfNotPresent
    tag: ""

  # Kubernetes service account configuration
  serviceAccount:
    # Create service account
    # Set to false if using an existing service account
    create: true
    
    # Automatically mount service account token
    automount: true
    
    # Annotations for service account
    # Used for cloud provider identity (IRSA, Workload Identity, etc.)
    # Examples:
    # AWS IRSA:
    #   eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT_ID:role/nvsentinel-janitor
    # GCP Workload Identity:
    #   iam.gke.io/gcp-service-account: nvsentinel-janitor@PROJECT_ID.iam.gserviceaccount.com
    # Azure Workload Identity:
    #   azure.workload.identity/client-id: "12345678-1234-1234-1234-123456789012"
    annotations: {}
    
    # Service account name
    # Leave empty to use generated name from chart
    name: ""

  podAnnotations: {}
  podLabels: {}
  podSecurityContext: {}
  securityContext: {}

  # Pod resource limits
  # Default is no limits - set based on cluster size
  resources: {}
  #   limits:
  #     cpu: 100m
  #     memory: 128Mi
  #   requests:
  #     cpu: 100m
  #     memory: 128Mi

  # Scheduling configuration
  nodeSelector: {}
  tolerations: []
  affinity: {}

  # Autoscaling configuration
  # Not typically needed for janitor (low request rate)
  autoscaling:
    enabled: false
    minReplicas: 1
    maxReplicas: 100
    targetCPUUtilizationPercentage: 80

  # Janitor controller configuration
  config:
    # Global timeout for all operations
    # Used as default for controllers that don't specify their own
    # Should account for cloud provider API call time + node boot time
    # Format: duration string like "25m", "1h", "30s"
    timeout: "25m"
    
    # Manual mode - when true, controllers don't send actual reboot/terminate signals
    # Useful for testing without affecting infrastructure
    # Controllers will create/update resources but won't call cloud provider APIs
    manualMode: false
    
    # HTTP port for runtime configuration endpoint
    # Exposes configuration and status information
    httpPort: 8082
    
    # Node exclusions - nodes matching these selectors are never rebooted/terminated
    # Use to protect critical nodes from automated maintenance
    # Supports both matchLabels and matchExpressions
    nodes:
      exclusions: []
      # Examples:
      # Exclude control plane nodes:
      # - matchLabels:
      #     node-role.kubernetes.io/control-plane: ""
      # Exclude nodes with specific labels:
      # - matchExpressions:
      #   - key: critical-workload
      #     operator: In
      #     values:
      #       - "true"
      #   - key: app
      #     operator: In
      #     values:
      #       - database
      #       - cache
    
    # Controller-specific configuration
    controllers:
      # Reboot node controller - handles RebootNode CRDs
      rebootNode:
        # Enable reboot node controller
        enabled: true
        
        # Timeout for reboot operations
        # Should account for:
        # 1. Cloud provider API call time
        # 2. Node shutdown time
        # 3. Node boot time
        # 4. Kubernetes node ready time
        # Leave empty to use config.timeout
        # 
        # Recommended values by platform:
        # - AWS/Azure/OCI: "25m" (includes 5min API cooldown)
        # - GCP: "20m" (faster API, tracks operation status)
        # - kind/kwok: "5m" (for testing)
        timeout: "25m"
      
      # Terminate node controller - handles TerminateNode CRDs
      terminateNode:
        # Enable terminate node controller
        enabled: true
        
        # Timeout for terminate operations
        # Should account for cloud provider API call time
        # Termination is typically faster than reboot
        # Leave empty to use config.timeout
        timeout: "25m"

  # Cloud Service Provider (CSP) configuration
  # Configure the provider that matches your Kubernetes cluster
  csp:
    # Provider type - select one:
    # - kind: Local development with kind clusters (simulated operations)
    # - kwok: Testing with kwok (simulated nodes)
    # - aws: Amazon Web Services EKS clusters
    # - gcp: Google Cloud Platform GKE clusters
    # - azure: Microsoft Azure AKS clusters
    # - oci: Oracle Cloud Infrastructure OKE clusters
    provider: "kind"
    
    # AWS-specific configuration (only needed when provider=aws)
    aws:
      # AWS region where EKS cluster is running
      # Required for EC2 API calls
      # Examples: us-east-1, us-west-2, eu-west-1, ap-southeast-1
      region: ""
      
      # AWS Account ID (12-digit number)
      # Required for IRSA (IAM Roles for Service Accounts)
      # Example: "123456789012"
      # Find with: aws sts get-caller-identity --query Account --output text
      accountId: ""
      
      # IAM Role name for janitor service account
      # Role must have ec2:RebootInstances and ec2:TerminateInstances permissions
      # Example: "nvsentinel-janitor-role"
      # Create with: aws iam create-role --role-name nvsentinel-janitor-role
      # Attach policy: AmazonEC2FullAccess (or custom policy with reboot/terminate)
      iamRoleName: ""
      
      # Setup steps for AWS:
      # 1. Create IAM role with trust relationship to OIDC provider
      # 2. Attach policy with ec2:RebootInstances permission
      # 3. Annotate service account with role ARN
      # 4. Janitor pods will use IRSA to assume the role
    
    # GCP-specific configuration (only needed when provider=gcp)
    gcp:
      # GCP project ID where GKE cluster is running
      # Required for Compute Engine API calls
      # Example: "my-gcp-project"
      # Find with: gcloud config get-value project
      project: ""
      
      # GCP zone where cluster nodes are located
      # Required for instance operations
      # Examples: us-central1-a, us-west1-b, europe-west1-c
      # Find with: gcloud container clusters describe CLUSTER_NAME --region REGION
      zone: ""
      
      # GCP Service Account email for Workload Identity
      # Provide just the name (without @project.iam.gserviceaccount.com)
      # Example: "nvsentinel-janitor"
      # Full email will be: nvsentinel-janitor@PROJECT_ID.iam.gserviceaccount.com
      # Service account must have compute.instances.reset permission
      serviceAccount: ""
      
      # Setup steps for GCP:
      # 1. Create GCP service account: gcloud iam service-accounts create nvsentinel-janitor
      # 2. Grant role: gcloud projects add-iam-policy-binding PROJECT_ID \
      #      --member serviceAccount:nvsentinel-janitor@PROJECT_ID.iam.gserviceaccount.com \
      #      --role roles/compute.instanceAdmin.v1
      # 3. Bind Kubernetes SA to GCP SA:
      #    gcloud iam service-accounts add-iam-policy-binding \
      #      nvsentinel-janitor@PROJECT_ID.iam.gserviceaccount.com \
      #      --member "serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/janitor]" \
      #      --role roles/iam.workloadIdentityUser
      # 4. Annotate Kubernetes service account with GCP SA email
    
    # Azure-specific configuration (only needed when provider=azure)
    azure:
      # Azure subscription ID
      # Required for Azure Resource Manager API calls
      # Format: 8-4-4-4-12 UUID
      # Find with: az account show --query id --output tsv
      subscriptionId: ""
      
      # Azure resource group containing the AKS cluster
      # Often auto-detected from node provider IDs
      # If not auto-detected, specify manually
      # Find with: az aks show --name CLUSTER_NAME --resource-group RG_NAME --query nodeResourceGroup
      resourceGroup: ""
      
      # Azure region where AKS cluster is deployed
      # Examples: eastus, westus2, westeurope, southeastasia
      # Find with: az aks show --name CLUSTER_NAME --resource-group RG_NAME --query location
      location: ""
      
      # Managed Identity Client ID for Workload Identity
      # Format: 8-4-4-4-12 UUID
      # Managed identity must have Virtual Machine Contributor role
      # Find with: az identity show --name IDENTITY_NAME --resource-group RG_NAME --query clientId
      clientId: ""
      
      # Setup steps for Azure:
      # 1. Create managed identity: az identity create --name nvsentinel-janitor
      # 2. Grant role: az role assignment create \
      #      --assignee CLIENT_ID \
      #      --role "Virtual Machine Contributor" \
      #      --scope /subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP
      # 3. Create federated identity credential:
      #    az identity federated-credential create \
      #      --name nvsentinel-janitor-credential \
      #      --identity-name nvsentinel-janitor \
      #      --resource-group RG_NAME \
      #      --issuer "https://oidc.prod-aks.azure.com/TENANT_ID/" \
      #      --subject "system:serviceaccount:NAMESPACE:janitor"
      # 4. Annotate service account with client ID
    
    # OCI-specific configuration (only needed when provider=oci)
    oci:
      # OCI region where OKE cluster is running
      # Examples: us-ashburn-1, us-phoenix-1, uk-london-1, ap-mumbai-1
      # Find with: oci iam region list
      region: ""
      
      # OCI compartment OCID containing cluster resources
      # Format: ocid1.compartment.oc1..aaa...
      # Find with: oci iam compartment list
      compartment: ""
      
      # Path to OCI credentials file (optional)
      # Only needed if not using workload identity
      # Default location: ~/.oci/config
      # Leave empty to use workload identity
      credentialsFile: ""
      
      # OCI profile name in credentials file
      # Default profile name in config file
      profile: "DEFAULT"
      
      # OCI Principal OCID for Workload Identity
      # Format: ocid1.principal.oc1..aaa...
      # This is the OCID of the dynamic group or service principal
      # Required for OKE Workload Identity
      principalId: ""
      
      # Setup steps for OCI:
      # 1. Create dynamic group with matching rule for the pod
      # 2. Create policy granting dynamic group manage instance-family permission
      # 3. Configure workload identity for the service account
      # 4. Janitor pods will use workload identity to call OCI APIs

  # Webhook configuration
  # Janitor uses admission webhooks to validate CRDs
  webhook:
    # Port for webhook server
    port: 9443
    
    # Directory containing webhook TLS certificates
    # Certificates should be mounted via cert-manager
    certDir: "/tmp/k8s-webhook-server/serving-certs"
    
    # Cert-manager issuer for webhook certificate
    # This issuer must exist in the same namespace
    # Create with: kubectl apply -f - <<EOF
    # apiVersion: cert-manager.io/v1
    # kind: Issuer
    # metadata:
    #   name: selfsigned-ca-issuer
    # spec:
    #   selfSigned: {}
    # EOF
    certIssuer: "selfsigned-ca-issuer"

  # Metrics configuration
  # Janitor exposes Prometheus metrics
  metrics:
    tls:
      # Enable TLS for metrics endpoint
      # Recommended for production
      enabled: false
      
      # Directory containing metrics server TLS certificates
      certDir: "/tmp/k8s-metrics-server/metrics-certs"
      
      # Cert-manager issuer for metrics certificate
      # Defaults to webhook.certIssuer if not set
      issuerName: ""
      
      # Cert-manager issuer kind
      # Options: Issuer (namespace-scoped), ClusterIssuer (cluster-scoped)
      issuerKind: "Issuer"
      
      # Cert-manager issuer group (usually cert-manager.io)
      issuerGroup: ""
      
      # Certificate validity duration
      # Example: "2160h" = 90 days
      duration: "2160h"
      
      # Certificate renewal time before expiry
      # Example: "720h" = 30 days before expiry
      renewBefore: "720h"
      
      # Organization name in certificate subject
      organization: "NVIDIA"

################################################################################
# MONGODB-STORE CONFIGURATION
#
# MongoDB database for storing health events.
# Uses Bitnami MongoDB chart with TLS enabled by default.
# Required by: fault-quarantine, node-drainer, fault-remediation
################################################################################
mongodb-store:
  # MongoDB replica configuration
  mongodb:
    # Number of MongoDB replicas
    # Recommend 1 for development, 3 for production
    replicaCount: 1
    
    # Node selector for MongoDB pods
    # Example - run on control plane:
    # nodeSelector:
    #   node-role.kubernetes.io/control-plane: ""
    nodeSelector: {}
    
    # Tolerations for MongoDB pods
    tolerations: []
    
    # Tolerations for MongoDB job pods (init jobs, backup jobs)
    jobTolerations: []
    
    # Container image configuration
    image:
      registry: "docker.io"
      repository: "bitnamilegacy/mongodb"
      tag: "8.0.3-debian-12-r1"
      pullPolicy: "IfNotPresent"
    
    # TLS configuration for MongoDB
    # Required for secure communication
    tls:
      replicaset:
        # Existing TLS secrets for each replica
        # Must be created before installing chart
        # See: distros/kubernetes/mongodb-certs/README.md
        existingSecrets:
          - "mongo-server-cert-0"
      
      # NGINX image for TLS sidecar
      image:
        registry: "docker.io"
        repository: "bitnamilegacy/nginx"
        tag: "1.27.2-debian-12-r2"
        pullPolicy: "IfNotPresent"
    
    # MongoDB metrics exporter
    # Exposes MongoDB metrics for Prometheus
    metrics:
      enabled: true
      image:
        registry: docker.io
        repository: bitnamilegacy/mongodb-exporter
        tag: 0.41.2-debian-12-r1

################################################################################
# EVENT-EXPORTER CONFIGURATION
#
# Exports health events from MongoDB to external endpoints in CloudEvents format.
# Streams events in real-time with automatic backfill and resume capability.
# Supports OIDC authentication for secure communication.
#
# Requires: MongoDB store enabled
################################################################################
event-exporter:
  # Number of pod replicas (recommend 1 for ordered event delivery)
  replicaCount: 1

  # Container image configuration
  image:
    repository: ghcr.io/nvidia/nvsentinel/event-exporter
    pullPolicy: IfNotPresent
    tag: ""

  # Pod annotations
  # Example - for Istio service mesh:
  # podAnnotations:
  #   sidecar.istio.io/inject: "false"
  podAnnotations: {}

  # Pod resource limits and requests
  # Adjust based on event volume
  resources:
    limits:
      cpu: "1"
      memory: "1Gi"
    requests:
      cpu: "500m"
      memory: "512Mi"

  # OIDC secret configuration
  # The secret must contain a key named 'oidc-client-secret' with the OAuth2 client secret
  # Create the secret before deploying:
  # kubectl create secret generic event-exporter-oidc-secret \
  #   --from-literal=oidc-client-secret='YOUR_CLIENT_SECRET' \
  #   -n nvsentinel
  oidcSecretName: "event-exporter-oidc-secret"

  # Event exporter configuration
  # Structured YAML configuration that gets converted to TOML format in the ConfigMap
  exporter:
    # Metadata configuration
    # Custom fields included in every CloudEvent's data.metadata section
    metadata:
      cluster: "my-cluster"
      environment: "production"
      # Add any custom metadata fields here - they will be included in CloudEvents data.metadata
      # Examples:
      # region: "us-west-2"
      # datacenter: "dc01"
      # tenant: "acme-corp"
      # zone: "zone-a"

    # Sink configuration - where events are sent
    sink:
      # HTTP endpoint URL that accepts CloudEvents 1.0 format
      # Must support POST requests with Content-Type: application/cloudevents+json
      endpoint: "https://events.example.com/api/v1/events"
      
      # Timeout for HTTP requests to sink
      # Format: duration string like "30s", "1m", "5m"
      timeout: "30s"
      
      # Skip TLS certificate verification (dev/testing only)
      # Set to true to skip verification of the sink's TLS certificate
      # WARNING: Only use in development/testing environments
      # Default: false
      insecureSkipVerify: false

    # OIDC authentication configuration
    oidc:
      # OAuth2 token endpoint URL
      # Used to obtain bearer tokens for authenticating with the sink
      tokenUrl: "https://auth.example.com/oauth2/token"
      
      # OAuth2 client ID
      # The client secret is loaded from the Kubernetes secret specified by oidcSecretName
      clientId: "nvsentinel-exporter"
      
      # OAuth2 scope
      # Requested scopes for the access token
      # Example: "events:write events:read"
      scope: "events:write"
      
      # Skip TLS certificate verification for OIDC endpoint (dev/testing only)
      # WARNING: Only use in development/testing environments
      # Default: false
      insecureSkipVerify: false

    # Backfill configuration - export historical events on startup
    backfill:
      # Enable backfill of historical events
      # When true, exports existing events from MongoDB on startup
      # When false, only exports new events from current time
      enabled: true
      
      # Maximum age of events to backfill
      # Only events newer than this duration are exported
      # Format: duration string like "24h", "168h" (7 days), "720h" (30 days)
      # Example: "720h" = export events from last 30 days
      maxAge: "720h"
      
      # Maximum number of events to backfill
      # Limits the total number of historical events exported
      # Useful to prevent overwhelming the sink on first startup
      maxEvents: 1000000
      
      # Batch size for backfill operations
      # Number of events to fetch from MongoDB in each query
      # Higher values = more memory, fewer queries
      # Lower values = less memory, more queries
      # Recommended: 100-1000
      batchSize: 500
      
      # Rate limit for backfill operations (events per second)
      # Controls how fast historical events are sent to sink
      # Prevents overwhelming the sink endpoint
      # Example: 1000 = send up to 1000 events per second
      rateLimit: 1000

    # Resume token configuration - enables resuming from last position
    resumeToken:
      # MongoDB database name for storing resume tokens
      # Resume tokens track the last processed MongoDB change stream position
      # Enables the exporter to resume from where it left off after restart
      database: "nvsentinel"
      
      # MongoDB collection name for storing resume tokens
      # Only one resume token is stored (upserted on updates)
      collection: "resumetokens"

    # Failure handling configuration - retry and backoff behavior
    failureHandling:
      # Maximum retry attempts for failed event publishing
      # After this many failures, the event is logged and skipped
      # Recommended: 17 retries = ~30 minutes with exponential backoff
      maxRetries: 17
      
      # Initial backoff duration before first retry
      # Format: duration string like "1s", "5s", "10s"
      initialBackoff: "1s"
      
      # Maximum backoff duration between retries
      # Backoff increases exponentially but caps at this value
      # Format: duration string like "1m", "5m", "10m"
      # Example: "5m" = wait at most 5 minutes between retries
      maxBackoff: "5m"
      
      # Backoff multiplier for exponential backoff
      # Each retry waits: previous_wait * backoff_multiplier
      # Example with multiplier=2.0:
      #   Retry 1: 1s, Retry 2: 2s, Retry 3: 4s, Retry 4: 8s, etc.
      # Recommended: 2.0 for standard exponential backoff
      backoffMultiplier: 2.0

################################################################################
# PREFLIGHT CONFIGURATION
#
# Mutating admission webhook that injects GPU diagnostic init containers
# (DCGM diagnostics, NCCL loopback, NCCL all-reduce) into GPU pods in
# labeled namespaces. Catches hardware issues before workloads start.
#
# Requires: cert-manager (or OpenShift service CA), DCGM hostengine
# See: docs/configuration/preflight.md
################################################################################
preflight:
  replicaCount: 1

  # Container image configuration for the webhook server
  image:
    repository: ghcr.io/nvidia/nvsentinel/preflight
    pullPolicy: IfNotPresent
    tag: ""

  imagePullSecrets: []

  # Service account for the preflight webhook (needs ClusterRole for
  # pods, configmaps, and scheduler CRDs when gang coordination is enabled)
  serviceAccount:
    create: true
    annotations: {}
    name: ""

  securityContext:
    allowPrivilegeEscalation: false
    readOnlyRootFilesystem: true
    runAsNonRoot: true
    runAsUser: 65532
    runAsGroup: 65532
    capabilities:
      drop: ["ALL"]

  # Webhook Kubernetes Service
  service:
    type: ClusterIP
    port: 443
    targetPort: 8443

  resources:
    requests:
      cpu: 250m
      memory: 512Mi
    limits:
      cpu: 500m
      memory: 1024Mi

  livenessProbe:
    httpGet:
      scheme: HTTPS
      path: /healthz
      port: 8443
    initialDelaySeconds: 5
    periodSeconds: 10

  readinessProbe:
    httpGet:
      scheme: HTTPS
      path: /healthz
      port: 8443
    initialDelaySeconds: 2
    periodSeconds: 5

  nodeSelector: {}
  tolerations: []
  affinity: {}

  ##############################################################################
  # WEBHOOK TLS CONFIGURATION
  ##############################################################################
  webhook:
    # HTTPS port for the webhook server
    port: 8443
    # failurePolicy: Fail blocks pod creation if the webhook is down
    # failurePolicy: Ignore allows pods through (graceful degradation)
    failurePolicy: Fail
    # Timeout for the admission review call
    timeoutSeconds: 10
    # Set to true to create a self-signed CA + certificate chain
    # Set to false to use an existing CA issuer (set certIssuer)
    createIssuer: true
    # Only used when createIssuer=false - name of existing cert-manager Issuer
    certIssuer: ""
    caCertificateName: ""
    # Certificate provider: "cert-manager" or "openshift-service-ca"
    # Use "openshift-service-ca" on OpenShift to avoid requiring cert-manager
    certProvider: cert-manager

  ##############################################################################
  # INIT CONTAINERS (PREFLIGHT CHECKS)
  #
  # Each entry becomes an init container injected into GPU pods.
  # Standard corev1.Container fields are supported (image, env, resources,
  # securityContext, volumeMounts, etc.).
  #
  # The webhook automatically injects these env vars on every init container:
  #   NODE_NAME, PLATFORM_CONNECTOR_SOCKET, PROCESSING_STRATEGY
  # And for gang-aware containers:
  #   GANG_ID, GANG_CONFIG_DIR, GANG_TIMEOUT_SECONDS, POD_NAME
  ##############################################################################
  initContainers:
    # DCGM diagnostic - runs DCGM diag against allocated GPUs
    # Env vars:
    #   DCGM_DIAG_LEVEL       - diagnostic depth 1-4 (default 2)
    #   DCGM_HOSTENGINE_ADDR  - hostengine gRPC endpoint (default nvidia-dcgm.gpu-operator.svc:5555)
    - name: preflight-dcgm-diag
      image: ghcr.io/nvidia/nvsentinel/preflight-dcgm-diag:latest
      volumeMounts:
        - name: nvsentinel-socket
          mountPath: /var/run
      # resources:
      #   limits:
      #     memory: 512Mi

    # NCCL loopback - intra-node GPU-to-GPU all-reduce (NVLink/PCIe)
    # Env vars:
    #   BW_THRESHOLD_GBPS  - minimum bus bandwidth in GB/s (default 150)
    #                        Set ~15 for PCIe interconnect
    #   TEST_SIZE_MB       - message size in MB (default 256)
    #   SKIP_BANDWIDTH_CHECK - "true" to pass if benchmark completes (default false)
    - name: preflight-nccl-loopback
      image: ghcr.io/nvidia/nvsentinel/preflight-nccl-loopback:latest
      env:
        - name: BW_THRESHOLD_GBPS
          value: "150"
        - name: TEST_SIZE_MB
          value: "256"
      volumeMounts:
        - name: nvsentinel-socket
          mountPath: /var/run

    # NCCL all-reduce - multi-node GPU communication test
    # Requires gangCoordination.enabled=true and a gang-aware scheduler
    # Env vars:
    #   BW_THRESHOLD_GBPS    - minimum bus bandwidth in GB/s (default 100)
    #   MESSAGE_SIZES        - comma-separated sizes e.g. "4G,8G" (default "4G,8G")
    #   BENCHMARK_ITERS      - timed iterations per size (default 20)
    #   WARMUP_ITERS         - warmup iterations (default 5)
    #   NCCL_REDUCE_OP       - reduction op: sum/prod/min/max/avg (default sum)
    #   SKIP_BANDWIDTH_CHECK - "true" to pass if benchmark completes (default false)
    #   NCCL_DEBUG           - NCCL log verbosity (e.g. INFO, WARN)
    #   NCCL_DEBUG_SUBSYS    - NCCL subsystems to log (e.g. INIT,NET)
    - name: preflight-nccl-allreduce
      image: ghcr.io/nvidia/nvsentinel/preflight-nccl-allreduce:latest
      securityContext:
        capabilities:
          add: ["IPC_LOCK"]  # Required for RDMA memory registration
      env:
        - name: BW_THRESHOLD_GBPS
          value: "100"
        - name: MESSAGE_SIZES
          value: "4G"
        - name: NCCL_DEBUG
          value: "INFO"
        - name: NCCL_DEBUG_SUBSYS
          value: "INIT,NET"
      volumeMounts:
        - name: nvsentinel-socket
          mountPath: /var/run

  ##############################################################################
  # NCCL FABRIC CONFIGURATION
  #
  # In production the webhook automatically copies NCCL env vars and volume
  # mounts from the pod's main container using glob patterns below.
  # For standalone testing, use ncclAllreduceExtraEnv and
  # gangCoordination.extraHostPathMounts instead.
  ##############################################################################

  # Glob patterns for env var names to copy from user containers
  ncclEnvPatterns:
    - "NCCL_*"
    - "FI_*"
    - "LD_LIBRARY_PATH"
    - "UCX_*"
    - "TORCH_NCCL_*"
    - "CUDA_DEVICE_ORDER"

  # Glob patterns for volume mount names to copy from user containers
  volumeMountPatterns:
    - "host-opt-amazon*"
    - "host-libefa*"
    - "host-libibverbs*"
    - "amazon-efa*"
    - "nvtcpxo-*"
    - "nccl-*"
    - "dev-shm"

  # Extra env vars appended to preflight-nccl-allreduce AFTER user-container
  # env vars are copied. Use for standalone testing or overrides.
  ncclAllreduceExtraEnv: []

  ##############################################################################
  # GPU AND NETWORK RESOURCE DETECTION
  ##############################################################################

  # Extended resource names that identify GPU pods for injection
  gpuResourceNames:
    - "nvidia.com/gpu"

  # Network resource names copied to init containers (RDMA, InfiniBand, etc.)
  networkResourceNames:
    - "nvidia.com/mlnxnics"
    - "vpc.amazonaws.com/efa"
    # GCP multi-network resources (TCPXO)
    - "networking.gke.io.networks/gpu-nic0"
    - "networking.gke.io.networks/gpu-nic0.IP"
    - "networking.gke.io.networks/gpu-nic1"
    - "networking.gke.io.networks/gpu-nic1.IP"
    - "networking.gke.io.networks/gpu-nic2"
    - "networking.gke.io.networks/gpu-nic2.IP"
    - "networking.gke.io.networks/gpu-nic3"
    - "networking.gke.io.networks/gpu-nic3.IP"
    - "networking.gke.io.networks/gpu-nic4"
    - "networking.gke.io.networks/gpu-nic4.IP"
    - "networking.gke.io.networks/gpu-nic5"
    - "networking.gke.io.networks/gpu-nic5.IP"
    - "networking.gke.io.networks/gpu-nic6"
    - "networking.gke.io.networks/gpu-nic6.IP"
    - "networking.gke.io.networks/gpu-nic7"
    - "networking.gke.io.networks/gpu-nic7.IP"

  ##############################################################################
  # GANG DISCOVERY
  #
  # Identifies pods belonging to the same scheduling group for multi-node checks.
  # Default (empty): K8s 1.35+ native WorkloadRef API.
  # For PodGroup-based schedulers, set name and other fields.
  # See: docs/configuration/preflight.md#gang-discovery
  ##############################################################################
  gangDiscovery: {}
    # name: "volcano"
    # annotationKeys: ["scheduling.k8s.io/group-name"]
    # labelKeys: []
    # podGroupGVR:
    #   group: "scheduling.volcano.sh"
    #   version: "v1beta1"
    #   resource: "podgroups"
    # minCountExpr: "podGroup.spec.minMember"

  ##############################################################################
  # GANG COORDINATION
  #
  # Manages ConfigMaps for multi-node init container coordination.
  # Required for nccl-allreduce. See: docs/configuration/preflight.md
  ##############################################################################
  gangCoordination:
    # Enable gang coordination (required for gang-wide checks)
    enabled: true
    # Maximum time to wait for all gang members to register
    timeout: "10m"
    # Port for PyTorch distributed TCP bootstrap (torchrun)
    masterPort: 29500
    # Path where gang ConfigMap is mounted in init containers
    configMapMountPath: "/etc/preflight"
    # NCCL topology ConfigMap name - required for Azure InfiniBand (NDv4/v5)
    # Without this, NCCL cannot map GPUs to IB HCAs and falls back to TCP.
    # Set explicitly for a pre-existing ConfigMap, or use ncclTopoShape below.
    ncclTopoConfigMap: ""
    # VM shape for auto-creating NCCL topology ConfigMap from bundled XML.
    # Supported: "ndv4", "ndv5" (files in charts/preflight/files/).
    ncclTopoShape: ""
    # Optional hostPath mounts for NCCL/OFI/CUDA libs on the host.
    # Fabric-specific; set via overlay files (e.g. values-mnnvl-efa.yaml).
    extraHostPathMounts: []
    # Mount pre-existing pod volumes (e.g. GCP TCPXO plugin emptyDir).
    # Does NOT create new volumes, only adds volumeMounts.
    extraVolumeMounts: []
    # Mirror pod-level DRA resource claims to init containers.
    # Ensures init containers get same device access (GPUs, RDMA NICs,
    # IMEX channels). Defaults to true when omitted.
    # mirrorResourceClaims: true

  ##############################################################################
  # NAMESPACE SELECTION
  ##############################################################################

  # Webhook only injects into namespaces matching this selector.
  # Enable a namespace: kubectl label namespace <name> nvsentinel.nvidia.com/preflight=enabled
  namespaceSelector:
    matchLabels:
      nvsentinel.nvidia.com/preflight: "enabled"

  # Network policy to allow API server -> webhook traffic
  networkPolicy:
    enabled: true

################################################################################
# END OF CONFIGURATION
################################################################################