# ───────────────────────────────────────────────────────────────────────────── # Prometheus configuration for Claude Code Agent Monitor # ───────────────────────────────────────────────────────────────────────────── global: scrape_interval: 15s scrape_timeout: 10s evaluation_interval: 15s external_labels: cluster: "${CLUSTER_NAME:agent-monitor}" environment: "${ENVIRONMENT:production}" # ── Rule files ────────────────────────────────────────────────────────────── rule_files: - /etc/prometheus/rules/*.rules.yaml # ── Alertmanager ──────────────────────────────────────────────────────────── alerting: alertmanagers: - static_configs: - targets: - alertmanager:9093 scheme: http timeout: 10s api_version: v2 # ── Scrape configs ────────────────────────────────────────────────────────── scrape_configs: # ── Agent Monitor application ───────────────────────────────────────────── - job_name: "agent-monitor" metrics_path: /api/health scrape_interval: 15s scrape_timeout: 5s scheme: http # Static target for standalone deployments static_configs: - targets: - "agent-monitor:4820" labels: app: agent-monitor component: server # Relabeling to add standard labels relabel_configs: - source_labels: [__address__] target_label: instance - target_label: __metrics_path__ replacement: /api/health metric_relabel_configs: - source_labels: [__name__] regex: "go_.*" action: drop # ── Agent Monitor MCP sidecar ───────────────────────────────────────────── - job_name: "agent-monitor-mcp" scrape_interval: 30s static_configs: - targets: - "agent-monitor-mcp:8819" labels: app: agent-monitor component: mcp # ── Kubernetes service discovery (pods) ─────────────────────────────────── - job_name: "kubernetes-pods" scrape_interval: 15s kubernetes_sd_configs: - role: pod namespaces: own_namespace: false names: - agent-monitor - agent-monitor-dev - agent-monitor-staging - agent-monitor-production relabel_configs: # Only scrape pods with annotation prometheus.io/scrape=true - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true # Use custom metrics path if annotated - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) # Use custom port if annotated - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 target_label: __address__ # Use custom scheme if annotated - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (.+) # Map pod labels to Prometheus labels - action: labelmap regex: __meta_kubernetes_pod_label_(.+) # Add namespace label - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace # Add pod name label - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod # Add node name label - source_labels: [__meta_kubernetes_pod_node_name] action: replace target_label: node # ── Kubernetes service discovery (services) ────────────────────────────── - job_name: "kubernetes-services" scrape_interval: 15s kubernetes_sd_configs: - role: service namespaces: names: - agent-monitor - agent-monitor-dev - agent-monitor-staging - agent-monitor-production relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 target_label: __address__ - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: service # ── Kubernetes nodes ────────────────────────────────────────────────────── - job_name: "kubernetes-nodes" scrape_interval: 30s scheme: https kubernetes_sd_configs: - role: node tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: false bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics # ── Kubernetes cadvisor (container metrics) ─────────────────────────────── - job_name: "kubernetes-cadvisor" scrape_interval: 15s scheme: https kubernetes_sd_configs: - role: node tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: false bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor metric_relabel_configs: # Keep only container metrics for agent-monitor pods - source_labels: [container] regex: "agent-monitor.*" action: keep # ── Prometheus self-monitoring ──────────────────────────────────────────── - job_name: "prometheus" static_configs: - targets: - "localhost:9090"