nntrivi2001
57dc91585d
Internal SmartGift build of a Claude Code monitoring dashboard. Lanes: a durable unit of parallel agent work, one per working directory, tracked across session restarts. Managed lanes are git worktrees the dashboard provisions and can reset or remove behind a three-check destroy guard and a counted preflight; adopted lanes are directories you already own and are never destroyable. Pipelines: a lane moves through pipeline stages. A stage the agent declares with evidence renders green; a stage inferred from the tool-event stream renders dashed amber and never counts as done. Detection is forward-only within a 30-minute window, and never writes the declared stage. Workspace: one page at /run with a lane grid, the selected lane's pipeline, and a full Claude console behind a disclosure.
195 lines
7.0 KiB
YAML
195 lines
7.0 KiB
YAML
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Coralogix Alert Definitions for Claude Code Agent Monitor
|
|
#
|
|
# These alerts can be imported via the Coralogix Alerts API or Terraform
|
|
# provider (coralogix/coralogix). They mirror the Prometheus/Alertmanager
|
|
# rules in ../prometheus/rules/ for consistency across observability stacks.
|
|
#
|
|
# API import:
|
|
# curl -X POST "https://api.coralogix.com/api/v1/external/alerts" \
|
|
# -H "Authorization: Bearer $CORALOGIX_API_KEY" \
|
|
# -H "Content-Type: application/json" \
|
|
# -d @deployments/monitoring/coralogix/alerts.yaml
|
|
#
|
|
# Terraform:
|
|
# See the coralogix_alert resources in coralogix-terraform.tf
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
alerts:
|
|
# ── Critical ───────────────────────────────────────────────────────────────
|
|
|
|
- name: "Agent Monitor Down"
|
|
description: "No metrics received from agent-monitor pods for > 2 minutes"
|
|
severity: critical
|
|
type: metric
|
|
condition:
|
|
metric_name: "up"
|
|
filter:
|
|
job: "agent-monitor"
|
|
threshold: 1
|
|
comparison: less_than
|
|
for_duration: "2m"
|
|
of_last: "5m"
|
|
notifications:
|
|
- integration: pagerduty
|
|
- integration: slack
|
|
channel: "#agent-monitor-critical"
|
|
labels:
|
|
service: agent-monitor
|
|
team: platform
|
|
|
|
- name: "High Error Rate"
|
|
description: "5xx error rate exceeds 5% of total requests for 5 minutes"
|
|
severity: critical
|
|
type: ratio
|
|
condition:
|
|
numerator:
|
|
query: 'http_requests_total{job="agent-monitor", status=~"5.."}'
|
|
denominator:
|
|
query: 'http_requests_total{job="agent-monitor"}'
|
|
threshold: 0.05
|
|
comparison: greater_than
|
|
for_duration: "5m"
|
|
notifications:
|
|
- integration: pagerduty
|
|
- integration: slack
|
|
channel: "#agent-monitor-critical"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "Pod Restart Loop"
|
|
description: "Agent Monitor pod has restarted > 5 times in 15 minutes"
|
|
severity: critical
|
|
type: metric
|
|
condition:
|
|
query: 'increase(kube_pod_container_status_restarts_total{namespace=~"agent-monitor.*", container="agent-monitor"}[15m])'
|
|
threshold: 5
|
|
comparison: greater_than
|
|
for_duration: "1m"
|
|
notifications:
|
|
- integration: pagerduty
|
|
- integration: slack
|
|
channel: "#agent-monitor-critical"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "Persistent Volume Nearly Full"
|
|
description: "SQLite persistent volume is > 90% full"
|
|
severity: critical
|
|
type: metric
|
|
condition:
|
|
query: '(kubelet_volume_stats_used_bytes{namespace=~"agent-monitor.*"} / kubelet_volume_stats_capacity_bytes{namespace=~"agent-monitor.*"}) * 100'
|
|
threshold: 90
|
|
comparison: greater_than
|
|
for_duration: "5m"
|
|
notifications:
|
|
- integration: pagerduty
|
|
- integration: slack
|
|
channel: "#agent-monitor-critical"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
# ── Warning ────────────────────────────────────────────────────────────────
|
|
|
|
- name: "High Latency"
|
|
description: "P95 request latency exceeds 2 seconds for 5 minutes"
|
|
severity: warning
|
|
type: metric
|
|
condition:
|
|
query: 'histogram_quantile(0.95, rate(http_request_duration_seconds_bucket{job="agent-monitor"}[5m]))'
|
|
threshold: 2
|
|
comparison: greater_than
|
|
for_duration: "5m"
|
|
notifications:
|
|
- integration: slack
|
|
channel: "#agent-monitor-alerts"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "WebSocket Connection Spike"
|
|
description: "Active WebSocket connections exceed 1000"
|
|
severity: warning
|
|
type: metric
|
|
condition:
|
|
metric_name: "websocket_connections_active"
|
|
filter:
|
|
job: "agent-monitor"
|
|
threshold: 1000
|
|
comparison: greater_than
|
|
for_duration: "2m"
|
|
notifications:
|
|
- integration: slack
|
|
channel: "#agent-monitor-alerts"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "High Memory Usage"
|
|
description: "Container memory usage exceeds 85% of limit"
|
|
severity: warning
|
|
type: metric
|
|
condition:
|
|
query: '(container_memory_working_set_bytes{namespace=~"agent-monitor.*", container="agent-monitor"} / container_spec_memory_limit_bytes{namespace=~"agent-monitor.*", container="agent-monitor"}) * 100'
|
|
threshold: 85
|
|
comparison: greater_than
|
|
for_duration: "5m"
|
|
notifications:
|
|
- integration: slack
|
|
channel: "#agent-monitor-alerts"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "High CPU Usage"
|
|
description: "Container CPU usage exceeds 80% for 10 minutes"
|
|
severity: warning
|
|
type: metric
|
|
condition:
|
|
query: '(rate(container_cpu_usage_seconds_total{namespace=~"agent-monitor.*", container="agent-monitor"}[5m]) / container_spec_cpu_quota{namespace=~"agent-monitor.*", container="agent-monitor"} * 100000)'
|
|
threshold: 80
|
|
comparison: greater_than
|
|
for_duration: "10m"
|
|
notifications:
|
|
- integration: slack
|
|
channel: "#agent-monitor-alerts"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "HPA Maxed Out"
|
|
description: "HPA replicas at max for 15 minutes — may need capacity increase"
|
|
severity: warning
|
|
type: metric
|
|
condition:
|
|
query: 'kube_horizontalpodautoscaler_status_current_replicas{namespace=~"agent-monitor.*"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"agent-monitor.*"}'
|
|
threshold: 1
|
|
comparison: greater_than_or_equal
|
|
for_duration: "15m"
|
|
notifications:
|
|
- integration: slack
|
|
channel: "#agent-monitor-alerts"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
- name: "Slow Database Queries"
|
|
description: "SQLite query duration exceeds 1 second"
|
|
severity: warning
|
|
type: metric
|
|
condition:
|
|
metric_name: "sqlite_query_duration_seconds"
|
|
filter:
|
|
job: "agent-monitor"
|
|
threshold: 1
|
|
comparison: greater_than
|
|
for_duration: "5m"
|
|
notifications:
|
|
- integration: slack
|
|
channel: "#agent-monitor-alerts"
|
|
labels:
|
|
service: agent-monitor
|
|
|
|
# ── Notification integrations ────────────────────────────────────────────────
|
|
# Configure these in Coralogix UI: Settings → Integrations → Outbound Webhooks
|
|
#
|
|
# Required integrations:
|
|
# - pagerduty: PagerDuty Events API v2 routing key
|
|
# - slack: Slack webhook for #agent-monitor-critical and #agent-monitor-alerts
|
|
# - email: (optional) Email notification group
|