Grafana Dashboard Best Practices

Row 1: Overview (stat panels — key numbers)
  [Uptime] [Error Rate] [P99 Latency] [Active Users] [CPU %]

Row 2: Traffic & Latency (time series)
  [Request Rate over time] [Latency percentiles over time]

Row 3: Errors (time series + table)
  [Error rate over time] [Top errors by endpoint]

Row 4: Resources (time series)
  [CPU usage] [Memory usage] [Disk I/O] [Network I/O]

Row 5: Logs (optional)
  [Recent error logs from Loki]

Variable: namespace
  Query: label_values(kube_pod_info, namespace)
  Multi-value: true

Variable: service
  Query: label_values(http_requests_total{namespace="$namespace"}, service)

Variable: instance
  Query: label_values(up{namespace="$namespace", job="$service"}, instance)

rate(http_requests_total{namespace="$namespace", service="$service"}[5m])

sum(rate(http_requests_total{namespace="$namespace"}[5m])) by (service)

sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
/ sum(rate(http_requests_total[5m])) by (service)
* 100

histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket{service="$service"}[5m])) by (le)
)

# Container CPU (percentage of request)
sum(rate(container_cpu_usage_seconds_total{namespace="$namespace", pod=~"$service.*"}[5m]))
/ sum(kube_pod_container_resource_requests{namespace="$namespace", pod=~"$service.*", resource="cpu"})
* 100

# Container memory (percentage of limit)
sum(container_memory_working_set_bytes{namespace="$namespace", pod=~"$service.*"})
/ sum(kube_pod_container_resource_limits{namespace="$namespace", pod=~"$service.*", resource="memory"})
* 100

{
  "thresholds": {
    "steps": [
      { "value": null, "color": "green" },
      { "value": 1, "color": "yellow" },
      { "value": 5, "color": "red" }
    ]
  }
}

# grafana/provisioning/dashboards/default.yml
apiVersion: 1
providers:
  - name: default
    type: file
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: true

# Export
curl -s -H "Authorization: Bearer $GRAFANA_TOKEN" \
  "http://grafana:3000/api/dashboards/uid/my-dashboard" \
  | jq '.dashboard' > dashboards/my-dashboard.json

# Import
curl -s -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" \
  -H "Content-Type: application/json" \
  -d "{\"dashboard\": $(cat dashboards/my-dashboard.json), \"overwrite\": true}" \
  "http://grafana:3000/api/dashboards/db"

local grafana = import 'grafonnet/grafana.libsonnet';
local dashboard = grafana.dashboard;
local prometheus = grafana.prometheus;
local graphPanel = grafana.graphPanel;

dashboard.new(
  'Service Overview',
  tags=['generated', 'prometheus'],
  time_from='now-1h',
)
.addPanel(
  graphPanel.new(
    'Request Rate',
    datasource='Prometheus',
  ).addTarget(
    prometheus.target(
      'sum(rate(http_requests_total[5m])) by (service)',
      legendFormat='{{service}}',
    )
  ),
  gridPos={ x: 0, y: 0, w: 12, h: 8 }
)

# Add deployment annotation
curl -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" \
  -H "Content-Type: application/json" \
  -d "{\"text\": \"Deployed v2.1.0\", \"tags\": [\"deploy\"]}" \
  "http://grafana:3000/api/annotations"

# Annotation query for deploys
changes(kube_deployment_status_observed_generation{deployment="my-app"}[5m]) > 0