Kubernetes Troubleshooting Checklist

# What's unhealthy?
kubectl get pods -A | grep -v Running | grep -v Completed

# Recent events (errors bubble up here)
kubectl get events --sort-by='.lastTimestamp' -A | tail -20

# Node health
kubectl get nodes
kubectl top nodes

# Check logs (current crash)
kubectl logs my-pod

# Previous crash logs
kubectl logs my-pod --previous

# Check exit code
kubectl describe pod my-pod | grep -A5 "Last State"

# OOMKilled — increase memory limit
kubectl patch deployment my-app -p '{"spec":{"template":{"spec":{"containers":[{"name":"app","resources":{"limits":{"memory":"512Mi"}}}]}}}}'

# Missing env var — check ConfigMap/Secret
kubectl get configmap my-config -o yaml
kubectl get secret my-secret -o yaml

# Debug interactively
kubectl run debug --image=my-app --rm -it -- sh

kubectl describe pod my-pod | grep -A5 "Events"

# Check image name
kubectl get pod my-pod -o jsonpath='{.spec.containers[0].image}'

# Create registry secret
kubectl create secret docker-registry regcred \
  --docker-server=registry.example.com \
  --docker-username=user \
  --docker-password=pass

# Add to deployment
kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"regcred"}]}'

kubectl describe pod my-pod | grep -A10 "Events"

# Check available resources
kubectl describe nodes | grep -A5 "Allocated resources"

# Check PVC status
kubectl get pvc

# Check taints
kubectl describe nodes | grep Taints

# Does the service exist?
kubectl get svc my-service

# Are endpoints populated?
kubectl get endpoints my-service
# Empty endpoints = selector doesn't match pod labels

# Check pod labels
kubectl get pods --show-labels | grep my-app

# Test from inside the cluster
kubectl run test --image=busybox --rm -it -- wget -qO- http://my-service:80

# Test DNS resolution
kubectl run dns-test --image=busybox --rm -it -- nslookup my-service

# Check CoreDNS
kubectl get pods -n kube-system -l k8s-app=kube-dns
kubectl logs -n kube-system -l k8s-app=kube-dns

# Check resolv.conf inside a pod
kubectl exec my-pod -- cat /etc/resolv.conf

# Test outbound connectivity
kubectl exec my-pod -- curl -v https://api.example.com

# Check NetworkPolicy
kubectl get networkpolicy -n my-namespace

# Check egress rules
kubectl describe networkpolicy -n my-namespace

kubectl describe pvc my-pvc

# Common fixes:
# 1. No StorageClass
kubectl get storageclass

# 2. No available PV (for static provisioning)
kubectl get pv

# 3. Wrong access mode
# ReadWriteOnce: single node
# ReadWriteMany: multiple nodes (needs NFS/EFS)

# Fix: Set fsGroup in securityContext
spec:
  securityContext:
    fsGroup: 1000
  containers:
    - name: app
      securityContext:
        runAsUser: 1000

# Check memory usage
kubectl top pods
kubectl describe pod my-pod | grep -i oom

# Fix: increase limits or fix memory leak
kubectl set resources deployment my-app --limits=memory=1Gi

# Check CPU usage vs limits
kubectl top pods

# Signs: high latency, slow responses
# Fix: increase CPU limit or optimize code
kubectl set resources deployment my-app --limits=cpu=2

kubectl rollout status deployment my-app
kubectl describe deployment my-app

# Check new ReplicaSet
kubectl get rs -l app=my-app

# Rollback
kubectl rollout undo deployment my-app

# Rollback to specific revision
kubectl rollout history deployment my-app
kubectl rollout undo deployment my-app --to-revision=3

# Interactive debug container (K8s 1.25+)
kubectl debug my-pod -it --image=nicolaka/netshoot

# Copy of pod with debug tools
kubectl debug my-pod -it --image=nicolaka/netshoot --copy-to=debug-pod

# Node-level debugging
kubectl debug node/my-node -it --image=ubuntu