Client → Authentication → Authorization (RBAC) → Admission (Mutating webhooks)
       → Validation → Admission (Validating webhooks) → Persist to etcd → Response

# Watch pods as raw events
kubectl get pods --watch -o json

# Inspect audit log (if enabled)
cat /var/log/kubernetes/audit.log | jq .

# Check API server health
kubectl get --raw /healthz
kubectl get --raw /readyz
kubectl get --raw /livez

# Keys are prefixed by resource path
/registry/pods/default/my-pod
/registry/deployments/production/my-app
/registry/services/endpoints/kube-system/kube-dns

# Read a key directly (requires etcdctl and certs)
ETCDCTL_API=3 etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  get /registry/pods/default/my-pod | strings | head -50

# List all keys
etcdctl get / --prefix --keys-only

# Snapshot backup
etcdctl snapshot save /backup/etcd-snapshot-$(date +%F).db

# Verify snapshot
etcdctl snapshot status /backup/etcd-snapshot.db --write-out=table

# Restore (run before starting etcd)
etcdctl snapshot restore /backup/etcd-snapshot.db \
  --data-dir=/var/lib/etcd-restored \
  --name=etcd-node1 \
  --initial-cluster=etcd-node1=https://10.0.0.1:2380 \
  --initial-advertise-peer-urls=https://10.0.0.1:2380

# Get current revision
etcdctl endpoint status --write-out=json | jq '.[0].Status.header.revision'

# Compact to current revision (removes old revisions)
etcdctl compact $(etcdctl endpoint status --write-out=json | jq '.[0].Status.header.revision')

# Defragment (reclaims disk space after compaction)
etcdctl defrag

New pod (nodeName="") detected
  │
  ▼
Filter plugins — eliminate nodes that cannot run the pod
  ├─ NodeUnschedulable      (node has NoSchedule taint / unschedulable flag)
  ├─ NodeResourcesFit       (insufficient CPU/memory)
  ├─ NodeAffinity           (nodeSelector, nodeAffinity)
  ├─ TaintToleration        (pod must tolerate node taints)
  ├─ PodTopologySpread      (spread constraints)
  └─ VolumeBinding          (PVCs that need specific node topology)
  │
  ▼
Score plugins — rank feasible nodes (0-100 each, weighted sum)
  ├─ NodeResourcesBalancedAllocation   (prefer balanced CPU/mem usage)
  ├─ LeastAllocated                    (prefer nodes with most free resources)
  ├─ ImageLocality                     (prefer nodes with image already pulled)
  └─ InterPodAffinity                  (prefer nodes satisfying pod affinity)
  │
  ▼
Select highest-score node → write Binding object → kubelet picks it up

# Add a taint to a node
kubectl taint nodes node1 dedicated=gpu:NoSchedule
kubectl taint nodes node1 dedicated=gpu:NoExecute

# Remove a taint
kubectl taint nodes node1 dedicated=gpu:NoSchedule-

# Pod toleration (YAML)
tolerations:
- key: "dedicated"
  operator: "Equal"
  value: "gpu"
  effect: "NoSchedule"

affinity:
  nodeAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
      nodeSelectorTerms:
      - matchExpressions:
        - key: topology.kubernetes.io/zone
          operator: In
          values: [us-east-1a, us-east-1b]
  podAntiAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    - labelSelector:
        matchLabels:
          app: my-app
      topologyKey: kubernetes.io/hostname   # one pod per host

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000
globalDefault: false
preemptionPolicy: PreemptLowerPriority   # or Never

loop:
  desired = read desired state from API object
  actual  = observe current state (list pods, check nodes, …)
  diff    = desired - actual
  if diff != empty:
    apply changes (create/update/delete resources)
  sleep resyncPeriod (or wait for watch event)

API Server watch stream
  │ ADDED/MODIFIED/DELETED events
  ▼
Informer (per resource type, shared across controllers)
  ├─ ThreadSafeStore (in-memory cache — avoids API calls for reads)
  └─ Event handlers → enqueue object key (namespace/name)
        │
        ▼
     Work Queue (rate-limited, deduplicating)
        │
        ▼
     Worker goroutine → Reconcile(key)
        └─ reads from cache (lister)
        └─ calls API server only for writes

Pod assigned to node (spec.nodeName set by scheduler)
  │
  ▼
Admit pod — check resources, enforce policies (CPU manager, topology)
  │
  ▼
Setup volumes (CNI calls for network, CSI for volumes)
  │
  ▼
Pull images (via CRI → container runtime → image service)
  │
  ▼
Create sandbox (pause container establishes network namespace)
  │
  ▼
Run init containers sequentially (each must succeed before next)
  │
  ▼
Run app containers (in parallel)
  │
  ├─ Execute postStart hook (if defined)
  ├─ Start liveness / readiness / startup probes
  └─ Report status back to API server
  │
  ▼ (on termination)
Send SIGTERM → wait terminationGracePeriodSeconds → SIGKILL
Execute preStop hook before SIGTERM

# Inspect containers via crictl (bypasses kubelet)
crictl ps                        # list running containers
crictl pods                      # list pods
crictl inspect <container-id>   # full container state
crictl logs <container-id>
crictl exec -it <container-id> sh

# Check which runtime is in use
kubectl get node -o wide         # CONTAINER-RUNTIME column
cat /var/lib/kubelet/config.yaml | grep containerRuntimeEndpoint

# Static pod manifests (control plane components on kubeadm clusters)
ls /etc/kubernetes/manifests/
# kube-apiserver.yaml  kube-controller-manager.yaml  kube-scheduler.yaml  etcd.yaml

# kubelet watches this directory; changes take effect immediately
# Static pods are mirrored as read-only objects in the API server

# Inspect kube-proxy iptables rules
iptables -t nat -L KUBE-SERVICES -n --line-numbers
iptables -t nat -L KUBE-SVC-<hash> -n       # per-service chain
iptables -t nat -L KUBE-SEP-<hash> -n       # per-endpoint chain (DNAT)

# Example flow for ClusterIP 10.96.0.10:80 with 3 endpoints
# KUBE-SERVICES → KUBE-SVC-xxx → statistic --probability 0.33 → KUBE-SEP-1 (DNAT pod1:8080)
#                                                                  → KUBE-SEP-2 (DNAT pod2:8080)
#                                                                  → KUBE-SEP-3 (DNAT pod3:8080)

# Check kube-proxy mode
kubectl -n kube-system get configmap kube-proxy -o yaml | grep mode

Node A (10.0.0.1)                     Node B (10.0.0.2)
├─ eth0: 10.0.0.1                     ├─ eth0: 10.0.0.2
├─ cni0 bridge: 10.244.0.1/24         ├─ cni0 bridge: 10.244.1.1/24
│   ├─ veth → Pod A1: 10.244.0.2      │   ├─ veth → Pod B1: 10.244.1.2
│   └─ veth → Pod A2: 10.244.0.3      │   └─ veth → Pod B2: 10.244.1.3
│                                     │
└─── Overlay / BGP routes ────────────┘
     (flannel VXLAN, Calico BGP, Cilium eBPF, …)

# DNS name formats
my-svc.my-ns.svc.cluster.local          # ClusterIP service
my-pod.my-ns.pod.cluster.local           # pod IP (dots replaced with dashes)
pod-0.my-svc.my-ns.svc.cluster.local    # StatefulSet pod via headless service

# Debug DNS from a pod
kubectl run -it --rm dnstest --image=busybox --restart=Never -- sh
/ # nslookup kubernetes.default.svc.cluster.local
/ # cat /etc/resolv.conf

# Check CoreDNS config
kubectl -n kube-system get configmap coredns -o yaml

# Default deny all ingress in a namespace
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-ingress
  namespace: production
spec:
  podSelector: {}        # select all pods
  policyTypes: [Ingress]
---
# Allow ingress only from app=frontend pods
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-frontend
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - port: 8080

StorageClass (provisioner, parameters, reclaimPolicy)
  │
  │ (dynamic provisioning)
  ▼
PersistentVolumeClaim created by user
  │  storageClassName: fast-ssd
  │  storage: 10Gi
  │  accessModes: [ReadWriteOnce]
  │
  ▼  (external-provisioner sidecar watches PVCs)
CSI driver called: CreateVolume RPC
  │
  ▼
PersistentVolume created (Bound to PVC)
  │
  ▼
Pod references PVC → kubelet calls NodeStageVolume + NodePublishVolume
  │
  ▼
Volume mounted at pod path

On PVC delete: ReclaimPolicy
  Retain  → PV remains (manual cleanup)
  Delete  → CSI DeleteVolume called, PV deleted

kubelet                     CSI driver pod
  │                           │
  │  NodeStageVolume (gRPC)   │  ← format + mount to staging path
  │──────────────────────────►│
  │  NodePublishVolume        │  ← bind-mount staging → pod path
  │──────────────────────────►│

External controller sidecar (runs alongside driver):
  provisioner  → CreateVolume / DeleteVolume
  attacher     → ControllerPublishVolume / ControllerUnpublishVolume
  resizer      → ControllerExpandVolume

kubectl set image deployment/my-app container=image:v2
  │
  ▼
Deployment controller creates new ReplicaSet (RS-v2, replicas=0)
  │
  ▼
Scale RS-v2 up by 1  (maxSurge=1 → can go 1 above desired)
  │
  ▼
Wait for new pod Ready
  │
  ▼
Scale RS-v1 down by 1  (maxUnavailable=1 → can be 1 below desired)
  │
  ▼
Repeat until RS-v2 = desired, RS-v1 = 0
  │
  ▼
Old RS kept (scale=0) for rollback history (revisionHistoryLimit)

HPA controller (in controller-manager) runs every --horizontal-pod-autoscaler-sync-period (15s)
  │
  ├─ Queries metrics-server (or custom/external metrics adapter)
  │    currentMetricValue = avg CPU across pods
  │
  ├─ desiredReplicas = ceil(currentReplicas × (currentValue / targetValue))
  │
  └─ Scales Deployment / ReplicaSet if outside [minReplicas, maxReplicas]
       Cooldown: --horizontal-pod-autoscaler-downscale-stabilization (5m default)

# Role (namespace-scoped)
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pod-reader
  namespace: default
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list", "watch"]
---
# RoleBinding: bind Role to a ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: read-pods
  namespace: default
subjects:
- kind: ServiceAccount
  name: my-app
  namespace: default
roleRef:
  kind: Role
  name: pod-reader
  apiGroup: rbac.authorization.k8s.io

# /etc/kubernetes/enc/encryption-config.yaml
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources: [secrets]
  providers:
  - aescbc:
      keys:
      - name: key1
        secret: <base64-encoded-32-byte-key>
  - identity: {}   # fallback: read unencrypted (for migration)

# Then restart kube-apiserver with:
# --encryption-provider-config=/etc/kubernetes/enc/encryption-config.yaml

# Rewrite all secrets to encrypt them
kubectl get secrets --all-namespaces -o json | kubectl replace -f -

# Enable PSA on a namespace via labels
kubectl label namespace production \
  pod-security.kubernetes.io/enforce=restricted \
  pod-security.kubernetes.io/warn=restricted \
  pod-security.kubernetes.io/audit=restricted

# Control plane component health
kubectl get componentstatuses
kubectl get --raw /healthz

# Node internals
kubectl describe node <node>          # conditions, capacity, allocatable, events
kubectl top node                       # requires metrics-server
kubectl get events --sort-by=.metadata.creationTimestamp

# Pod scheduling
kubectl describe pod <pod>            # Events: FailedScheduling, etc.
kubectl get pod -o wide               # which node, IP
kubectl get pod <pod> -o jsonpath='{.spec.nodeName}'

# Watch reconciliation
kubectl get rs --watch                # watch ReplicaSet convergence
kubectl rollout status deploy/my-app  # watch Deployment rollout
kubectl rollout history deploy/my-app
kubectl rollout undo deploy/my-app    # roll back