Improve monitoring for prod

infrastructure/kubernetes/base/components/monitoring/ha-policies.yaml

@@ -0,0 +1,100 @@
+---
+# PodDisruptionBudgets ensure minimum availability during voluntary disruptions
+# (node drains, rolling updates, etc.)
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: prometheus-pdb
+  namespace: monitoring
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: prometheus
+
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: alertmanager-pdb
+  namespace: monitoring
+spec:
+  minAvailable: 2
+  selector:
+    matchLabels:
+      app: alertmanager
+
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: grafana-pdb
+  namespace: monitoring
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: grafana
+
+---
+# ResourceQuota limits total resources in monitoring namespace
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: monitoring-quota
+  namespace: monitoring
+spec:
+  hard:
+    # Compute resources
+    requests.cpu: "10"
+    requests.memory: "16Gi"
+    limits.cpu: "20"
+    limits.memory: "32Gi"
+
+    # Storage
+    persistentvolumeclaims: "10"
+    requests.storage: "100Gi"
+
+    # Object counts
+    pods: "50"
+    services: "20"
+    configmaps: "30"
+    secrets: "20"
+
+---
+# LimitRange sets default resource limits for pods in monitoring namespace
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: monitoring-limits
+  namespace: monitoring
+spec:
+  limits:
+  # Default container limits
+  - max:
+      cpu: "2"
+      memory: "4Gi"
+    min:
+      cpu: "10m"
+      memory: "16Mi"
+    default:
+      cpu: "500m"
+      memory: "512Mi"
+    defaultRequest:
+      cpu: "100m"
+      memory: "128Mi"
+    type: Container
+
+  # Pod limits
+  - max:
+      cpu: "4"
+      memory: "8Gi"
+    type: Pod
+
+  # PVC limits
+  - max:
+      storage: "50Gi"
+    min:
+      storage: "1Gi"
+    type: PersistentVolumeClaim