>k port-forward svc/vmagent-tratata-prod-aks-us-1 8429

curl -X POST http://localhost:8429/api/v1/import/prometheus \
-d 'kube_pod_status_phase{phase="Pending",exported_namespace="test-namespace",pod="logical-backup-pgsql-test-pod",cluster="test-prod-cluster"} 1'

curl -X POST http://localhost:8429/api/v1/import/prometheus \
-d 'kube_pod_status_phase{phase="Pending",exported_namespace="test-namespace",pod="logical-backup-pgsql-test-pod",cluster="test-prod-cluster"} 0'

curl -X POST https://alertmanager.domain.com/api/v1/alerts \
-H "Content-Type: application/json" \
-d '[
 {
   "labels": {
     "alertname": "KubernetesPodNotHealthy",
     "cluster": "test-prod-cluster",
     "exported_namespace": "test-namespace",
     "pod": "logical-backup-pgsql-test-pod",
     "phase": "Pending",
     "emoji": "🔥",
     "incident_type": "outage"
   },
   "annotations": {
     "summary": "POD'"'"'s state is `Pending` for `test-prod-cluster`/`test-namespace`/`logical-backup-pgsql-test-pod`",
     "description": "Pod test-namespace/logical-backup-pgsql-test-pod has been in a non-running state for longer than 15 minutes.\n  VALUE = 1\n  LABELS = map[cluster:test-prod-cluster exported_namespace:test-namespace phase:Pending pod:logical-backup-pgsql-test-pod]"
   },
   "startsAt": "'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'",
   "endsAt": "'$(date -u -d "+3 minutes" +"%Y-%m-%dT%H:%M:%SZ")'"
 }
]'

The node had condition: [DiskPressure].
The node was low on resource: ephemeral-storage. Threshold quantity: 4281126156, available: 3438556Ki

resource "aws_launch_template" "node-launch-template" {
...
  block_device_mappings {
    device_name = "/dev/xvda"
    ebs {
      volume_size = 200
      volume_type = "gp3"
    }
  }
...

  lifecycle {
    ignore_changes 

kubectl get nodes -o yaml | grep -i -A6 ephemeral-storage

resource "aws_launch_template" "node-launch-template" {
...
# !!! вот тут много не надо, это для самой ноды, операционка, можно хоть 10
  block_device_mappings {
    device_name = "/dev/xvda"
    ebs {
      volume_size = 20
      volume_type = "gp3"
    }
  }
# !!! вот это добавляем, для POD-ов
  block_device_mappings {
    device_name = "/dev/xvdb"
    ebs {
      volume_size = 200
      volume_type = "gp3"
    }
  }

kubectl get nodes -o yaml | grep -i -A6 ephemeral-storage

---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
...
spec:
  cooldownPeriod: 300
  maxReplicaCount: 50
  minReplicaCount: 2
  pollingInterval: 30
  scaleTargetRef:
    name: application
  triggers:
  - authenticationRef:
      name: keda-aws-credentials
    metadata:
      awsRegion: us-east-1
      identityOwner: operator
      queueLength: "500"
      queueURL: https://sqs.us-east-1.amazonaws.com/123456789/sqsname
    type: aws-sqs-queue

---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
...
spec:
...
    metricType: Value

---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
...
spec:
  advanced:
    horizontalPodAutoscalerConfig:
      behavior:
        scaleDown:
          policies:
          - periodSeconds: 15
            type: Pods
            value: 1
          stabilizationWindowSeconds: 300
        scaleUp:
          policies:
          - periodSeconds: 15
            type: Pods
            value: 1
          selectPolicy: Max
          stabilizationWindowSeconds: 60
... (остальное так же)

---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
...
spec:
  advanced:
...(тут тоже самое)
    metadata:
...
      scaleOnInFlight: "false"   << - - вот это вот важное нам
...

- alert: PostgreSQLlaggingbehind
  expr: pg_replication_lag_seconds{} > 7200
  for: 5m
  labels:
    severity: warning
  annotations:
    summary: "PSQLDB `{{ $labels.pod }}`/`{{ $labels.cluster }}`/`{{ $labels.namespace }}` has lag of `{{ $value }}`"

patronictl list

patronictl reinit clustername instancename

# Allow Security rule in NSG (allows communication between pods on edge node pool)
resource "azurerm_network_security_rule" "edge_asg_allow_pod_to_pod_rule" {
  count                        = local.edge_node_pool_exists ? 1 : 0
  name                         = "allow-pod-to-pod-edge-asg"
  priority                     = 960
  direction                    = "Outbound"
  access                       = "Allow"
  protocol                     = "*"
  source_address_prefixes      = azurerm_subnet.edge.address_prefixes
  source_port_range            = "*"
  destination_address_prefixes = azurerm_subnet.edge.address_prefixes
  destination_port_range       = "*"
  resource_group_name          = "${var.kubernetes_cluster.name}-nrg"
  network_security_group_name  = data.azurerm_resources.nsg[0].resources.0.name
}

patronictl list

resource "aws_eks_node_group" "on-demand-group" {
...
  labels = {
    "node-type" = "on-demand"
...
  }
...
}


resource "aws_eks_node_group" "spot-group" {
...
  labels = {
    "node-type" = "spot"
...
  }
...
}

kubectl get nodes -L node-role,node-type

NAME                           STATUS   ROLES    AGE   VERSION               NODE-ROLE        NODE-TYPE
ip-10-1-123-114.ec2.internal   Ready    <none>   70d   v1.31.4-eks-aeac579   infrastructure   on-demand
ip-10-1-14-194.ec2.internal    Ready    <none>   42m   v1.31.6-eks-aad632c   applications     on-demand
ip-10-1-24-165.ec2.internal    Ready    <none>   52s   v1.31.6-eks-aad632c   applications     spot

kube-state-metrics:
...
  extraArgs:
    - --metric-labels-allowlist=nodes=[node-type,node-role]

sum(kube_node_labels) by (label_node_type)