locals {
  aws_ami_name         = "bottlerocket"
  aws_ami_architecture = "x86_64"
  aws_ami_type         = "${upper(local.aws_ami_name)}_${local.aws_ami_architecture}"
}

data "aws_ssm_parameter" "eks_ami_image_version" {
  name = "/aws/service/${local.aws_ami_name}/aws-k8s-${aws_eks_cluster.eks.version}/${local.aws_ami_architecture}/latest/image_version"
}

resource "aws_eks_node_group" "blue-green-group" {
...
  ami_type        = local.aws_ami_type
  release_version = data.aws_ssm_parameter.eks_ami_image_version.value
...

  lifecycle {
    ignore_changes        = [release_version]
...
  }

ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   10s     v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    Ready      <none>   13s     v1.30.5-eks-baa6d11

ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   5s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   10s     v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    Ready      <none>   14s     v1.30.5-eks-baa6d11

memory.available: "100Mi" 
nodefs.available: "10%" 
nodefs.inodesFree: "5%" 
imagefs.available: "15%"

apiclient -u /settings | jq

[settings.kubernetes]
api-server = "${endpoint}"
cluster-certificate = "${cluster_auth_base64}"
cluster-name = "${cluster_name}"
pod-pids-limit = 1024
image-gc-high-threshold-percent = 85
image-gc-low-threshold-percent = 80
container-log-max-files = 50
container-log-max-size = "100Mi"
[settings.kubernetes.system-reserved]
cpu = "200m"
ephemeral-storage = "1Gi"
memory = "1Gi"
[settings.kubernetes.kube-reserved]
memory = "893Mi"
[settings.kernel]
lockdown = "integrity"
[settings.host-containers.admin]
enabled = ${enable_admin_container}
[settings.host-containers.control]
enabled = ${enable_control_container}
[settings.kubernetes.eviction-hard]
"memory.available" = "1Gi"
"nodefs.available" = "20%"
"nodefs.inodesFree" = "5%"
"imagefs.available" = "15%"
[settings.kubernetes.eviction-soft]
"memory.available" = "10%"
[settings.kubernetes.eviction-soft-grace-period]
"memory.available" = "30s"

resource "aws_launch_template" "node-launch-template" {
...
  user_data = base64encode(templatefile("${path.module}/../../templates/bottlerocket.tpl",
    {
      "cluster_name"             = aws_eks_cluster.eks.name #обязательный параметр!!!
      "endpoint"                 = aws_eks_cluster.eks.endpoint #обязательный параметр!!!
      "cluster_auth_base64"      = aws_eks_cluster.eks.certificate_authority[0].data #обязательный параметр!!!
      "aws_region"               = var.AWS_DEFAULT_REGION #обязательный параметр!!!
      "enable_admin_container"   = false
      "enable_control_container" = true
    }
  ))
}

Switched terragrunt to version "0.72.3" 
time terragrunt plan
Acquiring state lock. This may take a few moments...

No changes. Your infrastructure matches the configuration.

real    0m54.200s
user    0m28.351s
sys     0m2.390s

Switched terragrunt to version "0.54.15"
time terragrunt plan
Acquiring state lock. This may take a few moments...

No changes. Your infrastructure matches the configuration.

real    0m43.498s
user    0m31.284s
sys     0m4.366s

Switched terragrunt to version "0.56.0"
time terragrunt plan
Acquiring state lock. This may take a few moments...

No changes. Your infrastructure matches the configuration.

real    3m29.356s
user    2m27.303s
sys     0m14.885s

---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: ${NAME}
  namespace: ${NAMESPACE}
spec:
  replicaCount: 1
  datasource:
    url: ${URL_DATASOURCE}
  notifier:
    url: ${URL_NOTIFIER}
  evaluationInterval: "30s"
  selectAllByDefault: true
  remoteWrite:
    url: http://vminsert-victoria-metrics-cluster.service:8480/insert/0/prometheus/

---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: vmrule-storage-iops
  namespace: ${NAMESPACE}
spec:
  groups:
    - name: storage_monitoring
      interval: 30s
      rules:
        - record: storage_class_iops_limits
          expr: |
            sum by(persistentvolume, storageclass, cluster) (
              (kube_persistentvolume_info{storageclass=~"azureblob-fuse-premium|azureblob-nfs-premium"} * 15000) or
              (kube_persistentvolume_info{storageclass=~"managed-premium|managed-csi-premium|managed-premium-burstable"} * 20000) or
              (kube_persistentvolume_info{storageclass=~"managed|managed-csi|default"} * 6000) or
              (kube_persistentvolume_info{storageclass="managed-csi-os-warm-nodes"} * 6000) or
              (kube_persistentvolume_info{storageclass=~"azurefile-premium|azurefile-csi-premium|azurefile-nonroot|azurefile-sc-fips|sc-rwm"} * 100000) or
              (kube_persistentvolume_info{storageclass=~"azurefile|azurefile-csi|azurefile-standart-nonroot"} * 10000) or
              (kube_persistentvolume_info * 500)
            )

        - record: pod_pvc_info
          expr: |
            max by(cluster, exported_namespace, pod, persistentvolumeclaim) (
              kube_pod_spec_volumes_persistentvolumeclaims_info
            )
        - record: pvc_storage_info
          expr: |
            max by(cluster, exported_namespace, persistentvolumeclaim, storageclass) (
              kube_persistentvolumeclaim_info
            )

        - record: pvc_current_iops
          expr: |
            sum by(cluster, pod, namespace, persistentvolumeclaim, storageclass) (
              (
                rate(container_fs_reads_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m]) +
                rate(container_fs_writes_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m])
              )
            )
            * on(cluster, pod) group_left(persistentvolumeclaim, exported_namespace)
            pod_pvc_info
            * on(cluster, exported_namespace, persistentvolumeclaim) group_left(storageclass)
            pvc_storage_info

        - record: pvc_iops_utilization_percent
          expr: |
            sum by(cluster, namespace, pod, persistentvolumeclaim, storageclass) (pvc_current_iops)
            * 100
            / on(cluster, storageclass) group_left()
            max by(cluster, storageclass) (storage_class_iops_limits)
---
- alert: HighIOPSUtilization
  expr: round(pvc_iops_utilization_percent) > 85
  for: 40m
  labels:
    severity: warning
  annotations:
    summary: "IOPS is `{{ $value }}%` on `{{ $labels.cluster }}`/`{{ $labels.namespace }}`/`{{ $labels.pod }}`."

apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
...
spec:
  groups:
...
    - name: "alertmanager"
      rules:
        - alert: Watchdog
          expr: vector(1)
          for: 5m
          labels:
            severity: info
          annotations:
            summary: "This is an Watchdog alert that is always firing."
...

route:
...
  routes:
    ##############################################################
    #                          Watchdog                          #
    ##############################################################
    # DO NOT REMOVE THIS ROUTE!!!
    # DO NOT ADD ROUTES ABOVE!!!
    - receiver: 'webhook-receiver'
      group_wait: 10s
      group_interval: 30s
      repeat_interval: 30s
      matchers:
        - alertname = "Watchdog"
      continue: true
...
receivers:
...
  - name: 'webhook-receiver'
    webhook_configs:
      - url: 'https://cross-alerting.domain.io/api/alert'
        send_resolved: true
....

function kvs() {
    kubectl get secret "$1" -o json | jq '.data | map_values(@base64d)'
}

> kvs grafana-monitoring

{
  "admin-password": "pupupu",
  "admin-user": "pupupu@pupupu.ded",
  "ldap-toml": ""
}