---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlert
metadata:
  name: ${NAME}
  namespace: ${NAMESPACE}
spec:
  replicaCount: 1
  datasource:
    url: ${URL_DATASOURCE}
  notifier:
    url: ${URL_NOTIFIER}
  evaluationInterval: "30s"
  selectAllByDefault: true
  remoteWrite:
    url: http://vminsert-victoria-metrics-cluster.service:8480/insert/0/prometheus/

---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
metadata:
  name: vmrule-storage-iops
  namespace: ${NAMESPACE}
spec:
  groups:
    - name: storage_monitoring
      interval: 30s
      rules:
        - record: storage_class_iops_limits
          expr: |
            sum by(persistentvolume, storageclass, cluster) (
              (kube_persistentvolume_info{storageclass=~"azureblob-fuse-premium|azureblob-nfs-premium"} * 15000) or
              (kube_persistentvolume_info{storageclass=~"managed-premium|managed-csi-premium|managed-premium-burstable"} * 20000) or
              (kube_persistentvolume_info{storageclass=~"managed|managed-csi|default"} * 6000) or
              (kube_persistentvolume_info{storageclass="managed-csi-os-warm-nodes"} * 6000) or
              (kube_persistentvolume_info{storageclass=~"azurefile-premium|azurefile-csi-premium|azurefile-nonroot|azurefile-sc-fips|sc-rwm"} * 100000) or
              (kube_persistentvolume_info{storageclass=~"azurefile|azurefile-csi|azurefile-standart-nonroot"} * 10000) or
              (kube_persistentvolume_info * 500)
            )

        - record: pod_pvc_info
          expr: |
            max by(cluster, exported_namespace, pod, persistentvolumeclaim) (
              kube_pod_spec_volumes_persistentvolumeclaims_info
            )
        - record: pvc_storage_info
          expr: |
            max by(cluster, exported_namespace, persistentvolumeclaim, storageclass) (
              kube_persistentvolumeclaim_info
            )

        - record: pvc_current_iops
          expr: |
            sum by(cluster, pod, namespace, persistentvolumeclaim, storageclass) (
              (
                rate(container_fs_reads_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m]) +
                rate(container_fs_writes_total{
                  device!~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)",
                  device=~"/dev/sd[b-z].*"
                }[5m])
              )
            )
            * on(cluster, pod) group_left(persistentvolumeclaim, exported_namespace)
            pod_pvc_info
            * on(cluster, exported_namespace, persistentvolumeclaim) group_left(storageclass)
            pvc_storage_info

        - record: pvc_iops_utilization_percent
          expr: |
            sum by(cluster, namespace, pod, persistentvolumeclaim, storageclass) (pvc_current_iops)
            * 100
            / on(cluster, storageclass) group_left()
            max by(cluster, storageclass) (storage_class_iops_limits)
---
- alert: HighIOPSUtilization
  expr: round(pvc_iops_utilization_percent) > 85
  for: 40m
  labels:
    severity: warning
  annotations:
    summary: "IOPS is `{{ $value }}%` on `{{ $labels.cluster }}`/`{{ $labels.namespace }}`/`{{ $labels.pod }}`."

apiVersion: operator.victoriametrics.com/v1beta1
kind: VMRule
...
spec:
  groups:
...
    - name: "alertmanager"
      rules:
        - alert: Watchdog
          expr: vector(1)
          for: 5m
          labels:
            severity: info
          annotations:
            summary: "This is an Watchdog alert that is always firing."
...

route:
...
  routes:
    ##############################################################
    #                          Watchdog                          #
    ##############################################################
    # DO NOT REMOVE THIS ROUTE!!!
    # DO NOT ADD ROUTES ABOVE!!!
    - receiver: 'webhook-receiver'
      group_wait: 10s
      group_interval: 30s
      repeat_interval: 30s
      matchers:
        - alertname = "Watchdog"
      continue: true
...
receivers:
...
  - name: 'webhook-receiver'
    webhook_configs:
      - url: 'https://cross-alerting.domain.io/api/alert'
        send_resolved: true
....

function kvs() {
    kubectl get secret "$1" -o json | jq '.data | map_values(@base64d)'
}

> kvs grafana-monitoring

{
  "admin-password": "pupupu",
  "admin-user": "pupupu@pupupu.ded",
  "ldap-toml": ""
}

      rules:
        - alert: SSLexpiresSoon
          expr: nginx_ingress_controller_ssl_expire_time_seconds - time() <= 7 * 24 * 60 * 60
          annotations:
            summary: "SSL for `{{ $labels.host }}` expires in {{ .Value | humanizeDuration }}"

token="glpat-***"
gitlab_url="https://gitlab***"

page=1
while true; do
    response=$(curl -s --header "PRIVATE-TOKEN: $token" "$gitlab_url/api/v4/personal_access_tokens?page=$page&per_page=50")
    if [[ -z "$response" || "$response" == "[]" ]]; then
        break
    fi
    echo "$response" | jq -r '.[] | select(.expires_at < "2025-01-01") | .id' >> expired_token_ids.txt
    ((page++))
done

# .gitlab-ci.yml
check-tokens:
  image: python:3.11-slim
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
    - if: $CI_PIPELINE_SOURCE == "web"
      when: always
  before_script:
    - pip install --no-cache-dir -r requirements.txt
  script:
    - python3 check_tokens.py
  variables:
    GITLAB_TOKEN: ${GITLAB_API_TOKEN}
    GITLAB_URL: "https://gitlab.****" #на самом деле тут есть CI дефолт переменная, надо переделать после написания заметки
    SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
  tags:
    - docker

python-gitlab>=3.15.0
requests>=2.31.0
python-dateutil>=2.8.2

GITLAB_API_TOKEN
SLACK_WEBHOOK_URL