git add .
git commit -m "fix" --no-verify
git push origin main --force
git tag -f prod
git push --force origin prod
git clean -fdx
git reset --hard HEAD~5
git push --force-with-lease
git reflog expire --expire=now --all
git gc --aggressive --prune=now

FROM ubuntu:latest
COPY ./ /app
WORKDIR /app
RUN apt-get update 
RUN apt-get install -y cowsay
RUN apt-get upgrade
RUN apt-get -y install libpq-dev imagemagick gsfonts ruby-full ssh supervisor
RUN gem install bundler
RUN echo "vm.swappiness=10" >> /etc/sysctl.conf && chmod -R 777 /tmp
RUN curl -sL https://deb.nodesource.com/setup_9.x | sudo bash -
RUN apt-get install -y nodejs
RUN bundle install --without development test --path vendor/bundle
RUN rm -rf /usr/local/bundle/cache/*.gem 
RUN apt-get clean 
RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
CMD [“/app/init.sh”]

# Alarm for Primary Node
resource "aws_cloudwatch_metric_alarm" "redis_cpu_primary" {
  alarm_name          = "${var.project_prefix}-${var.environment}-redis-cpu-primary"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = "300"
  metric_name         = "CPUUtilization"
  namespace          = "AWS/ElastiCache"
  period             = "60"
  statistic          = "Average"
  threshold          = "80"
  alarm_description  = "Redis CPU utilization is too high on primary node"
  alarm_actions      = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  ok_actions         = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  dimensions = {
    CacheClusterId = "${aws_elasticache_replication_group.******.id}-001"
  }
}

# Alarm for Replica Node
resource "aws_cloudwatch_metric_alarm" "redis_cpu_replica" {
  alarm_name          = "${var.project_prefix}-${var.environment}-redis-cpu-replica"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = "300"
  metric_name         = "CPUUtilization"
  namespace          = "AWS/ElastiCache"
  period             = "60"
  statistic          = "Average"
  threshold          = "80"
  alarm_description  = "Redis CPU utilization is too high on replica node"
  alarm_actions      = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  ok_actions         = ["arn:aws:sns:us-east-1:${data.aws_caller_identity.this.account_id}:aws-health-notification"]
  dimensions = {
    CacheClusterId = "${aws_elasticache_replication_group.******.id}-002"
  }
}

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

helm upgrade cloudwatch-exporter prometheus-community/prometheus-cloudwatch-exporter \
  --install \
  --version 0.25.0 \
  --namespace monitoring \
  --set aws.aws_access_key_id="${{ secrets.AWS_ACCESS_KEY_ID }}" \
  --set aws.aws_secret_access_key="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
  --values=EKS/cloudwatch-values.yaml

config: |-
  region: us-east-1
  period_seconds: 30
  delay_seconds: 30
  metrics:
  - aws_namespace: AWS/RDS
    aws_metric_name: CPUUtilization
    aws_dimensions: [DBInstanceIdentifier]
    aws_dimension_select_regex:
      DBInstanceIdentifier: [".*production.*"]
    aws_statistics: [Average]
  - aws_namespace: AWS/SQS
    aws_metric_name: ApproximateNumberOfMessagesVisible
    aws_dimensions: [QueueName]
    aws_dimension_select_regex:
      QueueName: [".*production.*"]
    aws_statistics: [Sum]
  - aws_namespace: AWS/ES
    aws_metric_name: FreeStorageSpace
    period_seconds: 60
    aws_dimensions: [DomainName, ClientId, NodeId]
    aws_statistics: [Minimum]
  - aws_namespace: AWS/ElastiCache
    aws_metric_name: CPUUtilization
    period_seconds: 60
    aws_dimensions: [CacheClusterId]
    aws_statistics: [Average]

- job_name: 'cloudwatch-exporter'
  scrape_interval: 30s
  static_configs:
    - targets: ['cloudwatch-exporter-prometheus-cloudwatch-exporter.monitoring.svc.cluster.local:9106']

- alert: REDIS_HIGH_CPU_USAGE
  expr: aws_elasticache_cpuutilization_average > 80
  for: 5m
  labels:
    severity: critical
    emoji: 🔥
  annotations:
    summary: '`{{ $labels.cacheclusterid }}` is using {{ $value }}% of CPU. <https://****.grafana-workspace.us-east-1.amazonaws.com/d/AWSERedis/aws-elasticache-redis?var-cacheclusterid={{ $labels.cacheclusterid }}|Dashboard> | <https://****.pagerduty.com/incidents|PD>'

- alert: RDS_HIGH_CPU_USAGE
  expr: round(avg_over_time(aws_rds_cpuutilization_average[5m]), 0.01) > 80
  for: 5m
  labels:
    severity: critical
    emoji: 🔥
  annotations:
    summary: '`{{ $labels.dbinstance_identifier }}` is using {{ $value }}% of CPU. <https://****.grafana-workspace.us-east-1.amazonaws.com/d/AWSRDSdbi/aws-rds?var-dbinstanceidentifier={{ $labels.dbinstance_identifier }}|Dashboard> | <https://***.pagerduty.com/incidents|PD>'

locals {
  aws_ami_name         = "bottlerocket"
  aws_ami_architecture = "x86_64"
  aws_ami_type         = "${upper(local.aws_ami_name)}_${local.aws_ami_architecture}"
}

data "aws_ssm_parameter" "eks_ami_image_version" {
  name = "/aws/service/${local.aws_ami_name}/aws-k8s-${aws_eks_cluster.eks.version}/${local.aws_ami_architecture}/latest/image_version"
}

resource "aws_eks_node_group" "blue-green-group" {
...
  ami_type        = local.aws_ami_type
  release_version = data.aws_ssm_parameter.eks_ami_image_version.value
...

  lifecycle {
    ignore_changes        = [release_version]
...
  }

ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   1s      v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    NotReady   <none>   10s     v1.30.5-eks-baa6d11
ip-10-1-33-89.ec2.internal    Ready      <none>   13s     v1.30.5-eks-baa6d11

ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   0s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   5s      v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    NotReady   <none>   10s     v1.30.5-eks-baa6d11
ip-10-1-34-52.ec2.internal    Ready      <none>   14s     v1.30.5-eks-baa6d11