# Install Prometheus for metrics collection - name: Add prometheus-community helm chart repo kubernetes.core.helm_repository: name: prometheus-community repo_url: "https://prometheus-community.github.io/helm-charts" - name: Install Prometheus Operator kubernetes.core.helm: name: prometheus-operator chart_ref: prometheus-community/kube-prometheus-stack release_namespace: monitoring create_namespace: true values: grafana: enabled: false prometheus: prometheusSpec: enableAdminAPI: true priorityClassName: system-cluster-critical podDisruptionBudget: enabled: enabled minAvailable: 1 loop: - karpenter - cas environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_' ~ cloud_provider | upper ~ '_' ~ (item | upper)) }}" AWS_ACCESS_KEY_ID: "{{ AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "{{ AWS_SECRET_ACCESS_KEY }}" AWS_DEFAULT_REGION: "us-east-1" # Configure image pull secret for IKS - name: Configure ICR pull secret for IKS block: - name: Get default ICR pull secret kubernetes.core.k8s_info: api_version: v1 kind: Secret name: all-icr-io namespace: default register: icr_secret environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_CAS') }}" - name: Copy ICR pull secret to kube-system namespace kubernetes.core.k8s: state: present definition: apiVersion: v1 kind: Secret metadata: name: all-icr-io namespace: kube-system type: "{{ icr_secret.resources[0].type }}" data: "{{ icr_secret.resources[0].data }}" environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_CAS') }}" when: cloud_provider == 'iks' # Configure IBM Cloud credentials for storage - name: Configure IBM Cloud credentials secret for storage block: - name: Create IBM Cloud credentials secret kubernetes.core.k8s: state: present definition: apiVersion: v1 kind: Secret metadata: name: ibm-cloud-credentials namespace: default type: Opaque stringData: "ibm-credentials.env": | IBMCLOUD_APIKEY={{ IBM_CLOUD_TOKEN }} environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_CAS') }}" when: cloud_provider == 'iks' # Install autoscaler based on provider - name: Configure autoscaler block: - name: Add autoscaler chart repo for EKS kubernetes.core.helm_repository: name: autoscaler_eks repo_url: "https://kubernetes.github.io/autoscaler" when: cloud_provider == 'eks' - name: Deploy Cluster Autoscaler for EKS kubernetes.core.helm: name: cas chart_ref: autoscaler_eks/cluster-autoscaler release_namespace: cas create_namespace: true values: autoDiscovery: clusterName: cas-eks awsRegion: us-east-1 rbac: create: true serviceAccount: create: true name: aws-cluster-autoscaler annotations: "eks.amazonaws.com/role-arn": "{{ terraform_cluster_result_eks.outputs.cluster_autoscaler_role_arn.value }}" extraArgs: skip-nodes-with-local-storage: false expander: least-waste scale-down-enabled: true scale-down-delay-after-add: 2m scale-down-unneeded-time: 2m workerpools: - default: enabled: true max: 6 min: 1 environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_EKS_CAS') }}" AWS_ACCESS_KEY_ID: "{{ AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "{{ AWS_SECRET_ACCESS_KEY }}" AWS_DEFAULT_REGION: "us-east-1" when: cloud_provider == 'eks' - name: Login to IBM Cloud ansible.builtin.command: cmd: ibmcloud login --apikey {{ IBM_CLOUD_TOKEN }} when: cloud_provider == 'iks' - name: Target IBM Cloud resource group ansible.builtin.command: cmd: ibmcloud target -g kscale-workload when: cloud_provider == 'iks' - name: Enable Cluster Autoscaler addon for IKS command: ibmcloud ks cluster addon enable cluster-autoscaler --cluster cas-iks register: enable_autoscaler when: cloud_provider == 'iks' - name: Wait for Cluster Autoscaler addon to be ready command: ibmcloud ks cluster addon ls --cluster cas-iks register: addon_status until: addon_status.stdout is search("cluster-autoscaler.*normal.*Addon Ready") retries: 60 delay: 30 when: cloud_provider == 'iks' - name: Configure worker pool autoscaling for IKS kubernetes.core.k8s: state: present definition: apiVersion: v1 kind: ConfigMap metadata: name: iks-ca-configmap namespace: kube-system data: workerPoolsConfig.json: | [ { "name": "default", "minSize": 1, "maxSize": 3, "enabled": true } ] expander: "least-waste" scaleDownEnabled: "true" scaleDownUnneededTime: "10m" scaleDownDelayAfterAdd: "10m" maxNodeProvisionTime: "120m" skipNodesWithSystemPods: "true" scanInterval: "1m" maxBulkSoftTaintCount: "1" maxNodeGroupBinpackingDuration: "10s" scaleDownUtilizationThreshold: "0.5" maxTotalUnreadyPercentage: "45" okTotalUnreadyCount: "3" maxEmptyBulkDelete: "10" prometheusScrape: "true" environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_CAS') }}" when: cloud_provider == 'iks' # Install Karpenter based on provider - name: Configure Karpenter block: - name: Add Karpenter chart repo for IKS kubernetes.core.helm_repository: name: pfeifferj repo_url: "https://pfeifferj.github.io/karpenter-provider-ibm-cloud" when: cloud_provider == 'iks' - name: Deploy Karpenter Autoscaler for IKS kubernetes.core.helm: name: karpenter chart_ref: pfeifferj/karpenter chart_version: "0.35.0" release_namespace: karpenter create_namespace: true values: credentials: ibm_api_key: "{{ IBM_CLOUD_TOKEN }}" vpc_api_key: "{{ IBM_VPC_API_KEY }}" region: "us-south" image: repository: ghcr.io/pfeifferj/karpenter-provider-ibm-cloud/controller tag: latest pullPolicy: IfNotPresent digest: "sha256:98e1441aa5d15f6e23e58a466c2526942e853d49281eefffa64e22361f7a19ad" environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_KARPENTER') }}" when: cloud_provider == 'iks' - name: Apply Karpenter NodeClass configuration kubernetes.core.k8s: state: present src: "{{ playbook_dir }}/files/karpenter-nodeclass.yaml" environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_KARPENTER') }}" when: cloud_provider == 'iks' - name: Apply Karpenter NodePool configuration kubernetes.core.k8s: state: present src: "{{ playbook_dir }}/files/karpenter-nodepool.yaml" environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_IKS_KARPENTER') }}" when: cloud_provider == 'iks' # Create service account for metrics access - name: Create service account for Prometheus kubernetes.core.k8s: api_version: v1 kind: ServiceAccount namespace: monitoring name: prometheus-sa loop: - karpenter - cas environment: KUBECONFIG: "{{ lookup('vars', 'KUBECONFIG_' ~ cloud_provider | upper ~ '_' ~ (item | upper)) }}" AWS_ACCESS_KEY_ID: "{{ AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "{{ AWS_SECRET_ACCESS_KEY }}" AWS_DEFAULT_REGION: "us-eas