All Pods Across Namespaces:
NAMESPACE         NAME                                                     READY   STATUS    RESTARTS      AGE   IP              NODE           NOMINATED NODE   READINESS GATES
calico-system     calico-kube-controllers-6c4746885c-72kcc                 1/1     Running   0             67m   172.17.21.66    10.10.10.222   <none>           <none>
calico-system     calico-node-kb5rb                                        1/1     Running   0             67m   10.10.12.225    10.10.12.225   <none>           <none>
calico-system     calico-node-vk2f2                                        1/1     Running   0             67m   10.10.10.222    10.10.10.222   <none>           <none>
calico-system     calico-node-xwdhl                                        1/1     Running   0             64m   10.10.11.230    10.10.11.230   <none>           <none>
calico-system     calico-typha-bdf5d7d5d-tlkl7                             1/1     Running   0             67m   10.10.10.222    10.10.10.222   <none>           <none>
calico-system     calico-typha-bdf5d7d5d-vztp2                             1/1     Running   0             64m   10.10.12.225    10.10.12.225   <none>           <none>
karpenter         karpenter-54c7b7d7b5-h7np6                               1/1     Running   0             25m   172.17.21.83    10.10.10.222   <none>           <none>
kube-system       coredns-5b8c669669-dj648                                 1/1     Running   0             66m   172.17.21.78    10.10.10.222   <none>           <none>
kube-system       coredns-5b8c669669-qszj8                                 1/1     Running   0             76m   172.17.21.70    10.10.10.222   <none>           <none>
kube-system       coredns-5b8c669669-tvwcj                                 1/1     Running   0             66m   172.17.21.77    10.10.10.222   <none>           <none>
kube-system       coredns-autoscaler-5cc6fb4bf9-jkmkf                      1/1     Running   0             76m   172.17.21.75    10.10.10.222   <none>           <none>
kube-system       dashboard-metrics-scraper-d66d64495-shvdp                1/1     Running   0             73m   172.17.21.68    10.10.10.222   <none>           <none>
kube-system       ibm-master-proxy-static-10.10.10.222                     2/2     Running   0             66m   10.10.10.222    10.10.10.222   <none>           <none>
kube-system       ibm-master-proxy-static-10.10.11.230                     2/2     Running   0             63m   10.10.11.230    10.10.11.230   <none>           <none>
kube-system       ibm-master-proxy-static-10.10.12.225                     2/2     Running   0             66m   10.10.12.225    10.10.12.225   <none>           <none>
kube-system       ibm-vpc-block-csi-controller-0                           7/7     Running   0             64m   172.17.60.197   10.10.12.225   <none>           <none>
kube-system       ibm-vpc-block-csi-node-ngz4k                             4/4     Running   0             67m   172.17.60.196   10.10.12.225   <none>           <none>
kube-system       ibm-vpc-block-csi-node-ntlpr                             4/4     Running   0             64m   172.17.51.130   10.10.11.230   <none>           <none>
kube-system       ibm-vpc-block-csi-node-qd2h9                             4/4     Running   0             67m   172.17.21.80    10.10.10.222   <none>           <none>
kube-system       ingress-cluster-healthcheck-6dd8ff858-rm8jk              1/1     Running   0             73m   172.17.21.73    10.10.10.222   <none>           <none>
kube-system       konnectivity-agent-762d4                                 1/1     Running   0             64m   172.17.51.129   10.10.11.230   <none>           <none>
kube-system       konnectivity-agent-bkvxd                                 1/1     Running   0             67m   172.17.21.76    10.10.10.222   <none>           <none>
kube-system       konnectivity-agent-z96fz                                 1/1     Running   0             67m   172.17.60.193   10.10.12.225   <none>           <none>
kube-system       kubernetes-dashboard-6796cf565d-w6n54                    1/1     Running   0             73m   172.17.21.69    10.10.10.222   <none>           <none>
kube-system       metrics-server-6876fc477-2jjb9                           3/3     Running   0             66m   172.17.60.195   10.10.12.225   <none>           <none>
kube-system       metrics-server-6876fc477-d9j62                           3/3     Running   0             66m   172.17.60.194   10.10.12.225   <none>           <none>
kube-system       public-crctear38d0or2lgurkrd0-alb1-6d4647c857-hg94n      0/1     Pending   0             43m   <none>          <none>         <none>           <none>
kube-system       public-crctear38d0or2lgurkrd0-alb1-6d4647c857-jk9td      1/1     Running   0             43m   172.17.60.198   10.10.12.225   <none>           <none>
kube-system       public-crctear38d0or2lgurkrd0-alb2-69b77dcb79-9ng8p      0/1     Pending   0             43m   <none>          <none>         <none>           <none>
kube-system       public-crctear38d0or2lgurkrd0-alb2-69b77dcb79-ctvm6      1/1     Running   0             43m   172.17.51.131   10.10.11.230   <none>           <none>
kube-system       public-crctear38d0or2lgurkrd0-alb3-859f5dcb65-jknvc      1/1     Running   0             43m   172.17.21.81    10.10.10.222   <none>           <none>
kube-system       public-crctear38d0or2lgurkrd0-alb3-859f5dcb65-nq7t5      0/1     Pending   0             43m   <none>          <none>         <none>           <none>
kube-system       snapshot-controller-5c8c675cd7-mg47g                     1/1     Running   0             75m   172.17.21.65    10.10.10.222   <none>           <none>
kube-system       snapshot-controller-5c8c675cd7-q5xfh                     1/1     Running   0             75m   172.17.21.67    10.10.10.222   <none>           <none>
kube-system       snapshot-controller-5c8c675cd7-x8wxf                     1/1     Running   0             75m   172.17.21.71    10.10.10.222   <none>           <none>
monitoring        alertmanager-prometheus-operator-kube-p-alertmanager-0   2/2     Running   0             35m   172.17.51.135   10.10.11.230   <none>           <none>
monitoring        prometheus-operator-kube-p-operator-6b795b97b6-qngtx     1/1     Running   0             35m   172.17.51.134   10.10.11.230   <none>           <none>
monitoring        prometheus-operator-kube-state-metrics-7d7756cc6-74qt2   1/1     Running   0             35m   172.17.51.133   10.10.11.230   <none>           <none>
monitoring        prometheus-operator-prometheus-node-exporter-n5nl7       1/1     Running   0             35m   10.10.12.225    10.10.12.225   <none>           <none>
monitoring        prometheus-operator-prometheus-node-exporter-th4fx       1/1     Running   0             35m   10.10.10.222    10.10.10.222   <none>           <none>
monitoring        prometheus-operator-prometheus-node-exporter-xw5rz       1/1     Running   0             35m   10.10.11.230    10.10.11.230   <none>           <none>
monitoring        prometheus-prometheus-operator-kube-p-prometheus-0       2/2     Running   0             35m   172.17.51.136   10.10.11.230   <none>           <none>
tigera-operator   tigera-operator-64465cf8c6-b26ws                         1/1     Running   1 (67m ago)   76m   10.10.10.222    10.10.10.222   <none>           <none>

Karpenter Pods Status:
NAMESPACE         NAME                                                     READY   STATUS    RESTARTS      AGE
calico-system     calico-kube-controllers-6c4746885c-72kcc                 1/1     Running   0             66m
calico-system     calico-node-kb5rb                                        1/1     Running   0             66m
calico-system     calico-node-vk2f2                                        1/1     Running   0             66m
calico-system     calico-node-xwdhl                                        1/1     Running   0             63m
calico-system     calico-typha-bdf5d7d5d-tlkl7                             1/1     Running   0             66m
calico-system     calico-typha-bdf5d7d5d-vztp2                             1/1     Running   0             63m
karpenter         karpenter-54c7b7d7b5-h7np6                               1/1     Running   0             24m
kube-system       coredns-5b8c669669-dj648                                 1/1     Running   0             65m
kube-system       coredns-5b8c669669-qszj8                                 1/1     Running   0             75m
kube-system       coredns-5b8c669669-tvwcj                                 1/1     Running   0             65m
kube-system       coredns-autoscaler-5cc6fb4bf9-jkmkf                      1/1     Running   0             75m
kube-system       dashboard-metrics-scraper-d66d64495-shvdp                1/1     Running   0             71m
kube-system       ibm-master-proxy-static-10.10.10.222                     2/2     Running   0             65m
kube-system       ibm-master-proxy-static-10.10.11.230                     2/2     Running   0             62m
kube-system       ibm-master-proxy-static-10.10.12.225                     2/2     Running   0             65m
kube-system       ibm-vpc-block-csi-controller-0                           7/7     Running   0             63m
kube-system       ibm-vpc-block-csi-node-ngz4k                             4/4     Running   0             66m
kube-system       ibm-vpc-block-csi-node-ntlpr                             4/4     Running   0             63m
kube-system       ibm-vpc-block-csi-node-qd2h9                             4/4     Running   0             66m
kube-system       ingress-cluster-healthcheck-6dd8ff858-rm8jk              1/1     Running   0             72m
kube-system       konnectivity-agent-762d4                                 1/1     Running   0             63m
kube-system       konnectivity-agent-bkvxd                                 1/1     Running   0             66m
kube-system       konnectivity-agent-z96fz                                 1/1     Running   0             66m
kube-system       kubernetes-dashboard-6796cf565d-w6n54                    1/1     Running   0             71m
kube-system       metrics-server-6876fc477-2jjb9                           3/3     Running   0             65m
kube-system       metrics-server-6876fc477-d9j62                           3/3     Running   0             65m
kube-system       public-crctear38d0or2lgurkrd0-alb1-6d4647c857-hg94n      0/1     Pending   0             42m
kube-system       public-crctear38d0or2lgurkrd0-alb1-6d4647c857-jk9td      1/1     Running   0             42m
kube-system       public-crctear38d0or2lgurkrd0-alb2-69b77dcb79-9ng8p      0/1     Pending   0             42m
kube-system       public-crctear38d0or2lgurkrd0-alb2-69b77dcb79-ctvm6      1/1     Running   0             42m
kube-system       public-crctear38d0or2lgurkrd0-alb3-859f5dcb65-jknvc      1/1     Running   0             42m
kube-system       public-crctear38d0or2lgurkrd0-alb3-859f5dcb65-nq7t5      0/1     Pending   0             42m
kube-system       snapshot-controller-5c8c675cd7-mg47g                     1/1     Running   0             74m
kube-system       snapshot-controller-5c8c675cd7-q5xfh                     1/1     Running   0             74m
kube-system       snapshot-controller-5c8c675cd7-x8wxf                     1/1     Running   0             74m
monitoring        alertmanager-prometheus-operator-kube-p-alertmanager-0   2/2     Running   0             34m
monitoring        prometheus-operator-kube-p-operator-6b795b97b6-qngtx     1/1     Running   0             34m
monitoring        prometheus-operator-kube-state-metrics-7d7756cc6-74qt2   1/1     Running   0             34m
monitoring        prometheus-operator-prometheus-node-exporter-n5nl7       1/1     Running   0             34m
monitoring        prometheus-operator-prometheus-node-exporter-th4fx       1/1     Running   0             34m
monitoring        prometheus-operator-prometheus-node-exporter-xw5rz       1/1     Running   0             34m
monitoring        prometheus-prometheus-operator-kube-p-prometheus-0       2/2     Running   0             34m
tigera-operator   tigera-operator-64465cf8c6-b26ws                         1/1     Running   1 (66m ago)   75m

CAS Pods Status:
NAMESPACE         NAME                                                     READY   STATUS    RESTARTS      AGE
calico-system     calico-kube-controllers-5868c566c5-q2m2k                 1/1     Running   0             64m
calico-system     calico-node-d4r4q                                        1/1     Running   0             19m
calico-system     calico-node-f6xsf                                        1/1     Running   0             64m
calico-system     calico-node-hwtsv                                        1/1     Running   0             64m
calico-system     calico-node-kjvxc                                        1/1     Running   0             64m
calico-system     calico-node-l7bl6                                        1/1     Running   0             18m
calico-system     calico-node-nhqkg                                        1/1     Running   0             17m
calico-system     calico-typha-79dfc7459f-5nfzk                            1/1     Running   0             18m
calico-system     calico-typha-79dfc7459f-6ngtx                            1/1     Running   0             64m
calico-system     calico-typha-79dfc7459f-mhb69                            1/1     Running   0             64m
kube-system       coredns-5b8c669669-c54jw                                 1/1     Running   0             64m
kube-system       coredns-5b8c669669-hmfb9                                 1/1     Running   0             64m
kube-system       coredns-5b8c669669-qj6lq                                 1/1     Running   0             73m
kube-system       coredns-autoscaler-5cc6fb4bf9-2ml5r                      1/1     Running   0             73m
kube-system       dashboard-metrics-scraper-d66d64495-8n6hs                1/1     Running   0             70m
kube-system       ibm-iks-cluster-autoscaler-b895685c4-tvx7w               2/2     Running   0             22m
kube-system       ibm-master-proxy-static-10.10.10.223                     2/2     Running   0             63m
kube-system       ibm-master-proxy-static-10.10.10.227                     2/2     Running   0             19m
kube-system       ibm-master-proxy-static-10.10.11.231                     2/2     Running   0             64m
kube-system       ibm-master-proxy-static-10.10.11.235                     2/2     Running   0             17m
kube-system       ibm-master-proxy-static-10.10.12.224                     2/2     Running   0             64m
kube-system       ibm-master-proxy-static-10.10.12.230                     2/2     Running   0             17m
kube-system       ibm-vpc-block-csi-controller-0                           7/7     Running   0             59m
kube-system       ibm-vpc-block-csi-node-6dlpl                             4/4     Running   0             65m
kube-system       ibm-vpc-block-csi-node-94cwp                             4/4     Running   1 (15m ago)   17m
kube-system       ibm-vpc-block-csi-node-9w6bl                             4/4     Running   1 (17m ago)   19m
kube-system       ibm-vpc-block-csi-node-f6mqt                             4/4     Running   0             65m
kube-system       ibm-vpc-block-csi-node-hpx4z                             4/4     Running   1 (17m ago)   18m
kube-system       ibm-vpc-block-csi-node-x6ncp                             4/4     Running   0             65m
kube-system       ingress-cluster-healthcheck-6dd8ff858-26cvw              1/1     Running   0             70m
kube-system       konnectivity-agent-7xjsv                                 1/1     Running   0             65m
kube-system       konnectivity-agent-8grzp                                 1/1     Running   0             19m
kube-system       konnectivity-agent-c9lq2                                 1/1     Running   0             65m
kube-system       konnectivity-agent-dd94s                                 1/1     Running   0             65m
kube-system       konnectivity-agent-dm5w5                                 1/1     Running   0             18m
kube-system       konnectivity-agent-swvqj                                 1/1     Running   0             17m
kube-system       kubernetes-dashboard-6796cf565d-2gmpd                    1/1     Running   0             70m
kube-system       metrics-server-86774d7dfd-lxtbp                          3/3     Running   0             64m
kube-system       metrics-server-86774d7dfd-n5jvb                          3/3     Running   0             64m
kube-system       public-crctear3td09chms9oo14g-alb1-6679bb5d7-rjj9s       1/1     Running   0             48m
kube-system       public-crctear3td09chms9oo14g-alb1-6679bb5d7-srzc9       1/1     Running   0             48m
kube-system       public-crctear3td09chms9oo14g-alb2-5dc54b7cc9-5rhd4      1/1     Running   0             40m
kube-system       public-crctear3td09chms9oo14g-alb2-5dc54b7cc9-sh7j6      1/1     Running   0             40m
kube-system       public-crctear3td09chms9oo14g-alb3-7cd8c58f6c-cjjvl      1/1     Running   0             32m
kube-system       public-crctear3td09chms9oo14g-alb3-7cd8c58f6c-dvvcz      1/1     Running   0             32m
kube-system       snapshot-controller-5c8c675cd7-bh4l7                     1/1     Running   0             72m
kube-system       snapshot-controller-5c8c675cd7-h9sds                     1/1     Running   0             72m
kube-system       snapshot-controller-5c8c675cd7-lmxf5                     1/1     Running   0             72m
monitoring        alertmanager-prometheus-operator-kube-p-alertmanager-0   2/2     Running   0             33m
monitoring        prometheus-operator-kube-p-operator-6b795b97b6-2smdj     1/1     Running   0             33m
monitoring        prometheus-operator-kube-state-metrics-7d7756cc6-ssjnd   1/1     Running   0             33m
monitoring        prometheus-operator-prometheus-node-exporter-2qbdq       1/1     Running   0             19m
monitoring        prometheus-operator-prometheus-node-exporter-c55sq       1/1     Running   0             33m
monitoring        prometheus-operator-prometheus-node-exporter-cv7j5       1/1     Running   0             33m
monitoring        prometheus-operator-prometheus-node-exporter-gt694       1/1     Running   0             17m
monitoring        prometheus-operator-prometheus-node-exporter-nbj4h       1/1     Running   0             18m
monitoring        prometheus-operator-prometheus-node-exporter-zb27p       1/1     Running   0             33m
monitoring        prometheus-prometheus-operator-kube-p-prometheus-0       2/2     Running   0             33m
tigera-operator   tigera-operator-64465cf8c6-92n6p                         1/1     Running   0             73m

Karpenter Pods in Namespace:
NAMESPACE   NAME                         READY   STATUS    RESTARTS   AGE
karpenter   karpenter-54c7b7d7b5-h7np6   1/1     Running   0          25m

CAS Pods in Namespace:


Karpenter Logs:
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:224
2024-12-13T23:17:11.087Z	ERROR	Reconciler error	{"controller": "ibmnodeclass", "controllerGroup": "karpenter.ibm.sh", "controllerKind": "IBMNodeClass", "IBMNodeClass": {"name":"auto-placement"}, "namespace": "", "name": "auto-placement", "reconcileID": "a2b77f0e-3f32-4583-bf15-a75540cd6a4d", "error": "listing instance types from catalog: getting IAM token: initializing IAM client: Authentication information was not properly configured."}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:224

CAS Logs:
Failed to get logs

Karpenter Pod Details:
Name:                   karpenter
Namespace:              karpenter
CreationTimestamp:      Fri, 13 Dec 2024 23:55:14 +0100
Labels:                 app.kubernetes.io/instance=karpenter
                        app.kubernetes.io/managed-by=Helm
                        app.kubernetes.io/name=karpenter
                        app.kubernetes.io/version=0.35.0
                        helm.sh/chart=karpenter-0.35.0
Annotations:            deployment.kubernetes.io/revision: 1
                        meta.helm.sh/release-name: karpenter
                        meta.helm.sh/release-namespace: karpenter
Selector:               app.kubernetes.io/instance=karpenter,app.kubernetes.io/name=karpenter
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app.kubernetes.io/instance=karpenter
                    app.kubernetes.io/name=karpenter
  Service Account:  karpenter
  Containers:
   karpenter:
    Image:       ghcr.io/pfeifferj/karpenter-provider-ibm-cloud/controller:latest@sha256:f0fdf8a404a38dfa79c48be965c883345b3dd61710f7f6324103699618acc50c
    Ports:       8080/TCP, 8081/TCP
    Host Ports:  0/TCP, 0/TCP
    Liveness:    http-get http://:8081/healthz delay=5s timeout=1s period=10s #success=1 #failure=3
    Readiness:   http-get http://:8081/readyz delay=5s timeout=1s period=10s #success=1 #failure=3
    Environment:
      KARPENTER_LOG_LEVEL:           debug
      KARPENTER_VERBOSE_LOG_LEVEL:   5
      KARPENTER_DEBUG:               true
      IBM_API_KEY:                   <set to the key 'ibm_api_key' in secret 'karpenter-credentials'>    Optional: false
      IBM_REGION:                    <set to the key 'region' in secret 'karpenter-credentials'>         Optional: false
      VPC_URL:                       <set to the key 'vpc_url' in secret 'karpenter-credentials'>        Optional: false
      VPC_AUTH_TYPE:                 <set to the key 'vpc_auth_type' in secret 'karpenter-credentials'>  Optional: false
      VPC_API_KEY:                   <set to the key 'vpc_api_key' in secret 'karpenter-credentials'>    Optional: false
      CONTROLLER_MANAGER_VERBOSITY:  4
      CONTROLLER_MANAGER_LOG_LEVEL:  debug
    Mounts:                          <none>
  Volumes:                           <none>
  Node-Selectors:                    <none>
  Tolerations:                       <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   karpenter-54c7b7d7b5 (1/1 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  25m   deployment-controller  Scaled up replica set karpenter-54c7b7d7b5 to 1

CAS Pod Details:
Failed to get pod details

Karpenter Resources:
