All Pods Across Namespaces:
NAMESPACE         NAME                                                     READY   STATUS    RESTARTS      AGE   IP               NODE           NOMINATED NODE   READINESS GATES
calico-system     calico-kube-controllers-77888f8f99-4gtx7                 1/1     Running   0             44m   172.17.78.139    10.10.11.246   <none>           <none>
calico-system     calico-node-g528h                                        1/1     Running   0             44m   10.10.11.246     10.10.11.246   <none>           <none>
calico-system     calico-node-n8vlk                                        1/1     Running   0             44m   10.10.10.236     10.10.10.236   <none>           <none>
calico-system     calico-node-p24qp                                        1/1     Running   0             44m   10.10.12.241     10.10.12.241   <none>           <none>
calico-system     calico-typha-78cc94cbb4-jhf28                            1/1     Running   0             44m   10.10.11.246     10.10.11.246   <none>           <none>
calico-system     calico-typha-78cc94cbb4-mst59                            1/1     Running   0             43m   10.10.10.236     10.10.10.236   <none>           <none>
karpenter         karpenter-6675446ddf-t7gfv                               1/1     Running   0             80s   172.17.78.145    10.10.11.246   <none>           <none>
kube-system       coredns-5b8c669669-2jmxz                                 1/1     Running   0             43m   172.17.113.130   10.10.10.236   <none>           <none>
kube-system       coredns-5b8c669669-8sxmj                                 1/1     Running   0             50m   172.17.78.135    10.10.11.246   <none>           <none>
kube-system       coredns-5b8c669669-bh2kv                                 1/1     Running   0             43m   172.17.113.131   10.10.10.236   <none>           <none>
kube-system       coredns-autoscaler-5cc6fb4bf9-xf68x                      1/1     Running   0             50m   172.17.78.131    10.10.11.246   <none>           <none>
kube-system       dashboard-metrics-scraper-d66d64495-9wb52                1/1     Running   0             47m   172.17.78.130    10.10.11.246   <none>           <none>
kube-system       ibm-master-proxy-static-10.10.10.236                     2/2     Running   0             44m   10.10.10.236     10.10.10.236   <none>           <none>
kube-system       ibm-master-proxy-static-10.10.11.246                     2/2     Running   0             43m   10.10.11.246     10.10.11.246   <none>           <none>
kube-system       ibm-master-proxy-static-10.10.12.241                     2/2     Running   0             42m   10.10.12.241     10.10.12.241   <none>           <none>
kube-system       ibm-vpc-block-csi-controller-0                           7/7     Running   0             39m   172.17.100.195   10.10.12.241   <none>           <none>
kube-system       ibm-vpc-block-csi-node-d9tfn                             4/4     Running   0             44m   172.17.78.142    10.10.11.246   <none>           <none>
kube-system       ibm-vpc-block-csi-node-lhtwn                             4/4     Running   0             44m   172.17.113.134   10.10.10.236   <none>           <none>
kube-system       ibm-vpc-block-csi-node-mgmh2                             4/4     Running   0             44m   172.17.100.194   10.10.12.241   <none>           <none>
kube-system       ingress-cluster-healthcheck-6dd8ff858-q82r2              1/1     Running   0             47m   172.17.78.134    10.10.11.246   <none>           <none>
kube-system       konnectivity-agent-75n2q                                 1/1     Running   0             44m   172.17.100.193   10.10.12.241   <none>           <none>
kube-system       konnectivity-agent-8zqmd                                 1/1     Running   0             44m   172.17.78.140    10.10.11.246   <none>           <none>
kube-system       konnectivity-agent-mcts6                                 1/1     Running   0             44m   172.17.113.129   10.10.10.236   <none>           <none>
kube-system       kubernetes-dashboard-6796cf565d-fh2h8                    1/1     Running   0             47m   172.17.78.137    10.10.11.246   <none>           <none>
kube-system       metrics-server-84dc85ff88-5jdl9                          3/3     Running   0             43m   172.17.113.133   10.10.10.236   <none>           <none>
kube-system       metrics-server-84dc85ff88-tsg4g                          3/3     Running   0             43m   172.17.113.132   10.10.10.236   <none>           <none>
kube-system       public-crctejhj5d01rdseshe2p0-alb1-6fcb9db4c9-7v5p7      1/1     Running   0             18m   172.17.100.196   10.10.12.241   <none>           <none>
kube-system       public-crctejhj5d01rdseshe2p0-alb1-6fcb9db4c9-qm98j      0/1     Pending   0             18m   <none>           <none>         <none>           <none>
kube-system       public-crctejhj5d01rdseshe2p0-alb2-86c786cffc-7fc2r      0/1     Pending   0             11m   <none>           <none>         <none>           <none>
kube-system       public-crctejhj5d01rdseshe2p0-alb2-86c786cffc-vxkt9      1/1     Running   0             11m   172.17.78.143    10.10.11.246   <none>           <none>
kube-system       public-crctejhj5d01rdseshe2p0-alb3-cbd889b6c-8nn9g       1/1     Running   0             11m   172.17.113.135   10.10.10.236   <none>           <none>
kube-system       public-crctejhj5d01rdseshe2p0-alb3-cbd889b6c-kb7cd       0/1     Pending   0             11m   <none>           <none>         <none>           <none>
kube-system       snapshot-controller-5c8c675cd7-4fzdz                     1/1     Running   0             50m   172.17.78.136    10.10.11.246   <none>           <none>
kube-system       snapshot-controller-5c8c675cd7-txlvz                     1/1     Running   0             50m   172.17.78.129    10.10.11.246   <none>           <none>
kube-system       snapshot-controller-5c8c675cd7-z26ch                     1/1     Running   0             50m   172.17.78.138    10.10.11.246   <none>           <none>
monitoring        alertmanager-prometheus-operator-kube-p-alertmanager-0   2/2     Running   0             11m   172.17.100.201   10.10.12.241   <none>           <none>
monitoring        prometheus-operator-kube-p-operator-6b795b97b6-6zh8r     1/1     Running   0             11m   172.17.100.199   10.10.12.241   <none>           <none>
monitoring        prometheus-operator-kube-state-metrics-7d7756cc6-k6bqt   1/1     Running   0             11m   172.17.100.198   10.10.12.241   <none>           <none>
monitoring        prometheus-operator-prometheus-node-exporter-bnzpf       1/1     Running   0             11m   10.10.10.236     10.10.10.236   <none>           <none>
monitoring        prometheus-operator-prometheus-node-exporter-ghkhp       1/1     Running   0             11m   10.10.11.246     10.10.11.246   <none>           <none>
monitoring        prometheus-operator-prometheus-node-exporter-rdmrn       1/1     Running   0             11m   10.10.12.241     10.10.12.241   <none>           <none>
monitoring        prometheus-prometheus-operator-kube-p-prometheus-0       2/2     Running   0             11m   172.17.100.200   10.10.12.241   <none>           <none>
tigera-operator   tigera-operator-64465cf8c6-cb2dj                         1/1     Running   1 (44m ago)   51m   10.10.11.246     10.10.11.246   <none>           <none>

Karpenter Pods Status:
NAMESPACE         NAME                                                     READY   STATUS    RESTARTS      AGE
calico-system     calico-kube-controllers-77888f8f99-4gtx7                 1/1     Running   0             43m
calico-system     calico-node-g528h                                        1/1     Running   0             43m
calico-system     calico-node-n8vlk                                        1/1     Running   0             43m
calico-system     calico-node-p24qp                                        1/1     Running   0             42m
calico-system     calico-typha-78cc94cbb4-jhf28                            1/1     Running   0             43m
calico-system     calico-typha-78cc94cbb4-mst59                            1/1     Running   0             42m
karpenter         karpenter-6675446ddf-t7gfv                               1/1     Running   0             13s
kube-system       coredns-5b8c669669-2jmxz                                 1/1     Running   0             42m
kube-system       coredns-5b8c669669-8sxmj                                 1/1     Running   0             49m
kube-system       coredns-5b8c669669-bh2kv                                 1/1     Running   0             42m
kube-system       coredns-autoscaler-5cc6fb4bf9-xf68x                      1/1     Running   0             49m
kube-system       dashboard-metrics-scraper-d66d64495-9wb52                1/1     Running   0             46m
kube-system       ibm-master-proxy-static-10.10.10.236                     2/2     Running   0             43m
kube-system       ibm-master-proxy-static-10.10.11.246                     2/2     Running   0             42m
kube-system       ibm-master-proxy-static-10.10.12.241                     2/2     Running   0             41m
kube-system       ibm-vpc-block-csi-controller-0                           7/7     Running   0             38m
kube-system       ibm-vpc-block-csi-node-d9tfn                             4/4     Running   0             43m
kube-system       ibm-vpc-block-csi-node-lhtwn                             4/4     Running   0             43m
kube-system       ibm-vpc-block-csi-node-mgmh2                             4/4     Running   0             42m
kube-system       ingress-cluster-healthcheck-6dd8ff858-q82r2              1/1     Running   0             46m
kube-system       konnectivity-agent-75n2q                                 1/1     Running   0             42m
kube-system       konnectivity-agent-8zqmd                                 1/1     Running   0             43m
kube-system       konnectivity-agent-mcts6                                 1/1     Running   0             43m
kube-system       kubernetes-dashboard-6796cf565d-fh2h8                    1/1     Running   0             46m
kube-system       metrics-server-84dc85ff88-5jdl9                          3/3     Running   0             42m
kube-system       metrics-server-84dc85ff88-tsg4g                          3/3     Running   0             42m
kube-system       public-crctejhj5d01rdseshe2p0-alb1-6fcb9db4c9-7v5p7      1/1     Running   0             17m
kube-system       public-crctejhj5d01rdseshe2p0-alb1-6fcb9db4c9-qm98j      0/1     Pending   0             17m
kube-system       public-crctejhj5d01rdseshe2p0-alb2-86c786cffc-7fc2r      0/1     Pending   0             10m
kube-system       public-crctejhj5d01rdseshe2p0-alb2-86c786cffc-vxkt9      1/1     Running   0             10m
kube-system       public-crctejhj5d01rdseshe2p0-alb3-cbd889b6c-8nn9g       1/1     Running   0             10m
kube-system       public-crctejhj5d01rdseshe2p0-alb3-cbd889b6c-kb7cd       0/1     Pending   0             10m
kube-system       snapshot-controller-5c8c675cd7-4fzdz                     1/1     Running   0             49m
kube-system       snapshot-controller-5c8c675cd7-txlvz                     1/1     Running   0             49m
kube-system       snapshot-controller-5c8c675cd7-z26ch                     1/1     Running   0             49m
monitoring        alertmanager-prometheus-operator-kube-p-alertmanager-0   2/2     Running   0             10m
monitoring        prometheus-operator-kube-p-operator-6b795b97b6-6zh8r     1/1     Running   0             10m
monitoring        prometheus-operator-kube-state-metrics-7d7756cc6-k6bqt   1/1     Running   0             10m
monitoring        prometheus-operator-prometheus-node-exporter-bnzpf       1/1     Running   0             10m
monitoring        prometheus-operator-prometheus-node-exporter-ghkhp       1/1     Running   0             10m
monitoring        prometheus-operator-prometheus-node-exporter-rdmrn       1/1     Running   0             10m
monitoring        prometheus-prometheus-operator-kube-p-prometheus-0       2/2     Running   0             10m
tigera-operator   tigera-operator-64465cf8c6-cb2dj                         1/1     Running   1 (43m ago)   50m

CAS Pods Status:
NAMESPACE         NAME                                                     READY   STATUS    RESTARTS      AGE
calico-system     calico-kube-controllers-796cfb86-sg6cg                   1/1     Running   0             45m
calico-system     calico-node-hsr5c                                        1/1     Running   0             45m
calico-system     calico-node-lfr9f                                        1/1     Running   0             45m
calico-system     calico-node-qjbkl                                        1/1     Running   0             45m
calico-system     calico-typha-6896df66cb-4g97k                            1/1     Running   0             45m
calico-system     calico-typha-6896df66cb-b62mg                            1/1     Running   0             45m
kube-system       coredns-5b8c669669-ldfn4                                 1/1     Running   0             44m
kube-system       coredns-5b8c669669-xd54k                                 1/1     Running   0             54m
kube-system       coredns-5b8c669669-zqjdt                                 1/1     Running   0             44m
kube-system       coredns-autoscaler-5cc6fb4bf9-25kfz                      1/1     Running   0             54m
kube-system       dashboard-metrics-scraper-d66d64495-psp5r                1/1     Running   0             51m
kube-system       ibm-iks-cluster-autoscaler-77d6bcb67c-59zq5              2/2     Running   0             50s
kube-system       ibm-master-proxy-static-10.10.10.235                     2/2     Running   0             45m
kube-system       ibm-master-proxy-static-10.10.11.245                     2/2     Running   0             44m
kube-system       ibm-master-proxy-static-10.10.12.240                     2/2     Running   0             46m
kube-system       ibm-vpc-block-csi-controller-0                           7/7     Running   0             43m
kube-system       ibm-vpc-block-csi-node-bz9sg                             4/4     Running   0             46m
kube-system       ibm-vpc-block-csi-node-mgl55                             4/4     Running   0             46m
kube-system       ibm-vpc-block-csi-node-vpwhm                             4/4     Running   0             46m
kube-system       ingress-cluster-healthcheck-6dd8ff858-kvbdl              1/1     Running   0             51m
kube-system       konnectivity-agent-87qll                                 1/1     Running   0             46m
kube-system       konnectivity-agent-8rbsr                                 1/1     Running   0             46m
kube-system       konnectivity-agent-rk5gn                                 1/1     Running   0             46m
kube-system       kubernetes-dashboard-6796cf565d-kpl7v                    1/1     Running   0             51m
kube-system       metrics-server-6f965958cd-jzzqm                          3/3     Running   0             44m
kube-system       metrics-server-6f965958cd-k6fhj                          3/3     Running   0             44m
kube-system       public-crctejh55d09ii42tsrleg-alb1-5696b59b98-4vdqk      0/1     Pending   0             38m
kube-system       public-crctejh55d09ii42tsrleg-alb1-5696b59b98-pjcxj      1/1     Running   0             38m
kube-system       public-crctejh55d09ii42tsrleg-alb2-7d48fdc9b9-j65s2      1/1     Running   0             26m
kube-system       public-crctejh55d09ii42tsrleg-alb2-7d48fdc9b9-qlzj6      0/1     Pending   0             26m
kube-system       public-crctejh55d09ii42tsrleg-alb3-fbd964459-2rwg2       1/1     Running   0             13m
kube-system       public-crctejh55d09ii42tsrleg-alb3-fbd964459-vdbqs       0/1     Pending   0             13m
kube-system       snapshot-controller-5c8c675cd7-frqkp                     1/1     Running   0             53m
kube-system       snapshot-controller-5c8c675cd7-lr4h9                     1/1     Running   0             53m
kube-system       snapshot-controller-5c8c675cd7-tfkn6                     1/1     Running   0             53m
monitoring        alertmanager-prometheus-operator-kube-p-alertmanager-0   2/2     Running   0             8m34s
monitoring        prometheus-operator-kube-p-operator-6b795b97b6-ns9f7     1/1     Running   0             8m40s
monitoring        prometheus-operator-kube-state-metrics-7d7756cc6-45xgd   1/1     Running   0             8m40s
monitoring        prometheus-operator-prometheus-node-exporter-hbm5v       1/1     Running   0             8m40s
monitoring        prometheus-operator-prometheus-node-exporter-jgbm4       1/1     Running   0             8m40s
monitoring        prometheus-operator-prometheus-node-exporter-vbw7t       1/1     Running   0             8m40s
monitoring        prometheus-prometheus-operator-kube-p-prometheus-0       2/2     Running   0             8m34s
tigera-operator   tigera-operator-64465cf8c6-4j896                         1/1     Running   1 (45m ago)   54m

Karpenter Pods in Namespace:
NAMESPACE   NAME                         READY   STATUS    RESTARTS   AGE
karpenter   karpenter-6675446ddf-t7gfv   1/1     Running   0          84s

CAS Pods in Namespace:


Karpenter Logs:
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:224
2024-12-14T08:52:29.537Z	ERROR	Reconciler error	{"controller": "ibmnodeclass", "controllerGroup": "karpenter.ibm.sh", "controllerKind": "IBMNodeClass", "IBMNodeClass": {"name":"auto-placement"}, "namespace": "", "name": "auto-placement", "reconcileID": "2d4ceefa-9107-4a18-b0d2-783191e064ac", "error": "listing instance types from catalog: getting IAM token: initializing IAM client: Authentication information was not properly configured."}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	sigs.k8s.io/controller-runtime@v0.19.1/pkg/internal/controller/controller.go:224

CAS Logs:
Failed to get logs

Karpenter Pod Details:
Name:                   karpenter
Namespace:              karpenter
CreationTimestamp:      Sat, 14 Dec 2024 09:51:42 +0100
Labels:                 app.kubernetes.io/instance=karpenter
                        app.kubernetes.io/managed-by=Helm
                        app.kubernetes.io/name=karpenter
                        app.kubernetes.io/version=0.35.0
                        helm.sh/chart=karpenter-0.35.0
Annotations:            deployment.kubernetes.io/revision: 1
                        meta.helm.sh/release-name: karpenter
                        meta.helm.sh/release-namespace: karpenter
Selector:               app.kubernetes.io/instance=karpenter,app.kubernetes.io/name=karpenter
Replicas:               1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:           app.kubernetes.io/instance=karpenter
                    app.kubernetes.io/name=karpenter
  Service Account:  karpenter
  Containers:
   karpenter:
    Image:       ghcr.io/pfeifferj/karpenter-provider-ibm-cloud/controller:latest@sha256:fd765fddf96d3a6b14a65e458dbb50e291b3823b7a02dc0cdd39ea144d2fa505
    Ports:       8080/TCP, 8081/TCP
    Host Ports:  0/TCP, 0/TCP
    Liveness:    http-get http://:8081/healthz delay=5s timeout=1s period=10s #success=1 #failure=3
    Readiness:   http-get http://:8081/readyz delay=5s timeout=1s period=10s #success=1 #failure=3
    Environment:
      KARPENTER_LOG_LEVEL:           debug
      KARPENTER_VERBOSE_LOG_LEVEL:   5
      KARPENTER_DEBUG:               true
      IBM_API_KEY:                   <set to the key 'ibm_api_key' in secret 'karpenter-credentials'>    Optional: false
      IBM_REGION:                    <set to the key 'region' in secret 'karpenter-credentials'>         Optional: false
      VPC_URL:                       <set to the key 'vpc_url' in secret 'karpenter-credentials'>        Optional: false
      VPC_AUTH_TYPE:                 <set to the key 'vpc_auth_type' in secret 'karpenter-credentials'>  Optional: false
      VPC_API_KEY:                   <set to the key 'vpc_api_key' in secret 'karpenter-credentials'>    Optional: false
      CONTROLLER_MANAGER_VERBOSITY:  4
      CONTROLLER_MANAGER_LOG_LEVEL:  debug
    Mounts:                          <none>
  Volumes:                           <none>
  Node-Selectors:                    <none>
  Tolerations:                       <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      True    MinimumReplicasAvailable
  Progressing    True    NewReplicaSetAvailable
OldReplicaSets:  <none>
NewReplicaSet:   karpenter-6675446ddf (1/1 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  88s   deployment-controller  Scaled up replica set karpenter-6675446ddf to 1

CAS Pod Details:
Failed to get pod details

Karpenter Resources:
