I have enabled the Observability add-on, but it seems to take an age to bring the various pods up, if they ever start. I have reinstalled microk8s and enabled the add-on on a clean install but no change. Has anyone actually got this add-on working? Any suggestions on what could be wrong with my install? Thanks.
$ microk8s version
MicroK8s v1.27.0 revision 5141
$ microk8s status
microk8s is running
high-availability: no
datastore master nodes: 127.0.0.1:19001
datastore standby nodes: none
addons:
enabled:
dns # (core) CoreDNS
ha-cluster # (core) Configure high availability on the current node
helm # (core) Helm - the package manager for Kubernetes
helm3 # (core) Helm 3 - the package manager for Kubernetes
hostpath-storage # (core) Storage class; allocates storage from host directory
ingress # (core) Ingress controller for external access
observability # (core) A lightweight observability stack for logs, traces and metrics
registry # (core) Private image registry exposed on localhost:32000
storage # (core) Alias to hostpath-storage add-on, deprecated
disabled:
cert-manager # (core) Cloud native certificate management
community # (core) The community addons repository
dashboard # (core) The Kubernetes dashboard
gpu # (core) Automatic enablement of Nvidia CUDA
host-access # (core) Allow Pods connecting to Host services smoothly
kube-ovn # (core) An advanced network fabric for Kubernetes
mayastor # (core) OpenEBS MayaStor
metallb # (core) Loadbalancer for your Kubernetes cluster
metrics-server # (core) K8s Metrics Server for API access to service metrics
minio # (core) MinIO object storage
prometheus # (core) Prometheus operator for monitoring and logging
rbac # (core) Role-Based Access Control for authorisation
$ kubectl get all -n=observability
NAME READY STATUS RESTARTS AGE
pod/kube-prom-stack-grafana-6c47f548d6-65svm 0/3 Completed 0 53m
pod/kube-prom-stack-grafana-6c47f548d6-72v89 0/3 Evicted 0 52m
pod/loki-0 0/1 Completed 0 53m
pod/kube-prom-stack-kube-prome-operator-64ffd55b77-shhxs 0/1 ContainerStatusUnknown 1 53m
pod/kube-prom-stack-kube-state-metrics-6c586bf4c8-tvm6v 0/1 ContainerStatusUnknown 1 53m
pod/kube-prom-stack-grafana-6c47f548d6-6ljjw 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-vg8jb 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-jmdlb 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-lstvb 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-c7th6 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-rpf7k 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-tlqgf 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-tgf26 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-kfjdc 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-v55c5 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-nv2vd 0/3 ContainerStatusUnknown 3 52m
pod/kube-prom-stack-grafana-6c47f548d6-fvg7l 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-ddzgh 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-cwcrq 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-trr72 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-bd2p5 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-5fhxg 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-b2sm8 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-gjlq2 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-78kxc 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-6z6c7 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-j6m9f 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-sjxsc 0/3 Evicted 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-2vwpk 0/3 Evicted 0 42m
pod/kube-prom-stack-kube-prome-operator-64ffd55b77-rlscb 0/1 Completed 0 51m
pod/kube-prom-stack-kube-state-metrics-6c586bf4c8-p7cwm 0/1 ContainerStatusUnknown 1 50m
pod/kube-prom-stack-kube-prome-operator-64ffd55b77-8hvnc 1/1 Running 0 42m
pod/kube-prom-stack-kube-state-metrics-6c586bf4c8-zt5xb 1/1 Running 0 42m
pod/kube-prom-stack-grafana-6c47f548d6-8mzz4 0/3 Evicted 0 35m
pod/kube-prom-stack-grafana-6c47f548d6-cb6m4 0/3 Completed 0 42m
pod/tempo-0 2/2 Running 0 34m
pod/alertmanager-kube-prom-stack-kube-prome-alertmanager-0 2/2 Running 1 (24m ago) 29m
pod/kube-prom-stack-grafana-6c47f548d6-nl8rq 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-7gtrb 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-6bm24 0/3 Completed 0 35m
pod/kube-prom-stack-grafana-6c47f548d6-674wc 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-7jlsp 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-c6tts 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-vpmns 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-lpmsh 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-2p5l8 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-8sjw4 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-26jlj 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-nxlt9 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-mxf42 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-nwtk4 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-fww2n 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-768ch 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-z75lm 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-z25nm 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-kqrlj 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-hcqw8 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-zfdsh 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-x5bl8 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-xqf5b 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-42tk7 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-7rn87 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-smk7m 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-k6zpj 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-56xbt 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-6jtn4 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-tt58f 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-6p9xv 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-blrxf 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-qr4dg 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-xh252 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-q2tm5 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-78sbm 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-brcjx 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-zj9db 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-sr4w8 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-6ccc8 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-nqfz6 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-9wpx7 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-9pkqt 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-fjcf2 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-st7cm 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-q6zzw 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-8wmbc 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-bmvlb 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-7tt95 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-xwvwq 0/3 Evicted 0 18m
pod/kube-prom-stack-grafana-6c47f548d6-jw7qs 0/3 Evicted 0 7m27s
pod/kube-prom-stack-grafana-6c47f548d6-vz2nt 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-s2498 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-jsz7q 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-9449x 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-ckbcf 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-xxxcx 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-4fzj2 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-mwsqj 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-rqfj4 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-lxnpc 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-5jsfz 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-wf9ls 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-zpmb5 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-qlstv 0/3 ContainerStatusUnknown 3 18m
pod/kube-prom-stack-grafana-6c47f548d6-bvvzr 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-dcvbw 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-7rhdz 0/3 Evicted 0 7m26s
pod/kube-prom-stack-grafana-6c47f548d6-b8r6g 0/3 Evicted 0 7m25s
pod/kube-prom-stack-grafana-6c47f548d6-8j458 0/3 Evicted 0 7m25s
pod/kube-prom-stack-grafana-6c47f548d6-vgzk7 0/3 Evicted 0 7m25s
pod/kube-prom-stack-grafana-6c47f548d6-6vfsc 0/3 Evicted 0 7m25s
pod/kube-prom-stack-grafana-6c47f548d6-f9wwc 0/3 Evicted 0 7m25s
pod/kube-prom-stack-grafana-6c47f548d6-5dwl5 0/3 Evicted 0 7m25s
pod/kube-prom-stack-grafana-6c47f548d6-svdc5 0/3 Evicted 0 7m24s
pod/kube-prom-stack-grafana-6c47f548d6-z6mzk 0/3 Evicted 0 7m24s
pod/kube-prom-stack-grafana-6c47f548d6-dfgb7 0/3 Evicted 0 7m24s
pod/kube-prom-stack-grafana-6c47f548d6-dd9gw 0/3 Evicted 0 7m24s
pod/kube-prom-stack-grafana-6c47f548d6-vw2wp 0/3 Evicted 0 7m24s
pod/kube-prom-stack-grafana-6c47f548d6-5qcl7 0/3 Evicted 0 7m24s
pod/kube-prom-stack-grafana-6c47f548d6-qlfj6 0/3 Evicted 0 7m23s
pod/kube-prom-stack-grafana-6c47f548d6-9gg9k 0/3 Evicted 0 7m23s
pod/kube-prom-stack-grafana-6c47f548d6-9j5ss 0/3 Evicted 0 7m23s
pod/kube-prom-stack-grafana-6c47f548d6-qqrqb 0/3 Evicted 0 7m23s
pod/kube-prom-stack-grafana-6c47f548d6-csf7s 0/3 Evicted 0 7m23s
pod/kube-prom-stack-grafana-6c47f548d6-dtxml 0/3 Evicted 0 7m23s
pod/kube-prom-stack-grafana-6c47f548d6-mxvz9 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-f57tj 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-sfskh 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-zzr4b 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-fc4l2 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-ndvq4 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-dll2x 0/3 Evicted 0 7m22s
pod/kube-prom-stack-grafana-6c47f548d6-j2tpp 0/3 Evicted 0 7m21s
pod/kube-prom-stack-grafana-6c47f548d6-lrzdg 0/3 Evicted 0 7m21s
pod/kube-prom-stack-grafana-6c47f548d6-pwjbp 0/3 Evicted 0 7m21s
pod/kube-prom-stack-grafana-6c47f548d6-5nscn 0/3 Evicted 0 7m21s
pod/kube-prom-stack-grafana-6c47f548d6-kkpgc 0/3 Evicted 0 7m21s
pod/kube-prom-stack-grafana-6c47f548d6-mdwqx 0/3 Evicted 0 7m21s
pod/kube-prom-stack-grafana-6c47f548d6-5g6rq 0/3 Evicted 0 7m20s
pod/kube-prom-stack-grafana-6c47f548d6-9sdjq 0/3 Evicted 0 7m20s
pod/kube-prom-stack-grafana-6c47f548d6-7j7jj 0/3 Evicted 0 7m20s
pod/kube-prom-stack-grafana-6c47f548d6-p8s8h 0/3 Evicted 0 7m20s
pod/kube-prom-stack-grafana-6c47f548d6-8g89z 0/3 Evicted 0 7m20s
pod/kube-prom-stack-grafana-6c47f548d6-27xcz 0/3 Evicted 0 7m20s
pod/kube-prom-stack-grafana-6c47f548d6-tjjqj 0/3 Evicted 0 7m19s
pod/kube-prom-stack-grafana-6c47f548d6-lqcqj 0/3 Evicted 0 7m19s
pod/kube-prom-stack-grafana-6c47f548d6-qz8ch 0/3 Evicted 0 7m19s
pod/kube-prom-stack-grafana-6c47f548d6-lz78m 0/3 Evicted 0 7m19s
pod/kube-prom-stack-grafana-6c47f548d6-zrk98 0/3 Evicted 0 7m19s
pod/kube-prom-stack-grafana-6c47f548d6-zkqvr 0/3 Evicted 0 7m19s
pod/loki-promtail-97msp 0/1 Evicted 0 4m3s
pod/kube-prom-stack-prometheus-node-exporter-hgrjk 0/1 Evicted 0 3m54s
pod/prometheus-kube-prom-stack-kube-prome-prometheus-0 0/2 Pending 0 2m1s
pod/kube-prom-stack-grafana-6c47f548d6-8czwd 3/3 Running 0 7m18s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kube-prom-stack-grafana ClusterIP 10.152.183.129 <none> 80/TCP 53m
service/kube-prom-stack-prometheus-node-exporter ClusterIP 10.152.183.161 <none> 9100/TCP 53m
service/kube-prom-stack-kube-state-metrics ClusterIP 10.152.183.222 <none> 8080/TCP 53m
service/kube-prom-stack-kube-prome-prometheus ClusterIP 10.152.183.206 <none> 9090/TCP 53m
service/kube-prom-stack-kube-prome-operator ClusterIP 10.152.183.180 <none> 443/TCP 53m
service/kube-prom-stack-kube-prome-alertmanager ClusterIP 10.152.183.139 <none> 9093/TCP 53m
service/alertmanager-operated ClusterIP None <none> 9093/TCP,9094/TCP,9094/UDP 53m
service/prometheus-operated ClusterIP None <none> 9090/TCP 53m
service/loki-headless ClusterIP None <none> 3100/TCP 53m
service/loki-memberlist ClusterIP None <none> 7946/TCP 53m
service/loki ClusterIP 10.152.183.79 <none> 3100/TCP 53m
service/tempo ClusterIP 10.152.183.250 <none> 3100/TCP,16687/TCP,16686/TCP,6831/UDP,6832/UDP,14268/TCP,14250/TCP,9411/TCP,55680/TCP,55681/TCP,4317/TCP,4318/TCP,55678/TCP 53m
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/loki-promtail 1 1 0 1 0 <none> 53m
daemonset.apps/kube-prom-stack-prometheus-node-exporter 1 1 0 1 0 <none> 53m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/kube-prom-stack-kube-prome-operator 1/1 1 1 53m
deployment.apps/kube-prom-stack-kube-state-metrics 1/1 1 1 53m
deployment.apps/kube-prom-stack-grafana 1/1 1 1 53m
NAME DESIRED CURRENT READY AGE
replicaset.apps/kube-prom-stack-kube-prome-operator-64ffd55b77 1 1 1 53m
replicaset.apps/kube-prom-stack-kube-state-metrics-6c586bf4c8 1 1 1 53m
replicaset.apps/kube-prom-stack-grafana-6c47f548d6 1 1 1 53m
NAME READY AGE
statefulset.apps/loki 0/1 53m
statefulset.apps/tempo 1/1 53m
statefulset.apps/alertmanager-kube-prom-stack-kube-prome-alertmanager 1/1 53m
statefulset.apps/prometheus-kube-prom-stack-kube-prome-prometheus 0/1 53m