add cluster monitoring
This commit is contained in:
parent
d2547d903b
commit
65d826b87e
|
|
@ -0,0 +1,84 @@
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: grafana-deployment
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: grafana
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: grafana
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: grafana
|
||||||
|
image: grafana/grafana
|
||||||
|
env:
|
||||||
|
- name: GF_AUTH_ANONYMOUS_ENABLED
|
||||||
|
value: "true"
|
||||||
|
- name: GF_AUTH_ANONYMOUS_ORG_ROLE
|
||||||
|
value: Admin
|
||||||
|
- name: GF_DATABASE_TYPE
|
||||||
|
value: mysql
|
||||||
|
- name: GF_DATABASE_HOST
|
||||||
|
value: mariadb.monitoring.svc:3306
|
||||||
|
- name: GF_DATABASE_NAME
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: mariadb-credentials-secret
|
||||||
|
key: database
|
||||||
|
- name: GF_DATABASE_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: mariadb-credentials-secret
|
||||||
|
key: username
|
||||||
|
- name: GF_DATABASE_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: mariadb-credentials-secret
|
||||||
|
key: password
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3000
|
||||||
|
volumeMounts:
|
||||||
|
- name: grafana-datasources
|
||||||
|
mountPath: /etc/grafana/provisioning/datasources
|
||||||
|
volumes:
|
||||||
|
- name: grafana-datasources
|
||||||
|
configMap:
|
||||||
|
name: grafana-datasources
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: grafana
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: grafana
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
protocol: TCP
|
||||||
|
port: 3000
|
||||||
|
targetPort: http
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1beta1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: grafana-ingress
|
||||||
|
annotations:
|
||||||
|
nginx.ingress.kubernetes.io/rewrite-target: /
|
||||||
|
cert-manager.io/cluster-issuer: letsencrypt
|
||||||
|
spec:
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- grafana.127.0.0.1.nip.io
|
||||||
|
secretName: letsencrypt-cert
|
||||||
|
rules:
|
||||||
|
- host: grafana.127.0.0.1.nip.io
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
backend:
|
||||||
|
serviceName: grafana
|
||||||
|
servicePort: http
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: grafana
|
||||||
|
labels:
|
||||||
|
monitor: main
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
managed-by: kustomize
|
||||||
|
app: grafana
|
||||||
|
endpoints:
|
||||||
|
- port: http
|
||||||
|
path: /metrics
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
bases:
|
||||||
|
- ../../mariadb
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- grafana-deployment.yaml
|
||||||
|
- grafana-servicemonitor.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: grafana-datasources
|
||||||
|
files:
|
||||||
|
- datasources.yaml=provision/datasources.yaml
|
||||||
|
|
||||||
|
secretGenerator:
|
||||||
|
- name: mariadb-credentials-secret
|
||||||
|
type: Opaque
|
||||||
|
behavior: replace
|
||||||
|
literals:
|
||||||
|
- database=grafana
|
||||||
|
- username=grafana
|
||||||
|
- password=replaceme
|
||||||
|
|
||||||
|
namespace: grafana
|
||||||
|
|
||||||
|
commonLabels:
|
||||||
|
app: grafana
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
apiVersion: 1
|
||||||
|
datasources:
|
||||||
|
- name: prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus-main.monitoring.svc:9090
|
||||||
|
is_default: true
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: kube-state-metrics
|
||||||
|
labels:
|
||||||
|
monitor: main
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
managed-by: kustomize
|
||||||
|
app: kube-state-metrics
|
||||||
|
endpoints:
|
||||||
|
- port: http-metrics
|
||||||
|
path: /metrics
|
||||||
|
honorLabels: true
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
apiVersion: badjware/v1
|
||||||
|
kind: RemoteResources
|
||||||
|
metadata:
|
||||||
|
name: kube-state-metrics
|
||||||
|
resources:
|
||||||
|
- url: https://raw.githubusercontent.com/kubernetes/kube-state-metrics/master/examples/standard/cluster-role-binding.yaml
|
||||||
|
sha256: 47d33684e45f1aa6a9f10af998e485e76608f92885f2558d436dbd91f87738aa
|
||||||
|
- url: https://raw.githubusercontent.com/kubernetes/kube-state-metrics/master/examples/standard/cluster-role.yaml
|
||||||
|
sha256: 26e6b749f1956fcab55f48f2a25b5195c686b814a3546640a3ed2f3485b5b0af
|
||||||
|
- url: https://raw.githubusercontent.com/kubernetes/kube-state-metrics/master/examples/standard/deployment.yaml
|
||||||
|
sha256: 716fa3a4bcb50daebd2040cdf3d9abd0978f4a0222e220b8b86ab020dca7d5aa
|
||||||
|
- url: https://raw.githubusercontent.com/kubernetes/kube-state-metrics/master/examples/standard/service-account.yaml
|
||||||
|
sha256: d356059416f2ead70b7d27a6fc9cc3e091a8f21f7022ce23da96ed5d871c018b
|
||||||
|
- url: https://raw.githubusercontent.com/kubernetes/kube-state-metrics/master/examples/standard/service.yaml
|
||||||
|
sha256: e110496c646d2d7b41311ef88d315f239d3505785d99ffd8d0a1f9ec9832eb84
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
resources:
|
||||||
|
- kube-state-metrics-servicemonitors.yaml
|
||||||
|
|
||||||
|
commonLabels:
|
||||||
|
app: kube-state-metrics
|
||||||
|
|
||||||
|
generators:
|
||||||
|
- kube-state-metrics.yaml
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
bases:
|
||||||
|
- kube-state-metrics
|
||||||
|
- node-exporter
|
||||||
|
- prometheus
|
||||||
|
- grafana
|
||||||
|
- prometheus-operator
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
|
||||||
|
namespace: monitoring
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: monitoring
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
resources:
|
||||||
|
- node-exporter-daemonset.yaml
|
||||||
|
- node-exporter-servicemonitor.yaml
|
||||||
|
|
||||||
|
commonLabels:
|
||||||
|
app: node-exporter
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: node-exporter
|
||||||
|
namespace: kube-monitoring
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: node-exporter
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: node-exporter
|
||||||
|
labels:
|
||||||
|
app: node-exporter
|
||||||
|
spec:
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
containers:
|
||||||
|
- name: node-exporter
|
||||||
|
image: prom/node-exporter:v0.18.1
|
||||||
|
args:
|
||||||
|
- "--path.procfs=/host/proc"
|
||||||
|
- "--path.sysfs=/host/sys"
|
||||||
|
ports:
|
||||||
|
- containerPort: 9100
|
||||||
|
name: http-metrics
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 20Mi
|
||||||
|
cpu: 100m
|
||||||
|
limits:
|
||||||
|
memory: 50Mi
|
||||||
|
cpu: 500m
|
||||||
|
volumeMounts:
|
||||||
|
- name: proc
|
||||||
|
readOnly: true
|
||||||
|
mountPath: /host/proc
|
||||||
|
- name: sys
|
||||||
|
readOnly: true
|
||||||
|
mountPath: /host/sys
|
||||||
|
tolerations:
|
||||||
|
- effect: NoSchedule
|
||||||
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- name: proc
|
||||||
|
hostPath:
|
||||||
|
path: /proc
|
||||||
|
- name: sys
|
||||||
|
hostPath:
|
||||||
|
path: /sys
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: nodes-exporter
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- name: http-metrics
|
||||||
|
port: 9100
|
||||||
|
protocol: TCP
|
||||||
|
targetPort: http-metrics
|
||||||
|
selector:
|
||||||
|
app: node-exporter
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: node-exporter
|
||||||
|
labels:
|
||||||
|
monitor: main
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
managed-by: kustomize
|
||||||
|
app: node-exporter
|
||||||
|
endpoints:
|
||||||
|
- port: http-metrics
|
||||||
|
path: /metrics
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
generators:
|
||||||
|
- prometheus-operator.yaml
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
apiVersion: badjware/v1
|
||||||
|
kind: RemoteResources
|
||||||
|
metadata:
|
||||||
|
name: prometheus-operator
|
||||||
|
resources:
|
||||||
|
- url: https://raw.githubusercontent.com/coreos/prometheus-operator/master/example/rbac/prometheus-operator/prometheus-operator-cluster-role-binding.yaml
|
||||||
|
sha256: 22c1f67d0417fc743660d3e55b2d2c19b767f62dfb5326bda97b5bbfba875b6f
|
||||||
|
- url: https://raw.githubusercontent.com/coreos/prometheus-operator/master/example/rbac/prometheus-operator/prometheus-operator-cluster-role.yaml
|
||||||
|
sha256: 6475a9fe2a044a3653d9047c9e57598afd6deefb72bbe6521e3a101f2148cada
|
||||||
|
- url: https://raw.githubusercontent.com/coreos/prometheus-operator/master/example/rbac/prometheus-operator/prometheus-operator-deployment.yaml
|
||||||
|
sha256: ceb21dc7274285143301ca9ab7ef4dec07c3d83a8b15113b5ccdbd73ecaad49a
|
||||||
|
- url: https://raw.githubusercontent.com/coreos/prometheus-operator/master/example/rbac/prometheus-operator/prometheus-operator-service-account.yaml
|
||||||
|
sha256: f9cb00698e38c3db99a3b750b4a27ba2bb07e6574ceaa13eb5aa976bb72ca5f6
|
||||||
|
- url: https://raw.githubusercontent.com/coreos/prometheus-operator/master/example/rbac/prometheus-operator/prometheus-operator-service-monitor.yaml
|
||||||
|
sha256: 26ceeabca8dd0c6069e1ad918e8b02df67b3667864c4cf985c48a3b07692c6ae
|
||||||
|
- url: https://raw.githubusercontent.com/coreos/prometheus-operator/master/example/rbac/prometheus-operator/prometheus-operator-service.yaml
|
||||||
|
sha256: f600f2aa9b4b2090d1ab52018d7cf74853e40edfd71532db4f2e9323cf6f3505
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
resources:
|
||||||
|
- prometheus-crd.yaml
|
||||||
|
- prometheus-rbac.yaml
|
||||||
|
- prometheus-servicemonitor.yaml
|
||||||
|
|
||||||
|
commonLabels:
|
||||||
|
app: prometheus
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: Prometheus
|
||||||
|
metadata:
|
||||||
|
name: main
|
||||||
|
spec:
|
||||||
|
podMetadata:
|
||||||
|
labels:
|
||||||
|
managed-by: kustomize
|
||||||
|
serviceAccountName: prometheus
|
||||||
|
serviceMonitorSelector:
|
||||||
|
matchLabels:
|
||||||
|
monitor: main
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 400Mi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: prometheus-main
|
||||||
|
labels:
|
||||||
|
prometheus: main
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
prometheus: main
|
||||||
|
ports:
|
||||||
|
- name: web
|
||||||
|
protocol: TCP
|
||||||
|
port: 9090
|
||||||
|
targetPort: web
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1beta1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: prometheus-ingress
|
||||||
|
annotations:
|
||||||
|
nginx.ingress.kubernetes.io/rewrite-target: /
|
||||||
|
cert-manager.io/cluster-issuer: letsencrypt
|
||||||
|
spec:
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- prometheus.127.0.0.1.nip.io
|
||||||
|
secretName: letsencrypt-cert
|
||||||
|
rules:
|
||||||
|
- host: prometheus.127.0.0.1.nip.io
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
backend:
|
||||||
|
serviceName: prometheus-main
|
||||||
|
servicePort: web
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources:
|
||||||
|
- nodes
|
||||||
|
- nodes/metrics
|
||||||
|
- services
|
||||||
|
- endpoints
|
||||||
|
- pods
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources:
|
||||||
|
- configmaps
|
||||||
|
verbs: ["get"]
|
||||||
|
- nonResourceURLs: ["/metrics"]
|
||||||
|
verbs: ["get"]
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: prometheus
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: prometheus
|
||||||
|
namespace: default
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
labels:
|
||||||
|
app: prometheus
|
||||||
|
monitor: main
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
managed-by: kustomize
|
||||||
|
app: prometheus
|
||||||
|
prometheus: main
|
||||||
|
endpoints:
|
||||||
|
- port: web
|
||||||
|
path: /metrics
|
||||||
|
---
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: kubelet
|
||||||
|
labels:
|
||||||
|
monitor: main
|
||||||
|
spec:
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- kube-system
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
k8s-app: kubelet
|
||||||
|
endpoints:
|
||||||
|
- port: https-metrics
|
||||||
|
scheme: https
|
||||||
|
path: /metrics
|
||||||
|
honorLabels: true
|
||||||
|
interval: 60s
|
||||||
|
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||||
|
tlsConfig:
|
||||||
|
caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||||
|
insecureSkipVerify: true
|
||||||
|
- port: https-metrics
|
||||||
|
scheme: https
|
||||||
|
path: /metrics/cadvisor
|
||||||
|
honorLabels: true
|
||||||
|
interval: 60s
|
||||||
|
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||||
|
tlsConfig:
|
||||||
|
caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||||
|
insecureSkipVerify: true
|
||||||
|
|
@ -5,6 +5,7 @@ bases:
|
||||||
- ../../base/kubernetes-dashboard
|
- ../../base/kubernetes-dashboard
|
||||||
- ../../base/gitea
|
- ../../base/gitea
|
||||||
- ../../base/nextcloud
|
- ../../base/nextcloud
|
||||||
|
- ../../base/monitoring
|
||||||
|
|
||||||
resources:
|
resources:
|
||||||
- cert-manager/clusterissuer.yaml
|
- cert-manager/clusterissuer.yaml
|
||||||
|
|
@ -14,7 +15,7 @@ secretGenerator:
|
||||||
type: Opaque
|
type: Opaque
|
||||||
namespace: cert-manager
|
namespace: cert-manager
|
||||||
literals:
|
literals:
|
||||||
- 'access-token=${ssm:/prod/digitalocean/api_token}'
|
- access-token=${ssm:/prod/digitalocean/api_token}
|
||||||
|
|
||||||
# - name: drone-gitea-oauth-secret
|
# - name: drone-gitea-oauth-secret
|
||||||
# type: Opaque
|
# type: Opaque
|
||||||
|
|
@ -37,6 +38,26 @@ patchesJson6902:
|
||||||
- op: replace
|
- op: replace
|
||||||
path: /spec/rules/0/host
|
path: /spec/rules/0/host
|
||||||
value: kubernetes-dashboard.staging.badjware.dev
|
value: kubernetes-dashboard.staging.badjware.dev
|
||||||
|
- target:
|
||||||
|
<<: *ingress_target
|
||||||
|
name: prometheus-ingress
|
||||||
|
patch: |-
|
||||||
|
- op: replace
|
||||||
|
path: /spec/tls/0/hosts/0
|
||||||
|
value: prometheus.staging.badjware.dev
|
||||||
|
- op: replace
|
||||||
|
path: /spec/rules/0/host
|
||||||
|
value: prometheus.staging.badjware.dev
|
||||||
|
- target:
|
||||||
|
<<: *ingress_target
|
||||||
|
name: grafana-ingress
|
||||||
|
patch: |-
|
||||||
|
- op: replace
|
||||||
|
path: /spec/tls/0/hosts/0
|
||||||
|
value: grafana.staging.badjware.dev
|
||||||
|
- op: replace
|
||||||
|
path: /spec/rules/0/host
|
||||||
|
value: grafana.staging.badjware.dev
|
||||||
- target:
|
- target:
|
||||||
<<: *ingress_target
|
<<: *ingress_target
|
||||||
name: nextcloud-ingress
|
name: nextcloud-ingress
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue