forward prom metrics to grafana-cloud

kustomize/bases/prometheus/kustomization.yaml

@@ -2,4 +2,15 @@ resources:
   - prometheus.yaml
   - prometheus-rbac.yaml
   - prometheus-ingress.yaml
-  - traefik-podmonitor.yaml
+  - traefik-podmonitor.yaml
+
+secretGenerator:
+  - name: grafana-cloud-credentials
+    type: Opaque
+    behavior: create
+    literals:
+      - username=changeme
+      - password=changeme
+
+configurations:
+  - kustomizeconfig/namereference.yaml

kustomize/bases/prometheus/kustomizeconfig/namereference.yaml

@@ -0,0 +1,7 @@
+nameReference:
+  - kind: Secret
+    fieldSpecs:
+      - kind: Prometheus
+        path: spec/remoteWrite/basicAuth/username/name
+      - kind: Prometheus
+        path: spec/remoteWrite/basicAuth/password/name

kustomize/bases/prometheus/prometheus.yaml

@@ -4,8 +4,8 @@ metadata:
   name: prometheus
 spec:
   serviceAccountName: prometheus
-  retention: 28d
-  retentionSize: 9GB
+  retention: 1d
+  retentionSize: 5GB
   ruleSelector:
     matchLabels:
       monitor: prometheus
@@ -18,6 +18,15 @@ spec:
   probeSelector:
     matchLabels:
       monitor: prometheus
+  remoteWrite:
+    - url: changeme
+      basicAuth:
+        username:
+          name: grafana-cloud-credentials
+          key: username
+        password:
+          name: grafana-cloud-credentials
+          key: password
   securityContext:
     runAsGroup: 65534
     runAsNonRoot: true
@@ -26,14 +35,7 @@ spec:
   resources:
     requests:
       cpu: 500m
-      memory: 1Gi
+      memory: 500Mi
     limits:
       cpu: 1000m
-      memory: 2Gi
-  storage:
-    volumeClaimTemplate:
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 10Gi
+      memory: 500Mi

kustomize/overlays/prod/kustomization.yaml

@@ -62,6 +62,13 @@ secretGenerator:
     behavior: replace
     literals:
       - password=${ssm:/k3s/prod/nextcloud/redis/password}
+  - name: grafana-cloud-credentials
+    type: Opaque
+    namespace: monitoring
+    behavior: replace
+    literals:
+      - username=${ssm:/k3s/prod/monitoring/grafana-cloud/username}
+      - password=${ssm:/k3s/prod/monitoring/grafana-cloud/password}
 
 # allow "kubectl apply -l managed-by=kustomize --prune ..."
 commonLabels:
@@ -77,6 +84,11 @@ patchesJson6902:
       kind: Deployment
       name: nextcloud
     path: patches/nextcloud-deployment-patch.yaml
+  - target:
+      version: v1
+      kind: Prometheus
+      name: prometheus
+    path: patches/prometheus-patch.yaml
 
 # patchesStrategicMerge:
 #   - patches/blackbox-exporter-probe-patch.yaml

kustomize/overlays/prod/patches/prometheus-patch.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: /spec/remoteWrite/0/url
+  value: https://prometheus-prod-10-prod-us-central-0.grafana.net/api/prom/push