remove node-exporter

kustomize/bases/blackbox-exporter/blackbox-exporter-deployment.yaml

@@ -24,7 +24,7 @@ spec:
               cpu: 100m
             limits:
               memory: 128Mi
-              cpu: 500m
+              cpu: 100m
           ports:
             - containerPort: 9115
               name: http-metrics

kustomize/bases/drone-runner/drone-runner-deployment.yaml

@@ -42,7 +42,7 @@ spec:
               memory: 100Mi
             limits:
               cpu: 100m
-              memory: 250Mi
+              memory: 100Mi
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

kustomize/bases/drone-server/drone-server-deployment.yaml

@@ -60,7 +60,7 @@ spec:
               memory: 100Mi
             limits:
               cpu: 250m
-              memory: 250Mi
+              memory: 100Mi
           volumeMounts:
             - mountPath: /data
               name: drone-server-pv

kustomize/bases/gitea/gitea-deployment.yaml

@@ -46,8 +46,8 @@ spec:
               cpu: 250m
               memory: 250Mi
             limits:
-              cpu: 500m
-              memory: 500Mi
+              cpu: 250m
+              memory: 250Mi
           volumeMounts:
             - name: gitea-pv
               mountPath: /data

kustomize/bases/grafana-agent/grafana-agent-volume.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-agent-wal-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

kustomize/bases/grafana-agent/kustomization.yaml

@@ -0,0 +1,18 @@
+resources:
+  - https://raw.githubusercontent.com/grafana/agent/v0.24.2/production/kubernetes/agent-bare.yaml
+  - grafana-agent-volume.yaml
+
+secretGenerator:
+  - name: grafana-agent
+    behavior: create
+    literals:
+      - agent.yaml=
+
+patchesJson6902:
+  - target:
+      version: v1
+      kind: StatefulSet
+      name: grafana-agent
+    path: patches/grafana-agent-statefulset-patch.yaml
+
+namespace: default

kustomize/bases/grafana-agent/patches/grafana-agent-statefulset-patch.yaml

@@ -0,0 +1,13 @@
+- op: remove
+  path: /spec/volumeClaimTemplates
+- op: add
+  path: /spec/template/spec/volumes
+  value:
+    - name: grafana-agent
+      secret:
+        secretName: grafana-agent
+    - name: agent-wal
+      persistentVolumeClaim:
+        claimName: grafana-agent-wal-pvc
+
+

kustomize/bases/grafana/grafana-deployment.yaml

@@ -61,8 +61,8 @@ spec:
               cpu: 250m
               memory: 250Mi
             limits:
-              cpu: 500m
-              memory: 500Mi
+              cpu: 250m
+              memory: 250Mi
           volumeMounts:
             - name: grafana-datasources
               mountPath: /etc/grafana/provisioning/datasources

kustomize/bases/helium-blockchain-exporter/helium-blockchain-exporter-deployment.yaml

@@ -1,50 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: helium-blockchain-exporter
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: helium-blockchain-exporter
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: helium-blockchain-exporter
-    spec:
-      containers:
-        - name: helium-blockchain-exporter
-          image: badjware/helium-blockchain-exporter
-          imagePullPolicy: Always
-          env:
-            - name: ACCOUNTS
-              valueFrom:
-                configMapKeyRef:
-                  name: helium-blockchain-exporter-config
-                  key: ACCOUNTS
-          args:
-            - -accounts
-            - $(ACCOUNTS)
-          resources:
-            requests:
-              cpu: 100m
-              memory: 100Mi
-            limits:
-              cpu: 100m
-              memory: 100Mi
-          ports:
-            - containerPort: 9865
-              name: http-metrics
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: helium-blockchain-exporter
-  labels:
-    app.kubernetes.io/name: helium-blockchain-exporter
-spec:
-  selector:
-    app.kubernetes.io/name: helium-blockchain-exporter
-  ports:
-    - name: http-metrics
-      port: 9865
-      targetPort: http-metrics

kustomize/bases/helium-blockchain-exporter/kustomization.yaml

@@ -1,8 +0,0 @@
-resources:
-  - helium-blockchain-exporter-deployment.yaml
-
-configMapGenerator:
-  - name: helium-blockchain-exporter-config
-    literals:
-      # set the helium accounts to scrape here
-      - ACCOUNTS=

kustomize/bases/longhorn/longhorn-helmchart.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   repo: https://charts.longhorn.io
   chart: longhorn
-  version: 1.2.3
+  version: 1.2.4
   targetNamespace: longhorn-system
   set:
     backupTargetCredentialSecret: s3-backupstore-credentials

kustomize/bases/nextcloud/nextcloud-deployment.yaml

@@ -72,8 +72,8 @@ spec:
               cpu: 1000m
               memory: 1Gi
             limits:
-              cpu: 2000m
-              memory: 2Gi
+              cpu: 1000m
+              memory: 1Gi
           volumeMounts:
             - name: nextcloud-pv
               mountPath: /var/www/html

kustomize/bases/node-exporter/node-exporter-daemonset.yaml

@@ -31,8 +31,8 @@ spec:
             memory: 20Mi
             cpu: 100m
           limits:
-            memory: 50Mi
-            cpu: 500m
+            memory: 20Mi
+            cpu: 100m
         volumeMounts:
         - name: proc
           readOnly:  true

kustomize/bases/postgres/postgres-statefulset.yaml

@@ -44,7 +44,7 @@ spec:
               cpu: 500m
               memory: 1Gi
             limits:
-              cpu: 1000m
+              cpu: 500m
               memory: 1Gi
           volumeMounts:
             - name: postgres-pvc

kustomize/bases/prometheus/kustomization.yaml

@@ -2,4 +2,15 @@ resources:
   - prometheus.yaml
   - prometheus-rbac.yaml
   - prometheus-ingress.yaml
-  - traefik-podmonitor.yaml
+  - traefik-podmonitor.yaml
+
+secretGenerator:
+  - name: grafana-cloud-credentials
+    type: Opaque
+    behavior: create
+    literals:
+      - username=changeme
+      - password=changeme
+
+configurations:
+  - kustomizeconfig/namereference.yaml

kustomize/bases/prometheus/kustomizeconfig/namereference.yaml

@@ -0,0 +1,7 @@
+nameReference:
+  - kind: Secret
+    fieldSpecs:
+      - kind: Prometheus
+        path: spec/remoteWrite/basicAuth/username/name
+      - kind: Prometheus
+        path: spec/remoteWrite/basicAuth/password/name

kustomize/bases/prometheus/prometheus.yaml

@@ -4,7 +4,7 @@ metadata:
   name: prometheus
 spec:
   serviceAccountName: prometheus
-  retention: 28d
+  retention: 30d
   retentionSize: 9GB
   ruleSelector:
     matchLabels:
@@ -18,6 +18,15 @@ spec:
   probeSelector:
     matchLabels:
       monitor: prometheus
+  # remoteWrite:
+  #   - url: changeme
+  #     basicAuth:
+  #       username:
+  #         name: grafana-cloud-credentials
+  #         key: username
+  #       password:
+  #         name: grafana-cloud-credentials
+  #         key: password
   securityContext:
     runAsGroup: 65534
     runAsNonRoot: true
@@ -25,15 +34,15 @@ spec:
     fsGroup: 65534
   resources:
     requests:
-      cpu: 500m
+      cpu: 1000m
       memory: 1Gi
     limits:
       cpu: 1000m
-      memory: 2Gi
+      memory: 1Gi
   storage:
     volumeClaimTemplate:
       spec:
         accessModes: ["ReadWriteOnce"]
         resources:
           requests:
-            storage: 10Gi
+            storage: 10Gi

kustomize/bases/redis/redis-deployment.yaml

@@ -32,7 +32,7 @@ spec:
               cpu: 250m
               memory: 250Mi
             limits:
-              cpu: 500m
+              cpu: 250m
               memory: 250Mi
 ---
 apiVersion: v1

kustomize/namespaces/monitoring/kustomization.yaml

@@ -1,18 +1,16 @@
 bases:
   - ../../bases/prometheus
-  - ../../bases/node-exporter
+  # - ../../bases/node-exporter
   - ../../bases/blackbox-exporter
-  - ../../bases/helium-blockchain-exporter
   - ../../bases/kube-state-metrics
+  - ../../bases/grafana-agent
   - servicemonitors/any-namespaces-http.yaml
   - servicemonitors/kubelet.yaml
   - servicemonitors/longhorn.yaml
   - servicemonitors/prometheus.yaml
   - servicemonitors/kube-state-metrics.yaml
-  - servicemonitors/helium-blockchain-exporter.yaml
-  - servicemonitors/node-exporter.yaml
+  # - servicemonitors/node-exporter.yaml
   - probes/blackbox-exporter.yaml
-  - prometheusrules/helium-blockchain-exporter.yaml
 
 resources:
   - namespace.yaml

kustomize/namespaces/monitoring/prometheusrules/helium-blockchain-exporter.yaml

@@ -1,34 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: helium-blockchain-exporter-rules
-  labels:
-    monitor: prometheus
-spec:
-  groups:
-    - name: helium-blockchain-exporter-rules
-      interval: 5m
-      rules:
-        # Accounts
-        - record: account:helium_account_deposits_hnt:increase30m
-          expr: sum by (account) (increase(helium_account_deposits_hnt_total[30m]))
-        - record: account:helium_account_withdrawals_hnt:increase30m
-          expr: -sum by (account) (increase(helium_account_withdrawals_hnt_total[30m]))
-        - record: type:helium_account_activity:floor_increase30m
-          expr: sum by (account, type) (floor(increase(helium_account_activity_total[30m])))
-        - record: account:helium_account_rewards_hnt:increase1h
-          expr: sum by (account) (increase(helium_account_rewards_hnt_total[1h]))
-        - record: account:helium_account_rewards_hnt:increase24h
-          expr: sum by (account) (increase(helium_account_rewards_hnt_total[24h]))
-
-        # Hotspots
-        - record: type:helium_hotspot_activity:floor_increase30m
-          expr: sum by (account, hotspot, hotspot_name, type) (floor(increase(helium_hotspot_activity_total[30m])))
-        - record: hotspot:helium_hotspot_rewards_hnt:increase30m
-          expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[30m]))
-        - record: hotspot:helium_hotspot_rewards_hnt:increase1h
-          expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[1h]))
-        - record: hotspot:helium_hotspot_rewards_hnt:increase24h
-          expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[24h]))
-        - record: hotspot:helium_hotspot_rewards_hnt:increase7d
-          expr: sum by (account, hotspot, hotspot_name) (increase(helium_hotspot_rewards_hnt_total[7d]))

kustomize/namespaces/monitoring/servicemonitors/helium-blockchain-exporter.yaml

@@ -1,15 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: helium-blockchain-exporter
-  labels:
-    app.kubernetes.io/name: helium-blockchain-exporter
-    monitor: prometheus
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: helium-blockchain-exporter
-  endpoints:
-    - port: http-metrics
-      interval: 60s
-      scrapeTimeout: 30s

kustomize/overlays/prod/configurations/grafana-agent/agent.yaml

@@ -0,0 +1,88 @@
+metrics:
+  wal_directory: /var/lib/agent/wal
+  global:
+    scrape_interval: 60s
+    external_labels:
+      cluster: cloud
+  configs:
+    - name: integrations
+      remote_write:
+        - url: https://prometheus-prod-10-prod-us-central-0.grafana.net/api/prom/push
+          basic_auth:
+            username: 443422
+            password: ${ssm:/k3s/prod/monitoring/grafana-cloud/password}
+      scrape_configs:
+        - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+          job_name: integrations/kubernetes/cadvisor
+          kubernetes_sd_configs:
+            - role: node
+          metric_relabel_configs:
+            - source_labels: [__name__]
+              regex: namespace_memory:kube_pod_container_resource_requests:sum|kubelet_running_containers|container_cpu_usage_seconds_total|kube_pod_container_info|container_network_receive_packets_dropped_total|kube_pod_status_phase|kubelet_pod_start_duration_seconds_count|kubelet_cgroup_manager_duration_seconds_bucket|kube_horizontalpodautoscaler_status_desired_replicas|cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits|node_namespace_pod_container:container_memory_swap|kube_statefulset_status_replicas_ready|kube_horizontalpodautoscaler_spec_max_replicas|cluster:namespace:pod_memory:active:kube_pod_container_resource_requests|process_cpu_seconds_total|process_resident_memory_bytes|kubelet_server_expiration_renew_errors|kube_daemonset.*|container_fs_reads_total|machine_memory_bytes|kubelet_volume_stats_inodes_used|volume_manager_total_volumes|kube_statefulset_status_replicas|namespace_cpu:kube_pod_container_resource_limits:sum|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kubelet_pod_worker_duration_seconds_count|namespace_workload_pod:kube_pod_owner:relabel|kubelet_cgroup_manager_duration_seconds_count|container_cpu_cfs_throttled_periods_total|kube_node_spec_taint|container_fs_reads_bytes_total|kubelet_certificate_manager_client_ttl_seconds|container_network_receive_bytes_total|kubelet_running_container_count|kube_daemonset_status_number_available|kube_node_status_allocatable|container_fs_writes_total|kube_namespace_status_phase|kubelet_volume_stats_available_bytes|kubelet_pleg_relist_duration_seconds_bucket|kubelet_runtime_operations_errors_total|kube_pod_container_status_waiting_reason|kube_replicaset_owner|kube_resourcequota|kube_pod_info|kubelet_pleg_relist_duration_seconds_count|kube_deployment_status_replicas_available|cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests|kubelet_running_pods|kube_statefulset_status_replicas_updated|kube_deployment_status_replicas_updated|kube_job_spec_completions|kube_daemonset_status_number_misscheduled|kubelet_certificate_manager_server_ttl_seconds|container_network_transmit_bytes_total|container_memory_cache|kubelet_volume_stats_capacity_bytes|node_namespace_pod_container:container_memory_cache|container_memory_rss|container_memory_swap|storage_operation_duration_seconds_count|kube_replicaset.*|kube_pod_owner|cluster:namespace:pod_memory:active:kube_pod_container_resource_limits|kubelet_volume_stats_inodes|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_updated_number_scheduled|kube_statefulset.*|kube_node_info|go_goroutines|kubelet_pod_worker_duration_seconds_bucket|kubelet_node_config_error|container_cpu_cfs_periods_total|kubelet_pleg_relist_interval_seconds_bucket|kube_job.*|container_network_receive_packets_total|container_fs_writes_bytes_total|kubelet_running_pod_count|kube_deployment_spec_replicas|up|kube_node_status_capacity|namespace_cpu:kube_pod_container_resource_requests:sum|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate|container_memory_working_set_bytes|kubelet_node_name|node_namespace_pod_container:container_memory_rss|storage_operation_errors_total|kube_statefulset_metadata_generation|container_network_transmit_packets_total|kubelet_runtime_operations_total|kube_statefulset_status_observed_generation|kube_horizontalpodautoscaler_status_current_replicas|kubernetes_build_info|kubelet_certificate_manager_client_expiration_renew_errors|kube_job_failed|namespace_workload_pod|node_namespace_pod_container:container_memory_working_set_bytes|kube_statefulset_replicas|kube_deployment_status_observed_generation|kube_pod_container_status_restarts_total|kube_daemonset_status_current_number_scheduled|kube_pod_start_time|namespace_memory:kube_pod_container_resource_limits:sum|container_network_transmit_packets_dropped_total|rest_client_requests_total|kube_deployment_metadata_generation|kube_statefulset_status_update_revision|kube_job_status_succeeded|kube_horizontalpodautoscaler_spec_min_replicas|kube_statefulset_status_current_revision|node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile|kube_node_status_condition
+              action: keep
+          relabel_configs:
+            - replacement: kubernetes.default.svc.cluster.local:443
+              target_label: __address__
+            - regex: (.+)
+              replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+              source_labels:
+                - __meta_kubernetes_node_name
+              target_label: __metrics_path__
+          scheme: https
+          tls_config:
+            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            insecure_skip_verify: false
+            server_name: kubernetes
+        - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+          job_name: integrations/kubernetes/kubelet
+          kubernetes_sd_configs:
+            - role: node
+          metric_relabel_configs:
+            - source_labels: [__name__]
+              regex: namespace_memory:kube_pod_container_resource_requests:sum|kubelet_running_containers|container_cpu_usage_seconds_total|kube_pod_container_info|container_network_receive_packets_dropped_total|kube_pod_status_phase|kubelet_pod_start_duration_seconds_count|kubelet_cgroup_manager_duration_seconds_bucket|kube_horizontalpodautoscaler_status_desired_replicas|cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits|node_namespace_pod_container:container_memory_swap|kube_statefulset_status_replicas_ready|kube_horizontalpodautoscaler_spec_max_replicas|cluster:namespace:pod_memory:active:kube_pod_container_resource_requests|process_cpu_seconds_total|process_resident_memory_bytes|kubelet_server_expiration_renew_errors|kube_daemonset.*|container_fs_reads_total|machine_memory_bytes|kubelet_volume_stats_inodes_used|volume_manager_total_volumes|kube_statefulset_status_replicas|namespace_cpu:kube_pod_container_resource_limits:sum|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kubelet_pod_worker_duration_seconds_count|namespace_workload_pod:kube_pod_owner:relabel|kubelet_cgroup_manager_duration_seconds_count|container_cpu_cfs_throttled_periods_total|kube_node_spec_taint|container_fs_reads_bytes_total|kubelet_certificate_manager_client_ttl_seconds|container_network_receive_bytes_total|kubelet_running_container_count|kube_daemonset_status_number_available|kube_node_status_allocatable|container_fs_writes_total|kube_namespace_status_phase|kubelet_volume_stats_available_bytes|kubelet_pleg_relist_duration_seconds_bucket|kubelet_runtime_operations_errors_total|kube_pod_container_status_waiting_reason|kube_replicaset_owner|kube_resourcequota|kube_pod_info|kubelet_pleg_relist_duration_seconds_count|kube_deployment_status_replicas_available|cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests|kubelet_running_pods|kube_statefulset_status_replicas_updated|kube_deployment_status_replicas_updated|kube_job_spec_completions|kube_daemonset_status_number_misscheduled|kubelet_certificate_manager_server_ttl_seconds|container_network_transmit_bytes_total|container_memory_cache|kubelet_volume_stats_capacity_bytes|node_namespace_pod_container:container_memory_cache|container_memory_rss|container_memory_swap|storage_operation_duration_seconds_count|kube_replicaset.*|kube_pod_owner|cluster:namespace:pod_memory:active:kube_pod_container_resource_limits|kubelet_volume_stats_inodes|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_updated_number_scheduled|kube_statefulset.*|kube_node_info|go_goroutines|kubelet_pod_worker_duration_seconds_bucket|kubelet_node_config_error|container_cpu_cfs_periods_total|kubelet_pleg_relist_interval_seconds_bucket|kube_job.*|container_network_receive_packets_total|container_fs_writes_bytes_total|kubelet_running_pod_count|kube_deployment_spec_replicas|up|kube_node_status_capacity|namespace_cpu:kube_pod_container_resource_requests:sum|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate|container_memory_working_set_bytes|kubelet_node_name|node_namespace_pod_container:container_memory_rss|storage_operation_errors_total|kube_statefulset_metadata_generation|container_network_transmit_packets_total|kubelet_runtime_operations_total|kube_statefulset_status_observed_generation|kube_horizontalpodautoscaler_status_current_replicas|kubernetes_build_info|kubelet_certificate_manager_client_expiration_renew_errors|kube_job_failed|namespace_workload_pod|node_namespace_pod_container:container_memory_working_set_bytes|kube_statefulset_replicas|kube_deployment_status_observed_generation|kube_pod_container_status_restarts_total|kube_daemonset_status_current_number_scheduled|kube_pod_start_time|namespace_memory:kube_pod_container_resource_limits:sum|container_network_transmit_packets_dropped_total|rest_client_requests_total|kube_deployment_metadata_generation|kube_statefulset_status_update_revision|kube_job_status_succeeded|kube_horizontalpodautoscaler_spec_min_replicas|kube_statefulset_status_current_revision|node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile|kube_node_status_condition
+              action: keep
+          relabel_configs:
+            - replacement: kubernetes.default.svc.cluster.local:443
+              target_label: __address__
+            - regex: (.+)
+              replacement: /api/v1/nodes/${1}/proxy/metrics
+              source_labels:
+                - __meta_kubernetes_node_name
+              target_label: __metrics_path__
+          scheme: https
+          tls_config:
+            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            insecure_skip_verify: false
+            server_name: kubernetes
+        - job_name: integrations/kubernetes/kube-state-metrics
+          kubernetes_sd_configs:
+            - role: pod
+          metric_relabel_configs:
+            - source_labels: [__name__]
+              regex: namespace_memory:kube_pod_container_resource_requests:sum|kubelet_running_containers|container_cpu_usage_seconds_total|kube_pod_container_info|container_network_receive_packets_dropped_total|kube_pod_status_phase|kubelet_pod_start_duration_seconds_count|kubelet_cgroup_manager_duration_seconds_bucket|kube_horizontalpodautoscaler_status_desired_replicas|cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits|node_namespace_pod_container:container_memory_swap|kube_statefulset_status_replicas_ready|kube_horizontalpodautoscaler_spec_max_replicas|cluster:namespace:pod_memory:active:kube_pod_container_resource_requests|process_cpu_seconds_total|process_resident_memory_bytes|kubelet_server_expiration_renew_errors|kube_daemonset.*|container_fs_reads_total|machine_memory_bytes|kubelet_volume_stats_inodes_used|volume_manager_total_volumes|kube_statefulset_status_replicas|namespace_cpu:kube_pod_container_resource_limits:sum|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kubelet_pod_worker_duration_seconds_count|namespace_workload_pod:kube_pod_owner:relabel|kubelet_cgroup_manager_duration_seconds_count|container_cpu_cfs_throttled_periods_total|kube_node_spec_taint|container_fs_reads_bytes_total|kubelet_certificate_manager_client_ttl_seconds|container_network_receive_bytes_total|kubelet_running_container_count|kube_daemonset_status_number_available|kube_node_status_allocatable|container_fs_writes_total|kube_namespace_status_phase|kubelet_volume_stats_available_bytes|kubelet_pleg_relist_duration_seconds_bucket|kubelet_runtime_operations_errors_total|kube_pod_container_status_waiting_reason|kube_replicaset_owner|kube_resourcequota|kube_pod_info|kubelet_pleg_relist_duration_seconds_count|kube_deployment_status_replicas_available|cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests|kubelet_running_pods|kube_statefulset_status_replicas_updated|kube_deployment_status_replicas_updated|kube_job_spec_completions|kube_daemonset_status_number_misscheduled|kubelet_certificate_manager_server_ttl_seconds|container_network_transmit_bytes_total|container_memory_cache|kubelet_volume_stats_capacity_bytes|node_namespace_pod_container:container_memory_cache|container_memory_rss|container_memory_swap|storage_operation_duration_seconds_count|kube_replicaset.*|kube_pod_owner|cluster:namespace:pod_memory:active:kube_pod_container_resource_limits|kubelet_volume_stats_inodes|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_updated_number_scheduled|kube_statefulset.*|kube_node_info|go_goroutines|kubelet_pod_worker_duration_seconds_bucket|kubelet_node_config_error|container_cpu_cfs_periods_total|kubelet_pleg_relist_interval_seconds_bucket|kube_job.*|container_network_receive_packets_total|container_fs_writes_bytes_total|kubelet_running_pod_count|kube_deployment_spec_replicas|up|kube_node_status_capacity|namespace_cpu:kube_pod_container_resource_requests:sum|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate|container_memory_working_set_bytes|kubelet_node_name|node_namespace_pod_container:container_memory_rss|storage_operation_errors_total|kube_statefulset_metadata_generation|container_network_transmit_packets_total|kubelet_runtime_operations_total|kube_statefulset_status_observed_generation|kube_horizontalpodautoscaler_status_current_replicas|kubernetes_build_info|kubelet_certificate_manager_client_expiration_renew_errors|kube_job_failed|namespace_workload_pod|node_namespace_pod_container:container_memory_working_set_bytes|kube_statefulset_replicas|kube_deployment_status_observed_generation|kube_pod_container_status_restarts_total|kube_daemonset_status_current_number_scheduled|kube_pod_start_time|namespace_memory:kube_pod_container_resource_limits:sum|container_network_transmit_packets_dropped_total|rest_client_requests_total|kube_deployment_metadata_generation|kube_statefulset_status_update_revision|kube_job_status_succeeded|kube_horizontalpodautoscaler_spec_min_replicas|kube_statefulset_status_current_revision|node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile|kube_node_status_condition
+              action: keep
+          relabel_configs:
+            - action: keep
+              regex: kube-state-metrics
+              source_labels:
+                - __meta_kubernetes_pod_label_app_kubernetes_io_name
+
+integrations:
+  eventhandler:
+    cache_path: /var/lib/agent/eventhandler.cache
+    logs_instance: integrations
+logs:
+  configs:
+    - name: integrations
+      clients:
+        - url: https://logs-prod3.grafana.net/loki/api/v1/push
+          basic_auth:
+            username: 220681
+            password: ${ssm:/k3s/prod/monitoring/grafana-cloud/password}
+          external_labels:
+            cluster: cloud
+            job: integrations/kubernetes/eventhandler
+      positions:
+        filename: /tmp/positions.yaml
+      target_config:
+        sync_period: 10s

kustomize/overlays/prod/kustomization.yaml

@@ -30,11 +30,7 @@ images:
   - name: redis
     newTag: 6.2.5
 
-configMapGenerator:
-  - name: helium-blockchain-exporter-config
-    behavior: replace
-    literals:
-      - ACCOUNTS=13ntuT3ZC5A1RKEUciXh7qxvQvZ38azqBPHSdUGTpun8evVVTVy
+configMapGenerator: []
 
 secretGenerator:
   - name: drone-secret
@@ -66,6 +62,18 @@ secretGenerator:
     behavior: replace
     literals:
       - password=${ssm:/k3s/prod/nextcloud/redis/password}
+  - name: grafana-agent
+    namespace: monitoring
+    behavior: replace
+    files:
+      - agent.yaml=configurations/grafana-agent/agent.yaml
+  # - name: grafana-cloud-credentials
+  #   type: Opaque
+  #   namespace: monitoring
+  #   behavior: replace
+  #   literals:
+  #     - username=${ssm:/k3s/prod/monitoring/grafana-cloud/username}
+  #     - password=${ssm:/k3s/prod/monitoring/grafana-cloud/password}
 
 # allow "kubectl apply -l managed-by=kustomize --prune ..."
 commonLabels:
@@ -81,6 +89,11 @@ patchesJson6902:
       kind: Deployment
       name: nextcloud
     path: patches/nextcloud-deployment-patch.yaml
+  # - target:
+  #     version: v1
+  #     kind: Prometheus
+  #     name: prometheus
+  #   path: patches/prometheus-patch.yaml
 
 # patchesStrategicMerge:
 #   - patches/blackbox-exporter-probe-patch.yaml

kustomize/overlays/prod/patches/prometheus-patch.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: /spec/remoteWrite/0/url
+  value: https://prometheus-prod-10-prod-us-central-0.grafana.net/api/prom/push