From fa95b03900ec1fcb5b074c77d62505033a099a36 Mon Sep 17 00:00:00 2001
From: Massaki Archambault <marchambault@badjware.dev>
Date: Mon, 5 Feb 2024 21:03:35 -0500
Subject: [PATCH] expose litellm and add api key authentication

---
 .../bases/litellm/litellm-deployment.yaml     |  1 +
 .../configurations/librechat/librechat.yaml   |  4 ++--
 .../prod/configurations/litellm/config.yml    | 10 +++++++++-
 kustomize/env/prod/kustomization.yaml         | 17 ++++++++++++++++
 .../patches/librechat-deployment-patch.yaml   |  8 ++++++++
 .../prod/patches/librechat-ingress-patch.yaml | 13 ++++++++++++
 .../patches/litellm-deployment-patch.yaml     |  8 ++++++++
 .../prometheus/prometheus-additional.yaml     |  4 ++++
 .../resources/litellm-externalsecret.yaml     | 20 +++++++++++++++++++
 .../prod/resources/litellm-stripprefix.yaml   |  9 +++++++++
 10 files changed, 91 insertions(+), 3 deletions(-)
 create mode 100644 kustomize/env/prod/patches/librechat-deployment-patch.yaml
 create mode 100644 kustomize/env/prod/patches/librechat-ingress-patch.yaml
 create mode 100644 kustomize/env/prod/patches/litellm-deployment-patch.yaml
 create mode 100644 kustomize/env/prod/patches/prometheus/prometheus-additional.yaml
 create mode 100644 kustomize/env/prod/resources/litellm-externalsecret.yaml
 create mode 100644 kustomize/env/prod/resources/litellm-stripprefix.yaml

diff --git a/kustomize/bases/litellm/litellm-deployment.yaml b/kustomize/bases/litellm/litellm-deployment.yaml
index 63af181..1f038e2 100644
--- a/kustomize/bases/litellm/litellm-deployment.yaml
+++ b/kustomize/bases/litellm/litellm-deployment.yaml
@@ -25,6 +25,7 @@ spec:
         - name: litellm
           image: ghcr.io/berriai/litellm:main-latest
           args: ['--config', '/config/config.yml']
+          env: []
           resources:
             requests:
               memory: 200Mi
diff --git a/kustomize/env/prod/configurations/librechat/librechat.yaml b/kustomize/env/prod/configurations/librechat/librechat.yaml
index 6cba07f..a6a9b00 100644
--- a/kustomize/env/prod/configurations/librechat/librechat.yaml
+++ b/kustomize/env/prod/configurations/librechat/librechat.yaml
@@ -4,10 +4,10 @@ endpoints:
   custom:
       # Example using Mistral AI API
     - name: "Mistral"
-      apiKey: "noUse"
+      apiKey: "${LITELLM_MASTER_KEY}"
       baseURL: "http://librechat-litellm.llm.svc:8000"
       models: 
-        default: ["mistral-7b", "mistral-openorca"]
+        default: ["mistral-7b", "mistral-openorca", "dolphin-mistral"]
       titleConvo: true
       titleModel: "mistral-7b" 
       summarize: true
diff --git a/kustomize/env/prod/configurations/litellm/config.yml b/kustomize/env/prod/configurations/litellm/config.yml
index d2141cb..2807bdf 100644
--- a/kustomize/env/prod/configurations/litellm/config.yml
+++ b/kustomize/env/prod/configurations/litellm/config.yml
@@ -8,4 +8,12 @@ model_list:
     litellm_params:
       model: ollama/mistral-openorca
       api_base: http://192.168.30.20:11434
-      # stream: True
\ No newline at end of file
+      # stream: True
+  - model_name: dolphin-mistral
+    litellm_params:
+      model: ollama/dolphin-mistral
+      api_base: http://192.168.30.20:11434
+      # stream: True
+
+general_settings:
+  master_key: "os.environ/MASTER_KEY"
\ No newline at end of file
diff --git a/kustomize/env/prod/kustomization.yaml b/kustomize/env/prod/kustomization.yaml
index 48038b8..d062563 100644
--- a/kustomize/env/prod/kustomization.yaml
+++ b/kustomize/env/prod/kustomization.yaml
@@ -13,6 +13,8 @@ resources:
   - ../../overlays/actual
   - ../../overlays/llm
   - probes/snmp-exporter.yaml
+  - resources/litellm-externalsecret.yaml
+  - resources/litellm-stripprefix.yaml
 
 # resources:
 #   - probes/external-services-bobcat-miner.yaml
@@ -103,6 +105,21 @@ patches:
       kind: Deployment
       name: deluge-server
     path: patches/deluge-deployment-patch.yaml
+  - target: 
+      version: v1
+      kind: Deployment
+      name: librechat-server
+    path: patches/librechat-deployment-patch.yaml
+  - target: 
+      version: v1
+      kind: Ingress
+      name: librechat-server
+    path: patches/librechat-ingress-patch.yaml
+  - target: 
+      version: v1
+      kind: Deployment
+      name: librechat-litellm
+    path: patches/litellm-deployment-patch.yaml
   # - target:
   #     version: v1
   #     kind: Prometheus
diff --git a/kustomize/env/prod/patches/librechat-deployment-patch.yaml b/kustomize/env/prod/patches/librechat-deployment-patch.yaml
new file mode 100644
index 0000000..78cf1e5
--- /dev/null
+++ b/kustomize/env/prod/patches/librechat-deployment-patch.yaml
@@ -0,0 +1,8 @@
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value:
+    name: LITELLM_MASTER_KEY
+    valueFrom:
+      secretKeyRef:
+        name: litellm-config
+        key: master_key
\ No newline at end of file
diff --git a/kustomize/env/prod/patches/librechat-ingress-patch.yaml b/kustomize/env/prod/patches/librechat-ingress-patch.yaml
new file mode 100644
index 0000000..1905cc0
--- /dev/null
+++ b/kustomize/env/prod/patches/librechat-ingress-patch.yaml
@@ -0,0 +1,13 @@
+- op: add
+  path: /metadata/annotations/traefik.ingress.kubernetes.io~1router.middlewares
+  value: llm-litellm-stripprefix@kubernetescrd
+- op: add
+  path: /spec/rules/0/http/paths/-
+  value:
+    path: /backend
+    pathType: Prefix
+    backend:
+      service:
+        name: librechat-litellm
+        port:
+          name: http
diff --git a/kustomize/env/prod/patches/litellm-deployment-patch.yaml b/kustomize/env/prod/patches/litellm-deployment-patch.yaml
new file mode 100644
index 0000000..7d2dcb3
--- /dev/null
+++ b/kustomize/env/prod/patches/litellm-deployment-patch.yaml
@@ -0,0 +1,8 @@
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value:
+    name: MASTER_KEY
+    valueFrom:
+      secretKeyRef:
+        name: litellm-config
+        key: master_key
\ No newline at end of file
diff --git a/kustomize/env/prod/patches/prometheus/prometheus-additional.yaml b/kustomize/env/prod/patches/prometheus/prometheus-additional.yaml
new file mode 100644
index 0000000..da94e1c
--- /dev/null
+++ b/kustomize/env/prod/patches/prometheus/prometheus-additional.yaml
@@ -0,0 +1,4 @@
+- job_name: additional/bastion-haproxy
+  static_configs:
+  - targets: ["192.168.20.10:8080"]
+  
\ No newline at end of file
diff --git a/kustomize/env/prod/resources/litellm-externalsecret.yaml b/kustomize/env/prod/resources/litellm-externalsecret.yaml
new file mode 100644
index 0000000..f2fdae5
--- /dev/null
+++ b/kustomize/env/prod/resources/litellm-externalsecret.yaml
@@ -0,0 +1,20 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: litellm-config
+  namespace: llm
+spec:
+  secretStoreRef:
+    name: aws-parameters-store
+    kind: ClusterSecretStore
+  target:
+    name: litellm-config
+    template:
+      metadata:
+        labels:
+          app.kubernetes.io/managed-by: external-secret
+        annotations: {}
+  data:
+    - secretKey: master_key
+      remoteRef:
+        key: /k3s/prod/llm/litellm/master_key
\ No newline at end of file
diff --git a/kustomize/env/prod/resources/litellm-stripprefix.yaml b/kustomize/env/prod/resources/litellm-stripprefix.yaml
new file mode 100644
index 0000000..8a4ce09
--- /dev/null
+++ b/kustomize/env/prod/resources/litellm-stripprefix.yaml
@@ -0,0 +1,9 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: litellm-stripprefix
+  namespace: llm
+spec:
+  stripPrefix:
+    prefixes:
+      - /backend
\ No newline at end of file