expose litellm and add api key authentication

kustomize/bases/librechat/kustomization.yaml

@@ -0,0 +1,29 @@
+resources:
+  - ../mongodb
+  - ../litellm
+  - librechat-deployment.yaml
+  - librechat-externalsecret.yaml
+  - librechat-ingress.yaml
+
+namePrefix: librechat-
+
+configMapGenerator:
+  - name: kustomize-generated-config
+    literals:
+      - LIBRECHAT_EXTERNAL_HOST=chat.badjware.dev
+      - LIBRECHAT_EXTERNAL_URL=https://chat.badjware.dev
+  - name: server-config
+    literals:
+      - librechat.yaml=
+
+replacements:
+  - source:
+      kind: ConfigMap
+      name: kustomize-generated-config
+      fieldPath: data.LIBRECHAT_EXTERNAL_HOST
+    targets:
+      - select:
+          kind: Ingress
+          name: server
+        fieldPaths:
+          - spec.rules.0.host

kustomize/bases/librechat/librechat-deployment.yaml

@@ -0,0 +1,152 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: server
+  labels:
+    app.kubernetes.io/name: librechat
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: librechat
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: librechat
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  # Image only supports amd64
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                      - amd64
+      containers:
+        - name: librechat
+          image: ghcr.io/danny-avila/librechat
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: DOMAIN_CLIENT
+              valueFrom:
+                configMapKeyRef:
+                  name: kustomize-generated-config
+                  key: LIBRECHAT_EXTERNAL_URL
+            - name: DOMAIN_SERVER
+              valueFrom:
+                configMapKeyRef:
+                  name: kustomize-generated-config
+                  key: LIBRECHAT_EXTERNAL_URL
+            - name: MONGO_URI
+              value: mongodb://librechat-mongodb.$(NAMESPACE).svc:27017/LibreChat
+            - name: SEARCH
+              value: 'false' # TODO
+            - name: CREDS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: server-tokens
+                  key: creds_key
+            - name: CREDS_IV
+              valueFrom:
+                secretKeyRef:
+                  name: server-tokens
+                  key: creds_iv
+            - name: JWT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: server-tokens
+                  key: jwt_secret
+            - name: JWT_REFRESH_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: server-tokens
+                  key: jwt_refresh_secret
+            - name: ALLOW_EMAIL_LOGIN
+              value: 'false'
+            - name: ALLOW_REGISTRATION
+              value: 'false'
+            - name: ALLOW_SOCIAL_LOGIN
+              value: 'true'
+            - name: ALLOW_SOCIAL_REGISTRATION
+              value: 'true'
+            - name: OPENID_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: server-openid-config
+                  key: openid_client_id
+            - name: OPENID_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: server-openid-config
+                  key: openid_client_secret
+            - name: OPENID_ISSUER
+              valueFrom:
+                secretKeyRef:
+                  name: server-openid-config
+                  key: openid_issuer
+            - name: OPENID_SESSION_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: server-openid-config
+                  key: openid_session_secret
+            - name: OPENID_SCOPE
+              value: openid profile email
+            - name: OPENID_CALLBACK_URL
+              value: /oauth/openid/callback
+          ports:
+            - name: http
+              containerPort: 3080
+          resources:
+            requests:
+              cpu: 250m
+              memory: 500Mi
+            limits:
+              cpu: 1000m
+              memory: 500Mi
+          volumeMounts:
+            - name: server-images-pv
+              mountPath: /app/client/public/images
+            - name: server-config
+              mountPath: /app/librechat.yaml
+              subPath: librechat.yaml
+      volumes:
+        - name: server-images-pv
+          persistentVolumeClaim:
+            claimName: server-images-pvc
+        - name: server-config
+          configMap:
+            name: server-config
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: server-images-pvc
+  labels:
+    app.kubernetes.io/name: librechat
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: server
+  labels:
+    app.kubernetes.io/name: librechat
+spec:
+  selector:
+    app.kubernetes.io/name: librechat
+  ports:
+    - name: http
+      port: 3080
+      targetPort: http

kustomize/bases/librechat/librechat-externalsecret.yaml

@@ -0,0 +1,57 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: server-tokens
+spec:
+  secretStoreRef:
+    name: aws-parameters-store
+    kind: ClusterSecretStore
+  target:
+    name: server-tokens
+    template:
+      metadata:
+        labels:
+          app.kubernetes.io/managed-by: external-secret
+        annotations: {}
+  data:
+    - secretKey: creds_key
+      remoteRef:
+        key: /k3s/prod/llm/librechat/creds_key
+    - secretKey: creds_iv
+      remoteRef:
+        key: /k3s/prod/llm/librechat/creds_iv
+    - secretKey: jwt_secret
+      remoteRef:
+        key: /k3s/prod/llm/librechat/jwt_secret
+    - secretKey: jwt_refresh_secret
+      remoteRef:
+        key: /k3s/prod/llm/librechat/jwt_refresh_secret
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: server-openid-config
+spec:
+  secretStoreRef:
+    name: aws-parameters-store
+    kind: ClusterSecretStore
+  target:
+    name: server-openid-config
+    template:
+      metadata:
+        labels:
+          app.kubernetes.io/managed-by: external-secret
+        annotations: {}
+  data:
+    - secretKey: openid_client_id
+      remoteRef:
+        key: /k3s/prod/llm/librechat/openid_client_id
+    - secretKey: openid_client_secret
+      remoteRef:
+        key: /k3s/prod/llm/librechat/openid_client_secret
+    - secretKey: openid_issuer
+      remoteRef:
+        key: /k3s/prod/llm/librechat/openid_issuer
+    - secretKey: openid_session_secret
+      remoteRef:
+        key: /k3s/prod/llm/librechat/openid_session_secret

kustomize/bases/librechat/librechat-ingress.yaml

@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: server
+  labels:
+    app.kubernetes.io/name: librechat
+    probe: blackbox-http
+spec:
+  rules:
+  - host: ${LIBRECHAT_EXTERNAL_HOST}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: server
+            port:
+              name: http

kustomize/bases/litellm/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - litellm-deployment.yaml
+
+commonLabels:
+  app.kubernetes.io/component: litellm
+
+configMapGenerator:
+  - name: litellm-config
+    literals:
+      - config.yml=

kustomize/bases/litellm/litellm-deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: litellm
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: litellm
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: litellm
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  # Image only supports amd64
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values:
+                      - amd64
+      containers:
+        - name: litellm
+          image: ghcr.io/berriai/litellm:main-latest
+          args: ['--config', '/config/config.yml']
+          env: []
+          resources:
+            requests:
+              memory: 200Mi
+              cpu: 200m
+            limits:
+              memory: 200Mi
+              cpu: 200m
+          ports:
+            - containerPort: 8000
+              name: http
+          volumeMounts:
+            - name: litellm-config
+              mountPath: /config
+      volumes:
+        - name: litellm-config
+          configMap:
+            name: litellm-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: litellm
+  labels:
+    app.kubernetes.io/component: litellm
+spec:
+  selector:
+    app.kubernetes.io/component: litellm
+  ports:
+    - name: http
+      port: 8000
+      targetPort: http

kustomize/bases/mongodb/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - mongodb-statefulset.yaml
+
+commonLabels:
+  app.kubernetes.io/component: mongodb

kustomize/bases/mongodb/mongodb-statefulset.yaml

@@ -0,0 +1,56 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mongodb
+  labels:
+    app.kubernetes.io/component: mongodb
+spec:
+  serviceName: mongodb
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: mongodb
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: mongodb
+    spec:
+      containers:
+        - name: mongodb
+          image: mongo:4.4.18 # NOTE: this is the last version with raspberry pi 4 support
+          ports:
+            - name: mongodb
+              containerPort: 27017
+          resources:
+            requests:
+              cpu: 500m
+              memory: 500Mi
+            limits:
+              cpu: 500m
+              memory: 500Mi
+          volumeMounts:
+            - name: mongodb-pvc
+              mountPath: /data/db
+  volumeClaimTemplates:
+    - metadata:
+        name: mongodb-pvc
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 5Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mongodb
+  labels:
+    app.kubernetes.io/component: mongodb
+spec:
+  selector:
+    app.kubernetes.io/component: mongodb
+  ports:
+    - name: mongodb
+      port: 27017
+      targetPort: mongodb

kustomize/env/prod/configurations/librechat/librechat.yaml

@@ -0,0 +1,20 @@
+version: 1.0.1
+cache: true
+endpoints:
+  custom:
+      # Example using Mistral AI API
+    - name: "Mistral"
+      apiKey: "${LITELLM_MASTER_KEY}"
+      baseURL: "http://librechat-litellm.llm.svc:8000"
+      models: 
+        default: ["mistral-7b", "mistral-openorca", "dolphin-mistral"]
+      titleConvo: true
+      titleModel: "mistral-7b" 
+      summarize: true
+      summaryModel: "mistral-7b" 
+      forcePrompt: false 
+      modelDisplayLabel: "Mistral"
+      # addParams:
+      #   safe_prompt: true
+      # NOTE: For Mistral, it is necessary to drop the following parameters or you will encounter a 422 Error:
+      dropParams: ["stop", "user", "frequency_penalty", "presence_penalty"]

kustomize/env/prod/configurations/litellm/config.yml

@@ -0,0 +1,19 @@
+model_list:
+  - model_name: mistral-7b
+    litellm_params:
+      model: ollama/mistral
+      api_base: http://192.168.30.20:11434
+      # stream: True
+  - model_name: mistral-openorca
+    litellm_params:
+      model: ollama/mistral-openorca
+      api_base: http://192.168.30.20:11434
+      # stream: True
+  - model_name: dolphin-mistral
+    litellm_params:
+      model: ollama/dolphin-mistral
+      api_base: http://192.168.30.20:11434
+      # stream: True
+
+general_settings:
+  master_key: "os.environ/MASTER_KEY"

kustomize/env/prod/kustomization.yaml

@@ -11,7 +11,10 @@ resources:
   - ../../overlays/jellyfin
   - ../../overlays/deluge
   - ../../overlays/actual
+  - ../../overlays/llm
   - probes/snmp-exporter.yaml
+  - resources/litellm-externalsecret.yaml
+  - resources/litellm-stripprefix.yaml
 
 # resources:
 #   - probes/external-services-bobcat-miner.yaml
@@ -52,7 +55,17 @@ images:
   - name: bitnami/kubectl
     newTag: "1.26"
 
-# configMapGenerator:
+configMapGenerator:
+  - name: librechat-server-config
+    namespace: llm
+    behavior: replace
+    files:
+      - librechat.yaml=configurations/librechat/librechat.yaml
+  - name: librechat-litellm-config
+    namespace: llm
+    behavior: replace
+    files:
+      - config.yml=configurations/litellm/config.yml
 #   - name: home-assistant-server-config
 #     namespace: home-assistant
 #     behavior: replace
@@ -92,6 +105,21 @@ patches:
       kind: Deployment
       name: deluge-server
     path: patches/deluge-deployment-patch.yaml
+  - target: 
+      version: v1
+      kind: Deployment
+      name: librechat-server
+    path: patches/librechat-deployment-patch.yaml
+  - target: 
+      version: v1
+      kind: Ingress
+      name: librechat-server
+    path: patches/librechat-ingress-patch.yaml
+  - target: 
+      version: v1
+      kind: Deployment
+      name: librechat-litellm
+    path: patches/litellm-deployment-patch.yaml
   # - target:
   #     version: v1
   #     kind: Prometheus

kustomize/env/prod/patches/librechat-deployment-patch.yaml

@@ -0,0 +1,8 @@
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value:
+    name: LITELLM_MASTER_KEY
+    valueFrom:
+      secretKeyRef:
+        name: litellm-config
+        key: master_key

kustomize/env/prod/patches/librechat-ingress-patch.yaml

@@ -0,0 +1,13 @@
+- op: add
+  path: /metadata/annotations/traefik.ingress.kubernetes.io~1router.middlewares
+  value: llm-litellm-stripprefix@kubernetescrd
+- op: add
+  path: /spec/rules/0/http/paths/-
+  value:
+    path: /backend
+    pathType: Prefix
+    backend:
+      service:
+        name: librechat-litellm
+        port:
+          name: http

kustomize/env/prod/patches/litellm-deployment-patch.yaml

@@ -0,0 +1,8 @@
+- op: add
+  path: /spec/template/spec/containers/0/env/-
+  value:
+    name: MASTER_KEY
+    valueFrom:
+      secretKeyRef:
+        name: litellm-config
+        key: master_key

kustomize/env/prod/patches/prometheus/prometheus-additional.yaml

@@ -0,0 +1,4 @@
+- job_name: additional/bastion-haproxy
+  static_configs:
+  - targets: ["192.168.20.10:8080"]
+  

kustomize/env/prod/resources/litellm-externalsecret.yaml

@@ -0,0 +1,20 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: litellm-config
+  namespace: llm
+spec:
+  secretStoreRef:
+    name: aws-parameters-store
+    kind: ClusterSecretStore
+  target:
+    name: litellm-config
+    template:
+      metadata:
+        labels:
+          app.kubernetes.io/managed-by: external-secret
+        annotations: {}
+  data:
+    - secretKey: master_key
+      remoteRef:
+        key: /k3s/prod/llm/litellm/master_key

kustomize/env/prod/resources/litellm-stripprefix.yaml

@@ -0,0 +1,9 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: litellm-stripprefix
+  namespace: llm
+spec:
+  stripPrefix:
+    prefixes:
+      - /backend

kustomize/overlays/llm/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - namespace.yaml
+  - ../../bases/librechat
+
+namespace: llm

kustomize/overlays/llm/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: llm