From 2806142962ba1bc1a19814b0a98e048e7fa11622 Mon Sep 17 00:00:00 2001
From: Massaki Archambault <marchambault@badjware.dev>
Date: Wed, 7 Aug 2024 21:08:02 -0400
Subject: [PATCH] add ingress for ollama and remote litellm

---
 kustomize/bases/ollama/kustomization.yaml     | 19 +++++++++++++++++++
 kustomize/bases/ollama/ollama-deployment.yaml |  4 +++-
 kustomize/bases/ollama/ollama-ingress.yaml    | 19 +++++++++++++++++++
 .../bases/openwebui/openwebui-deployment.yaml |  9 ++++++++-
 .../prod/configurations/litellm/config.yaml   | 12 ++----------
 kustomize/env/prod/kustomization.yaml         | 15 +++++++++------
 kustomize/overlays/llm/kustomization.yaml     |  2 +-
 7 files changed, 61 insertions(+), 19 deletions(-)
 create mode 100644 kustomize/bases/ollama/ollama-ingress.yaml

diff --git a/kustomize/bases/ollama/kustomization.yaml b/kustomize/bases/ollama/kustomization.yaml
index f61f906..39655b4 100644
--- a/kustomize/bases/ollama/kustomization.yaml
+++ b/kustomize/bases/ollama/kustomization.yaml
@@ -1,7 +1,26 @@
 resources:
   - ollama-deployment.yaml
+  - ollama-ingress.yaml
 
 namePrefix: ollama-
 
 commonLabels:
   app.kubernetes.io/name: ollama
+
+configMapGenerator:
+  - name: kustomize-generated-config
+    literals:
+      - OLLAMA_EXTERNAL_HOST=ollama.badjnet.home
+      - OLLAMA_EXTERNAL_URL=http://ollama.badjnet.home
+
+replacements:
+  - source:
+      kind: ConfigMap
+      name: kustomize-generated-config
+      fieldPath: data.OLLAMA_EXTERNAL_HOST
+    targets:
+      - select:
+          kind: Ingress
+          name: server
+        fieldPaths:
+          - spec.rules.0.host
diff --git a/kustomize/bases/ollama/ollama-deployment.yaml b/kustomize/bases/ollama/ollama-deployment.yaml
index c7aad52..8a1bee4 100644
--- a/kustomize/bases/ollama/ollama-deployment.yaml
+++ b/kustomize/bases/ollama/ollama-deployment.yaml
@@ -30,7 +30,9 @@ spec:
         - name: server
           image: ollama/ollama
           # image: badjware/ollama-tweak
-          # env:
+          env:
+            - name: OLLAMA_KEEP_ALIVE
+              value: 36h
             # - name: HSA_OVERRIDE_GFX_VERSION
             #   value: 10.1.0
             # - name: HSA_ENABLE_SDMA
diff --git a/kustomize/bases/ollama/ollama-ingress.yaml b/kustomize/bases/ollama/ollama-ingress.yaml
new file mode 100644
index 0000000..8cf7956
--- /dev/null
+++ b/kustomize/bases/ollama/ollama-ingress.yaml
@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: server
+  labels:
+    app.kubernetes.io/name: ollama
+    probe: blackbox-http
+spec:
+  rules:
+  - host: ${OLLAMA_EXTERNAL_HOST}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: server
+            port:
+              name: http
diff --git a/kustomize/bases/openwebui/openwebui-deployment.yaml b/kustomize/bases/openwebui/openwebui-deployment.yaml
index 42383c7..be7adcc 100644
--- a/kustomize/bases/openwebui/openwebui-deployment.yaml
+++ b/kustomize/bases/openwebui/openwebui-deployment.yaml
@@ -17,6 +17,7 @@ spec:
       containers:
         - name: server
           image: ghcr.io/open-webui/open-webui:main
+          tty: true
           env: 
             - name: NAMESPACE
               valueFrom:
@@ -24,10 +25,16 @@ spec:
                   fieldPath: metadata.namespace
             - name: OLLAMA_BASE_URL
               value: http://ollama-server.$(NAMESPACE).svc:11434
+            - name: RAG_EMBEDDING_ENGINE
+              value: ollama
+            - name: AUDIO_STT_ENGINE
+              value: openai
+            # - name: GLOBAL_LOG_LEVEL
+            #   value: DEBUG
           resources:
             requests:
               cpu: 250m
-              memory: 1Gi
+              memory: 500Mi
             limits:
               cpu: 500m
               memory: 1Gi
diff --git a/kustomize/env/prod/configurations/litellm/config.yaml b/kustomize/env/prod/configurations/litellm/config.yaml
index 74ef45a..1634e22 100644
--- a/kustomize/env/prod/configurations/litellm/config.yaml
+++ b/kustomize/env/prod/configurations/litellm/config.yaml
@@ -1,13 +1,5 @@
 model_list:
-  - model_name: llama3-8b
+  - model_name: "*"
     litellm_params:
-      model: ollama_chat/llama3
+      model: "ollama_chat/*"
       api_base: http://ollama-server:11434
-  - model_name: gemma2-9b
-    litellm_params:
-      model: ollama_chat/gemma2
-      api_base: http://ollama-server:11434
-  - model_name: phi3-3b
-    litellm_params:
-      model: ollama_chat/phi3:mini
-      api_base: http://ollama-server:11434
\ No newline at end of file
diff --git a/kustomize/env/prod/kustomization.yaml b/kustomize/env/prod/kustomization.yaml
index 6c7bea8..efca3e7 100644
--- a/kustomize/env/prod/kustomization.yaml
+++ b/kustomize/env/prod/kustomization.yaml
@@ -54,15 +54,18 @@ images:
     newTag: "1.29"
   - name: ollama/ollama
     newTag: 0.3.3
-    # newTag: 0.3.3-rocm
+  - name: ghcr.io/berriai/litellm
+    newTag: main-v1.43.1
+  - name: ghcr.io/open-webui/open-webui
+    newTag: v0.3.10 # https://github.com/open-webui/open-webui/discussions/4331
 
 
 configMapGenerator:
-  - name: litellm-proxy-config
-    namespace: llm
-    behavior: replace
-    files:
-      - config.yaml=./configurations/litellm/config.yaml
+  # - name: litellm-proxy-config
+  #   namespace: llm
+  #   behavior: replace
+  #   files:
+  #     - config.yaml=./configurations/litellm/config.yaml
 #   - name: home-assistant-server-config
 #     namespace: home-assistant
 #     behavior: replace
diff --git a/kustomize/overlays/llm/kustomization.yaml b/kustomize/overlays/llm/kustomization.yaml
index e19c0e6..699275c 100644
--- a/kustomize/overlays/llm/kustomization.yaml
+++ b/kustomize/overlays/llm/kustomization.yaml
@@ -1,7 +1,7 @@
 resources:
   - namespace.yaml
   - ../../bases/ollama
-  - ../../bases/litellm
+  # - ../../bases/litellm
   - ../../bases/openwebui
   - ../../bases/sillytavern