working prototype using ollama

.env

@@ -0,0 +1,326 @@
+# HF_TOKEN=
+PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.9,max_split_size_mb:512
+HSA_OVERRIDE_GFX_VERSION=11.0.0 # GFX1101/Navi32 (RX 7800XT)
+
+#=============================================================#
+#                   LibreChat Configuration                   #
+#=============================================================#
+# Please refer to the reference documentation for assistance  #
+# with configuring your LibreChat environment. The guide is   #
+# available both online and within your local LibreChat       #
+# directory:                                                  #
+# Online: https://docs.librechat.ai/install/dotenv.html       #
+# Locally: ./docs/install/dotenv.md                           #
+#=============================================================#
+
+#==================================================#
+#               Server Configuration               #
+#==================================================#
+
+APP_TITLE=LibreChat
+# CUSTOM_FOOTER="My custom footer"
+
+HOST=localhost
+PORT=3080
+
+MONGO_URI=mongodb://127.0.0.1:27017/LibreChat
+
+DOMAIN_CLIENT=http://localhost:3080
+DOMAIN_SERVER=http://localhost:3080
+
+NO_INDEX=true
+
+#===============#
+# Debug Logging #
+#===============#
+
+# DEBUG_LOGGING=true
+# DEBUG_CONSOLE=false
+
+#=============#
+# Permissions #
+#=============#
+
+# UID=1000
+# GID=1000
+
+#===================================================#
+#                     Endpoints                     #
+#===================================================#
+
+# ENDPOINTS=openAI,azureOpenAI,bingAI,chatGPTBrowser,google,gptPlugins,anthropic
+
+# PROXY=
+
+#============#
+# Anthropic  #
+#============#
+
+# ANTHROPIC_API_KEY=user_provided
+# ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2
+# ANTHROPIC_REVERSE_PROXY=
+
+#============#
+# Azure      #
+#============#
+
+# AZURE_API_KEY=
+# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4
+# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo
+# PLUGINS_USE_AZURE="true"
+
+# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE
+
+# AZURE_OPENAI_API_INSTANCE_NAME=
+# AZURE_OPENAI_API_DEPLOYMENT_NAME=
+# AZURE_OPENAI_API_VERSION=
+# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME=
+# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=
+
+#============#
+# BingAI     #
+#============#
+
+# BINGAI_TOKEN=user_provided
+# BINGAI_HOST=https://cn.bing.com
+
+#============#
+# ChatGPT    #
+#============#
+
+# CHATGPT_TOKEN=
+# CHATGPT_MODELS=text-davinci-002-render-sha
+# CHATGPT_REVERSE_PROXY=
+
+#============#
+# Google     #
+#============#
+
+# GOOGLE_KEY=user_provided
+# GOOGLE_MODELS=gemini-pro,gemini-pro-vision,chat-bison,chat-bison-32k,codechat-bison,codechat-bison-32k,text-bison,text-bison-32k,text-unicorn,code-gecko,code-bison,code-bison-32k
+# GOOGLE_REVERSE_PROXY=
+
+#============#
+# OpenAI     #
+#============#
+
+# OPENAI_API_KEY=user_provided
+# OPENAI_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k
+
+# DEBUG_OPENAI=false
+
+# TITLE_CONVO=false
+# OPENAI_TITLE_MODEL=gpt-3.5-turbo
+
+# OPENAI_SUMMARIZE=true
+# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo
+
+# OPENAI_FORCE_PROMPT=true
+
+# OPENAI_REVERSE_PROXY=
+
+# OPENAI_ORGANIZATION= 
+
+#============#
+# OpenRouter #
+#============#
+
+# OPENROUTER_API_KEY=
+
+#============#
+# Plugins    #
+#============#
+
+# PLUGIN_MODELS=gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613
+
+# DEBUG_PLUGINS=true
+
+CREDS_KEY=f34be427ebb29de8d88c107a71546019685ed8b241d8f2ed00c3df97ad2566f0
+CREDS_IV=e2341419ec3dd3d19b13a1a87fafcbfb
+
+# Azure AI Search
+#-----------------
+# AZURE_AI_SEARCH_SERVICE_ENDPOINT=
+# AZURE_AI_SEARCH_INDEX_NAME=
+# AZURE_AI_SEARCH_API_KEY=
+
+# AZURE_AI_SEARCH_API_VERSION=
+# AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE=
+# AZURE_AI_SEARCH_SEARCH_OPTION_TOP=
+# AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
+
+# DALL·E
+#----------------
+# DALLE_API_KEY=
+# DALLE3_API_KEY=
+# DALLE2_API_KEY=
+# DALLE3_SYSTEM_PROMPT=
+# DALLE2_SYSTEM_PROMPT=
+# DALLE_REVERSE_PROXY=
+# DALLE3_BASEURL=
+# DALLE2_BASEURL=
+
+# DALL·E (via Azure OpenAI)
+# Note: requires some of the variables above to be set
+#----------------
+# DALLE3_AZURE_API_VERSION=
+# DALLE2_AZURE_API_VERSION=
+
+# Google
+#-----------------
+# GOOGLE_API_KEY=
+# GOOGLE_CSE_ID=
+
+# SerpAPI
+#-----------------
+# SERPAPI_API_KEY=
+
+# Stable Diffusion
+#-----------------
+# SD_WEBUI_URL=http://host.docker.internal:7860
+
+# WolframAlpha
+#-----------------
+# WOLFRAM_APP_ID=
+
+# Zapier
+#-----------------
+# ZAPIER_NLA_API_KEY=
+
+#==================================================#
+#                      Search                      #
+#==================================================#
+
+SEARCH=true
+MEILI_NO_ANALYTICS=true
+MEILI_HOST=http://0.0.0.0:7700
+MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
+
+#===================================================#
+#                    User System                    #
+#===================================================#
+
+#========================#
+# Moderation             #
+#========================#
+
+# OPENAI_MODERATION=false
+# OPENAI_MODERATION_API_KEY=
+# # OPENAI_MODERATION_REVERSE_PROXY=
+
+# BAN_VIOLATIONS=true
+# BAN_DURATION=1000 * 60 * 60 * 2
+# BAN_INTERVAL=20
+
+# LOGIN_VIOLATION_SCORE=1
+# REGISTRATION_VIOLATION_SCORE=1
+# CONCURRENT_VIOLATION_SCORE=1
+# MESSAGE_VIOLATION_SCORE=1
+# NON_BROWSER_VIOLATION_SCORE=20
+
+# LOGIN_MAX=7
+# LOGIN_WINDOW=5
+# REGISTER_MAX=5
+# REGISTER_WINDOW=60
+
+# LIMIT_CONCURRENT_MESSAGES=true
+# CONCURRENT_MESSAGE_MAX=2
+
+# LIMIT_MESSAGE_IP=true
+# MESSAGE_IP_MAX=40
+# MESSAGE_IP_WINDOW=1
+
+# LIMIT_MESSAGE_USER=false
+# MESSAGE_USER_MAX=40
+# MESSAGE_USER_WINDOW=1
+
+#========================#
+# Balance                #
+#========================#
+
+# CHECK_BALANCE=false
+
+#========================#
+# Registration and Login #
+#========================#
+
+ALLOW_EMAIL_LOGIN=true
+ALLOW_REGISTRATION=true
+ALLOW_SOCIAL_LOGIN=false
+ALLOW_SOCIAL_REGISTRATION=false
+
+SESSION_EXPIRY=1000 * 60 * 15
+REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7
+
+JWT_SECRET=16f8c0ef4a5d391b26034086c628469d3f9f497f08163ab9b40137092f2909ef
+JWT_REFRESH_SECRET=eaa5191f2914e30b9387fd84e254e4ba6fc51b4654968a9b0803b456a54b8418
+
+# Discord
+# DISCORD_CLIENT_ID=
+# DISCORD_CLIENT_SECRET=
+# DISCORD_CALLBACK_URL=/oauth/discord/callback
+
+# # Facebook
+# FACEBOOK_CLIENT_ID=
+# FACEBOOK_CLIENT_SECRET=
+# FACEBOOK_CALLBACK_URL=/oauth/facebook/callback
+
+# # GitHub
+# GITHUB_CLIENT_ID=
+# GITHUB_CLIENT_SECRET=
+# GITHUB_CALLBACK_URL=/oauth/github/callback
+
+# # Google
+# GOOGLE_CLIENT_ID=
+# GOOGLE_CLIENT_SECRET=
+# GOOGLE_CALLBACK_URL=/oauth/google/callback
+
+# # OpenID
+# OPENID_CLIENT_ID=
+# OPENID_CLIENT_SECRET=
+# OPENID_ISSUER=
+# OPENID_SESSION_SECRET=
+# OPENID_SCOPE="openid profile email"
+# OPENID_CALLBACK_URL=/oauth/openid/callback
+
+# OPENID_BUTTON_LABEL=
+# OPENID_IMAGE_URL=
+
+# #========================#
+# # Email Password Reset   #
+# #========================#
+
+# EMAIL_SERVICE=                  
+# EMAIL_HOST=                     
+# EMAIL_PORT=25                   
+# EMAIL_ENCRYPTION=               
+# EMAIL_ENCRYPTION_HOSTNAME=      
+# EMAIL_ALLOW_SELFSIGNED=         
+# EMAIL_USERNAME=                 
+# EMAIL_PASSWORD=                 
+# EMAIL_FROM_NAME=                
+# EMAIL_FROM=noreply@librechat.ai
+
+# #========================#
+# # Firebase CDN           #
+# #========================#
+
+# FIREBASE_API_KEY=
+# FIREBASE_AUTH_DOMAIN=
+# FIREBASE_PROJECT_ID=
+# FIREBASE_STORAGE_BUCKET=
+# FIREBASE_MESSAGING_SENDER_ID=
+# FIREBASE_APP_ID=
+
+#==================================================#
+#                      Others                      #
+#==================================================#
+#   You should leave the following commented out   #
+
+# NODE_ENV=
+
+# REDIS_URI=
+# USE_REDIS=
+
+# E2E_USER_EMAIL=
+# E2E_USER_PASSWORD=

.gitignore

@@ -0,0 +1 @@
+.env.override

docker-compose.yml

@@ -0,0 +1,71 @@
+version: "3"
+services:
+  ollama:
+    image: ollama/ollama:0.1.22-rocm
+    restart: unless-stopped
+    entrypoint: /bootstrap.sh
+    env_file:
+      - .env
+    ports:
+     - 11434:11434
+    devices:
+      - /dev/kfd
+      - /dev/dri
+    group_add:
+      - video
+    ipc: host
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp=unconfined
+    volumes:
+      - ./ollama/bootstrap.sh:/bootstrap.sh:ro
+      - ollama:/root/.ollama
+
+  litellm:
+    image: ghcr.io/berriai/litellm:main-latest
+    command: --config /config.yaml
+    ports:
+     - 8000:8000
+    volumes:
+      - ./litellm/config.yaml:/config.yaml:ro
+
+  librechat:
+    image: ghcr.io/danny-avila/librechat
+    ports:
+      - 3080:3080
+    depends_on:
+      - mongodb
+    restart: unless-stopped
+    # user: "${UID}:${GID}"
+    env_file:
+      - .env
+    environment:
+      - HOST=0.0.0.0
+      - MONGO_URI=mongodb://mongodb:27017/LibreChat
+      - MEILI_HOST=http://meilisearch:7700
+    volumes:
+      - ./librechat/librechat.yaml:/app/librechat.yaml:ro
+      - librechat_images:/app/client/public/images
+  mongodb:
+    image: mongo
+    restart: unless-stopped
+    # user: "${UID}:${GID}"
+    volumes:
+      - librechat_mongodb_data:/data/db
+    command: mongod --noauth
+  meilisearch:
+    image: getmeili/meilisearch:v1.5
+    restart: unless-stopped
+    # user: "${UID}:${GID}"
+    environment:
+      - MEILI_HOST=http://meilisearch:7700
+      - MEILI_NO_ANALYTICS=true
+    volumes:
+      - librechat_meilisearch_data:/meili_data
+
+volumes:
+  ollama:
+  librechat_images:
+  librechat_mongodb_data:
+  librechat_meilisearch_data:

librechat/librechat.yaml

@@ -0,0 +1,20 @@
+version: 1.0.1
+cache: true
+endpoints:
+  custom:
+      # Example using Mistral AI API
+    - name: "Mistral"
+      apiKey: "noUse"
+      baseURL: "http://litellm:8000"
+      models: 
+        default: ["mistral-7b"]
+      titleConvo: true
+      titleModel: "mistral-7b" 
+      summarize: true
+      summaryModel: "mistral-7b" 
+      forcePrompt: false 
+      modelDisplayLabel: "Mistral"
+      # addParams:
+      #   safe_prompt: true
+      # NOTE: For Mistral, it is necessary to drop the following parameters or you will encounter a 422 Error:
+      dropParams: ["stop", "user", "frequency_penalty", "presence_penalty"]

litellm/config.yaml

@@ -0,0 +1,6 @@
+model_list:
+  - model_name: mistral-7b
+    litellm_params:
+      model: ollama/mistral
+      api_base: http://ollama:11434
+      # stream: True

ollama/bootstrap.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -x
+
+ollama serve &
+
+sleep 1
+
+for model in ${@:-mistral}; do
+    ollama pull "$model"
+done
+
+wait