From a3f6259cc949f10b393beb229488f0bf146e82c4 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Fri, 2 Feb 2024 22:27:22 -0500 Subject: [PATCH] working prototype using ollama --- .env | 326 +++++++++++++++++++++++++++++++++++++++ .gitignore | 1 + docker-compose.yml | 71 +++++++++ librechat/librechat.yaml | 20 +++ litellm/config.yaml | 6 + ollama/bootstrap.sh | 11 ++ 6 files changed, 435 insertions(+) create mode 100644 .env create mode 100644 .gitignore create mode 100644 docker-compose.yml create mode 100644 librechat/librechat.yaml create mode 100644 litellm/config.yaml create mode 100755 ollama/bootstrap.sh diff --git a/.env b/.env new file mode 100644 index 0000000..2710a0f --- /dev/null +++ b/.env @@ -0,0 +1,326 @@ +# HF_TOKEN= +PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.9,max_split_size_mb:512 +HSA_OVERRIDE_GFX_VERSION=11.0.0 # GFX1101/Navi32 (RX 7800XT) + +#=============================================================# +# LibreChat Configuration # +#=============================================================# +# Please refer to the reference documentation for assistance # +# with configuring your LibreChat environment. The guide is # +# available both online and within your local LibreChat # +# directory: # +# Online: https://docs.librechat.ai/install/dotenv.html # +# Locally: ./docs/install/dotenv.md # +#=============================================================# + +#==================================================# +# Server Configuration # +#==================================================# + +APP_TITLE=LibreChat +# CUSTOM_FOOTER="My custom footer" + +HOST=localhost +PORT=3080 + +MONGO_URI=mongodb://127.0.0.1:27017/LibreChat + +DOMAIN_CLIENT=http://localhost:3080 +DOMAIN_SERVER=http://localhost:3080 + +NO_INDEX=true + +#===============# +# Debug Logging # +#===============# + +# DEBUG_LOGGING=true +# DEBUG_CONSOLE=false + +#=============# +# Permissions # +#=============# + +# UID=1000 +# GID=1000 + +#===================================================# +# Endpoints # +#===================================================# + +# ENDPOINTS=openAI,azureOpenAI,bingAI,chatGPTBrowser,google,gptPlugins,anthropic + +# PROXY= + +#============# +# Anthropic # +#============# + +# ANTHROPIC_API_KEY=user_provided +# ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2 +# ANTHROPIC_REVERSE_PROXY= + +#============# +# Azure # +#============# + +# AZURE_API_KEY= +# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 +# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo +# PLUGINS_USE_AZURE="true" + +# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE + +# AZURE_OPENAI_API_INSTANCE_NAME= +# AZURE_OPENAI_API_DEPLOYMENT_NAME= +# AZURE_OPENAI_API_VERSION= +# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= +# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= + +#============# +# BingAI # +#============# + +# BINGAI_TOKEN=user_provided +# BINGAI_HOST=https://cn.bing.com + +#============# +# ChatGPT # +#============# + +# CHATGPT_TOKEN= +# CHATGPT_MODELS=text-davinci-002-render-sha +# CHATGPT_REVERSE_PROXY= + +#============# +# Google # +#============# + +# GOOGLE_KEY=user_provided +# GOOGLE_MODELS=gemini-pro,gemini-pro-vision,chat-bison,chat-bison-32k,codechat-bison,codechat-bison-32k,text-bison,text-bison-32k,text-unicorn,code-gecko,code-bison,code-bison-32k +# GOOGLE_REVERSE_PROXY= + +#============# +# OpenAI # +#============# + +# OPENAI_API_KEY=user_provided +# OPENAI_MODELS=gpt-3.5-turbo-0125,gpt-3.5-turbo-0301,gpt-3.5-turbo,gpt-4,gpt-4-0613,gpt-4-vision-preview,gpt-3.5-turbo-0613,gpt-3.5-turbo-16k-0613,gpt-4-0125-preview,gpt-4-turbo-preview,gpt-4-1106-preview,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-instruct-0914,gpt-3.5-turbo-16k + +# DEBUG_OPENAI=false + +# TITLE_CONVO=false +# OPENAI_TITLE_MODEL=gpt-3.5-turbo + +# OPENAI_SUMMARIZE=true +# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo + +# OPENAI_FORCE_PROMPT=true + +# OPENAI_REVERSE_PROXY= + +# OPENAI_ORGANIZATION= + +#============# +# OpenRouter # +#============# + +# OPENROUTER_API_KEY= + +#============# +# Plugins # +#============# + +# PLUGIN_MODELS=gpt-4,gpt-4-turbo-preview,gpt-4-0125-preview,gpt-4-1106-preview,gpt-4-0613,gpt-3.5-turbo,gpt-3.5-turbo-0125,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613 + +# DEBUG_PLUGINS=true + +CREDS_KEY=f34be427ebb29de8d88c107a71546019685ed8b241d8f2ed00c3df97ad2566f0 +CREDS_IV=e2341419ec3dd3d19b13a1a87fafcbfb + +# Azure AI Search +#----------------- +# AZURE_AI_SEARCH_SERVICE_ENDPOINT= +# AZURE_AI_SEARCH_INDEX_NAME= +# AZURE_AI_SEARCH_API_KEY= + +# AZURE_AI_SEARCH_API_VERSION= +# AZURE_AI_SEARCH_SEARCH_OPTION_QUERY_TYPE= +# AZURE_AI_SEARCH_SEARCH_OPTION_TOP= +# AZURE_AI_SEARCH_SEARCH_OPTION_SELECT= + +# DALL·E +#---------------- +# DALLE_API_KEY= +# DALLE3_API_KEY= +# DALLE2_API_KEY= +# DALLE3_SYSTEM_PROMPT= +# DALLE2_SYSTEM_PROMPT= +# DALLE_REVERSE_PROXY= +# DALLE3_BASEURL= +# DALLE2_BASEURL= + +# DALL·E (via Azure OpenAI) +# Note: requires some of the variables above to be set +#---------------- +# DALLE3_AZURE_API_VERSION= +# DALLE2_AZURE_API_VERSION= + +# Google +#----------------- +# GOOGLE_API_KEY= +# GOOGLE_CSE_ID= + +# SerpAPI +#----------------- +# SERPAPI_API_KEY= + +# Stable Diffusion +#----------------- +# SD_WEBUI_URL=http://host.docker.internal:7860 + +# WolframAlpha +#----------------- +# WOLFRAM_APP_ID= + +# Zapier +#----------------- +# ZAPIER_NLA_API_KEY= + +#==================================================# +# Search # +#==================================================# + +SEARCH=true +MEILI_NO_ANALYTICS=true +MEILI_HOST=http://0.0.0.0:7700 +MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt + +#===================================================# +# User System # +#===================================================# + +#========================# +# Moderation # +#========================# + +# OPENAI_MODERATION=false +# OPENAI_MODERATION_API_KEY= +# # OPENAI_MODERATION_REVERSE_PROXY= + +# BAN_VIOLATIONS=true +# BAN_DURATION=1000 * 60 * 60 * 2 +# BAN_INTERVAL=20 + +# LOGIN_VIOLATION_SCORE=1 +# REGISTRATION_VIOLATION_SCORE=1 +# CONCURRENT_VIOLATION_SCORE=1 +# MESSAGE_VIOLATION_SCORE=1 +# NON_BROWSER_VIOLATION_SCORE=20 + +# LOGIN_MAX=7 +# LOGIN_WINDOW=5 +# REGISTER_MAX=5 +# REGISTER_WINDOW=60 + +# LIMIT_CONCURRENT_MESSAGES=true +# CONCURRENT_MESSAGE_MAX=2 + +# LIMIT_MESSAGE_IP=true +# MESSAGE_IP_MAX=40 +# MESSAGE_IP_WINDOW=1 + +# LIMIT_MESSAGE_USER=false +# MESSAGE_USER_MAX=40 +# MESSAGE_USER_WINDOW=1 + +#========================# +# Balance # +#========================# + +# CHECK_BALANCE=false + +#========================# +# Registration and Login # +#========================# + +ALLOW_EMAIL_LOGIN=true +ALLOW_REGISTRATION=true +ALLOW_SOCIAL_LOGIN=false +ALLOW_SOCIAL_REGISTRATION=false + +SESSION_EXPIRY=1000 * 60 * 15 +REFRESH_TOKEN_EXPIRY=(1000 * 60 * 60 * 24) * 7 + +JWT_SECRET=16f8c0ef4a5d391b26034086c628469d3f9f497f08163ab9b40137092f2909ef +JWT_REFRESH_SECRET=eaa5191f2914e30b9387fd84e254e4ba6fc51b4654968a9b0803b456a54b8418 + +# Discord +# DISCORD_CLIENT_ID= +# DISCORD_CLIENT_SECRET= +# DISCORD_CALLBACK_URL=/oauth/discord/callback + +# # Facebook +# FACEBOOK_CLIENT_ID= +# FACEBOOK_CLIENT_SECRET= +# FACEBOOK_CALLBACK_URL=/oauth/facebook/callback + +# # GitHub +# GITHUB_CLIENT_ID= +# GITHUB_CLIENT_SECRET= +# GITHUB_CALLBACK_URL=/oauth/github/callback + +# # Google +# GOOGLE_CLIENT_ID= +# GOOGLE_CLIENT_SECRET= +# GOOGLE_CALLBACK_URL=/oauth/google/callback + +# # OpenID +# OPENID_CLIENT_ID= +# OPENID_CLIENT_SECRET= +# OPENID_ISSUER= +# OPENID_SESSION_SECRET= +# OPENID_SCOPE="openid profile email" +# OPENID_CALLBACK_URL=/oauth/openid/callback + +# OPENID_BUTTON_LABEL= +# OPENID_IMAGE_URL= + +# #========================# +# # Email Password Reset # +# #========================# + +# EMAIL_SERVICE= +# EMAIL_HOST= +# EMAIL_PORT=25 +# EMAIL_ENCRYPTION= +# EMAIL_ENCRYPTION_HOSTNAME= +# EMAIL_ALLOW_SELFSIGNED= +# EMAIL_USERNAME= +# EMAIL_PASSWORD= +# EMAIL_FROM_NAME= +# EMAIL_FROM=noreply@librechat.ai + +# #========================# +# # Firebase CDN # +# #========================# + +# FIREBASE_API_KEY= +# FIREBASE_AUTH_DOMAIN= +# FIREBASE_PROJECT_ID= +# FIREBASE_STORAGE_BUCKET= +# FIREBASE_MESSAGING_SENDER_ID= +# FIREBASE_APP_ID= + +#==================================================# +# Others # +#==================================================# +# You should leave the following commented out # + +# NODE_ENV= + +# REDIS_URI= +# USE_REDIS= + +# E2E_USER_EMAIL= +# E2E_USER_PASSWORD= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d7a3a1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env.override diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..13fc3ae --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,71 @@ +version: "3" +services: + ollama: + image: ollama/ollama:0.1.22-rocm + restart: unless-stopped + entrypoint: /bootstrap.sh + env_file: + - .env + ports: + - 11434:11434 + devices: + - /dev/kfd + - /dev/dri + group_add: + - video + ipc: host + cap_add: + - SYS_PTRACE + security_opt: + - seccomp=unconfined + volumes: + - ./ollama/bootstrap.sh:/bootstrap.sh:ro + - ollama:/root/.ollama + + litellm: + image: ghcr.io/berriai/litellm:main-latest + command: --config /config.yaml + ports: + - 8000:8000 + volumes: + - ./litellm/config.yaml:/config.yaml:ro + + librechat: + image: ghcr.io/danny-avila/librechat + ports: + - 3080:3080 + depends_on: + - mongodb + restart: unless-stopped + # user: "${UID}:${GID}" + env_file: + - .env + environment: + - HOST=0.0.0.0 + - MONGO_URI=mongodb://mongodb:27017/LibreChat + - MEILI_HOST=http://meilisearch:7700 + volumes: + - ./librechat/librechat.yaml:/app/librechat.yaml:ro + - librechat_images:/app/client/public/images + mongodb: + image: mongo + restart: unless-stopped + # user: "${UID}:${GID}" + volumes: + - librechat_mongodb_data:/data/db + command: mongod --noauth + meilisearch: + image: getmeili/meilisearch:v1.5 + restart: unless-stopped + # user: "${UID}:${GID}" + environment: + - MEILI_HOST=http://meilisearch:7700 + - MEILI_NO_ANALYTICS=true + volumes: + - librechat_meilisearch_data:/meili_data + +volumes: + ollama: + librechat_images: + librechat_mongodb_data: + librechat_meilisearch_data: \ No newline at end of file diff --git a/librechat/librechat.yaml b/librechat/librechat.yaml new file mode 100644 index 0000000..59aa389 --- /dev/null +++ b/librechat/librechat.yaml @@ -0,0 +1,20 @@ +version: 1.0.1 +cache: true +endpoints: + custom: + # Example using Mistral AI API + - name: "Mistral" + apiKey: "noUse" + baseURL: "http://litellm:8000" + models: + default: ["mistral-7b"] + titleConvo: true + titleModel: "mistral-7b" + summarize: true + summaryModel: "mistral-7b" + forcePrompt: false + modelDisplayLabel: "Mistral" + # addParams: + # safe_prompt: true + # NOTE: For Mistral, it is necessary to drop the following parameters or you will encounter a 422 Error: + dropParams: ["stop", "user", "frequency_penalty", "presence_penalty"] \ No newline at end of file diff --git a/litellm/config.yaml b/litellm/config.yaml new file mode 100644 index 0000000..8fea7b5 --- /dev/null +++ b/litellm/config.yaml @@ -0,0 +1,6 @@ +model_list: + - model_name: mistral-7b + litellm_params: + model: ollama/mistral + api_base: http://ollama:11434 + # stream: True \ No newline at end of file diff --git a/ollama/bootstrap.sh b/ollama/bootstrap.sh new file mode 100755 index 0000000..714c659 --- /dev/null +++ b/ollama/bootstrap.sh @@ -0,0 +1,11 @@ +#!/bin/bash -x + +ollama serve & + +sleep 1 + +for model in ${@:-mistral}; do + ollama pull "$model" +done + +wait \ No newline at end of file