Created DEFAULT_NUM_CTX VAR with a deafult of 32768

2024-11-18 20:48:35 +00:00
parent 233d22e080
commit bfaaf86c69
2 changed files with 12 additions and 1 deletions
--- a/.env.example
+++ b/.env.example
@@ -56,3 +56,10 @@ XAI_API_KEY=

 # Include this environment variable if you want more logging for debugging locally
 VITE_LOG_LEVEL=debug
+
+# Example Context Values for qwen2.5-coder:32b
+# 
+# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM
+# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM
+# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM
+# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM