From bfaaf86c69aa558dda35dc638e30108e5d58dc69 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Mon, 18 Nov 2024 20:48:35 +0000 Subject: [PATCH 01/15] Created DEFAULT_NUM_CTX VAR with a deafult of 32768 --- .env.example | 7 +++++++ app/lib/.server/llm/model.ts | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 46a21e8..386d407 100644 --- a/.env.example +++ b/.env.example @@ -56,3 +56,10 @@ XAI_API_KEY= # Include this environment variable if you want more logging for debugging locally VITE_LOG_LEVEL=debug + +# Example Context Values for qwen2.5-coder:32b +# +# DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM +# DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM +# DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index 6be9d11..266dd69 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -8,6 +8,10 @@ import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? + parseInt(process.env.DEFAULT_NUM_CTX, 10) : + 32768; + export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ apiKey, @@ -58,7 +62,7 @@ export function getGroqModel(apiKey: string, model: string) { export function getOllamaModel(baseURL: string, model: string) { let Ollama = ollama(model, { - numCtx: 32768, + numCtx: DEFAULT_NUM_CTX, }); Ollama.config.baseURL = `${baseURL}/api`; From 53594234819346a3d8940b9101ebd9ec69b25641 Mon Sep 17 00:00:00 2001 From: Aaron Bolton Date: Tue, 19 Nov 2024 07:46:51 +0000 Subject: [PATCH 02/15] DEFAULT_NUM_CTX additions adding further changes for DEFAULT_NUM_CTX, including docs --- .env.example | 3 ++- CONTRIBUTING.md | 16 ++++++++++++++++ Dockerfile | 8 ++++++-- docker-compose.yaml | 2 ++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 386d407..9710a8e 100644 --- a/.env.example +++ b/.env.example @@ -62,4 +62,5 @@ VITE_LOG_LEVEL=debug # DEFAULT_NUM_CTX=32768 # Consumes 36GB of VRAM # DEFAULT_NUM_CTX=24576 # Consumes 32GB of VRAM # DEFAULT_NUM_CTX=12288 # Consumes 26GB of VRAM -# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM \ No newline at end of file +# DEFAULT_NUM_CTX=6144 # Consumes 24GB of VRAM +DEFAULT_NUM_CTX= \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bf3bfb..23f2b8d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,7 @@ # Contributing to Bolt.new Fork +## DEFAULT_NUM_CTX + +The `DEFAULT_NUM_CTX` environment variable can be used to limit the maximum number of context values used by the qwen2.5-coder model. For example, to limit the context to 24576 values (which uses 32GB of VRAM), set `DEFAULT_NUM_CTX=24576` in your `.env.local` file. First off, thank you for considering contributing to Bolt.new! This fork aims to expand the capabilities of the original project by integrating multiple LLM providers and enhancing functionality. Every contribution helps make Bolt.new a better tool for developers worldwide. @@ -80,6 +83,19 @@ ANTHROPIC_API_KEY=XXX ```bash VITE_LOG_LEVEL=debug ``` + + - Optionally set context size: +```bash +DEFAULT_NUM_CTX=32768 +``` + +Some Example Context Values for the qwen2.5-coder:32b models are. + +* DEFAULT_NUM_CTX=32768 - Consumes 36GB of VRAM +* DEFAULT_NUM_CTX=24576 - Consumes 32GB of VRAM +* DEFAULT_NUM_CTX=12288 - Consumes 26GB of VRAM +* DEFAULT_NUM_CTX=6144 - Consumes 24GB of VRAM + **Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore. ### 🚀 Running the Development Server diff --git a/Dockerfile b/Dockerfile index 3b5a74c..1d68673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV WRANGLER_SEND_METRICS=false \ GROQ_API_KEY=${GROQ_API_KEY} \ @@ -33,7 +34,8 @@ ENV WRANGLER_SEND_METRICS=false \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} # Pre-configure wrangler to disable metrics RUN mkdir -p /root/.config/.wrangler && \ @@ -54,6 +56,7 @@ ARG OPEN_ROUTER_API_KEY ARG GOOGLE_GENERATIVE_AI_API_KEY ARG OLLAMA_API_BASE_URL ARG VITE_LOG_LEVEL=debug +ARG DEFAULT_NUM_CTX ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPENAI_API_KEY=${OPENAI_API_KEY} \ @@ -61,7 +64,8 @@ ENV GROQ_API_KEY=${GROQ_API_KEY} \ OPEN_ROUTER_API_KEY=${OPEN_ROUTER_API_KEY} \ GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} \ OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} \ - VITE_LOG_LEVEL=${VITE_LOG_LEVEL} + VITE_LOG_LEVEL=${VITE_LOG_LEVEL} \ + DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX} RUN mkdir -p ${WORKDIR}/run CMD pnpm run dev --host diff --git a/docker-compose.yaml b/docker-compose.yaml index c391dd7..6fbd704 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,6 +20,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" @@ -46,6 +47,7 @@ services: - GOOGLE_GENERATIVE_AI_API_KEY=${GOOGLE_GENERATIVE_AI_API_KEY} - OLLAMA_API_BASE_URL=${OLLAMA_API_BASE_URL} - VITE_LOG_LEVEL=${VITE_LOG_LEVEL:-debug} + - DEFAULT_NUM_CTX=${DEFAULT_NUM_CTX:-32768} - RUNNING_IN_DOCKER=true extra_hosts: - "host.docker.internal:host-gateway" From 7a03b2647375844c076d0a4c72923b9deffc857c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stig-=C3=98rjan=20Smelror?= Date: Tue, 19 Nov 2024 20:21:27 +0100 Subject: [PATCH 03/15] Update the Google Gemini models list --- app/utils/constants.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/utils/constants.ts b/app/utils/constants.ts index fbcf226..42441b3 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -58,7 +58,11 @@ const PROVIDER_LIST: ProviderInfo[] = [ name: 'Google', staticModels: [ { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' } + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash', provider: 'Google' }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash 8b', provider: 'Google' }, + { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro', provider: 'Google' }, + { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' }, { From b945ec8d2ca57996685e74a85da4e965daab7ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stig-=C3=98rjan=20Smelror?= Date: Tue, 19 Nov 2024 20:24:52 +0100 Subject: [PATCH 04/15] Fix the list of names to include the correct model --- app/utils/constants.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 42441b3..b7aed65 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -58,10 +58,10 @@ const PROVIDER_LIST: ProviderInfo[] = [ name: 'Google', staticModels: [ { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash 8b', provider: 'Google' }, + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google' }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google' }, { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro', provider: 'Google' }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google' }, { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' From 4ac0af4cdbbe998240f7497cb8b489e29c51c390 Mon Sep 17 00:00:00 2001 From: Sujal Shah Date: Wed, 20 Nov 2024 17:48:41 +0530 Subject: [PATCH 05/15] fix: enhance prompt "Invalid or missing provider" bad request error --- app/lib/hooks/usePromptEnhancer.ts | 37 +++++++++++++++--------------- app/routes/api.enhancer.ts | 33 +++++++++++++++----------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/app/lib/hooks/usePromptEnhancer.ts b/app/lib/hooks/usePromptEnhancer.ts index ee44999..6275ef3 100644 --- a/app/lib/hooks/usePromptEnhancer.ts +++ b/app/lib/hooks/usePromptEnhancer.ts @@ -1,4 +1,5 @@ import { useState } from 'react'; +import type { ProviderInfo } from '~/types/model'; import { createScopedLogger } from '~/utils/logger'; const logger = createScopedLogger('usePromptEnhancement'); @@ -13,54 +14,54 @@ export function usePromptEnhancer() { }; const enhancePrompt = async ( - input: string, + input: string, setInput: (value: string) => void, model: string, - provider: string, - apiKeys?: Record + provider: ProviderInfo, + apiKeys?: Record, ) => { setEnhancingPrompt(true); setPromptEnhanced(false); - + const requestBody: any = { message: input, model, provider, }; - + if (apiKeys) { requestBody.apiKeys = apiKeys; } - + const response = await fetch('/api/enhancer', { method: 'POST', body: JSON.stringify(requestBody), }); - + const reader = response.body?.getReader(); - + const originalInput = input; - + if (reader) { const decoder = new TextDecoder(); - + let _input = ''; let _error; - + try { setInput(''); - + while (true) { const { value, done } = await reader.read(); - + if (done) { break; } - + _input += decoder.decode(value); - + logger.trace('Set input', _input); - + setInput(_input); } } catch (error) { @@ -70,10 +71,10 @@ export function usePromptEnhancer() { if (_error) { logger.error(_error); } - + setEnhancingPrompt(false); setPromptEnhanced(true); - + setTimeout(() => { setInput(_input); }); diff --git a/app/routes/api.enhancer.ts b/app/routes/api.enhancer.ts index 7040b89..77e6f2f 100644 --- a/app/routes/api.enhancer.ts +++ b/app/routes/api.enhancer.ts @@ -2,7 +2,7 @@ import { type ActionFunctionArgs } from '@remix-run/cloudflare'; import { StreamingTextResponse, parseStreamPart } from 'ai'; import { streamText } from '~/lib/.server/llm/stream-text'; import { stripIndents } from '~/utils/stripIndent'; -import type { StreamingOptions } from '~/lib/.server/llm/stream-text'; +import type { ProviderInfo } from '~/types/model'; const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -12,25 +12,27 @@ export async function action(args: ActionFunctionArgs) { } async function enhancerAction({ context, request }: ActionFunctionArgs) { - const { message, model, provider, apiKeys } = await request.json<{ + const { message, model, provider, apiKeys } = await request.json<{ message: string; model: string; - provider: string; + provider: ProviderInfo; apiKeys?: Record; }>(); - // Validate 'model' and 'provider' fields + const { name: providerName } = provider; + + // validate 'model' and 'provider' fields if (!model || typeof model !== 'string') { throw new Response('Invalid or missing model', { status: 400, - statusText: 'Bad Request' + statusText: 'Bad Request', }); } - if (!provider || typeof provider !== 'string') { + if (!providerName || typeof providerName !== 'string') { throw new Response('Invalid or missing provider', { status: 400, - statusText: 'Bad Request' + statusText: 'Bad Request', }); } @@ -39,7 +41,9 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) { [ { role: 'user', - content: `[Model: ${model}]\n\n[Provider: ${provider}]\n\n` + stripIndents` + content: + `[Model: ${model}]\n\n[Provider: ${providerName}]\n\n` + + stripIndents` I want you to improve the user prompt that is wrapped in \`\` tags. IMPORTANT: Only respond with the improved prompt and nothing else! @@ -52,23 +56,24 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) { ], context.cloudflare.env, undefined, - apiKeys + apiKeys, ); const transformStream = new TransformStream({ transform(chunk, controller) { const text = decoder.decode(chunk); - const lines = text.split('\n').filter(line => line.trim() !== ''); - + const lines = text.split('\n').filter((line) => line.trim() !== ''); + for (const line of lines) { try { const parsed = parseStreamPart(line); + if (parsed.type === 'text') { controller.enqueue(encoder.encode(parsed.value)); } } catch (e) { - // Skip invalid JSON lines - console.warn('Failed to parse stream part:', line); + // skip invalid JSON lines + console.warn('Failed to parse stream part:', line, e); } } }, @@ -83,7 +88,7 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) { if (error instanceof Error && error.message?.includes('API key')) { throw new Response('Invalid or missing API key', { status: 401, - statusText: 'Unauthorized' + statusText: 'Unauthorized', }); } From 50de8d0e1b26e3198427aab6b1fe1302d904a99f Mon Sep 17 00:00:00 2001 From: Cole Medin Date: Wed, 20 Nov 2024 08:27:21 -0600 Subject: [PATCH 06/15] Updating README with new features and a link to our community --- README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5288923..6bcacd4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ [![Bolt.new: AI-Powered Full-Stack Web Development in the Browser](./public/social_preview_index.jpg)](https://bolt.new) -# Bolt.new Fork by Cole Medin +# Bolt.new Fork by Cole Medin - oTToDev -This fork of Bolt.new allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. +This fork of Bolt.new (oTToDev) allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, LMStudio, Mistral, xAI, HuggingFace, DeepSeek, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models. + +Join the community for oTToDev! + +https://thinktank.ottomator.ai # Requested Additions to this Fork - Feel Free to Contribute!! @@ -20,21 +24,23 @@ This fork of Bolt.new allows you to choose the LLM that you use for each prompt! - ✅ Publish projects directly to GitHub (@goncaloalves) - ✅ Ability to enter API keys in the UI (@ali00209) - ✅ xAI Grok Beta Integration (@milutinke) +- ✅ LM Studio Integration (@karrot0) +- ✅ HuggingFace Integration (@ahsan3219) +- ✅ Bolt terminal to see the output of LLM run commands (@thecodacus) +- ✅ Streaming of code output (@thecodacus) +- ✅ Ability to revert code to earlier version (@wonderwhy-er) - ⬜ **HIGH PRIORITY** - Prevent Bolt from rewriting files as often (file locking and diffs) - ⬜ **HIGH PRIORITY** - Better prompting for smaller LLMs (code window sometimes doesn't start) -- ⬜ **HIGH PRIORITY** Load local projects into the app +- ⬜ **HIGH PRIORITY** - Load local projects into the app - ⬜ **HIGH PRIORITY** - Attach images to prompts - ⬜ **HIGH PRIORITY** - Run agents in the backend as opposed to a single model call - ⬜ Mobile friendly -- ⬜ LM Studio Integration - ⬜ Together Integration - ⬜ Azure Open AI API Integration -- ⬜ HuggingFace Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration - ⬜ Cohere Integration - ⬜ Deploy directly to Vercel/Netlify/other similar platforms -- ⬜ Ability to revert code to earlier version - ⬜ Prompt caching - ⬜ Better prompt enhancing - ⬜ Have LLM plan the project in a MD file for better results/transparency From b39e48b7da7c49ddc838b583a60391a975dee23e Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Sun, 17 Nov 2024 15:52:38 +0200 Subject: [PATCH 07/15] .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b43105b..3303fba 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ dist-ssr *.sln *.sw? +/.history /.cache /build .env.local From b0b6760136a4156ff73b900162a8c28de9f1a10d Mon Sep 17 00:00:00 2001 From: eduardruzga Date: Wed, 20 Nov 2024 16:47:19 +0200 Subject: [PATCH 08/15] Add background for chat window --- app/components/chat/BaseChat.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 629c5cb..902cb23 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -192,9 +192,11 @@ export const BaseChat = React.forwardRef( }}
Date: Wed, 20 Nov 2024 15:08:42 +0000 Subject: [PATCH 09/15] Cohere support added --- .env.example | 4 +++ README.md | 2 +- app/lib/.server/llm/api-key.ts | 2 ++ app/lib/.server/llm/constants.ts | 2 +- app/lib/.server/llm/model.ts | 14 ++++++++++ app/utils/constants.ts | 16 +++++++++++ package.json | 1 + pnpm-lock.yaml | 47 ++++++++++++++++++++++++++++++++ 8 files changed, 86 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 00a0fd9..473f05a 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,10 @@ OPENAI_LIKE_API_KEY= # You only need this environment variable set if you want to use Mistral models MISTRAL_API_KEY= +# Get the Cohere Api key by following these instructions - +# https://dashboard.cohere.com/api-keys +# You only need this environment variable set if you want to use Cohere models +COHERE_API_KEY= # Get LMStudio Base URL from LM Studio Developer Console # Make sure to enable CORS diff --git a/README.md b/README.md index 6bcacd4..7a3957a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ https://thinktank.ottomator.ai - ⬜ Azure Open AI API Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ⬜ Cohere Integration +- ✅ Cohere Integration - ⬜ Deploy directly to Vercel/Netlify/other similar platforms - ⬜ Prompt caching - ⬜ Better prompt enhancing diff --git a/app/lib/.server/llm/api-key.ts b/app/lib/.server/llm/api-key.ts index f282aa8..7d8d2f9 100644 --- a/app/lib/.server/llm/api-key.ts +++ b/app/lib/.server/llm/api-key.ts @@ -35,6 +35,8 @@ export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Re return env.OPENAI_LIKE_API_KEY || cloudflareEnv.OPENAI_LIKE_API_KEY; case "xAI": return env.XAI_API_KEY || cloudflareEnv.XAI_API_KEY; + case "Cohere": + return env.COHERE_API_KEY; default: return ""; } diff --git a/app/lib/.server/llm/constants.ts b/app/lib/.server/llm/constants.ts index 7b3a0f2..1f993c9 100644 --- a/app/lib/.server/llm/constants.ts +++ b/app/lib/.server/llm/constants.ts @@ -1,5 +1,5 @@ // see https://docs.anthropic.com/en/docs/about-claude/models -export const MAX_TOKENS = 8000; +export const MAX_TOKENS = 4096; // limits the number of model responses that can be returned in a single request export const MAX_RESPONSE_SEGMENTS = 2; diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts index e07f2bb..2e7c568 100644 --- a/app/lib/.server/llm/model.ts +++ b/app/lib/.server/llm/model.ts @@ -7,6 +7,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'; import { ollama } from 'ollama-ai-provider'; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { createMistral } from '@ai-sdk/mistral'; +import { createCohere } from '@ai-sdk/cohere' export function getAnthropicModel(apiKey: string, model: string) { const anthropic = createAnthropic({ @@ -23,6 +24,15 @@ export function getOpenAILikeModel(baseURL:string,apiKey: string, model: string) return openai(model); } + +export function getCohereAIModel(apiKey:string, model: string){ + const cohere = createCohere({ + apiKey, + }); + + return cohere(model); +} + export function getOpenAIModel(apiKey: string, model: string) { const openai = createOpenAI({ apiKey, @@ -108,6 +118,8 @@ export function getXAIModel(apiKey: string, model: string) { return openai(model); } + + export function getModel(provider: string, model: string, env: Env, apiKeys?: Record) { const apiKey = getAPIKey(env, provider, apiKeys); const baseURL = getBaseURL(env, provider); @@ -135,6 +147,8 @@ export function getModel(provider: string, model: string, env: Env, apiKeys?: Re return getLMStudioModel(baseURL, model); case 'xAI': return getXAIModel(apiKey, model); + case 'Cohere': + return getCohereAIModel(apiKey, model); default: return getOllamaModel(baseURL, model); } diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 672c7a8..fe76932 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -33,6 +33,22 @@ const PROVIDER_LIST: ProviderInfo[] = [ staticModels: [], getDynamicModels: getOpenAILikeModels }, + { + name: 'Cohere', + staticModels: [ + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, + { name: 'command-r', label: 'Command R', provider: 'Cohere' }, + { name: 'command', label: 'Command', provider: 'Cohere' }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere' }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, + ], + getApiKeyLink: 'https://dashboard.cohere.com/api-keys' + }, { name: 'OpenRouter', staticModels: [ diff --git a/package.json b/package.json index 40ede0f..cc1c256 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@ai-sdk/anthropic": "^0.0.39", + "@ai-sdk/cohere": "^1.0.1", "@ai-sdk/google": "^0.0.52", "@ai-sdk/mistral": "^0.0.43", "@ai-sdk/openai": "^0.0.66", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4158d19..951d1a4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@ai-sdk/anthropic': specifier: ^0.0.39 version: 0.0.39(zod@3.23.8) + '@ai-sdk/cohere': + specifier: ^1.0.1 + version: 1.0.1(zod@3.23.8) '@ai-sdk/google': specifier: ^0.0.52 version: 0.0.52(zod@3.23.8) @@ -279,6 +282,12 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/cohere@1.0.1': + resolution: {integrity: sha512-xLaSYl/hs9EqfpvT9PvqZrDWjJPQPZBd0iT32T6812vN6kwuEQ6sSgQvqHWczIqxeej2GNRgMQwDL6Lh0L5pZw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/google@0.0.52': resolution: {integrity: sha512-bfsA/1Ae0SQ6NfLwWKs5SU4MBwlzJjVhK6bTVBicYFjUxg9liK/W76P1Tq/qK9OlrODACz3i1STOIWsFPpIOuQ==} engines: {node: '>=18'} @@ -324,6 +333,15 @@ packages: zod: optional: true + '@ai-sdk/provider-utils@2.0.1': + resolution: {integrity: sha512-TNg7rPhRtETB2Z9F0JpOvpGii9Fs8EWM8nYy1jEkvSXkrPJ6b/9zVnDdaJsmLFDyrMbOsPJlkblYtmYEQou36w==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider@0.0.12': resolution: {integrity: sha512-oOwPQD8i2Ynpn22cur4sk26FW3mSy6t6/X/K1Ay2yGBKYiSpRyLfObhOrZEGsXDx+3euKy4nEZ193R36NM+tpQ==} engines: {node: '>=18'} @@ -336,6 +354,10 @@ packages: resolution: {integrity: sha512-XMsNGJdGO+L0cxhhegtqZ8+T6nn4EoShS819OvCgI2kLbYTIvk0GWFGD0AXJmxkxs3DrpsJxKAFukFR7bvTkgQ==} engines: {node: '>=18'} + '@ai-sdk/provider@1.0.0': + resolution: {integrity: sha512-Sj29AzooJ7SYvhPd+AAWt/E7j63E9+AzRnoMHUaJPRYzOd/WDrVNxxv85prF9gDcQ7XPVlSk9j6oAZV9/DXYpA==} + engines: {node: '>=18'} + '@ai-sdk/react@0.0.62': resolution: {integrity: sha512-1asDpxgmeHWL0/EZPCLENxfOHT+0jce0z/zasRhascodm2S6f6/KZn5doLG9jdmarcb+GjMjFmmwyOVXz3W1xg==} engines: {node: '>=18'} @@ -3033,6 +3055,10 @@ packages: resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} engines: {node: '>=14.18'} + eventsource-parser@3.0.0: + resolution: {integrity: sha512-T1C0XCUimhxVQzW4zFipdx0SficT651NnkR0ZSH3yQwh+mFMdLfgjABVi4YtMTtaL4s168593DaoaRLMqryavA==} + engines: {node: '>=18.0.0'} + evp_bytestokey@1.0.3: resolution: {integrity: sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==} @@ -5687,6 +5713,12 @@ snapshots: '@ai-sdk/provider-utils': 1.0.9(zod@3.23.8) zod: 3.23.8 + '@ai-sdk/cohere@1.0.1(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.0 + '@ai-sdk/provider-utils': 2.0.1(zod@3.23.8) + zod: 3.23.8 + '@ai-sdk/google@0.0.52(zod@3.23.8)': dependencies: '@ai-sdk/provider': 0.0.24 @@ -5733,6 +5765,15 @@ snapshots: optionalDependencies: zod: 3.23.8 + '@ai-sdk/provider-utils@2.0.1(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 1.0.0 + eventsource-parser: 3.0.0 + nanoid: 3.3.7 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.23.8 + '@ai-sdk/provider@0.0.12': dependencies: json-schema: 0.4.0 @@ -5745,6 +5786,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@1.0.0': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/react@0.0.62(react@18.3.1)(zod@3.23.8)': dependencies: '@ai-sdk/provider-utils': 1.0.20(zod@3.23.8) @@ -8751,6 +8796,8 @@ snapshots: eventsource-parser@1.1.2: {} + eventsource-parser@3.0.0: {} + evp_bytestokey@1.0.3: dependencies: md5.js: 1.3.5 From eabfbb2220c58b9b54535f6167f3c8a60deaa8b4 Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:41:51 +0530 Subject: [PATCH 10/15] max token is now dynamically handle for each model --- app/lib/.server/llm/constants.ts | 2 +- app/lib/.server/llm/stream-text.ts | 17 ++-- app/utils/constants.ts | 123 +++++++++++++++-------------- app/utils/types.ts | 1 + 4 files changed, 77 insertions(+), 66 deletions(-) diff --git a/app/lib/.server/llm/constants.ts b/app/lib/.server/llm/constants.ts index 1f993c9..7b3a0f2 100644 --- a/app/lib/.server/llm/constants.ts +++ b/app/lib/.server/llm/constants.ts @@ -1,5 +1,5 @@ // see https://docs.anthropic.com/en/docs/about-claude/models -export const MAX_TOKENS = 4096; +export const MAX_TOKENS = 8000; // limits the number of model responses that can be returned in a single request export const MAX_RESPONSE_SEGMENTS = 2; diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 8bfd797..dba5a61 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -41,10 +41,9 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } - export function streamText( - messages: Messages, - env: Env, + messages: Messages, + env: Env, options?: StreamingOptions, apiKeys?: Record ) { @@ -64,13 +63,21 @@ export function streamText( return { ...message, content }; } - return message; // No changes for non-user messages + return message; }); + const modelDetails = MODEL_LIST.find((m) => m.name === currentModel); + + const dynamicMaxTokens = + modelDetails && modelDetails.maxTokenAllowed + ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) + : MAX_TOKENS; + console.log(dynamicMaxTokens) + return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), system: getSystemPrompt(), - maxTokens: MAX_TOKENS, + maxTokens: dynamicMaxTokens, messages: convertToCoreMessages(processedMessages), ...options, }); diff --git a/app/utils/constants.ts b/app/utils/constants.ts index fe76932..942afc4 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -12,12 +12,12 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'Anthropic', staticModels: [ - { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic' }, - { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic' }, - { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic' }, - { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic' }, - { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic' }, - { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic' } + { name: 'claude-3-5-sonnet-latest', label: 'Claude 3.5 Sonnet (new)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-5-sonnet-20240620', label: 'Claude 3.5 Sonnet (old)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-5-haiku-latest', label: 'Claude 3.5 Haiku (new)', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 }, + { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 } ], getApiKeyLink: "https://console.anthropic.com/settings/keys", }, @@ -36,36 +36,37 @@ const PROVIDER_LIST: ProviderInfo[] = [ { name: 'Cohere', staticModels: [ - { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere' }, - { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere' }, - { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere' }, - { name: 'command-r', label: 'Command R', provider: 'Cohere' }, - { name: 'command', label: 'Command', provider: 'Cohere' }, - { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere' }, - { name: 'command-light', label: 'Command Light', provider: 'Cohere' }, - { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere' }, - { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere' }, - { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere' }, + { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 }, + { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 }, ], getApiKeyLink: 'https://dashboard.cohere.com/api-keys' }, { name: 'OpenRouter', staticModels: [ - { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI' }, + { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 }, { name: 'anthropic/claude-3.5-sonnet', label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', provider: 'OpenRouter' + , maxTokenAllowed: 8000 }, - { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter' }, - { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter' }, - { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter' }, - { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter' }, - { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter' }, - { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter' }, - { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter' }, - { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter' } + { name: 'anthropic/claude-3-haiku', label: 'Anthropic: Claude 3 Haiku (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'deepseek/deepseek-coder', label: 'Deepseek-Coder V2 236B (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'google/gemini-flash-1.5', label: 'Google Gemini Flash 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'google/gemini-pro-1.5', label: 'Google Gemini Pro 1.5 (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'mistralai/mistral-nemo', label: 'OpenRouter Mistral Nemo (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'qwen/qwen-110b-chat', label: 'OpenRouter Qwen 110b Chat (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 }, + { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 } ], getDynamicModels: getOpenRouterModels, getApiKeyLink: 'https://openrouter.ai/settings/keys', @@ -73,70 +74,70 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Google', staticModels: [ - { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google' }, - { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google' }, - { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google' }, - { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google' }, - { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google' }, - { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google' } + { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 }, + { name: 'gemini-exp-1114', label: 'Gemini exp-1114', provider: 'Google', maxTokenAllowed: 8192 } ], getApiKeyLink: 'https://aistudio.google.com/app/apikey' }, { name: 'Groq', staticModels: [ - { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq' }, - { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq' }, - { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq' } + { name: 'llama-3.1-70b-versatile', label: 'Llama 3.1 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 }, + { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://console.groq.com/keys' }, { name: 'HuggingFace', staticModels: [ - { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace' }, - { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace' }, - { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace' }, - { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace' } + { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: '01-ai/Yi-1.5-34B-Chat', label: 'Yi-1.5-34B-Chat (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: 'codellama/CodeLlama-34b-Instruct-hf', label: 'CodeLlama-34b-Instruct (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 }, + { name: 'NousResearch/Hermes-3-Llama-3.1-8B', label: 'Hermes-3-Llama-3.1-8B (HuggingFace)', provider: 'HuggingFace', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://huggingface.co/settings/tokens' }, - + { name: 'OpenAI', staticModels: [ - { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI' }, - { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI' }, - { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI' }, - { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI' } + { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 }, + { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 } ], getApiKeyLink: "https://platform.openai.com/api-keys", }, { name: 'xAI', staticModels: [ - { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI' } + { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key' }, { name: 'Deepseek', staticModels: [ - { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek' }, - { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek' } + { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 }, + { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://platform.deepseek.com/api_keys' }, { name: 'Mistral', staticModels: [ - { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral' }, - { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral' }, - { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral' }, - { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral' }, - { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral' }, - { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral' }, - { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral' }, - { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral' }, - { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral' } + { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 }, + { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 } ], getApiKeyLink: 'https://console.mistral.ai/api-keys/' }, { @@ -180,7 +181,8 @@ async function getOllamaModels(): Promise { return data.models.map((model: OllamaModel) => ({ name: model.name, label: `${model.name} (${model.details.parameter_size})`, - provider: 'Ollama' + provider: 'Ollama', + maxTokenAllowed:8000, })); } catch (e) { return []; @@ -233,8 +235,9 @@ async function getOpenRouterModels(): Promise { name: m.id, label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed( 2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor( - m.context_length / 1000)}k`, - provider: 'OpenRouter' + m.context_length / 1000)}k`, + provider: 'OpenRouter', + maxTokenAllowed:8000, })); } diff --git a/app/utils/types.ts b/app/utils/types.ts index a5f9fc1..171edc3 100644 --- a/app/utils/types.ts +++ b/app/utils/types.ts @@ -25,6 +25,7 @@ export interface ModelInfo { name: string; label: string; provider: string; + maxTokenAllowed: number; } export interface ProviderInfo { From 934044f73549cbc2a94e7888af172cc6a4ab743c Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:47:43 +0530 Subject: [PATCH 11/15] console message removed --- app/lib/.server/llm/stream-text.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index dba5a61..0a9c8e2 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -72,7 +72,6 @@ export function streamText( modelDetails && modelDetails.maxTokenAllowed ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) : MAX_TOKENS; - console.log(dynamicMaxTokens) return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), From 4269d57ea8c1b0f8e3142936af74dbeb6e7b2e6c Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Wed, 20 Nov 2024 21:52:08 +0530 Subject: [PATCH 12/15] README.md updated --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a3957a..f952e0a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ https://thinktank.ottomator.ai - ⬜ Azure Open AI API Integration - ⬜ Perplexity Integration - ⬜ Vertex AI Integration -- ✅ Cohere Integration +- ✅ Cohere Integration (@hasanraiyan) +- ✅ Dynamic model max token length (@hasanraiyan) - ⬜ Deploy directly to Vercel/Netlify/other similar platforms - ⬜ Prompt caching - ⬜ Better prompt enhancing From e1c3d603dfa9d31d1aa955a86264916a1a56e28a Mon Sep 17 00:00:00 2001 From: Raiyan Hasan Date: Thu, 21 Nov 2024 11:49:09 +0530 Subject: [PATCH 13/15] Update stream-text.ts dynamic model max Token updated --- app/lib/.server/llm/stream-text.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 0a9c8e2..4fedccb 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -68,10 +68,12 @@ export function streamText( const modelDetails = MODEL_LIST.find((m) => m.name === currentModel); - const dynamicMaxTokens = - modelDetails && modelDetails.maxTokenAllowed - ? Math.min(MAX_TOKENS, modelDetails.maxTokenAllowed) - : MAX_TOKENS; + + + const dynamicMaxTokens = +modelDetails && modelDetails.maxTokenAllowed + ? modelDetails.maxTokenAllowed + : MAX_TOKENS; return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), From f644066189333512eec77b24b9c515f734c6b354 Mon Sep 17 00:00:00 2001 From: Qwikode Date: Thu, 21 Nov 2024 15:12:33 +0200 Subject: [PATCH 14/15] fix(ui): mobile friendly --- app/components/chat/APIKeyManager.tsx | 47 +++++---- app/components/chat/BaseChat.tsx | 36 +++---- app/components/chat/Messages.client.tsx | 97 ++++++++++--------- .../header/HeaderActionButtons.client.tsx | 5 +- app/components/workbench/Workbench.client.tsx | 7 +- app/lib/hooks/index.ts | 1 + app/lib/hooks/useViewport.ts | 18 ++++ 7 files changed, 124 insertions(+), 87 deletions(-) create mode 100644 app/lib/hooks/useViewport.ts diff --git a/app/components/chat/APIKeyManager.tsx b/app/components/chat/APIKeyManager.tsx index 5b2c85e..c61e466 100644 --- a/app/components/chat/APIKeyManager.tsx +++ b/app/components/chat/APIKeyManager.tsx @@ -10,11 +10,7 @@ interface APIKeyManagerProps { labelForGetApiKey?: string; } -export const APIKeyManager: React.FC = ({ - provider, - apiKey, - setApiKey, - }) => { +export const APIKeyManager: React.FC = ({ provider, apiKey, setApiKey }) => { const [isEditing, setIsEditing] = useState(false); const [tempKey, setTempKey] = useState(apiKey); @@ -24,15 +20,29 @@ export const APIKeyManager: React.FC = ({ }; return ( -
- {provider?.name} API Key: +
+
+ {provider?.name} API Key: + {!isEditing && ( +
+ + {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} + + setIsEditing(true)} title="Edit API Key"> +
+ +
+ )} +
+ {isEditing ? ( - <> +
setTempKey(e.target.value)} - className="flex-1 p-1 text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" + className="flex-1 px-2 py-1 text-xs lg:text-sm rounded border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus" />
@@ -40,20 +50,15 @@ export const APIKeyManager: React.FC = ({ setIsEditing(false)} title="Cancel">
- +
) : ( <> - - {apiKey ? '••••••••' : 'Not set (will still work if set in .env file)'} - - setIsEditing(true)} title="Edit API Key"> -
- - - {provider?.getApiKeyLink && window.open(provider?.getApiKeyLink)} title="Edit API Key"> - {provider?.labelForGetApiKey || 'Get API Key'} -
- } + {provider?.getApiKeyLink && ( + window.open(provider?.getApiKeyLink)} title="Edit API Key"> + {provider?.labelForGetApiKey || 'Get API Key'} +
+ + )} )}
diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 902cb23..396fb01 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -27,9 +27,9 @@ const EXAMPLE_PROMPTS = [ const providerList = PROVIDER_LIST; -const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList }) => { +const ModelSelector = ({ model, setModel, provider, setProvider, modelList, providerList, apiKeys }) => { return ( -
+