iBoss/bolt-diy
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: refactored LLM Providers: Adapting Modular Approach (#832)

Browse Source 
* refactor: Refactoring Providers to have providers as modules

* updated package and lock file

* added grok model back

* updated registry system
This commit is contained in:
Anirban Kar
2024-12-21 11:45:17 +05:30
committed by GitHub
parent 63abf52000
commit 7295352a98
30 changed files with 1621 additions and 961 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
app/lib/.server/llm/api-key.ts
View File

@@ -1,111 +0,0 @@
import { env } from 'node:process';
import type { IProviderSetting } from '~/types/model';
import { getProviderBaseUrlAndKey } from '~/utils/constants';
export function getAPIKey(cloudflareEnv: Env, provider: string, userApiKeys?: Record<string, string>) {
/**
* The `cloudflareEnv` is only used when deployed or when previewing locally.
* In development the environment variables are available through `env`.
*/
// First check user-provided API keys
if (userApiKeys?.[provider]) {
return userApiKeys[provider];
}
const { apiKey } = getProviderBaseUrlAndKey({
provider,
apiKeys: userApiKeys,
providerSettings: undefined,
serverEnv: cloudflareEnv as any,
defaultBaseUrlKey: '',
defaultApiTokenKey: '',
});
if (apiKey) {
return apiKey;
}
// Fall back to hardcoded environment variables names
switch (provider) {
case 'Anthropic':
return env.ANTHROPIC_API_KEY || cloudflareEnv.ANTHROPIC_API_KEY;
case 'OpenAI':
return env.OPENAI_API_KEY || cloudflareEnv.OPENAI_API_KEY;
case 'Google':
return env.GOOGLE_GENERATIVE_AI_API_KEY || cloudflareEnv.GOOGLE_GENERATIVE_AI_API_KEY;
case 'Groq':
return env.GROQ_API_KEY || cloudflareEnv.GROQ_API_KEY;
case 'HuggingFace':
return env.HuggingFace_API_KEY || cloudflareEnv.HuggingFace_API_KEY;
case 'OpenRouter':
return env.OPEN_ROUTER_API_KEY || cloudflareEnv.OPEN_ROUTER_API_KEY;
case 'Deepseek':
return env.DEEPSEEK_API_KEY || cloudflareEnv.DEEPSEEK_API_KEY;
case 'Mistral':
return env.MISTRAL_API_KEY || cloudflareEnv.MISTRAL_API_KEY;
case 'OpenAILike':
return env.OPENAI_LIKE_API_KEY || cloudflareEnv.OPENAI_LIKE_API_KEY;
case 'Together':
return env.TOGETHER_API_KEY || cloudflareEnv.TOGETHER_API_KEY;
case 'xAI':
return env.XAI_API_KEY || cloudflareEnv.XAI_API_KEY;
case 'Perplexity':
return env.PERPLEXITY_API_KEY || cloudflareEnv.PERPLEXITY_API_KEY;
case 'Cohere':
return env.COHERE_API_KEY;
case 'AzureOpenAI':
return env.AZURE_OPENAI_API_KEY;
default:
return '';
}
}
export function getBaseURL(cloudflareEnv: Env, provider: string, providerSettings?: Record<string, IProviderSetting>) {
const { baseUrl } = getProviderBaseUrlAndKey({
provider,
apiKeys: {},
providerSettings,
serverEnv: cloudflareEnv as any,
defaultBaseUrlKey: '',
defaultApiTokenKey: '',
});
if (baseUrl) {
return baseUrl;
}
let settingBaseUrl = providerSettings?.[provider].baseUrl;
if (settingBaseUrl && settingBaseUrl.length == 0) {
settingBaseUrl = undefined;
}
switch (provider) {
case 'Together':
return (
settingBaseUrl ||
env.TOGETHER_API_BASE_URL ||
cloudflareEnv.TOGETHER_API_BASE_URL ||
'https://api.together.xyz/v1'
);
case 'OpenAILike':
return settingBaseUrl || env.OPENAI_LIKE_API_BASE_URL || cloudflareEnv.OPENAI_LIKE_API_BASE_URL;
case 'LMStudio':
return (
settingBaseUrl || env.LMSTUDIO_API_BASE_URL || cloudflareEnv.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'
);
case 'Ollama': {
let baseUrl =
settingBaseUrl || env.OLLAMA_API_BASE_URL || cloudflareEnv.OLLAMA_API_BASE_URL || 'http://localhost:11434';
if (env.RUNNING_IN_DOCKER === 'true') {
baseUrl = baseUrl.replace('localhost', 'host.docker.internal');
}
return baseUrl;
}
default:
return '';
}
}

190
app/lib/.server/llm/model.ts
View File

@@ -1,190 +0,0 @@
/*
* @ts-nocheck
* Preventing TS checks with files presented in the video for a better presentation.
*/
import { getAPIKey, getBaseURL } from '~/lib/.server/llm/api-key';
import { createAnthropic } from '@ai-sdk/anthropic';
import { createOpenAI } from '@ai-sdk/openai';
import { createGoogleGenerativeAI } from '@ai-sdk/google';
import { ollama } from 'ollama-ai-provider';
import { createOpenRouter } from '@openrouter/ai-sdk-provider';
import { createMistral } from '@ai-sdk/mistral';
import { createCohere } from '@ai-sdk/cohere';
import type { LanguageModelV1 } from 'ai';
import type { IProviderSetting } from '~/types/model';
export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768;
type OptionalApiKey = string | undefined;
export function getAnthropicModel(apiKey: OptionalApiKey, model: string) {
const anthropic = createAnthropic({
apiKey,
});
return anthropic(model);
}
export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) {
const openai = createOpenAI({
baseURL,
apiKey,
});
return openai(model);
}
export function getCohereAIModel(apiKey: OptionalApiKey, model: string) {
const cohere = createCohere({
apiKey,
});
return cohere(model);
}
export function getOpenAIModel(apiKey: OptionalApiKey, model: string) {
const openai = createOpenAI({
apiKey,
});
return openai(model);
}
export function getMistralModel(apiKey: OptionalApiKey, model: string) {
const mistral = createMistral({
apiKey,
});
return mistral(model);
}
export function getGoogleModel(apiKey: OptionalApiKey, model: string) {
const google = createGoogleGenerativeAI({
apiKey,
});
return google(model);
}
export function getGroqModel(apiKey: OptionalApiKey, model: string) {
const openai = createOpenAI({
baseURL: 'https://api.groq.com/openai/v1',
apiKey,
});
return openai(model);
}
export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) {
const openai = createOpenAI({
baseURL: 'https://api-inference.huggingface.co/v1/',
apiKey,
});
return openai(model);
}
export function getOllamaModel(baseURL: string, model: string) {
const ollamaInstance = ollama(model, {
numCtx: DEFAULT_NUM_CTX,
}) as LanguageModelV1 & { config: any };
ollamaInstance.config.baseURL = `${baseURL}/api`;
return ollamaInstance;
}
export function getDeepseekModel(apiKey: OptionalApiKey, model: string) {
const openai = createOpenAI({
baseURL: 'https://api.deepseek.com/beta',
apiKey,
});
return openai(model);
}
export function getOpenRouterModel(apiKey: OptionalApiKey, model: string) {
const openRouter = createOpenRouter({
apiKey,
});
return openRouter.chat(model);
}
export function getLMStudioModel(baseURL: string, model: string) {
const lmstudio = createOpenAI({
baseUrl: `${baseURL}/v1`,
apiKey: '',
});
return lmstudio(model);
}
export function getXAIModel(apiKey: OptionalApiKey, model: string) {
const openai = createOpenAI({
baseURL: 'https://api.x.ai/v1',
apiKey,
});
return openai(model);
}
export function getPerplexityModel(apiKey: OptionalApiKey, model: string) {
const perplexity = createOpenAI({
baseURL: 'https://api.perplexity.ai/',
apiKey,
});
return perplexity(model);
}
export function getModel(
provider: string,
model: string,
serverEnv: Env,
apiKeys?: Record<string, string>,
providerSettings?: Record<string, IProviderSetting>,
) {
/*
* let apiKey; // Declare first
* let baseURL;
*/
// console.log({provider,model});
const apiKey = getAPIKey(serverEnv, provider, apiKeys); // Then assign
const baseURL = getBaseURL(serverEnv, provider, providerSettings);
// console.log({apiKey,baseURL});
switch (provider) {
case 'Anthropic':
return getAnthropicModel(apiKey, model);
case 'OpenAI':
return getOpenAIModel(apiKey, model);
case 'Groq':
return getGroqModel(apiKey, model);
case 'HuggingFace':
return getHuggingFaceModel(apiKey, model);
case 'OpenRouter':
return getOpenRouterModel(apiKey, model);
case 'Google':
return getGoogleModel(apiKey, model);
case 'OpenAILike':
return getOpenAILikeModel(baseURL, apiKey, model);
case 'Together':
return getOpenAILikeModel(baseURL, apiKey, model);
case 'Deepseek':
return getDeepseekModel(apiKey, model);
case 'Mistral':
return getMistralModel(apiKey, model);
case 'LMStudio':
return getLMStudioModel(baseURL, model);
case 'xAI':
return getXAIModel(apiKey, model);
case 'Cohere':
return getCohereAIModel(apiKey, model);
case 'Perplexity':
return getPerplexityModel(apiKey, model);
default:
return getOllamaModel(baseURL, model);
}
}

11
app/lib/.server/llm/stream-text.ts
View File

@@ -1,5 +1,4 @@
import { convertToCoreMessages, streamText as _streamText } from 'ai';
import { getModel } from '~/lib/.server/llm/model';
import { MAX_TOKENS } from './constants';
import { getSystemPrompt } from '~/lib/common/prompts/prompts';
import {
@@ -8,6 +7,7 @@ import {
getModelList,
MODEL_REGEX,
MODIFICATIONS_TAG_NAME,
PROVIDER_LIST,
PROVIDER_REGEX,
WORK_DIR,
} from '~/utils/constants';
@@ -184,6 +184,8 @@ export async function streamText(props: {
const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS;
const provider = PROVIDER_LIST.find((p) => p.name === currentProvider) || DEFAULT_PROVIDER;
let systemPrompt =
PromptLibrary.getPropmtFromLibrary(promptId || 'default', {
cwd: WORK_DIR,
@@ -199,7 +201,12 @@ export async function streamText(props: {
}
return _streamText({
model: getModel(currentProvider, currentModel, serverEnv, apiKeys, providerSettings) as any,
model: provider.getModelInstance({
model: currentModel,
serverEnv,
apiKeys,
providerSettings,
}),
system: systemPrompt,
maxTokens: dynamicMaxTokens,
messages: convertToCoreMessages(processedMessages as any),