feat: configure dynamic providers via .env (#1108)

* Use backend API route to fetch dynamic models

# Conflicts:
#	app/components/chat/BaseChat.tsx

* Override ApiKeys if provided in frontend

* Remove obsolete artifact

* Transport api keys from client to server in header

* Cache static provider information

* Restore reading provider settings from cookie

* Reload only a single provider on api key change

* Transport apiKeys and providerSettings via cookies.

While doing this, introduce a simple helper function for cookies

app/routes/api.enhancer.ts

@@ -1,34 +1,13 @@
 import { type ActionFunctionArgs } from '@remix-run/cloudflare';
-
-//import { StreamingTextResponse, parseStreamPart } from 'ai';
 import { streamText } from '~/lib/.server/llm/stream-text';
 import { stripIndents } from '~/utils/stripIndent';
-import type { IProviderSetting, ProviderInfo } from '~/types/model';
+import type { ProviderInfo } from '~/types/model';
+import { getApiKeysFromCookie, getProviderSettingsFromCookie } from '~/lib/api/cookies';
 
 export async function action(args: ActionFunctionArgs) {
   return enhancerAction(args);
 }
 
-function parseCookies(cookieHeader: string) {
-  const cookies: any = {};
-
-  // Split the cookie string by semicolons and spaces
-  const items = cookieHeader.split(';').map((cookie) => cookie.trim());
-
-  items.forEach((item) => {
-    const [name, ...rest] = item.split('=');
-
-    if (name && rest) {
-      // Decode the name and value, and join value parts in case it contains '='
-      const decodedName = decodeURIComponent(name.trim());
-      const decodedValue = decodeURIComponent(rest.join('=').trim());
-      cookies[decodedName] = decodedValue;
-    }
-  });
-
-  return cookies;
-}
-
 async function enhancerAction({ context, request }: ActionFunctionArgs) {
   const { message, model, provider } = await request.json<{
     message: string;
@@ -55,12 +34,8 @@ async function enhancerAction({ context, request }: ActionFunctionArgs) {
   }
 
   const cookieHeader = request.headers.get('Cookie');
-
-  // Parse the cookie's value (returns an object or null if no cookie exists)
-  const apiKeys = JSON.parse(parseCookies(cookieHeader || '').apiKeys || '{}');
-  const providerSettings: Record<string, IProviderSetting> = JSON.parse(
-    parseCookies(cookieHeader || '').providers || '{}',
-  );
+  const apiKeys = getApiKeysFromCookie(cookieHeader);
+  const providerSettings = getProviderSettingsFromCookie(cookieHeader);
 
   try {
     const result = await streamText({

app/routes/api.llmcall.ts

@@ -1,34 +1,24 @@
 import { type ActionFunctionArgs } from '@remix-run/cloudflare';
-
-//import { StreamingTextResponse, parseStreamPart } from 'ai';
 import { streamText } from '~/lib/.server/llm/stream-text';
 import type { IProviderSetting, ProviderInfo } from '~/types/model';
 import { generateText } from 'ai';
-import { getModelList, PROVIDER_LIST } from '~/utils/constants';
+import { PROVIDER_LIST } from '~/utils/constants';
 import { MAX_TOKENS } from '~/lib/.server/llm/constants';
+import { LLMManager } from '~/lib/modules/llm/manager';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import { getApiKeysFromCookie, getProviderSettingsFromCookie } from '~/lib/api/cookies';
 
 export async function action(args: ActionFunctionArgs) {
   return llmCallAction(args);
 }
 
-function parseCookies(cookieHeader: string) {
-  const cookies: any = {};
-
-  // Split the cookie string by semicolons and spaces
-  const items = cookieHeader.split(';').map((cookie) => cookie.trim());
-
-  items.forEach((item) => {
-    const [name, ...rest] = item.split('=');
-
-    if (name && rest) {
-      // Decode the name and value, and join value parts in case it contains '='
-      const decodedName = decodeURIComponent(name.trim());
-      const decodedValue = decodeURIComponent(rest.join('=').trim());
-      cookies[decodedName] = decodedValue;
-    }
-  });
-
-  return cookies;
+async function getModelList(options: {
+  apiKeys?: Record<string, string>;
+  providerSettings?: Record<string, IProviderSetting>;
+  serverEnv?: Record<string, string>;
+}) {
+  const llmManager = LLMManager.getInstance(import.meta.env);
+  return llmManager.updateModelList(options);
 }
 
 async function llmCallAction({ context, request }: ActionFunctionArgs) {
@@ -58,12 +48,8 @@ async function llmCallAction({ context, request }: ActionFunctionArgs) {
   }
 
   const cookieHeader = request.headers.get('Cookie');
-
-  // Parse the cookie's value (returns an object or null if no cookie exists)
-  const apiKeys = JSON.parse(parseCookies(cookieHeader || '').apiKeys || '{}');
-  const providerSettings: Record<string, IProviderSetting> = JSON.parse(
-    parseCookies(cookieHeader || '').providers || '{}',
-  );
+  const apiKeys = getApiKeysFromCookie(cookieHeader);
+  const providerSettings = getProviderSettingsFromCookie(cookieHeader);
 
   if (streamOutput) {
     try {
@@ -105,8 +91,8 @@ async function llmCallAction({ context, request }: ActionFunctionArgs) {
     }
   } else {
     try {
-      const MODEL_LIST = await getModelList({ apiKeys, providerSettings, serverEnv: context.cloudflare.env as any });
-      const modelDetails = MODEL_LIST.find((m) => m.name === model);
+      const models = await getModelList({ apiKeys, providerSettings, serverEnv: context.cloudflare.env as any });
+      const modelDetails = models.find((m: ModelInfo) => m.name === model);
 
       if (!modelDetails) {
         throw new Error('Model not found');

app/routes/api.models.$provider.ts

@@ -0,0 +1,2 @@
+import { loader } from './api.models';
+export { loader };

app/routes/api.models.ts

@@ -1,6 +1,84 @@
 import { json } from '@remix-run/cloudflare';
-import { MODEL_LIST } from '~/utils/constants';
+import { LLMManager } from '~/lib/modules/llm/manager';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { ProviderInfo } from '~/types/model';
+import { getApiKeysFromCookie, getProviderSettingsFromCookie } from '~/lib/api/cookies';
 
-export async function loader() {
-  return json(MODEL_LIST);
+interface ModelsResponse {
+  modelList: ModelInfo[];
+  providers: ProviderInfo[];
+  defaultProvider: ProviderInfo;
+}
+
+let cachedProviders: ProviderInfo[] | null = null;
+let cachedDefaultProvider: ProviderInfo | null = null;
+
+function getProviderInfo(llmManager: LLMManager) {
+  if (!cachedProviders) {
+    cachedProviders = llmManager.getAllProviders().map((provider) => ({
+      name: provider.name,
+      staticModels: provider.staticModels,
+      getApiKeyLink: provider.getApiKeyLink,
+      labelForGetApiKey: provider.labelForGetApiKey,
+      icon: provider.icon,
+    }));
+  }
+
+  if (!cachedDefaultProvider) {
+    const defaultProvider = llmManager.getDefaultProvider();
+    cachedDefaultProvider = {
+      name: defaultProvider.name,
+      staticModels: defaultProvider.staticModels,
+      getApiKeyLink: defaultProvider.getApiKeyLink,
+      labelForGetApiKey: defaultProvider.labelForGetApiKey,
+      icon: defaultProvider.icon,
+    };
+  }
+
+  return { providers: cachedProviders, defaultProvider: cachedDefaultProvider };
+}
+
+export async function loader({
+  request,
+  params,
+}: {
+  request: Request;
+  params: { provider?: string };
+}): Promise<Response> {
+  const llmManager = LLMManager.getInstance(import.meta.env);
+
+  // Get client side maintained API keys and provider settings from cookies
+  const cookieHeader = request.headers.get('Cookie');
+  const apiKeys = getApiKeysFromCookie(cookieHeader);
+  const providerSettings = getProviderSettingsFromCookie(cookieHeader);
+
+  const { providers, defaultProvider } = getProviderInfo(llmManager);
+
+  let modelList: ModelInfo[] = [];
+
+  if (params.provider) {
+    // Only update models for the specific provider
+    const provider = llmManager.getProvider(params.provider);
+
+    if (provider) {
+      const staticModels = provider.staticModels;
+      const dynamicModels = provider.getDynamicModels
+        ? await provider.getDynamicModels(apiKeys, providerSettings, import.meta.env)
+        : [];
+      modelList = [...staticModels, ...dynamicModels];
+    }
+  } else {
+    // Update all models
+    modelList = await llmManager.updateModelList({
+      apiKeys,
+      providerSettings,
+      serverEnv: import.meta.env,
+    });
+  }
+
+  return json<ModelsResponse>({
+    modelList,
+    providers,
+    defaultProvider,
+  });
 }