From e78a5b0a050818454344020b1682c07c1051f932 Mon Sep 17 00:00:00 2001 From: Andrew Trokhymenko Date: Mon, 18 Nov 2024 19:55:28 -0500 Subject: [PATCH 01/29] image-upload --- app/components/chat/BaseChat.tsx | 121 ++++++++++++++++------ app/components/chat/Chat.client.tsx | 42 +++++++- app/components/chat/FilePreview.tsx | 40 +++++++ app/components/chat/SendButton.client.tsx | 5 +- app/components/chat/UserMessage.tsx | 27 ++++- app/components/sidebar/Menu.client.tsx | 2 +- app/components/workbench/EditorPanel.tsx | 9 +- app/components/workbench/FileTree.tsx | 2 +- app/lib/.server/llm/stream-text.ts | 41 ++++++-- app/routes/api.chat.ts | 17 ++- 10 files changed, 244 insertions(+), 62 deletions(-) create mode 100644 app/components/chat/FilePreview.tsx diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 629c5cb..e0cd928 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -17,6 +17,8 @@ import Cookies from 'js-cookie'; import styles from './BaseChat.module.scss'; import type { ProviderInfo } from '~/utils/types'; +import FilePreview from './FilePreview'; + const EXAMPLE_PROMPTS = [ { text: 'Build a todo app in React using Tailwind' }, { text: 'Build a simple blog using Astro' }, @@ -33,7 +35,7 @@ const ModelSelector = ({ model, setModel, provider, setProvider, modelList, prov { - setProvider(providerList.find(p => p.name === e.target.value)); + setProvider(providerList.find((p) => p.name === e.target.value)); const firstModel = [...modelList].find((m) => m.provider == e.target.value); setModel(firstModel ? firstModel.name : ''); }} @@ -51,7 +51,7 @@ const ModelSelector = ({ model, setModel, provider, setProvider, modelList, prov key={provider?.name} value={model} onChange={(e) => setModel(e.target.value)} - style={{ maxWidth: "70%" }} + style={{ maxWidth: '70%' }} className="flex-1 p-2 rounded-lg border border-bolt-elements-borderColor bg-bolt-elements-prompt-background text-bolt-elements-textPrimary focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus transition-all" > {[...modelList] @@ -93,32 +93,34 @@ interface BaseChatProps { setImageDataList?: (dataList: string[]) => void; } export const BaseChat = React.forwardRef( - ({ - textareaRef, - messageRef, - scrollRef, - showChat, - chatStarted = false, - isStreaming = false, - model, - setModel, - provider, - setProvider, - input = '', - enhancingPrompt, - handleInputChange, - promptEnhanced, - enhancePrompt, - sendMessage, - handleStop, - uploadedFiles, - setUploadedFiles, - imageDataList, - setImageDataList, - messages, - children, // Add this - }, ref) => { - console.log(provider); + ( + { + textareaRef, + messageRef, + scrollRef, + showChat = true, + chatStarted = false, + isStreaming = false, + model, + setModel, + provider, + setProvider, + input = '', + enhancingPrompt, + handleInputChange, + promptEnhanced, + enhancePrompt, + sendMessage, + handleStop, + uploadedFiles, + setUploadedFiles, + imageDataList, + setImageDataList, + messages, + children, // Add this + }, + ref, + ) => { const TEXTAREA_MAX_HEIGHT = chatStarted ? 400 : 200; const [apiKeys, setApiKeys] = useState>({}); const [modelList, setModelList] = useState(MODEL_LIST); @@ -139,7 +141,7 @@ export const BaseChat = React.forwardRef( Cookies.remove('apiKeys'); } - initializeModelList().then(modelList => { + initializeModelList().then((modelList) => { setModelList(modelList); }); }, []); @@ -239,12 +241,13 @@ export const BaseChat = React.forwardRef( setProvider={setProvider} providerList={PROVIDER_LIST} /> - {provider && + {provider && ( updateApiKey(provider.name, key)} - />} + /> + )} ( className="transition-all" onClick={() => handleFileUpload()} > -
+
( ); }, ); - - - diff --git a/app/components/chat/FilePreview.tsx b/app/components/chat/FilePreview.tsx index 378ada6..31fd11b 100644 --- a/app/components/chat/FilePreview.tsx +++ b/app/components/chat/FilePreview.tsx @@ -1,23 +1,22 @@ -// FilePreview.tsx +// Remove the lucide-react import import React from 'react'; -import { X } from 'lucide-react'; +// Rest of the interface remains the same interface FilePreviewProps { files: File[]; - imageDataList: string[]; // or imagePreviews: string[] + imageDataList: string[]; onRemove: (index: number) => void; } const FilePreview: React.FC = ({ files, imageDataList, onRemove }) => { if (!files || files.length === 0) { - return null; // Or render a placeholder if desired + return null; } return ( -
{/* Add horizontal scrolling if needed */} +
{files.map((file, index) => (
- {/* Display image preview or file icon */} {imageDataList[index] && (
{file.name} @@ -26,7 +25,7 @@ const FilePreview: React.FC = ({ files, imageDataList, onRemov className="absolute -top-2 -right-2 z-10 bg-white rounded-full p-1 shadow-md hover:bg-gray-100" >
- +
diff --git a/app/components/chat/UserMessage.tsx b/app/components/chat/UserMessage.tsx index a57d4fa..d7e1228 100644 --- a/app/components/chat/UserMessage.tsx +++ b/app/components/chat/UserMessage.tsx @@ -21,9 +21,6 @@ export function UserMessage({ content }: UserMessageProps) { ); } -// function sanitizeUserMessage(content: string) { -// return content.replace(modificationsRegex, '').replace(MODEL_REGEX, 'Using: $1').replace(PROVIDER_REGEX, ' ($1)\n\n').trim(); -// } function sanitizeUserMessage(content: string | Array<{type: string, text?: string, image_url?: {url: string}}>) { if (Array.isArray(content)) { return content.map(item => { diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index a50c28e..1603396 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -45,12 +45,12 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid if (item.type === 'text') { return { type: 'text', - text: item.text?.replace(/\[Model:.*?\]\n\n/, '').replace(/\[Provider:.*?\]\n\n/, '') + text: item.text?.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '') }; } return item; // Preserve image_url and other types as is }) - : textContent.replace(/\[Model:.*?\]\n\n/, '').replace(/\[Provider:.*?\]\n\n/, ''); + : textContent.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, ''); return { model, provider, content: cleanedContent }; } @@ -80,16 +80,6 @@ export function streamText( return message; // No changes for non-user messages }); - // const modelConfig = getModel(currentProvider, currentModel, env, apiKeys); - // const coreMessages = convertToCoreMessages(processedMessages); - - // console.log('Debug streamText:', JSON.stringify({ - // model: modelConfig, - // messages: processedMessages, - // coreMessages: coreMessages, - // system: getSystemPrompt() - // }, null, 2)); - return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), system: getSystemPrompt(), diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 284fccb..8fdb3d7 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -30,15 +30,15 @@ function parseCookies(cookieHeader) { } async function chatAction({ context, request }: ActionFunctionArgs) { - // console.log('=== API CHAT LOGGING START ==='); - // console.log('Request received:', request.url); - + const { messages, imageData } = await request.json<{ messages: Messages, imageData?: string[] }>(); const cookieHeader = request.headers.get("Cookie"); + + // Parse the cookie's value (returns an object or null if no cookie exists) const apiKeys = JSON.parse(parseCookies(cookieHeader).apiKeys || "{}"); const stream = new SwitchableStream(); @@ -71,13 +71,6 @@ async function chatAction({ context, request }: ActionFunctionArgs) { const result = await streamText(messages, context.cloudflare.env, options, apiKeys); - // console.log('=== API CHAT LOGGING START ==='); - // console.log('StreamText:', JSON.stringify({ - // messages, - // result, - // }, null, 2)); - // console.log('=== API CHAT LOGGING END ==='); - stream.switchSource(result.toAIStream()); return new Response(stream.readable, { diff --git a/package.json b/package.json index 56a9e72..40ede0f 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,6 @@ "jose": "^5.6.3", "js-cookie": "^3.0.5", "jszip": "^3.10.1", - "lucide-react": "^0.460.0", "nanostores": "^0.10.3", "ollama-ai-provider": "^0.15.2", "react": "^18.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ead147e..4158d19 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -155,9 +155,6 @@ importers: jszip: specifier: ^3.10.1 version: 3.10.1 - lucide-react: - specifier: ^0.460.0 - version: 0.460.0(react@18.3.1) nanostores: specifier: ^0.10.3 version: 0.10.3 @@ -3674,11 +3671,6 @@ packages: resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} engines: {node: '>=12'} - lucide-react@0.460.0: - resolution: {integrity: sha512-BVtq/DykVeIvRTJvRAgCsOwaGL8Un3Bxh8MbDxMhEWlZay3T4IpEKDEpwt5KZ0KJMHzgm6jrltxlT5eXOWXDHg==} - peerDependencies: - react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc - magic-string@0.25.9: resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==} @@ -9492,10 +9484,6 @@ snapshots: lru-cache@7.18.3: {} - lucide-react@0.460.0(react@18.3.1): - dependencies: - react: 18.3.1 - magic-string@0.25.9: dependencies: sourcemap-codec: 1.4.8 From 76713c2e6e5b2518f9d8d995647204e4ab66c6b9 Mon Sep 17 00:00:00 2001 From: Andrew Trokhymenko Date: Wed, 20 Nov 2024 17:56:07 -0500 Subject: [PATCH 04/29] fixes for PR #332 --- app/components/chat/UserMessage.tsx | 21 +++++---------------- app/components/workbench/EditorPanel.tsx | 9 +++++---- app/lib/.server/llm/stream-text.ts | 5 +++++ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/app/components/chat/UserMessage.tsx b/app/components/chat/UserMessage.tsx index d7e1228..c5d9c9b 100644 --- a/app/components/chat/UserMessage.tsx +++ b/app/components/chat/UserMessage.tsx @@ -9,10 +9,7 @@ interface UserMessageProps { } export function UserMessage({ content }: UserMessageProps) { - const sanitizedContent = sanitizeUserMessage(content); - const textContent = Array.isArray(sanitizedContent) - ? sanitizedContent.find(item => item.type === 'text')?.text || '' - : sanitizedContent; + const textContent = sanitizeUserMessage(content); return (
@@ -23,17 +20,9 @@ export function UserMessage({ content }: UserMessageProps) { function sanitizeUserMessage(content: string | Array<{type: string, text?: string, image_url?: {url: string}}>) { if (Array.isArray(content)) { - return content.map(item => { - if (item.type === 'text') { - return { - type: 'text', - text: item.text?.replace(/\[Model:.*?\]\n\n/, '').replace(/\[Provider:.*?\]\n\n/, '') - }; - } - return item; // Keep image_url items unchanged - }); + const textItem = content.find(item => item.type === 'text'); + return textItem?.text?.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '') || ''; } - // Handle legacy string content - return content.replace(/\[Model:.*?\]\n\n/, '').replace(/\[Provider:.*?\]\n\n/, ''); -} + return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, ''); +} \ No newline at end of file diff --git a/app/components/workbench/EditorPanel.tsx b/app/components/workbench/EditorPanel.tsx index e789f1d..a9c9d33 100644 --- a/app/components/workbench/EditorPanel.tsx +++ b/app/components/workbench/EditorPanel.tsx @@ -23,6 +23,7 @@ import { isMobile } from '~/utils/mobile'; import { FileBreadcrumb } from './FileBreadcrumb'; import { FileTree } from './FileTree'; import { Terminal, type TerminalRef } from './terminal/Terminal'; +import React from 'react'; interface EditorPanelProps { files?: FileMap; @@ -203,7 +204,7 @@ export const EditorPanel = memo( const isActive = activeTerminal === index; return ( - <> + {index == 0 ? ( - + )} - + ); })} {terminalCount < MAX_TERMINALS && } diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 1603396..7951512 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -80,6 +80,11 @@ export function streamText( return message; // No changes for non-user messages }); + console.log('Stream Text:', JSON.stringify({ + model: getModel(currentProvider, currentModel, env, apiKeys), + messages: convertToCoreMessages(processedMessages), + })); + return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), system: getSystemPrompt(), From 302cd28775b5abe214817c2b7790a313b2f8f34f Mon Sep 17 00:00:00 2001 From: Andrew Trokhymenko Date: Wed, 20 Nov 2024 19:35:57 -0500 Subject: [PATCH 05/29] . --- app/lib/.server/llm/stream-text.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 7951512..1603396 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -80,11 +80,6 @@ export function streamText( return message; // No changes for non-user messages }); - console.log('Stream Text:', JSON.stringify({ - model: getModel(currentProvider, currentModel, env, apiKeys), - messages: convertToCoreMessages(processedMessages), - })); - return _streamText({ model: getModel(currentProvider, currentModel, env, apiKeys), system: getSystemPrompt(), From 937ba7e61b9ba45d5283fbac3e8a34bd39f7d641 Mon Sep 17 00:00:00 2001 From: Andrew Trokhymenko Date: Thu, 21 Nov 2024 00:17:06 -0500 Subject: [PATCH 06/29] model pickup --- app/lib/.server/llm/stream-text.ts | 2 ++ app/routes/api.chat.ts | 8 ++++++-- app/utils/constants.ts | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 1603396..3b563ea 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -64,6 +64,8 @@ export function streamText( let currentModel = DEFAULT_MODEL; let currentProvider = DEFAULT_PROVIDER; + console.log('StreamText:', JSON.stringify(messages)); + const processedMessages = messages.map((message) => { if (message.role === 'user') { const { model, provider, content } = extractPropertiesFromMessage(message); diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 8fdb3d7..d622b46 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -31,11 +31,14 @@ function parseCookies(cookieHeader) { async function chatAction({ context, request }: ActionFunctionArgs) { - const { messages, imageData } = await request.json<{ + const { messages, imageData, model } = await request.json<{ messages: Messages, - imageData?: string[] + imageData?: string[], + model: string }>(); + console.log('ChatAction:', JSON.stringify(messages)); + const cookieHeader = request.headers.get("Cookie"); // Parse the cookie's value (returns an object or null if no cookie exists) @@ -47,6 +50,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { const options: StreamingOptions = { toolChoice: 'none', apiKeys, + model, onFinish: async ({ text: content, finishReason }) => { if (finishReason !== 'length') { return stream.close(); diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 308832b..501a87e 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -30,13 +30,15 @@ const PROVIDER_LIST: ProviderInfo[] = [ icon: "i-ph:cloud-arrow-down", }, { name: 'OpenAILike', - staticModels: [], + staticModels: [ + { name: 'o1-mini', label: 'o1-mini', provider: 'OpenAILike' }, + ], getDynamicModels: getOpenAILikeModels }, { name: 'OpenRouter', staticModels: [ - { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI' }, + { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenRouter' }, { name: 'anthropic/claude-3.5-sonnet', label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)', From 36b7d94bdd0ab8b92d780d581889c4dc75607cd7 Mon Sep 17 00:00:00 2001 From: navyseal4000 Date: Wed, 13 Nov 2024 21:36:50 -0600 Subject: [PATCH 07/29] Added speech to text capability --- app/components/chat/BaseChat.tsx | 79 ++++++++++++++++++++++++++++++++ app/types/global.d.ts | 2 + 2 files changed, 81 insertions(+) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 902cb23..fee28c0 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -87,6 +87,35 @@ interface BaseChatProps { enhancePrompt?: () => void; } +const SpeechRecognitionButton = ({ + isListening, + onStart, + onStop, + disabled +}: { + isListening: boolean; + onStart: () => void; + onStop: () => void; + disabled: boolean; +}) => { + return ( + + {isListening ? ( +
+ ) : ( +
+ )} + + ); +}; + export const BaseChat = React.forwardRef( ( { @@ -114,6 +143,8 @@ export const BaseChat = React.forwardRef( const TEXTAREA_MAX_HEIGHT = chatStarted ? 400 : 200; const [apiKeys, setApiKeys] = useState>({}); const [modelList, setModelList] = useState(MODEL_LIST); + const [isListening, setIsListening] = useState(false); + const [recognition, setRecognition] = useState(null); useEffect(() => { // Load API keys from cookies on component mount @@ -134,8 +165,49 @@ export const BaseChat = React.forwardRef( initializeModelList().then((modelList) => { setModelList(modelList); }); + if (typeof window !== 'undefined' && ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window)) { + const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; + const recognition = new SpeechRecognition(); + recognition.continuous = true; + recognition.interimResults = true; + + recognition.onresult = (event) => { + const transcript = Array.from(event.results) + .map(result => result[0]) + .map(result => result.transcript) + .join(''); + + if (handleInputChange) { + const syntheticEvent = { + target: { value: transcript }, + } as React.ChangeEvent; + handleInputChange(syntheticEvent); + } + }; + + recognition.onerror = (event) => { + console.error('Speech recognition error:', event.error); + setIsListening(false); + }; + + setRecognition(recognition); + } }, []); + const startListening = () => { + if (recognition) { + recognition.start(); + setIsListening(true); + } + }; + + const stopListening = () => { + if (recognition) { + recognition.stop(); + setIsListening(false); + } + }; + const updateApiKey = (provider: string, key: string) => { try { const updatedApiKeys = { ...apiKeys, [provider]: key }; @@ -284,6 +356,13 @@ export const BaseChat = React.forwardRef( )} + +
{input.length > 3 ? (
diff --git a/app/types/global.d.ts b/app/types/global.d.ts index a1f6789..193c65d 100644 --- a/app/types/global.d.ts +++ b/app/types/global.d.ts @@ -1,3 +1,5 @@ interface Window { showDirectoryPicker(): Promise; + webkitSpeechRecognition: typeof SpeechRecognition; + SpeechRecognition: typeof SpeechRecognition; } From a896f3f312bcecf9b9df588b63a4a3b78efbe06d Mon Sep 17 00:00:00 2001 From: navyseal4000 Date: Thu, 21 Nov 2024 07:55:53 -0600 Subject: [PATCH 08/29] Clear speech to text, listening upon submission --- app/components/chat/BaseChat.tsx | 38 ++++++++++++++++++++++++++++---- app/utils/constants.ts | 2 +- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index fee28c0..dde0cca 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -145,6 +145,7 @@ export const BaseChat = React.forwardRef( const [modelList, setModelList] = useState(MODEL_LIST); const [isListening, setIsListening] = useState(false); const [recognition, setRecognition] = useState(null); + const [transcript, setTranscript] = useState(''); useEffect(() => { // Load API keys from cookies on component mount @@ -177,6 +178,9 @@ export const BaseChat = React.forwardRef( .map(result => result.transcript) .join(''); + setTranscript(transcript); + + if (handleInputChange) { const syntheticEvent = { target: { value: transcript }, @@ -208,6 +212,25 @@ export const BaseChat = React.forwardRef( } }; + const handleSendMessage = (event: React.UIEvent, messageInput?: string) => { + if (sendMessage) { + sendMessage(event, messageInput); + if (recognition) { + recognition.abort(); // Stop current recognition + setTranscript(''); // Clear transcript + setIsListening(false); + + // Clear the input by triggering handleInputChange with empty value + if (handleInputChange) { + const syntheticEvent = { + target: { value: '' }, + } as React.ChangeEvent; + handleInputChange(syntheticEvent); + } + } + } + }; + const updateApiKey = (provider: string, key: string) => { try { const updatedApiKeys = { ...apiKeys, [provider]: key }; @@ -301,8 +324,11 @@ export const BaseChat = React.forwardRef( } event.preventDefault(); - - sendMessage?.(event); + if (isStreaming) { + handleStop?.(); + return; + } + handleSendMessage?.(event); } }} value={input} @@ -327,7 +353,7 @@ export const BaseChat = React.forwardRef( return; } - sendMessage?.(event); + handleSendMessage?.(event); }} /> )} @@ -384,7 +410,11 @@ export const BaseChat = React.forwardRef(
)} From 34a8a39a3c39f25eb0269942b98f0b6ca6c11044 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Mon, 2 Dec 2024 18:30:21 +0530 Subject: [PATCH 14/29] added artifact bundling for custom long artifacts like uploading folder --- app/components/chat/Artifact.tsx | 13 +++++++++++-- app/components/chat/ImportFolderButton.tsx | 2 +- app/lib/runtime/message-parser.ts | 2 ++ app/lib/stores/workbench.ts | 4 +++- app/types/artifact.ts | 1 + 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/app/components/chat/Artifact.tsx b/app/components/chat/Artifact.tsx index 682a4c7..f26b1ee 100644 --- a/app/components/chat/Artifact.tsx +++ b/app/components/chat/Artifact.tsx @@ -59,6 +59,14 @@ export const Artifact = memo(({ messageId }: ArtifactProps) => { workbenchStore.showWorkbench.set(!showWorkbench); }} > + {artifact.type == 'bundled' && ( + <> +
+
+
+
+ + )}
{artifact?.title}
Click to open Workbench
@@ -66,7 +74,7 @@ export const Artifact = memo(({ messageId }: ArtifactProps) => {
- {actions.length && ( + {actions.length && artifact.type !== 'bundled' && ( {
- {showActions && actions.length > 0 && ( + {artifact.type !== 'bundled' && showActions && actions.length > 0 && ( { transition={{ duration: 0.15 }} >
+
diff --git a/app/components/chat/ImportFolderButton.tsx b/app/components/chat/ImportFolderButton.tsx index 5f822ee..b28ca51 100644 --- a/app/components/chat/ImportFolderButton.tsx +++ b/app/components/chat/ImportFolderButton.tsx @@ -79,7 +79,7 @@ ${content} role: 'assistant', content: `I'll help you set up these files.${binaryFilesMessage} - + ${fileArtifacts.join('\n\n')} `, id: generateId(), diff --git a/app/lib/runtime/message-parser.ts b/app/lib/runtime/message-parser.ts index 48f3f52..ab6b695 100644 --- a/app/lib/runtime/message-parser.ts +++ b/app/lib/runtime/message-parser.ts @@ -192,6 +192,7 @@ export class StreamingMessageParser { const artifactTag = input.slice(i, openTagEnd + 1); const artifactTitle = this.#extractAttribute(artifactTag, 'title') as string; + const type = this.#extractAttribute(artifactTag, 'type') as string; const artifactId = this.#extractAttribute(artifactTag, 'id') as string; if (!artifactTitle) { @@ -207,6 +208,7 @@ export class StreamingMessageParser { const currentArtifact = { id: artifactId, title: artifactTitle, + type, } satisfies BoltArtifactData; state.currentArtifact = currentArtifact; diff --git a/app/lib/stores/workbench.ts b/app/lib/stores/workbench.ts index cbb3f8a..a1de47e 100644 --- a/app/lib/stores/workbench.ts +++ b/app/lib/stores/workbench.ts @@ -18,6 +18,7 @@ import { extractRelativePath } from '~/utils/diff'; export interface ArtifactState { id: string; title: string; + type?: string; closed: boolean; runner: ActionRunner; } @@ -229,7 +230,7 @@ export class WorkbenchStore { // TODO: what do we wanna do and how do we wanna recover from this? } - addArtifact({ messageId, title, id }: ArtifactCallbackData) { + addArtifact({ messageId, title, id,type }: ArtifactCallbackData) { const artifact = this.#getArtifact(messageId); if (artifact) { @@ -244,6 +245,7 @@ export class WorkbenchStore { id, title, closed: false, + type, runner: new ActionRunner(webcontainer, () => this.boltTerminal), }); } diff --git a/app/types/artifact.ts b/app/types/artifact.ts index e35697a..660729c 100644 --- a/app/types/artifact.ts +++ b/app/types/artifact.ts @@ -1,4 +1,5 @@ export interface BoltArtifactData { id: string; title: string; + type?: string; } From 0ab334126a9520907a58383f36209cfa8208bfbd Mon Sep 17 00:00:00 2001 From: Andrew Trokhymenko Date: Mon, 2 Dec 2024 14:08:41 -0500 Subject: [PATCH 15/29] adding to display the image in the chat conversation. and paste image too. tnx to @Stijnus --- app/components/chat/BaseChat.tsx | 30 +++++++++++++++++++++++ app/components/chat/UserMessage.tsx | 38 +++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index f004b3d..5c086d4 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -190,6 +190,35 @@ export const BaseChat = React.forwardRef( input.click(); }; + const handlePaste = async (e: React.ClipboardEvent) => { + const items = e.clipboardData?.items; + + if (!items) { + return; + } + + for (const item of items) { + if (item.type.startsWith('image/')) { + e.preventDefault(); + + const file = item.getAsFile(); + + if (file) { + const reader = new FileReader(); + + reader.onload = (e) => { + const base64Image = e.target?.result as string; + setUploadedFiles?.([...uploadedFiles, file]); + setImageDataList?.([...imageDataList, base64Image]); + }; + reader.readAsDataURL(file); + } + + break; + } + } + }; + const baseChat = (
( onChange={(event) => { handleInputChange?.(event); }} + onPaste={handlePaste} style={{ minHeight: TEXTAREA_MIN_HEIGHT, maxHeight: TEXTAREA_MAX_HEIGHT, diff --git a/app/components/chat/UserMessage.tsx b/app/components/chat/UserMessage.tsx index 167ce87..3e6485b 100644 --- a/app/components/chat/UserMessage.tsx +++ b/app/components/chat/UserMessage.tsx @@ -6,10 +6,39 @@ import { MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; import { Markdown } from './Markdown'; interface UserMessageProps { - content: string; + content: string | Array<{ type: string; text?: string; image?: string }>; } export function UserMessage({ content }: UserMessageProps) { + if (Array.isArray(content)) { + const textItem = content.find((item) => item.type === 'text'); + const textContent = sanitizeUserMessage(textItem?.text || ''); + const images = content.filter((item) => item.type === 'image' && item.image); + + return ( +
+
+
+ {textContent} +
+ {images.length > 0 && ( +
+ {images.map((item, index) => ( +
+ {`Uploaded +
+ ))} +
+ )} +
+
+ ); + } + const textContent = sanitizeUserMessage(content); return ( @@ -19,11 +48,6 @@ export function UserMessage({ content }: UserMessageProps) { ); } -function sanitizeUserMessage(content: string | Array<{ type: string; text?: string; image_url?: { url: string } }>) { - if (Array.isArray(content)) { - const textItem = content.find((item) => item.type === 'text'); - return textItem?.text?.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '') || ''; - } - +function sanitizeUserMessage(content: string) { return content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, ''); } From 1589d2a8f57143a05b7d2c9e7a03fc7fc1eca27c Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Tue, 3 Dec 2024 02:13:33 +0530 Subject: [PATCH 16/29] feat(Dynamic Models): together AI Dynamic Models --- app/lib/.server/llm/stream-text.ts | 18 +++---- app/routes/api.chat.ts | 14 ++--- app/types/model.ts | 2 +- app/utils/constants.ts | 83 ++++++++++++++++++++++++++++-- app/utils/parseCookies.ts | 19 +++++++ vite.config.ts | 2 +- 6 files changed, 113 insertions(+), 25 deletions(-) create mode 100644 app/utils/parseCookies.ts diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 3ef8792..bfb88c2 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -1,11 +1,8 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck – TODO: Provider proper types - import { convertToCoreMessages, streamText as _streamText } from 'ai'; import { getModel } from '~/lib/.server/llm/model'; import { MAX_TOKENS } from './constants'; import { getSystemPrompt } from './prompts'; -import { DEFAULT_MODEL, DEFAULT_PROVIDER, MODEL_LIST, MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; +import { DEFAULT_MODEL, DEFAULT_PROVIDER, getModelList, MODEL_REGEX, PROVIDER_REGEX } from '~/utils/constants'; interface ToolResult { toolCallId: string; @@ -32,7 +29,7 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid // Extract provider const providerMatch = message.content.match(PROVIDER_REGEX); - const provider = providerMatch ? providerMatch[1] : DEFAULT_PROVIDER; + const provider = providerMatch ? providerMatch[1] : DEFAULT_PROVIDER.name; // Remove model and provider lines from content const cleanedContent = message.content.replace(MODEL_REGEX, '').replace(PROVIDER_REGEX, '').trim(); @@ -40,10 +37,10 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } -export function streamText(messages: Messages, env: Env, options?: StreamingOptions, apiKeys?: Record) { +export async function streamText(messages: Messages, env: Env, options?: StreamingOptions,apiKeys?: Record) { let currentModel = DEFAULT_MODEL; - let currentProvider = DEFAULT_PROVIDER; - + let currentProvider = DEFAULT_PROVIDER.name; + const MODEL_LIST = await getModelList(apiKeys||{}); const processedMessages = messages.map((message) => { if (message.role === 'user') { const { model, provider, content } = extractPropertiesFromMessage(message); @@ -51,7 +48,6 @@ export function streamText(messages: Messages, env: Env, options?: StreamingOpti if (MODEL_LIST.find((m) => m.name === model)) { currentModel = model; } - currentProvider = provider; return { ...message, content }; @@ -65,10 +61,10 @@ export function streamText(messages: Messages, env: Env, options?: StreamingOpti const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS; return _streamText({ - model: getModel(currentProvider, currentModel, env, apiKeys), + model: getModel(currentProvider, currentModel, env, apiKeys) as any, system: getSystemPrompt(), maxTokens: dynamicMaxTokens, - messages: convertToCoreMessages(processedMessages), + messages: convertToCoreMessages(processedMessages as any), ...options, }); } diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index ac35a22..017ae92 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -1,6 +1,3 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck – TODO: Provider proper types - import { type ActionFunctionArgs } from '@remix-run/cloudflare'; import { MAX_RESPONSE_SEGMENTS, MAX_TOKENS } from '~/lib/.server/llm/constants'; import { CONTINUE_PROMPT } from '~/lib/.server/llm/prompts'; @@ -11,8 +8,8 @@ export async function action(args: ActionFunctionArgs) { return chatAction(args); } -function parseCookies(cookieHeader) { - const cookies = {}; +function parseCookies(cookieHeader:string) { + const cookies:any = {}; // Split the cookie string by semicolons and spaces const items = cookieHeader.split(';').map((cookie) => cookie.trim()); @@ -39,14 +36,13 @@ async function chatAction({ context, request }: ActionFunctionArgs) { const cookieHeader = request.headers.get('Cookie'); // Parse the cookie's value (returns an object or null if no cookie exists) - const apiKeys = JSON.parse(parseCookies(cookieHeader).apiKeys || '{}'); + const apiKeys = JSON.parse(parseCookies(cookieHeader||"").apiKeys || '{}'); const stream = new SwitchableStream(); try { const options: StreamingOptions = { toolChoice: 'none', - apiKeys, onFinish: async ({ text: content, finishReason }) => { if (finishReason !== 'length') { return stream.close(); @@ -63,7 +59,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { messages.push({ role: 'assistant', content }); messages.push({ role: 'user', content: CONTINUE_PROMPT }); - const result = await streamText(messages, context.cloudflare.env, options); + const result = await streamText(messages, context.cloudflare.env, options,apiKeys); return stream.switchSource(result.toAIStream()); }, @@ -79,7 +75,7 @@ async function chatAction({ context, request }: ActionFunctionArgs) { contentType: 'text/plain; charset=utf-8', }, }); - } catch (error) { + } catch (error:any) { console.log(error); if (error.message?.includes('API key')) { diff --git a/app/types/model.ts b/app/types/model.ts index 32522c6..29bff2e 100644 --- a/app/types/model.ts +++ b/app/types/model.ts @@ -3,7 +3,7 @@ import type { ModelInfo } from '~/utils/types'; export type ProviderInfo = { staticModels: ModelInfo[]; name: string; - getDynamicModels?: () => Promise; + getDynamicModels?: (apiKeys?: Record) => Promise; getApiKeyLink?: string; labelForGetApiKey?: string; icon?: string; diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 1120bc1..81a6457 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -1,3 +1,5 @@ +import Cookies from 'js-cookie'; +import { parseCookies } from './parseCookies'; import type { ModelInfo, OllamaApiResponse, OllamaModel } from './types'; import type { ProviderInfo } from '~/types/model'; @@ -262,6 +264,7 @@ const PROVIDER_LIST: ProviderInfo[] = [ }, { name: 'Together', + getDynamicModels: getTogetherModels, staticModels: [ { name: 'Qwen/Qwen2.5-Coder-32B-Instruct', @@ -293,6 +296,61 @@ const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat( export let MODEL_LIST: ModelInfo[] = [...staticModels]; + +export async function getModelList(apiKeys: Record) { + MODEL_LIST = [ + ...( + await Promise.all( + PROVIDER_LIST.filter( + (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, + ).map((p) => p.getDynamicModels(apiKeys)), + ) + ).flat(), + ...staticModels, + ]; + return MODEL_LIST; +} + +async function getTogetherModels(apiKeys?: Record): Promise { + try { + let baseUrl = import.meta.env.TOGETHER_API_BASE_URL || ''; + let provider='Together' + + if (!baseUrl) { + return []; + } + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? '' + + if (apiKeys && apiKeys[provider]) { + apiKey = apiKeys[provider]; + } + + if (!apiKey) { + return []; + } + + const response = await fetch(`${baseUrl}/models`, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + const res = (await response.json()) as any; + let data: any[] = (res || []).filter((model: any) => model.type=='chat') + return data.map((m: any) => ({ + name: m.id, + label: `${m.display_name} - in:$${(m.pricing.input).toFixed( + 2, + )} out:$${(m.pricing.output).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`, + provider: provider, + maxTokenAllowed: 8000, + })); +} catch (e) { + console.error('Error getting OpenAILike models:', e); + return []; +} +} + + const getOllamaBaseUrl = () => { const defaultBaseUrl = import.meta.env.OLLAMA_API_BASE_URL || 'http://localhost:11434'; @@ -339,8 +397,13 @@ async function getOpenAILikeModels(): Promise { if (!baseUrl) { return []; } + let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; + + let apikeys = JSON.parse(Cookies.get('apiKeys')||'{}') + if (apikeys && apikeys['OpenAILike']){ + apiKey = apikeys['OpenAILike']; + } - const apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? ''; const response = await fetch(`${baseUrl}/models`, { headers: { Authorization: `Bearer ${apiKey}`, @@ -396,7 +459,6 @@ async function getLMStudioModels(): Promise { if (typeof window === 'undefined') { return []; } - try { const baseUrl = import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234'; const response = await fetch(`${baseUrl}/v1/models`); @@ -414,12 +476,27 @@ async function getLMStudioModels(): Promise { } async function initializeModelList(): Promise { + let apiKeys: Record = {}; + try { + const storedApiKeys = Cookies.get('apiKeys'); + + if (storedApiKeys) { + const parsedKeys = JSON.parse(storedApiKeys); + + if (typeof parsedKeys === 'object' && parsedKeys !== null) { + apiKeys = parsedKeys; + } + } + + } catch (error) { + + } MODEL_LIST = [ ...( await Promise.all( PROVIDER_LIST.filter( (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels, - ).map((p) => p.getDynamicModels()), + ).map((p) => p.getDynamicModels(apiKeys)), ) ).flat(), ...staticModels, diff --git a/app/utils/parseCookies.ts b/app/utils/parseCookies.ts new file mode 100644 index 0000000..235adc3 --- /dev/null +++ b/app/utils/parseCookies.ts @@ -0,0 +1,19 @@ +export function parseCookies(cookieHeader: string) { + const cookies: any = {}; + + // Split the cookie string by semicolons and spaces + const items = cookieHeader.split(';').map((cookie) => cookie.trim()); + + items.forEach((item) => { + const [name, ...rest] = item.split('='); + + if (name && rest) { + // Decode the name and value, and join value parts in case it contains '=' + const decodedName = decodeURIComponent(name.trim()); + const decodedValue = decodeURIComponent(rest.join('=').trim()); + cookies[decodedName] = decodedValue; + } + }); + + return cookies; +} \ No newline at end of file diff --git a/vite.config.ts b/vite.config.ts index 5f86830..1bb08d3 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -28,7 +28,7 @@ export default defineConfig((config) => { chrome129IssuePlugin(), config.mode === 'production' && optimizeCssModules({ apply: 'build' }), ], - envPrefix:["VITE_","OPENAI_LIKE_API_","OLLAMA_API_BASE_URL","LMSTUDIO_API_BASE_URL"], + envPrefix: ["VITE_", "OPENAI_LIKE_API_", "OLLAMA_API_BASE_URL", "LMSTUDIO_API_BASE_URL","TOGETHER_API_BASE_URL"], css: { preprocessorOptions: { scss: { From 7192690c1cec4a87275d031df05c6861e43f84d9 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Tue, 3 Dec 2024 02:19:30 +0530 Subject: [PATCH 17/29] clean up --- app/utils/constants.ts | 1 - app/utils/parseCookies.ts | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100644 app/utils/parseCookies.ts diff --git a/app/utils/constants.ts b/app/utils/constants.ts index 81a6457..7109a93 100644 --- a/app/utils/constants.ts +++ b/app/utils/constants.ts @@ -1,5 +1,4 @@ import Cookies from 'js-cookie'; -import { parseCookies } from './parseCookies'; import type { ModelInfo, OllamaApiResponse, OllamaModel } from './types'; import type { ProviderInfo } from '~/types/model'; diff --git a/app/utils/parseCookies.ts b/app/utils/parseCookies.ts deleted file mode 100644 index 235adc3..0000000 --- a/app/utils/parseCookies.ts +++ /dev/null @@ -1,19 +0,0 @@ -export function parseCookies(cookieHeader: string) { - const cookies: any = {}; - - // Split the cookie string by semicolons and spaces - const items = cookieHeader.split(';').map((cookie) => cookie.trim()); - - items.forEach((item) => { - const [name, ...rest] = item.split('='); - - if (name && rest) { - // Decode the name and value, and join value parts in case it contains '=' - const decodedName = decodeURIComponent(name.trim()); - const decodedValue = decodeURIComponent(rest.join('=').trim()); - cookies[decodedName] = decodedValue; - } - }); - - return cookies; -} \ No newline at end of file From 5adc0f681c60b15e69da97dadfccf8efd3707259 Mon Sep 17 00:00:00 2001 From: Andrew Trokhymenko Date: Mon, 2 Dec 2024 20:27:10 -0500 Subject: [PATCH 18/29] adding drag and drop images to text area --- app/components/chat/BaseChat.tsx | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index 233aa66..749af6c 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -351,9 +351,41 @@ export const BaseChat = React.forwardRef( >