Merge pull request #1849 from xKevIsDev/mcp-token-usage

fix: add text sanitization function to clean user and assistant messages of new parts object

app/lib/.server/llm/stream-text.ts

@@ -26,6 +26,14 @@ export interface StreamingOptions extends Omit<Parameters<typeof _streamText>[0]
 
 const logger = createScopedLogger('stream-text');
 
+function sanitizeText(text: string): string {
+  let sanitized = text.replace(/<div class=\\"__boltThought__\\">.*?<\/div>/s, '');
+  sanitized = sanitized.replace(/<think>.*?<\/think>/s, '');
+  sanitized = sanitized.replace(/<boltAction type="file" filePath="package-lock\.json">[\s\S]*?<\/boltAction>/g, '');
+
+  return sanitized.trim();
+}
+
 export async function streamText(props: {
   messages: Omit<Message, 'id'>[];
   env?: Env;
@@ -58,30 +66,25 @@ export async function streamText(props: {
   let currentModel = DEFAULT_MODEL;
   let currentProvider = DEFAULT_PROVIDER.name;
   let processedMessages = messages.map((message) => {
+    const newMessage = { ...message };
+
     if (message.role === 'user') {
       const { model, provider, content } = extractPropertiesFromMessage(message);
       currentModel = model;
       currentProvider = provider;
-
+      newMessage.content = sanitizeText(content);
-      return { ...message, content };
     } else if (message.role == 'assistant') {
-      let content = message.content;
+      newMessage.content = sanitizeText(message.content);
-      content = content.replace(/<div class=\\"__boltThought__\\">.*?<\/div>/s, '');
-      content = content.replace(/<think>.*?<\/think>/s, '');
-
-      // Remove package-lock.json content specifically keeping token usage MUCH lower
-      content = content.replace(
-        /<boltAction type="file" filePath="package-lock\.json">[\s\S]*?<\/boltAction>/g,
-        '[package-lock.json content removed]',
-      );
-
-      // Trim whitespace potentially left after removals
-      content = content.trim();
-
-      return { ...message, content };
     }
 
-    return message;
+    // Sanitize all text parts in parts array, if present
+    if (Array.isArray(message.parts)) {
+      newMessage.parts = message.parts.map((part) =>
+        part.type === 'text' ? { ...part, text: sanitizeText(part.text) } : part,
+      );
+    }
+
+    return newMessage;
   });
 
   const provider = PROVIDER_LIST.find((p) => p.name === currentProvider) || DEFAULT_PROVIDER;

app/lib/common/prompts/new-prompt.ts

@@ -65,72 +65,78 @@ The year is 2025.
       : ''
   }
 
-  Create .env file if it doesn't exist${
+
+  ${
     supabase?.isConnected &&
     supabase?.hasSelectedProject &&
     supabase?.credentials?.supabaseUrl &&
     supabase?.credentials?.anonKey
-      ? ` with:
+      ? `
-    VITE_SUPABASE_URL=${supabase.credentials.supabaseUrl}
+    Create .env file if it doesn't exist${
-    VITE_SUPABASE_ANON_KEY=${supabase.credentials.anonKey}`
+      supabase?.isConnected &&
-      : '.'
+      supabase?.hasSelectedProject &&
+      supabase?.credentials?.supabaseUrl &&
+      supabase?.credentials?.anonKey
+        ? ` with:
+      VITE_SUPABASE_URL=${supabase.credentials.supabaseUrl}
+      VITE_SUPABASE_ANON_KEY=${supabase.credentials.anonKey}`
+        : '.'
+    }
+    DATA PRESERVATION REQUIREMENTS:
+      - DATA INTEGRITY IS HIGHEST PRIORITY - users must NEVER lose data
+      - FORBIDDEN: Destructive operations (DROP, DELETE) that could cause data loss
+      - FORBIDDEN: Transaction control (BEGIN, COMMIT, ROLLBACK, END)
+        Note: DO $$ BEGIN ... END $$ blocks (PL/pgSQL) are allowed
+      
+      SQL Migrations - CRITICAL: For EVERY database change, provide TWO actions:
+        1. Migration File: <boltAction type="supabase" operation="migration" filePath="/supabase/migrations/name.sql">
+        2. Query Execution: <boltAction type="supabase" operation="query" projectId="\${projectId}">
+      
+      Migration Rules:
+        - NEVER use diffs, ALWAYS provide COMPLETE file content
+        - Create new migration file for each change in /home/project/supabase/migrations
+        - NEVER update existing migration files
+        - Descriptive names without number prefix (e.g., create_users.sql)
+        - ALWAYS enable RLS: alter table users enable row level security;
+        - Add appropriate RLS policies for CRUD operations
+        - Use default values: DEFAULT false/true, DEFAULT 0, DEFAULT '', DEFAULT now()
+        - Start with markdown summary in multi-line comment explaining changes
+        - Use IF EXISTS/IF NOT EXISTS for safe operations
+      
+      Example migration:
+      /*
+        # Create users table
+        1. New Tables: users (id uuid, email text, created_at timestamp)
+        2. Security: Enable RLS, add read policy for authenticated users
+      */
+      CREATE TABLE IF NOT EXISTS users (
+        id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+        email text UNIQUE NOT NULL,
+        created_at timestamptz DEFAULT now()
+      );
+      ALTER TABLE users ENABLE ROW LEVEL SECURITY;
+      CREATE POLICY "Users read own data" ON users FOR SELECT TO authenticated USING (auth.uid() = id);
+    
+    Client Setup:
+      - Use @supabase/supabase-js
+      - Create singleton client instance
+      - Use environment variables from .env
+    
+    Authentication:
+      - ALWAYS use email/password signup
+      - FORBIDDEN: magic links, social providers, SSO (unless explicitly stated)
+      - FORBIDDEN: custom auth systems, ALWAYS use Supabase's built-in auth
+      - Email confirmation ALWAYS disabled unless stated
+    
+    Security:
+      - ALWAYS enable RLS for every new table
+      - Create policies based on user authentication
+      - One migration per logical change
+      - Use descriptive policy names
+      - Add indexes for frequently queried columns
+  `
+      : ''
   }
-  
-  NEVER modify Supabase config or .env files apart from creating .env.
-  Do not generate types for supabase.
-  
-  DATA PRESERVATION REQUIREMENTS:
-    - DATA INTEGRITY IS HIGHEST PRIORITY - users must NEVER lose data
-    - FORBIDDEN: Destructive operations (DROP, DELETE) that could cause data loss
-    - FORBIDDEN: Transaction control (BEGIN, COMMIT, ROLLBACK, END)
-      Note: DO $$ BEGIN ... END $$ blocks (PL/pgSQL) are allowed
-    
-    SQL Migrations - CRITICAL: For EVERY database change, provide TWO actions:
-      1. Migration File: <boltAction type="supabase" operation="migration" filePath="/supabase/migrations/name.sql">
-      2. Query Execution: <boltAction type="supabase" operation="query" projectId="\${projectId}">
-    
-    Migration Rules:
-      - NEVER use diffs, ALWAYS provide COMPLETE file content
-      - Create new migration file for each change in /home/project/supabase/migrations
-      - NEVER update existing migration files
-      - Descriptive names without number prefix (e.g., create_users.sql)
-      - ALWAYS enable RLS: alter table users enable row level security;
-      - Add appropriate RLS policies for CRUD operations
-      - Use default values: DEFAULT false/true, DEFAULT 0, DEFAULT '', DEFAULT now()
-      - Start with markdown summary in multi-line comment explaining changes
-      - Use IF EXISTS/IF NOT EXISTS for safe operations
-    
-    Example migration:
-    /*
-      # Create users table
-      1. New Tables: users (id uuid, email text, created_at timestamp)
-      2. Security: Enable RLS, add read policy for authenticated users
-    */
-    CREATE TABLE IF NOT EXISTS users (
-      id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-      email text UNIQUE NOT NULL,
-      created_at timestamptz DEFAULT now()
-    );
-    ALTER TABLE users ENABLE ROW LEVEL SECURITY;
-    CREATE POLICY "Users read own data" ON users FOR SELECT TO authenticated USING (auth.uid() = id);
-  
-  Client Setup:
-    - Use @supabase/supabase-js
-    - Create singleton client instance
-    - Use environment variables from .env
-  
-  Authentication:
-    - ALWAYS use email/password signup
-    - FORBIDDEN: magic links, social providers, SSO (unless explicitly stated)
-    - FORBIDDEN: custom auth systems, ALWAYS use Supabase's built-in auth
-    - Email confirmation ALWAYS disabled unless stated
-  
-  Security:
-    - ALWAYS enable RLS for every new table
-    - Create policies based on user authentication
-    - One migration per logical change
-    - Use descriptive policy names
-    - Add indexes for frequently queried columns
 </database_instructions>
 
 <artifact_instructions>

app/lib/stores/files.ts

@@ -597,7 +597,11 @@ export class FilesStore {
 
     // Set up file watcher
     webcontainer.internal.watchPaths(
-      { include: [`${WORK_DIR}/**`], exclude: ['**/node_modules', '.git'], includeContent: true },
+      {
+        include: [`${WORK_DIR}/**`],
+        exclude: ['**/node_modules', '.git', '**/package-lock.json'],
+        includeContent: true,
+      },
       bufferWatchEvents(100, this.#processEventBuffer.bind(this)),
     );
 

app/lib/stores/previews.ts

@@ -151,45 +151,6 @@ export class PreviewsStore {
       this._broadcastStorageSync();
     });
 
-    try {
-      // Watch for file changes
-      webcontainer.internal.watchPaths(
-        {
-          // Only watch specific file types that affect the preview
-          include: ['**/*.html', '**/*.css', '**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx', '**/*.json'],
-          exclude: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/coverage/**'],
-        },
-        async (_events) => {
-          const previews = this.previews.get();
-
-          for (const preview of previews) {
-            const previewId = this.getPreviewId(preview.baseUrl);
-
-            if (previewId) {
-              this.broadcastFileChange(previewId);
-            }
-          }
-        },
-      );
-
-      // Watch for DOM changes that might affect storage
-      if (typeof window !== 'undefined') {
-        const observer = new MutationObserver((_mutations) => {
-          // Broadcast storage changes when DOM changes
-          this._broadcastStorageSync();
-        });
-
-        observer.observe(document.body, {
-          childList: true,
-          subtree: true,
-          characterData: true,
-          attributes: true,
-        });
-      }
-    } catch (error) {
-      console.error('[Preview] Error setting up watchers:', error);
-    }
-
     // Listen for port events
     webcontainer.on('port', (port, type, url) => {
       let previewInfo = this.#availablePreviews.get(port);