feat: submit file changes to the llm (#11)

packages/bolt/app/lib/stores/editor.ts

@@ -10,7 +10,7 @@ export class EditorStore {
   #filesStore: FilesStore;
 
   selectedFile: SelectedFile = import.meta.hot?.data.selectedFile ?? atom<string | undefined>();
-  documents: MapStore<EditorDocuments> = import.meta.hot?.data.documents ?? map<EditorDocuments>({});
+  documents: MapStore<EditorDocuments> = import.meta.hot?.data.documents ?? map({});
 
   currentDocument = computed([this.documents, this.selectedFile], (documents, selectedFile) => {
     if (!selectedFile) {
@@ -74,7 +74,7 @@ export class EditorStore {
     });
   }
 
-  updateFile(filePath: string, newContent: string | Uint8Array) {
+  updateFile(filePath: string, newContent: string) {
     const documents = this.documents.get();
     const documentState = documents[filePath];
 

packages/bolt/app/lib/stores/files.ts

@@ -1,17 +1,22 @@
 import type { PathWatcherEvent, WebContainer } from '@webcontainer/api';
+import { getEncoding } from 'istextorbinary';
 import { map, type MapStore } from 'nanostores';
+import { Buffer } from 'node:buffer';
 import * as nodePath from 'node:path';
 import { bufferWatchEvents } from '~/utils/buffer';
 import { WORK_DIR } from '~/utils/constants';
+import { computeFileModifications } from '~/utils/diff';
 import { createScopedLogger } from '~/utils/logger';
+import { unreachable } from '~/utils/unreachable';
 
 const logger = createScopedLogger('FilesStore');
 
-const textDecoder = new TextDecoder('utf8', { fatal: true });
+const utf8TextDecoder = new TextDecoder('utf8', { fatal: true });
 
 export interface File {
   type: 'file';
-  content: string | Uint8Array;
+  content: string;
+  isBinary: boolean;
 }
 
 export interface Folder {
@@ -30,6 +35,16 @@ export class FilesStore {
    */
   #size = 0;
 
+  /**
+   * @note Keeps track all modified files with their original content since the last user message.
+   * Needs to be reset when the user sends another message and all changes have to be submitted
+   * for the model to be aware of the changes.
+   */
+  #modifiedFiles: Map<string, string> = import.meta.hot?.data.modifiedFiles ?? new Map();
+
+  /**
+   * Map of files that matches the state of WebContainer.
+   */
   files: MapStore<FileMap> = import.meta.hot?.data.files ?? map({});
 
   get filesCount() {
@@ -41,6 +56,7 @@ export class FilesStore {
 
     if (import.meta.hot) {
       import.meta.hot.data.files = this.files;
+      import.meta.hot.data.modifiedFiles = this.#modifiedFiles;
     }
 
     this.#init();
@@ -56,7 +72,15 @@ export class FilesStore {
     return dirent;
   }
 
-  async saveFile(filePath: string, content: string | Uint8Array) {
+  getFileModifications() {
+    return computeFileModifications(this.files.get(), this.#modifiedFiles);
+  }
+
+  resetFileModifications() {
+    this.#modifiedFiles.clear();
+  }
+
+  async saveFile(filePath: string, content: string) {
     const webcontainer = await this.#webcontainer;
 
     try {
@@ -66,9 +90,20 @@ export class FilesStore {
         throw new Error(`EINVAL: invalid file path, write '${relativePath}'`);
       }
 
+      const oldContent = this.getFile(filePath)?.content;
+
+      if (!oldContent) {
+        unreachable('Expected content to be defined');
+      }
+
       await webcontainer.fs.writeFile(relativePath, content);
 
-      this.files.setKey(filePath, { type: 'file', content });
+      if (!this.#modifiedFiles.has(filePath)) {
+        this.#modifiedFiles.set(filePath, oldContent);
+      }
+
+      // we immediately update the file and don't rely on the `change` event coming from the watcher
+      this.files.setKey(filePath, { type: 'file', content, isBinary: false });
 
       logger.info('File updated');
     } catch (error) {
@@ -117,7 +152,21 @@ export class FilesStore {
             this.#size++;
           }
 
-          this.files.setKey(sanitizedPath, { type: 'file', content: this.#decodeFileContent(buffer) });
+          let content = '';
+
+          /**
+           * @note This check is purely for the editor. The way we detect this is not
+           * bullet-proof and it's a best guess so there might be false-positives.
+           * The reason we do this is because we don't want to display binary files
+           * in the editor nor allow to edit them.
+           */
+          const isBinary = isBinaryFile(buffer);
+
+          if (!isBinary) {
+            content = this.#decodeFileContent(buffer);
+          }
+
+          this.files.setKey(sanitizedPath, { type: 'file', content, isBinary });
 
           break;
         }
@@ -140,10 +189,32 @@ export class FilesStore {
     }
 
     try {
-      return textDecoder.decode(buffer);
+      return utf8TextDecoder.decode(buffer);
     } catch (error) {
       console.log(error);
       return '';
     }
   }
 }
+
+function isBinaryFile(buffer: Uint8Array | undefined) {
+  if (buffer === undefined) {
+    return false;
+  }
+
+  return getEncoding(convertToBuffer(buffer), { chunkLength: 100 }) === 'binary';
+}
+
+/**
+ * Converts a `Uint8Array` into a Node.js `Buffer` by copying the prototype.
+ * The goal is to  avoid expensive copies. It does create a new typed array
+ * but that's generally cheap as long as it uses the same underlying
+ * array buffer.
+ */
+function convertToBuffer(view: Uint8Array): Buffer {
+  const buffer = new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
+
+  Object.setPrototypeOf(buffer, Buffer.prototype);
+
+  return buffer as Buffer;
+}

packages/bolt/app/lib/stores/workbench.ts

@@ -70,7 +70,7 @@ export class WorkbenchStore {
     this.showWorkbench.set(show);
   }
 
-  setCurrentDocumentContent(newContent: string | Uint8Array) {
+  setCurrentDocumentContent(newContent: string) {
     const filePath = this.currentDocument.get()?.filePath;
 
     if (!filePath) {
@@ -119,6 +119,22 @@ export class WorkbenchStore {
     this.#editorStore.setSelectedFile(filePath);
   }
 
+  async saveFile(filePath: string) {
+    const documents = this.#editorStore.documents.get();
+    const document = documents[filePath];
+
+    if (document === undefined) {
+      return;
+    }
+
+    await this.#filesStore.saveFile(filePath, document.value);
+
+    const newUnsavedFiles = new Set(this.unsavedFiles.get());
+    newUnsavedFiles.delete(filePath);
+
+    this.unsavedFiles.set(newUnsavedFiles);
+  }
+
   async saveCurrentDocument() {
     const currentDocument = this.currentDocument.get();
 
@@ -126,14 +142,7 @@ export class WorkbenchStore {
       return;
     }
 
-    const { filePath } = currentDocument;
-
-    await this.#filesStore.saveFile(filePath, currentDocument.value);
-
-    const newUnsavedFiles = new Set(this.unsavedFiles.get());
-    newUnsavedFiles.delete(filePath);
-
-    this.unsavedFiles.set(newUnsavedFiles);
+    await this.saveFile(currentDocument.filePath);
   }
 
   resetCurrentDocument() {
@@ -153,6 +162,20 @@ export class WorkbenchStore {
     this.setCurrentDocumentContent(file.content);
   }
 
+  async saveAllFiles() {
+    for (const filePath of this.unsavedFiles.get()) {
+      await this.saveFile(filePath);
+    }
+  }
+
+  getFileModifcations() {
+    return this.#filesStore.getFileModifications();
+  }
+
+  resetAllFileModifications() {
+    this.#filesStore.resetFileModifications();
+  }
+
   abortAllActions() {
     // TODO: what do we wanna do and how do we wanna recover from this?
   }