test (#4796)

server/queues/tasks/ExportDocumentTreeTask.ts

@@ -55,10 +55,10 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
     await Promise.all(
       attachments.map(async (attachment) => {
         try {
-          const img = await getFileByKey(attachment.key);
+          const stream = getFileByKey(attachment.key);
           const dir = path.dirname(pathInZip);
-          if (img) {
-            zip.file(path.join(dir, attachment.key), img as Blob, {
+          if (stream) {
+            zip.file(path.join(dir, attachment.key), stream, {
               createFolders: true,
             });
           }

server/queues/tasks/ImportMarkdownZipTask.test.ts

@@ -10,9 +10,9 @@ setupTestDatabase();
 describe("ImportMarkdownZipTask", () => {
   it("should import the documents, attachments", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "buffer", {
+    Object.defineProperty(fileOperation, "stream", {
       get() {
-        return fs.readFileSync(
+        return fs.createReadStream(
           path.resolve(__dirname, "..", "..", "test", "fixtures", "outline.zip")
         );
       },
@@ -33,9 +33,9 @@ describe("ImportMarkdownZipTask", () => {
 
   it("should throw an error with corrupt zip", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "buffer", {
+    Object.defineProperty(fileOperation, "stream", {
       get() {
-        return fs.readFileSync(
+        return fs.createReadStream(
           path.resolve(__dirname, "..", "..", "test", "fixtures", "corrupt.zip")
         );
       },
@@ -59,9 +59,9 @@ describe("ImportMarkdownZipTask", () => {
 
   it("should throw an error with empty collection in zip", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "buffer", {
+    Object.defineProperty(fileOperation, "stream", {
       get() {
-        return fs.readFileSync(
+        return fs.createReadStream(
           path.resolve(__dirname, "..", "..", "test", "fixtures", "empty.zip")
         );
       },

server/queues/tasks/ImportMarkdownZipTask.ts

@@ -10,10 +10,10 @@ import ImportTask, { StructuredImportData } from "./ImportTask";
 
 export default class ImportMarkdownZipTask extends ImportTask {
   public async parseData(
-    buffer: Buffer,
+    stream: NodeJS.ReadableStream,
     fileOperation: FileOperation
   ): Promise<StructuredImportData> {
-    const zip = await JSZip.loadAsync(buffer);
+    const zip = await JSZip.loadAsync(stream);
     const tree = ZipHelper.toFileTree(zip);
 
     return this.parseFileTree({ fileOperation, zip, tree });

server/queues/tasks/ImportNotionTask.test.ts

@@ -10,9 +10,9 @@ setupTestDatabase();
 describe("ImportNotionTask", () => {
   it("should import successfully from a Markdown export", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "buffer", {
+    Object.defineProperty(fileOperation, "stream", {
       get() {
-        return fs.readFileSync(
+        return fs.createReadStream(
           path.resolve(
             __dirname,
             "..",
@@ -45,9 +45,9 @@ describe("ImportNotionTask", () => {
 
   it("should import successfully from a HTML export", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "buffer", {
+    Object.defineProperty(fileOperation, "stream", {
       get() {
-        return fs.readFileSync(
+        return fs.createReadStream(
           path.resolve(
             __dirname,
             "..",

server/queues/tasks/ImportNotionTask.ts

@@ -11,10 +11,10 @@ import ImportTask, { StructuredImportData } from "./ImportTask";
 
 export default class ImportNotionTask extends ImportTask {
   public async parseData(
-    buffer: Buffer,
+    stream: NodeJS.ReadableStream,
     fileOperation: FileOperation
   ): Promise<StructuredImportData> {
-    const zip = await JSZip.loadAsync(buffer);
+    const zip = await JSZip.loadAsync(stream);
     const tree = ZipHelper.toFileTree(zip);
     return this.parseFileTree({ fileOperation, zip, tree });
   }

server/queues/tasks/ImportTask.ts

@@ -161,15 +161,27 @@ export default abstract class ImportTask extends BaseTask<Props> {
   }
 
   /**
-   * Fetch the remote data needed for the import, by default this will download
-   * any file associated with the FileOperation, save it to a temporary file,
-   * and return the path.
+   * Fetch the remote data associated with the file operation as a Buffer.
    *
    * @param fileOperation The FileOperation to fetch data for
-   * @returns string
+   * @returns A promise that resolves to the data as a buffer.
    */
-  protected async fetchData(fileOperation: FileOperation) {
-    return fileOperation.buffer;
+  protected async fetchData(fileOperation: FileOperation): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+      const bufs: Buffer[] = [];
+      const stream = fileOperation.stream;
+      if (!stream) {
+        return reject(new Error("No stream available"));
+      }
+
+      stream.on("data", function (d) {
+        bufs.push(d);
+      });
+      stream.on("error", reject);
+      stream.on("end", () => {
+        resolve(Buffer.concat(bufs));
+      });
+    });
   }
 
   /**