fix: Remove unzipper as it cannot handle zip within zip (#6162)

server/queues/tasks/ImportJSONTask.ts

@@ -53,7 +53,11 @@ export default class ImportJSONTask extends ImportTask {
         rootPath = path.dirname(node.path);
       }
       if (node.path === "metadata.json") {
-        metadata = JSON.parse(await fs.readFile(node.path, "utf8"));
+        try {
+          metadata = JSON.parse(await fs.readFile(node.path, "utf8"));
+        } catch (err) {
+          throw new Error(`Could not parse metadata.json. ${err.message}`);
+        }
       }
     }
 
@@ -116,9 +120,12 @@ export default class ImportJSONTask extends ImportTask {
         continue;
       }
 
-      const item: CollectionJSONExport = JSON.parse(
-        await fs.readFile(node.path, "utf8")
-      );
+      let item: CollectionJSONExport;
+      try {
+        item = JSON.parse(await fs.readFile(node.path, "utf8"));
+      } catch (err) {
+        throw new Error(`Could not parse ${node.path}. ${err.message}`);
+      }
 
       const collectionId = uuidv4();
       output.collections.push({

server/queues/tasks/ImportMarkdownZipTask.test.ts

@@ -1,4 +1,4 @@
-import fs from "fs";
+/* eslint-disable @typescript-eslint/no-empty-function */
 import path from "path";
 import { FileOperation } from "@server/models";
 import { buildFileOperation } from "@server/test/factories";
@@ -7,11 +7,19 @@ import ImportMarkdownZipTask from "./ImportMarkdownZipTask";
 describe("ImportMarkdownZipTask", () => {
   it("should import the documents, attachments", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "stream", {
+    Object.defineProperty(fileOperation, "handle", {
       get() {
-        return fs.createReadStream(
-          path.resolve(__dirname, "..", "..", "test", "fixtures", "outline.zip")
-        );
+        return {
+          path: path.resolve(
+            __dirname,
+            "..",
+            "..",
+            "test",
+            "fixtures",
+            "outline.zip"
+          ),
+          cleanup: async () => {},
+        };
       },
     });
     jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
@@ -30,11 +38,19 @@ describe("ImportMarkdownZipTask", () => {
 
   it("should throw an error with corrupt zip", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "stream", {
+    Object.defineProperty(fileOperation, "handle", {
       get() {
-        return fs.createReadStream(
-          path.resolve(__dirname, "..", "..", "test", "fixtures", "corrupt.zip")
-        );
+        return {
+          path: path.resolve(
+            __dirname,
+            "..",
+            "..",
+            "test",
+            "fixtures",
+            "corrupt.zip"
+          ),
+          cleanup: async () => {},
+        };
       },
     });
     jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
@@ -56,11 +72,19 @@ describe("ImportMarkdownZipTask", () => {
 
   it("should throw an error with empty collection in zip", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "stream", {
+    Object.defineProperty(fileOperation, "handle", {
       get() {
-        return fs.createReadStream(
-          path.resolve(__dirname, "..", "..", "test", "fixtures", "empty.zip")
-        );
+        return {
+          path: path.resolve(
+            __dirname,
+            "..",
+            "..",
+            "test",
+            "fixtures",
+            "empty.zip"
+          ),
+          cleanup: async () => {},
+        };
       },
     });
     jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);

server/queues/tasks/ImportNotionTask.test.ts

@@ -1,4 +1,4 @@
-import fs from "fs";
+/* eslint-disable @typescript-eslint/no-empty-function */
 import path from "path";
 import { FileOperation } from "@server/models";
 import { buildFileOperation } from "@server/test/factories";
@@ -7,18 +7,19 @@ import ImportNotionTask from "./ImportNotionTask";
 describe("ImportNotionTask", () => {
   it("should import successfully from a Markdown export", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "stream", {
+    Object.defineProperty(fileOperation, "handle", {
       get() {
-        return fs.createReadStream(
-          path.resolve(
+        return {
+          path: path.resolve(
             __dirname,
             "..",
             "..",
             "test",
             "fixtures",
             "notion-markdown.zip"
-          )
-        );
+          ),
+          cleanup: async () => {},
+        };
       },
     });
     jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);
@@ -44,18 +45,19 @@ describe("ImportNotionTask", () => {
 
   it("should import successfully from a HTML export", async () => {
     const fileOperation = await buildFileOperation();
-    Object.defineProperty(fileOperation, "stream", {
+    Object.defineProperty(fileOperation, "handle", {
       get() {
-        return fs.createReadStream(
-          path.resolve(
+        return {
+          path: path.resolve(
             __dirname,
             "..",
             "..",
             "test",
             "fixtures",
             "notion-html.zip"
-          )
-        );
+          ),
+          cleanup: async () => {},
+        };
       },
     });
     jest.spyOn(FileOperation, "findByPk").mockResolvedValue(fileOperation);

server/queues/tasks/ImportTask.ts

@@ -2,7 +2,6 @@ import path from "path";
 import { rm } from "fs-extra";
 import truncate from "lodash/truncate";
 import tmp from "tmp";
-import unzipper from "unzipper";
 import {
   AttachmentPreset,
   CollectionPermission,
@@ -13,7 +12,6 @@ import { CollectionValidation } from "@shared/validations";
 import attachmentCreator from "@server/commands/attachmentCreator";
 import documentCreator from "@server/commands/documentCreator";
 import { serializer } from "@server/editor";
-import env from "@server/env";
 import { InternalError, ValidationError } from "@server/errors";
 import Logger from "@server/logging/Logger";
 import {
@@ -25,6 +23,7 @@ import {
   Attachment,
 } from "@server/models";
 import { sequelize } from "@server/storage/database";
+import ZipHelper from "@server/utils/ZipHelper";
 import BaseTask, { TaskPriority } from "./BaseTask";
 
 type Props = {
@@ -198,30 +197,44 @@ export default abstract class ImportTask extends BaseTask<Props> {
   protected async fetchAndExtractData(
     fileOperation: FileOperation
   ): Promise<string> {
-    return new Promise((resolve, reject) => {
-      const stream = fileOperation.stream;
-      if (!stream) {
-        return reject(new Error("No stream available"));
-      }
+    let cleanup;
+    let filePath: string;
 
-      tmp.dir((err, path) => {
-        if (err) {
-          return reject(err);
-        }
+    try {
+      const res = await fileOperation.handle;
+      filePath = res.path;
+      cleanup = res.cleanup;
 
-        const dest = unzipper
-          .Extract({ path, verbose: env.isDevelopment })
-          .on("error", reject)
-          .on("close", () => resolve(path));
+      const path = await new Promise<string>((resolve, reject) => {
+        tmp.dir((err, tmpDir) => {
+          if (err) {
+            Logger.error("Could not create temporary directory", err);
+            return reject(err);
+          }
 
-        stream
-          .on("error", (err) => {
-            dest.end();
-            reject(err);
-          })
-          .pipe(dest);
+          Logger.debug(
+            "task",
+            `ImportTask extracting data for ${fileOperation.id}`
+          );
+
+          void ZipHelper.extract(filePath, tmpDir)
+            .then(() => resolve(tmpDir))
+            .catch((err) => {
+              Logger.error("Could not extract zip file", err);
+              reject(err);
+            });
+        });
       });
-    });
+
+      return path;
+    } finally {
+      Logger.debug(
+        "task",
+        `ImportTask cleaning up temporary data for ${fileOperation.id}`
+      );
+
+      await cleanup?.();
+    }
   }
 
   /**