fix: Long collection description prevents import (#3847)

* fix: Long collection description prevents import
fix: Parallelize attachment upload during import

* fix: Improve Notion image import matching

* chore: Bump JSZIP (perf)

* fix: Allow redirect from /doc/<id> to canonical url

* fix: Importing document with only title duplicates title in body

server/commands/documentImporter.test.ts

@@ -148,6 +148,21 @@ describe("documentImporter", () => {
     expect(response.title).toEqual("Heading 1");
   });
 
+  it("should handle only title", async () => {
+    const user = await buildUser();
+    const fileName = "markdown.md";
+    const content = `# Title`;
+    const response = await documentImporter({
+      user,
+      mimeType: "text/plain",
+      fileName,
+      content,
+      ip,
+    });
+    expect(response.text).toEqual("");
+    expect(response.title).toEqual("Title");
+  });
+
   it("should fallback to extension if mimetype unknown", async () => {
     const user = await buildUser();
     const fileName = "markdown.md";

server/commands/documentImporter.ts

@@ -190,7 +190,7 @@ async function documentImporter({
   if (text.startsWith("# ")) {
     const result = parseTitle(text);
     title = result.title;
-    text = text.replace(`# ${title}\n`, "");
+    text = text.replace(`# ${title}`, "").trimStart();
   }
 
   // If we parsed an emoji from _above_ the title then add it back at prefixing

server/models/Collection.ts

@@ -24,6 +24,7 @@ import isUUID from "validator/lib/isUUID";
 import { MAX_TITLE_LENGTH } from "@shared/constants";
 import { sortNavigationNodes } from "@shared/utils/collections";
 import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
+import { CollectionValidation } from "@shared/validations";
 import slugify from "@server/utils/slugify";
 import { NavigationNode, CollectionSort } from "~/types";
 import CollectionGroup from "./CollectionGroup";
@@ -150,8 +151,8 @@ class Collection extends ParanoidModel {
   name: string;
 
   @Length({
-    max: 1000,
-    msg: `description must be 1000 characters or less`,
+    max: CollectionValidation.maxDescriptionLength,
+    msg: `description must be ${CollectionValidation.maxDescriptionLength} characters or less`,
   })
   @Column
   description: string;

server/queues/tasks/ImportNotionTask.ts

@@ -139,13 +139,19 @@ export default class ImportNotionTask extends ImportTask {
 
       for (const image of imagesInText) {
         const name = path.basename(image.src);
-        const attachment = output.attachments.find((att) => att.name === name);
+        const attachment = output.attachments.find(
+          (att) =>
+            att.path.endsWith(image.src) ||
+            encodeURI(att.path).endsWith(image.src)
+        );
 
         if (!attachment) {
-          Logger.info(
-            "task",
-            `Could not find referenced attachment with name ${name} and src ${image.src}`
-          );
+          if (!image.src.startsWith("http")) {
+            Logger.info(
+              "task",
+              `Could not find referenced attachment with name ${name} and src ${image.src}`
+            );
+          }
         } else {
           text = text.replace(
             new RegExp(escapeRegExp(image.src), "g"),

server/queues/tasks/ImportTask.ts

@@ -1,4 +1,5 @@
 import { truncate } from "lodash";
+import { CollectionValidation } from "@shared/validations";
 import attachmentCreator from "@server/commands/attachmentCreator";
 import documentCreator from "@server/commands/documentCreator";
 import { sequelize } from "@server/database/sequelize";
@@ -206,22 +207,26 @@ export default abstract class ImportTask extends BaseTask<Props> {
       const ip = user.lastActiveIp || undefined;
 
       // Attachments
-      for (const item of data.attachments) {
-        const attachment = await attachmentCreator({
-          source: "import",
-          id: item.id,
-          name: item.name,
-          type: item.mimeType,
-          buffer: item.buffer,
-          user,
-          ip,
-          transaction,
-        });
-        attachments.set(item.id, attachment);
-      }
+      await Promise.all(
+        data.attachments.map(async (item) => {
+          Logger.debug("task", `ImportTask persisting attachment ${item.id}`);
+          const attachment = await attachmentCreator({
+            source: "import",
+            id: item.id,
+            name: item.name,
+            type: item.mimeType,
+            buffer: item.buffer,
+            user,
+            ip,
+            transaction,
+          });
+          attachments.set(item.id, attachment);
+        })
+      );
 
       // Collections
       for (const item of data.collections) {
+        Logger.debug("task", `ImportTask persisting collection ${item.id}`);
         let description = item.description;
 
         if (description) {
@@ -258,7 +263,9 @@ export default abstract class ImportTask extends BaseTask<Props> {
           },
           defaults: {
             id: item.id,
-            description,
+            description: truncate(description, {
+              length: CollectionValidation.maxDescriptionLength,
+            }),
             createdById: fileOperation.userId,
             permission: "read_write",
           },
@@ -307,6 +314,7 @@ export default abstract class ImportTask extends BaseTask<Props> {
 
       // Documents
       for (const item of data.documents) {
+        Logger.debug("task", `ImportTask persisting document ${item.id}`);
         let text = item.text;
 
         // Check all of the attachments we've created against urls in the text