fix: Long collection description prevents import (#3847)

* fix: Long collection description prevents import
fix: Parallelize attachment upload during import

* fix: Improve Notion image import matching

* chore: Bump JSZIP (perf)

* fix: Allow redirect from /doc/<id> to canonical url

* fix: Importing document with only title duplicates title in body

app/scenes/Document/components/DataLoader.tsx

@@ -45,7 +45,12 @@ function DataLoader({ match, children }: Props) {
   const { team } = auth;
   const [error, setError] = React.useState<Error | null>(null);
   const { revisionId, shareId, documentSlug } = match.params;
-  const document = documents.getByUrl(match.params.documentSlug);
+
+  // Allows loading by /doc/slug-<urlId> or /doc/<id>
+  const document =
+    documents.getByUrl(match.params.documentSlug) ??
+    documents.get(match.params.documentSlug);
+
   const revision = revisionId ? revisions.get(revisionId) : undefined;
   const sharedTree = document
     ? documents.getSharedTree(document.id)

package.json

@@ -108,7 +108,7 @@
     "is-printable-key-event": "^1.0.0",
     "json-loader": "0.5.4",
     "jsonwebtoken": "^8.5.0",
-    "jszip": "^3.7.1",
+    "jszip": "^3.10.0",
     "kbar": "0.1.0-beta.28",
     "koa": "^2.13.4",
     "koa-body": "^4.2.0",

server/commands/documentImporter.test.ts

@@ -148,6 +148,21 @@ describe("documentImporter", () => {
     expect(response.title).toEqual("Heading 1");
   });
 
+  it("should handle only title", async () => {
+    const user = await buildUser();
+    const fileName = "markdown.md";
+    const content = `# Title`;
+    const response = await documentImporter({
+      user,
+      mimeType: "text/plain",
+      fileName,
+      content,
+      ip,
+    });
+    expect(response.text).toEqual("");
+    expect(response.title).toEqual("Title");
+  });
+
   it("should fallback to extension if mimetype unknown", async () => {
     const user = await buildUser();
     const fileName = "markdown.md";

server/commands/documentImporter.ts

@@ -190,7 +190,7 @@ async function documentImporter({
   if (text.startsWith("# ")) {
     const result = parseTitle(text);
     title = result.title;
-    text = text.replace(`# ${title}\n`, "");
+    text = text.replace(`# ${title}`, "").trimStart();
   }
 
   // If we parsed an emoji from _above_ the title then add it back at prefixing

server/models/Collection.ts

@@ -24,6 +24,7 @@ import isUUID from "validator/lib/isUUID";
 import { MAX_TITLE_LENGTH } from "@shared/constants";
 import { sortNavigationNodes } from "@shared/utils/collections";
 import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
+import { CollectionValidation } from "@shared/validations";
 import slugify from "@server/utils/slugify";
 import { NavigationNode, CollectionSort } from "~/types";
 import CollectionGroup from "./CollectionGroup";
@@ -150,8 +151,8 @@ class Collection extends ParanoidModel {
   name: string;
 
   @Length({
-    max: 1000,
-    msg: `description must be 1000 characters or less`,
+    max: CollectionValidation.maxDescriptionLength,
+    msg: `description must be ${CollectionValidation.maxDescriptionLength} characters or less`,
   })
   @Column
   description: string;

server/queues/tasks/ImportNotionTask.ts

@@ -139,13 +139,19 @@ export default class ImportNotionTask extends ImportTask {
 
       for (const image of imagesInText) {
         const name = path.basename(image.src);
-        const attachment = output.attachments.find((att) => att.name === name);
+        const attachment = output.attachments.find(
+          (att) =>
+            att.path.endsWith(image.src) ||
+            encodeURI(att.path).endsWith(image.src)
+        );
 
         if (!attachment) {
-          Logger.info(
-            "task",
-            `Could not find referenced attachment with name ${name} and src ${image.src}`
-          );
+          if (!image.src.startsWith("http")) {
+            Logger.info(
+              "task",
+              `Could not find referenced attachment with name ${name} and src ${image.src}`
+            );
+          }
         } else {
           text = text.replace(
             new RegExp(escapeRegExp(image.src), "g"),

server/queues/tasks/ImportTask.ts

@@ -1,4 +1,5 @@
 import { truncate } from "lodash";
+import { CollectionValidation } from "@shared/validations";
 import attachmentCreator from "@server/commands/attachmentCreator";
 import documentCreator from "@server/commands/documentCreator";
 import { sequelize } from "@server/database/sequelize";
@@ -206,22 +207,26 @@ export default abstract class ImportTask extends BaseTask<Props> {
       const ip = user.lastActiveIp || undefined;
 
       // Attachments
-      for (const item of data.attachments) {
-        const attachment = await attachmentCreator({
-          source: "import",
-          id: item.id,
-          name: item.name,
-          type: item.mimeType,
-          buffer: item.buffer,
-          user,
-          ip,
-          transaction,
-        });
-        attachments.set(item.id, attachment);
-      }
+      await Promise.all(
+        data.attachments.map(async (item) => {
+          Logger.debug("task", `ImportTask persisting attachment ${item.id}`);
+          const attachment = await attachmentCreator({
+            source: "import",
+            id: item.id,
+            name: item.name,
+            type: item.mimeType,
+            buffer: item.buffer,
+            user,
+            ip,
+            transaction,
+          });
+          attachments.set(item.id, attachment);
+        })
+      );
 
       // Collections
       for (const item of data.collections) {
+        Logger.debug("task", `ImportTask persisting collection ${item.id}`);
         let description = item.description;
 
         if (description) {
@@ -258,7 +263,9 @@ export default abstract class ImportTask extends BaseTask<Props> {
           },
           defaults: {
             id: item.id,
-            description,
+            description: truncate(description, {
+              length: CollectionValidation.maxDescriptionLength,
+            }),
             createdById: fileOperation.userId,
             permission: "read_write",
           },
@@ -307,6 +314,7 @@ export default abstract class ImportTask extends BaseTask<Props> {
 
       // Documents
       for (const item of data.documents) {
+        Logger.debug("task", `ImportTask persisting document ${item.id}`);
         let text = item.text;
 
         // Check all of the attachments we've created against urls in the text

shared/validations.ts

@@ -0,0 +1,4 @@
+export const CollectionValidation = {
+  /* The maximum length of the collection description */
+  maxDescriptionLength: 1000,
+};

yarn.lock

@@ -9628,15 +9628,15 @@ jsonwebtoken@^8.5.0:
     array-includes "^3.1.1"
     object.assign "^4.1.1"
 
-jszip@^3.7.1:
-  version "3.7.1"
-  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.7.1.tgz#bd63401221c15625a1228c556ca8a68da6fda3d9"
-  integrity sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==
+jszip@^3.10.0, jszip@^3.7.1:
+  version "3.10.0"
+  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.0.tgz#faf3db2b4b8515425e34effcdbb086750a346061"
+  integrity sha512-LDfVtOLtOxb9RXkYOwPyNBTQDL4eUbqahtoY6x07GiDJHwSYvn8sHHIw8wINImV3MqbMNve2gSuM1DDqEKk09Q==
   dependencies:
     lie "~3.3.0"
     pako "~1.0.2"
     readable-stream "~2.3.6"
-    set-immediate-shim "~1.0.1"
+    setimmediate "^1.0.5"
 
 jwa@^1.4.1:
   version "1.4.1"
@@ -13272,10 +13272,10 @@ set-value@^2.0.0, set-value@^2.0.1:
     is-plain-object "^2.0.3"
     split-string "^3.0.1"
 
-setimmediate@^1.0.4:
+setimmediate@^1.0.4, setimmediate@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
-  integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=
+  integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==
 
 setprototypeof@1.1.0:
   version "1.1.0"