chore: Refactor data import (#3434)

* Complete refactor of import

* feat: Notion data import (#3442)

server/models/Collection.ts

@@ -520,7 +520,7 @@ class Collection extends ParanoidModel {
    */
   updateDocument = async function (
     updatedDocument: Document,
-    options?: { transaction: Transaction }
+    options?: { transaction?: Transaction | null }
   ) {
     if (!this.documentStructure) {
       return;

server/models/Document.ts

@@ -9,6 +9,7 @@ import {
   FindOptions,
   ScopeOptions,
   WhereOptions,
+  SaveOptions,
 } from "sequelize";
 import {
   ForeignKey,
@@ -238,7 +239,10 @@ class Document extends ParanoidModel {
   // hooks
 
   @BeforeSave
-  static async updateTitleInCollectionStructure(model: Document) {
+  static async updateTitleInCollectionStructure(
+    model: Document,
+    { transaction }: SaveOptions<Document>
+  ) {
     // templates, drafts, and archived documents don't appear in the structure
     // and so never need to be updated when the title changes
     if (
@@ -250,18 +254,16 @@ class Document extends ParanoidModel {
       return;
     }
 
-    return this.sequelize!.transaction(async (transaction: Transaction) => {
-      const collection = await Collection.findByPk(model.collectionId, {
-        transaction,
-        lock: transaction.LOCK.UPDATE,
-      });
-      if (!collection) {
-        return;
-      }
-
-      await collection.updateDocument(model, { transaction });
-      model.collection = collection;
+    const collection = await Collection.findByPk(model.collectionId, {
+      transaction,
+      lock: Transaction.LOCK.UPDATE,
     });
+    if (!collection) {
+      return;
+    }
+
+    await collection.updateDocument(model, { transaction });
+    model.collection = collection;
   }
 
   @AfterCreate
@@ -801,30 +803,28 @@ class Document extends ParanoidModel {
     return this.save(options);
   };
 
-  publish = async (userId: string) => {
+  publish = async (userId: string, { transaction }: SaveOptions<Document>) => {
     // If the document is already published then calling publish should act like
     // a regular save
     if (this.publishedAt) {
-      return this.save();
+      return this.save({ transaction });
     }
 
-    await this.sequelize.transaction(async (transaction: Transaction) => {
-      if (!this.template) {
-        const collection = await Collection.findByPk(this.collectionId, {
-          transaction,
-          lock: transaction.LOCK.UPDATE,
-        });
+    if (!this.template) {
+      const collection = await Collection.findByPk(this.collectionId, {
+        transaction,
+        lock: Transaction.LOCK.UPDATE,
+      });
 
-        if (collection) {
-          await collection.addDocumentToStructure(this, 0, { transaction });
-          this.collection = collection;
-        }
+      if (collection) {
+        await collection.addDocumentToStructure(this, 0, { transaction });
+        this.collection = collection;
       }
-    });
+    }
 
     this.lastModifiedById = userId;
     this.publishedAt = new Date();
-    return this.save();
+    return this.save({ transaction });
   };
 
   unpublish = async (userId: string) => {

server/models/Event.ts

@@ -1,3 +1,4 @@
+import { SaveOptions } from "sequelize";
 import {
   ForeignKey,
   AfterSave,
@@ -45,8 +46,12 @@ class Event extends BaseModel {
   }
 
   @AfterSave
-  static async enqueue(model: Event) {
-    globalEventQueue.add(model);
+  static async enqueue(model: Event, options: SaveOptions<Event>) {
+    if (options.transaction) {
+      options.transaction.afterCommit(() => void globalEventQueue.add(model));
+      return;
+    }
+    void globalEventQueue.add(model);
   }
 
   // associations

server/models/FileOperation.ts

@@ -7,13 +7,31 @@ import {
   Table,
   DataType,
 } from "sequelize-typescript";
-import { deleteFromS3 } from "@server/utils/s3";
+import { deleteFromS3, getFileByKey } from "@server/utils/s3";
 import Collection from "./Collection";
 import Team from "./Team";
 import User from "./User";
 import BaseModel from "./base/BaseModel";
 import Fix from "./decorators/Fix";
 
+export enum FileOperationType {
+  Import = "import",
+  Export = "export",
+}
+
+export enum FileOperationFormat {
+  MarkdownZip = "outline-markdown",
+  Notion = "notion",
+}
+
+export enum FileOperationState {
+  Creating = "creating",
+  Uploading = "uploading",
+  Complete = "complete",
+  Error = "error",
+  Expired = "expired",
+}
+
 @DefaultScope(() => ({
   include: [
     {
@@ -32,12 +50,15 @@ import Fix from "./decorators/Fix";
 @Fix
 class FileOperation extends BaseModel {
   @Column(DataType.ENUM("import", "export"))
-  type: "import" | "export";
+  type: FileOperationType;
+
+  @Column(DataType.STRING)
+  format: FileOperationFormat;
 
   @Column(
     DataType.ENUM("creating", "uploading", "complete", "error", "expired")
   )
-  state: "creating" | "uploading" | "complete" | "error" | "expired";
+  state: FileOperationState;
 
   @Column
   key: string;
@@ -57,6 +78,10 @@ class FileOperation extends BaseModel {
     await this.save();
   };
 
+  get buffer() {
+    return getFileByKey(this.key);
+  }
+
   // hooks
 
   @BeforeDestroy

server/models/Team.ts

@@ -173,45 +173,55 @@ class Team extends ParanoidModel {
     return subdomain;
   };
 
-  provisionFirstCollection = async function (userId: string) {
-    const collection = await Collection.create({
-      name: "Welcome",
-      description:
-        "This collection is a quick guide to what Outline is all about. Feel free to delete this collection once your team is up to speed with the basics!",
-      teamId: this.id,
-      createdById: userId,
-      sort: Collection.DEFAULT_SORT,
-      permission: "read_write",
-    });
-
-    // For the first collection we go ahead and create some intitial documents to get
-    // the team started. You can edit these in /server/onboarding/x.md
-    const onboardingDocs = [
-      "Integrations & API",
-      "Our Editor",
-      "Getting Started",
-      "What is Outline",
-    ];
-
-    for (const title of onboardingDocs) {
-      const text = await readFile(
-        path.join(process.cwd(), "server", "onboarding", `${title}.md`),
-        "utf8"
+  provisionFirstCollection = async (userId: string) => {
+    await this.sequelize!.transaction(async (transaction) => {
+      const collection = await Collection.create(
+        {
+          name: "Welcome",
+          description:
+            "This collection is a quick guide to what Outline is all about. Feel free to delete this collection once your team is up to speed with the basics!",
+          teamId: this.id,
+          createdById: userId,
+          sort: Collection.DEFAULT_SORT,
+          permission: "read_write",
+        },
+        {
+          transaction,
+        }
       );
-      const document = await Document.create({
-        version: 2,
-        isWelcome: true,
-        parentDocumentId: null,
-        collectionId: collection.id,
-        teamId: collection.teamId,
-        userId: collection.createdById,
-        lastModifiedById: collection.createdById,
-        createdById: collection.createdById,
-        title,
-        text,
-      });
-      await document.publish(collection.createdById);
-    }
+
+      // For the first collection we go ahead and create some intitial documents to get
+      // the team started. You can edit these in /server/onboarding/x.md
+      const onboardingDocs = [
+        "Integrations & API",
+        "Our Editor",
+        "Getting Started",
+        "What is Outline",
+      ];
+
+      for (const title of onboardingDocs) {
+        const text = await readFile(
+          path.join(process.cwd(), "server", "onboarding", `${title}.md`),
+          "utf8"
+        );
+        const document = await Document.create(
+          {
+            version: 2,
+            isWelcome: true,
+            parentDocumentId: null,
+            collectionId: collection.id,
+            teamId: collection.teamId,
+            userId: collection.createdById,
+            lastModifiedById: collection.createdById,
+            createdById: collection.createdById,
+            title,
+            text,
+          },
+          { transaction }
+        );
+        await document.publish(collection.createdById, { transaction });
+      }
+    });
   };
 
   collectionIds = async function (paranoid = true) {