feat: Import improvements (#3064)

* feat: Split and simplify import/export pages in prep for more options

* minor fixes

* File operations for imports

* test

* icons

server/queues/processors/exports.ts

@@ -19,60 +19,62 @@ export default class ExportsProcessor {
         const user = await User.findByPk(actorId);
         invariant(user, "user operation not found");
 
-        const exportData = await FileOperation.findByPk(event.modelId);
-        invariant(exportData, "exportData not found");
+        const fileOperation = await FileOperation.findByPk(event.modelId);
+        invariant(fileOperation, "fileOperation not found");
 
         const collectionIds =
-          // @ts-expect-error ts-migrate(2339) FIXME: Property 'collectionId' does not exist on type 'Co... Remove this comment to see the full error message
-          event.collectionId || (await user.collectionIds());
+          "collectionId" in event
+            ? event.collectionId
+            : await user.collectionIds();
         const collections = await Collection.findAll({
           where: {
             id: collectionIds,
           },
         });
-        this.updateFileOperation(exportData, actorId, teamId, {
+        this.updateFileOperation(fileOperation, actorId, teamId, {
           state: "creating",
         });
         // heavy lifting of creating the zip file
         Logger.info(
           "processor",
-          `Archiving collections for file operation ${exportData.id}`
+          `Archiving collections for file operation ${fileOperation.id}`
         );
         const filePath = await archiveCollections(collections);
-        let url, state;
+        let url;
+        let state: any = "creating";
 
         try {
           // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
           const readBuffer = await fs.promises.readFile(filePath);
           // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
           const stat = await fs.promises.stat(filePath);
-          this.updateFileOperation(exportData, actorId, teamId, {
+          this.updateFileOperation(fileOperation, actorId, teamId, {
             state: "uploading",
             size: stat.size,
           });
           Logger.info(
             "processor",
-            `Uploading archive for file operation ${exportData.id}`
+            `Uploading archive for file operation ${fileOperation.id}`
           );
           url = await uploadToS3FromBuffer(
             readBuffer,
             "application/zip",
-            exportData.key,
+            fileOperation.key,
             "private"
           );
           Logger.info(
             "processor",
-            `Upload complete for file operation ${exportData.id}`
+            `Upload complete for file operation ${fileOperation.id}`
           );
           state = "complete";
         } catch (error) {
           Logger.error("Error exporting collection data", error, {
-            fileOperationId: exportData.id,
+            fileOperationId: fileOperation.id,
           });
           state = "error";
-          url = null;
+          url = undefined;
         } finally {
-          this.updateFileOperation(exportData, actorId, teamId, {
+          this.updateFileOperation(fileOperation, actorId, teamId, {
             state,
             url,
           });
@@ -85,7 +87,7 @@ export default class ExportsProcessor {
           } else {
             mailer.sendTemplate("exportSuccess", {
               to: user.email,
-              id: exportData.id,
+              id: fileOperation.id,
               teamUrl: team.url,
             });
           }
@@ -101,15 +103,14 @@ export default class ExportsProcessor {
     fileOperation: FileOperation,
     actorId: string,
     teamId: string,
-    data: Record<string, any>
+    data: Partial<FileOperation>
   ) {
     await fileOperation.update(data);
     await Event.add({
       name: "fileOperations.update",
       teamId,
       actorId,
-      // @ts-expect-error dataValues exists
-      data: fileOperation.dataValues,
+      modelId: fileOperation.id,
     });
   }
 }

server/queues/processors/imports.ts

@@ -3,13 +3,14 @@ import os from "os";
 import File from "formidable/lib/file";
 import invariant from "invariant";
 import collectionImporter from "@server/commands/collectionImporter";
-import { Attachment, User } from "@server/models";
-import { Event } from "../../types";
+import { Event, FileOperation, Attachment, User } from "@server/models";
+import { Event as TEvent } from "../../types";
 
 export default class ImportsProcessor {
-  async on(event: Event) {
+  async on(event: TEvent) {
     switch (event.name) {
       case "collections.import": {
+        let state, error;
         const { type } = event.data;
         const attachment = await Attachment.findByPk(event.modelId);
         invariant(attachment, "attachment not found");
@@ -17,22 +18,55 @@ export default class ImportsProcessor {
         const user = await User.findByPk(event.actorId);
         invariant(user, "user not found");
 
-        const buffer: any = await attachment.buffer;
-        const tmpDir = os.tmpdir();
-        const tmpFilePath = `${tmpDir}/upload-${event.modelId}`;
-        await fs.promises.writeFile(tmpFilePath, buffer);
-        const file = new File({
-          name: attachment.name,
-          type: attachment.contentType,
-          path: tmpFilePath,
+        const fileOperation = await FileOperation.create({
+          type: "import",
+          state: "creating",
+          size: attachment.size,
+          key: attachment.key,
+          userId: user.id,
+          teamId: user.teamId,
         });
-        await collectionImporter({
-          file,
-          user,
-          type,
-          ip: event.ip,
+
+        await Event.add({
+          name: "fileOperations.create",
+          modelId: fileOperation.id,
+          teamId: user.teamId,
+          actorId: user.id,
         });
-        await attachment.destroy();
+
+        try {
+          const buffer = await attachment.buffer;
+          const tmpDir = os.tmpdir();
+          const tmpFilePath = `${tmpDir}/upload-${event.modelId}`;
+          await fs.promises.writeFile(tmpFilePath, buffer as Uint8Array);
+          const file = new File({
+            name: attachment.name,
+            type: attachment.contentType,
+            path: tmpFilePath,
+          });
+
+          await collectionImporter({
+            file,
+            user,
+            type,
+            ip: event.ip,
+          });
+          await attachment.destroy();
+
+          state = "complete";
+        } catch (err) {
+          state = "error";
+          error = err.message;
+        } finally {
+          await fileOperation.update({ state, error });
+          await Event.add({
+            name: "fileOperations.update",
+            modelId: fileOperation.id,
+            teamId: user.teamId,
+            actorId: user.id,
+          });
+        }
+
         return;
       }
 

server/queues/processors/websockets.ts

@@ -3,13 +3,18 @@ import { Op } from "sequelize";
 import {
   Document,
   Collection,
+  FileOperation,
   Group,
   CollectionGroup,
   GroupUser,
   Pin,
   Star,
 } from "@server/models";
-import { presentPin, presentStar } from "@server/presenters";
+import {
+  presentFileOperation,
+  presentPin,
+  presentStar,
+} from "@server/presenters";
 import { Event } from "../../types";
 
 export default class WebsocketsProcessor {
@@ -354,10 +359,14 @@ export default class WebsocketsProcessor {
         return;
       }
 
+      case "fileOperations.create":
       case "fileOperations.update": {
-        return socketio
-          .to(`user-${event.actorId}`)
-          .emit("fileOperations.update", event.data);
+        const fileOperation = await FileOperation.findByPk(event.modelId);
+        if (!fileOperation) {
+          return;
+        }
+        const data = await presentFileOperation(fileOperation);
+        return socketio.to(`user-${event.actorId}`).emit(event.name, data);
       }
 
       case "pins.create":