feat: Allow data imports larger than the standard attachment size (#4449)

* feat: Allow data imports larger than the standard attachment size * Use correct preset for data imports * Cleanup of expired attachments * lint
2022-11-20 09:22:57 -08:00
parent 1f49bd167d
commit 6e36ffb706
18 changed files with 375 additions and 92 deletions
--- a/server/routes/api/attachments.test.ts
+++ b/server/routes/api/attachments.test.ts
@@ -1,3 +1,4 @@
+import { AttachmentPreset } from "@shared/types";
 import Attachment from "@server/models/Attachment";
 import {
  buildUser,
@@ -34,6 +35,42 @@ describe("#attachments.create", () => {
      expect(res.status).toEqual(200);
    });

+    it("should allow upload using avatar preset", async () => {
+      const user = await buildUser();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.png",
+          contentType: "image/png",
+          size: 1000,
+          preset: AttachmentPreset.Avatar,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(200);
+
+      const body = await res.json();
+      const attachment = await Attachment.findByPk(body.data.attachment.id);
+      expect(attachment!.expiresAt).toBeNull();
+    });
+
+    it("should create expiring attachment using import preset", async () => {
+      const user = await buildUser();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.zip",
+          contentType: "application/zip",
+          size: 10000,
+          preset: AttachmentPreset.Import,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(200);
+
+      const body = await res.json();
+      const attachment = await Attachment.findByPk(body.data.attachment.id);
+      expect(attachment!.expiresAt).toBeTruthy();
+    });
+
    it("should not allow file upload for public attachments", async () => {
      const user = await buildUser();
      const res = await server.post("/api/attachments.create", {
@@ -47,6 +84,20 @@ describe("#attachments.create", () => {
      });
      expect(res.status).toEqual(400);
    });
+
+    it("should not allow file upload for avatar preset", async () => {
+      const user = await buildUser();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.pdf",
+          contentType: "application/pdf",
+          size: 1000,
+          preset: AttachmentPreset.Avatar,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(400);
+    });
  });

  describe("viewer", () => {
@@ -63,6 +114,20 @@ describe("#attachments.create", () => {
      });
      expect(res.status).toEqual(200);
    });
+
+    it("should allow upload using avatar preset", async () => {
+      const user = await buildViewer();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.png",
+          contentType: "image/png",
+          size: 1000,
+          preset: AttachmentPreset.Avatar,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(200);
+    });
  });
 });

--- a/server/routes/api/attachments.ts
+++ b/server/routes/api/attachments.ts
@@ -1,81 +1,87 @@
 import Router from "koa-router";
 import { v4 as uuidv4 } from "uuid";
+import { AttachmentPreset } from "@shared/types";
 import { bytesToHumanReadable } from "@shared/utils/files";
 import { AttachmentValidation } from "@shared/validations";
-import { sequelize } from "@server/database/sequelize";
 import { AuthorizationError, ValidationError } from "@server/errors";
 import auth from "@server/middlewares/authentication";
-import { Attachment, Document, Event } from "@server/models";
-import { authorize } from "@server/policies";
-import { ContextWithState } from "@server/types";
 import {
-  getPresignedPost,
-  publicS3Endpoint,
-  getSignedUrl,
-} from "@server/utils/s3";
+  transaction,
+  TransactionContext,
+} from "@server/middlewares/transaction";
+import { Attachment, Document, Event } from "@server/models";
+import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
+import { authorize } from "@server/policies";
+import { presentAttachment } from "@server/presenters";
+import { ContextWithState } from "@server/types";
+import { getPresignedPost, publicS3Endpoint } from "@server/utils/s3";
 import { assertIn, assertPresent, assertUuid } from "@server/validation";

 const router = new Router();
-const AWS_S3_ACL = process.env.AWS_S3_ACL || "private";

-router.post("attachments.create", auth(), async (ctx) => {
-  const {
-    name,
-    documentId,
-    contentType = "application/octet-stream",
-    size,
-    public: isPublic,
-  } = ctx.request.body;
-  assertPresent(name, "name is required");
-  assertPresent(size, "size is required");
+router.post(
+  "attachments.create",
+  auth(),
+  transaction(),
+  async (ctx: TransactionContext) => {
+    const {
+      name,
+      documentId,
+      contentType = "application/octet-stream",
+      size,
+      // 'public' is now deprecated and can be removed on December 1 2022.
+      public: isPublicDeprecated,
+      preset = isPublicDeprecated
+        ? AttachmentPreset.Avatar
+        : AttachmentPreset.DocumentAttachment,
+    } = ctx.request.body;
+    const { user, transaction } = ctx.state;

-  const { user } = ctx.state;
+    assertPresent(name, "name is required");
+    assertPresent(size, "size is required");

-  // Public attachments are only used for avatars, so this is loosely coupled –
-  // all user types can upload an avatar so no additional authorization is needed.
-  if (isPublic) {
-    assertIn(contentType, AttachmentValidation.avatarContentTypes);
-  } else {
-    authorize(user, "createAttachment", user.team);
-  }
+    // Public attachments are only used for avatars, so this is loosely coupled –
+    // all user types can upload an avatar so no additional authorization is needed.
+    if (preset === AttachmentPreset.Avatar) {
+      assertIn(contentType, AttachmentValidation.avatarContentTypes);
+    } else {
+      authorize(user, "createAttachment", user.team);
+    }

-  if (
-    process.env.AWS_S3_UPLOAD_MAX_SIZE &&
-    size > process.env.AWS_S3_UPLOAD_MAX_SIZE
-  ) {
-    throw ValidationError(
-      `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable(
-        parseInt(process.env.AWS_S3_UPLOAD_MAX_SIZE, 10)
-      )}`
-    );
-  }
+    const maxUploadSize = AttachmentHelper.presetToMaxUploadSize(preset);

-  const modelId = uuidv4();
-  const acl =
-    isPublic === undefined ? AWS_S3_ACL : isPublic ? "public-read" : "private";
-  const bucket = acl === "public-read" ? "public" : "uploads";
-  const keyPrefix = `${bucket}/${user.id}/${modelId}`;
-  const key = `${keyPrefix}/${name}`;
-  const presignedPost = await getPresignedPost(key, acl, contentType);
-  const endpoint = publicS3Endpoint();
-  const url = `${endpoint}/${keyPrefix}/${encodeURIComponent(name)}`;
+    if (size > maxUploadSize) {
+      throw ValidationError(
+        `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable(
+          maxUploadSize
+        )}`
+      );
+    }

-  if (documentId !== undefined) {
-    assertUuid(documentId, "documentId must be a uuid");
-    const document = await Document.findByPk(documentId, {
+    if (documentId !== undefined) {
+      assertUuid(documentId, "documentId must be a uuid");
+      const document = await Document.findByPk(documentId, {
+        userId: user.id,
+      });
+      authorize(user, "update", document);
+    }
+
+    const modelId = uuidv4();
+    const acl = AttachmentHelper.presetToAcl(preset);
+    const key = AttachmentHelper.getKey({
+      acl,
+      id: modelId,
+      name,
      userId: user.id,
    });
-    authorize(user, "update", document);
-  }

-  const attachment = await sequelize.transaction(async (transaction) => {
    const attachment = await Attachment.create(
      {
        id: modelId,
        key,
        acl,
        size,
-        url,
+        expiresAt: AttachmentHelper.presetToExpiry(preset),
        contentType,
        documentId,
        teamId: user.teamId,
@@ -97,29 +103,26 @@ router.post("attachments.create", auth(), async (ctx) => {
      { transaction }
    );

-    return attachment;
-  });
+    const presignedPost = await getPresignedPost(
+      key,
+      acl,
+      maxUploadSize,
+      contentType
+    );

-  ctx.body = {
-    data: {
-      maxUploadSize: process.env.AWS_S3_UPLOAD_MAX_SIZE,
-      uploadUrl: endpoint,
-      form: {
-        "Cache-Control": "max-age=31557600",
-        "Content-Type": contentType,
-        ...presignedPost.fields,
+    ctx.body = {
+      data: {
+        uploadUrl: publicS3Endpoint(),
+        form: {
+          "Cache-Control": "max-age=31557600",
+          "Content-Type": contentType,
+          ...presignedPost.fields,
+        },
+        attachment: presentAttachment(attachment),
      },
-      attachment: {
-        documentId,
-        contentType,
-        name,
-        id: attachment.id,
-        url: isPublic ? url : attachment.redirectUrl,
-        size,
-      },
-    },
-  };
-});
+    };
+  }
+);

 router.post("attachments.delete", auth(), async (ctx) => {
  const { id } = ctx.request.body;
@@ -168,8 +171,7 @@ const handleAttachmentsRedirect = async (ctx: ContextWithState) => {
  });

  if (attachment.isPrivate) {
-    const accessUrl = await getSignedUrl(attachment.key);
-    ctx.redirect(accessUrl);
+    ctx.redirect(await attachment.signedUrl);
  } else {
    ctx.redirect(attachment.canonicalUrl);
  }
--- a/server/routes/api/cron.ts
+++ b/server/routes/api/cron.ts
@@ -5,6 +5,7 @@ import env from "@server/env";
 import { AuthenticationError } from "@server/errors";
 import CleanupDeletedDocumentsTask from "@server/queues/tasks/CleanupDeletedDocumentsTask";
 import CleanupDeletedTeamsTask from "@server/queues/tasks/CleanupDeletedTeamsTask";
+import CleanupExpiredAttachmentsTask from "@server/queues/tasks/CleanupExpiredAttachmentsTask";
 import CleanupExpiredFileOperationsTask from "@server/queues/tasks/CleanupExpiredFileOperationsTask";
 import CleanupWebhookDeliveriesTask from "@server/queues/tasks/CleanupWebhookDeliveriesTask";
 import InviteReminderTask from "@server/queues/tasks/InviteReminderTask";
@@ -37,6 +38,8 @@ const cronHandler = async (ctx: Context) => {

  await CleanupExpiredFileOperationsTask.schedule({ limit });

+  await CleanupExpiredAttachmentsTask.schedule({ limit });
+
  await CleanupDeletedTeamsTask.schedule({ limit });

  await CleanupWebhookDeliveriesTask.schedule({ limit });
--- a/server/routes/api/documents.ts
+++ b/server/routes/api/documents.ts
@@ -1114,10 +1114,10 @@ router.post("documents.import", auth(), async (ctx) => {
    throw InvalidRequestError("Request must include a file parameter");
  }

-  if (env.MAXIMUM_IMPORT_SIZE && file.size > env.MAXIMUM_IMPORT_SIZE) {
+  if (file.size > env.AWS_S3_UPLOAD_MAX_SIZE) {
    throw InvalidRequestError(
      `The selected file was larger than the ${bytesToHumanReadable(
-        env.MAXIMUM_IMPORT_SIZE
+        env.AWS_S3_UPLOAD_MAX_SIZE
      )} maximum size`
    );
  }