feat: Allow data imports larger than the standard attachment size (#4449)

* feat: Allow data imports larger than the standard attachment size

* Use correct preset for data imports

* Cleanup of expired attachments

* lint

server/routes/api/attachments.ts

@@ -1,81 +1,87 @@
 import Router from "koa-router";
 import { v4 as uuidv4 } from "uuid";
+import { AttachmentPreset } from "@shared/types";
 import { bytesToHumanReadable } from "@shared/utils/files";
 import { AttachmentValidation } from "@shared/validations";
-import { sequelize } from "@server/database/sequelize";
 import { AuthorizationError, ValidationError } from "@server/errors";
 import auth from "@server/middlewares/authentication";
-import { Attachment, Document, Event } from "@server/models";
-import { authorize } from "@server/policies";
-import { ContextWithState } from "@server/types";
 import {
-  getPresignedPost,
-  publicS3Endpoint,
-  getSignedUrl,
-} from "@server/utils/s3";
+  transaction,
+  TransactionContext,
+} from "@server/middlewares/transaction";
+import { Attachment, Document, Event } from "@server/models";
+import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
+import { authorize } from "@server/policies";
+import { presentAttachment } from "@server/presenters";
+import { ContextWithState } from "@server/types";
+import { getPresignedPost, publicS3Endpoint } from "@server/utils/s3";
 import { assertIn, assertPresent, assertUuid } from "@server/validation";
 
 const router = new Router();
-const AWS_S3_ACL = process.env.AWS_S3_ACL || "private";
 
-router.post("attachments.create", auth(), async (ctx) => {
-  const {
-    name,
-    documentId,
-    contentType = "application/octet-stream",
-    size,
-    public: isPublic,
-  } = ctx.request.body;
-  assertPresent(name, "name is required");
-  assertPresent(size, "size is required");
+router.post(
+  "attachments.create",
+  auth(),
+  transaction(),
+  async (ctx: TransactionContext) => {
+    const {
+      name,
+      documentId,
+      contentType = "application/octet-stream",
+      size,
+      // 'public' is now deprecated and can be removed on December 1 2022.
+      public: isPublicDeprecated,
+      preset = isPublicDeprecated
+        ? AttachmentPreset.Avatar
+        : AttachmentPreset.DocumentAttachment,
+    } = ctx.request.body;
+    const { user, transaction } = ctx.state;
 
-  const { user } = ctx.state;
+    assertPresent(name, "name is required");
+    assertPresent(size, "size is required");
 
-  // Public attachments are only used for avatars, so this is loosely coupled –
-  // all user types can upload an avatar so no additional authorization is needed.
-  if (isPublic) {
-    assertIn(contentType, AttachmentValidation.avatarContentTypes);
-  } else {
-    authorize(user, "createAttachment", user.team);
-  }
+    // Public attachments are only used for avatars, so this is loosely coupled –
+    // all user types can upload an avatar so no additional authorization is needed.
+    if (preset === AttachmentPreset.Avatar) {
+      assertIn(contentType, AttachmentValidation.avatarContentTypes);
+    } else {
+      authorize(user, "createAttachment", user.team);
+    }
 
-  if (
-    process.env.AWS_S3_UPLOAD_MAX_SIZE &&
-    size > process.env.AWS_S3_UPLOAD_MAX_SIZE
-  ) {
-    throw ValidationError(
-      `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable(
-        parseInt(process.env.AWS_S3_UPLOAD_MAX_SIZE, 10)
-      )}`
-    );
-  }
+    const maxUploadSize = AttachmentHelper.presetToMaxUploadSize(preset);
 
-  const modelId = uuidv4();
-  const acl =
-    isPublic === undefined ? AWS_S3_ACL : isPublic ? "public-read" : "private";
-  const bucket = acl === "public-read" ? "public" : "uploads";
-  const keyPrefix = `${bucket}/${user.id}/${modelId}`;
-  const key = `${keyPrefix}/${name}`;
-  const presignedPost = await getPresignedPost(key, acl, contentType);
-  const endpoint = publicS3Endpoint();
-  const url = `${endpoint}/${keyPrefix}/${encodeURIComponent(name)}`;
+    if (size > maxUploadSize) {
+      throw ValidationError(
+        `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable(
+          maxUploadSize
+        )}`
+      );
+    }
 
-  if (documentId !== undefined) {
-    assertUuid(documentId, "documentId must be a uuid");
-    const document = await Document.findByPk(documentId, {
+    if (documentId !== undefined) {
+      assertUuid(documentId, "documentId must be a uuid");
+      const document = await Document.findByPk(documentId, {
+        userId: user.id,
+      });
+      authorize(user, "update", document);
+    }
+
+    const modelId = uuidv4();
+    const acl = AttachmentHelper.presetToAcl(preset);
+    const key = AttachmentHelper.getKey({
+      acl,
+      id: modelId,
+      name,
       userId: user.id,
     });
-    authorize(user, "update", document);
-  }
 
-  const attachment = await sequelize.transaction(async (transaction) => {
     const attachment = await Attachment.create(
       {
         id: modelId,
         key,
         acl,
         size,
-        url,
+        expiresAt: AttachmentHelper.presetToExpiry(preset),
         contentType,
         documentId,
         teamId: user.teamId,
@@ -97,29 +103,26 @@ router.post("attachments.create", auth(), async (ctx) => {
       { transaction }
     );
 
-    return attachment;
-  });
+    const presignedPost = await getPresignedPost(
+      key,
+      acl,
+      maxUploadSize,
+      contentType
+    );
 
-  ctx.body = {
-    data: {
-      maxUploadSize: process.env.AWS_S3_UPLOAD_MAX_SIZE,
-      uploadUrl: endpoint,
-      form: {
-        "Cache-Control": "max-age=31557600",
-        "Content-Type": contentType,
-        ...presignedPost.fields,
+    ctx.body = {
+      data: {
+        uploadUrl: publicS3Endpoint(),
+        form: {
+          "Cache-Control": "max-age=31557600",
+          "Content-Type": contentType,
+          ...presignedPost.fields,
+        },
+        attachment: presentAttachment(attachment),
       },
-      attachment: {
-        documentId,
-        contentType,
-        name,
-        id: attachment.id,
-        url: isPublic ? url : attachment.redirectUrl,
-        size,
-      },
-    },
-  };
-});
+    };
+  }
+);
 
 router.post("attachments.delete", auth(), async (ctx) => {
   const { id } = ctx.request.body;
@@ -168,8 +171,7 @@ const handleAttachmentsRedirect = async (ctx: ContextWithState) => {
   });
 
   if (attachment.isPrivate) {
-    const accessUrl = await getSignedUrl(attachment.key);
-    ctx.redirect(accessUrl);
+    ctx.redirect(await attachment.signedUrl);
   } else {
     ctx.redirect(attachment.canonicalUrl);
   }