feat: Allow data imports larger than the standard attachment size (#4449)

* feat: Allow data imports larger than the standard attachment size

* Use correct preset for data imports

* Cleanup of expired attachments

* lint

app/components/Editor.tsx

@@ -8,6 +8,7 @@ import { mergeRefs } from "react-merge-refs";
 import { Optional } from "utility-types";
 import insertFiles from "@shared/editor/commands/insertFiles";
 import { Heading } from "@shared/editor/lib/getHeadings";
+import { AttachmentPreset } from "@shared/types";
 import { getDataTransferFiles } from "@shared/utils/files";
 import parseDocumentSlug from "@shared/utils/parseDocumentSlug";
 import { isInternalUrl } from "@shared/utils/urls";
@@ -135,6 +136,7 @@ function Editor(props: Props, ref: React.RefObject<SharedEditor> | null) {
     async (file: File) => {
       const result = await uploadFile(file, {
         documentId: id,
+        preset: AttachmentPreset.DocumentAttachment,
       });
       return result.url;
     },

app/scenes/Settings/components/DropToImport.tsx

@@ -4,6 +4,7 @@ import * as React from "react";
 import Dropzone from "react-dropzone";
 import { useTranslation } from "react-i18next";
 import styled from "styled-components";
+import { AttachmentPreset } from "@shared/types";
 import Flex from "~/components/Flex";
 import LoadingIndicator from "~/components/LoadingIndicator";
 import useStores from "~/hooks/useStores";
@@ -39,6 +40,7 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) {
       try {
         const attachment = await uploadFile(file, {
           name: file.name,
+          preset: AttachmentPreset.Import,
         });
         await collections.import(attachment.id, format);
         onSubmit();

app/scenes/Settings/components/ImageUpload.tsx

@@ -5,6 +5,7 @@ import * as React from "react";
 import AvatarEditor from "react-avatar-editor";
 import Dropzone from "react-dropzone";
 import styled from "styled-components";
+import { AttachmentPreset } from "@shared/types";
 import { AttachmentValidation } from "@shared/validations";
 import RootStore from "~/stores/RootStore";
 import Button from "~/components/Button";
@@ -67,7 +68,7 @@ class ImageUpload extends React.Component<RootStore & Props> {
       });
       const attachment = await uploadFile(compressed, {
         name: this.file.name,
-        public: true,
+        preset: AttachmentPreset.Avatar,
       });
       this.props.onSuccess(attachment.url);
     } catch (err) {

app/utils/files.ts

@@ -1,4 +1,5 @@
 import invariant from "invariant";
+import { AttachmentPreset } from "@shared/types";
 import { client } from "./ApiClient";
 import Logger from "./Logger";
 
@@ -7,8 +8,8 @@ type UploadOptions = {
   name?: string;
   /** The document that this file was uploaded in, if any */
   documentId?: string;
-  /** Whether the file should be public in cloud storage */
+  /** The preset to use for attachment configuration */
-  public?: boolean;
+  preset: AttachmentPreset;
   /** Callback will be passed a number between 0-1 as upload progresses */
   onProgress?: (fractionComplete: number) => void;
 };
@@ -17,11 +18,12 @@ export const uploadFile = async (
   file: File | Blob,
   options: UploadOptions = {
     name: "",
+    preset: AttachmentPreset.DocumentAttachment,
   }
 ) => {
   const name = file instanceof File ? file.name : options.name;
   const response = await client.post("/attachments.create", {
-    public: options.public,
+    preset: options.preset,
     documentId: options.documentId,
     contentType: file.type,
     size: file.size,

server/env.ts

@@ -534,6 +534,20 @@ export class Environment {
   public RATE_LIMITER_DURATION_WINDOW =
     this.toOptionalNumber(process.env.RATE_LIMITER_DURATION_WINDOW) ?? 60;
 
+  /**
+   * Set max allowed upload size for file attachments.
+   */
+  @IsOptional()
+  @IsNumber()
+  public AWS_S3_UPLOAD_MAX_SIZE =
+    this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? 10000000000;
+
+  /**
+   * Set default AWS S3 ACL for file attachments.
+   */
+  @IsOptional()
+  public AWS_S3_ACL = process.env.AWS_S3_ACL ?? "private";
+
   private toOptionalString(value: string | undefined) {
     return value ? value : undefined;
   }

server/migrations/20221120151710-attachment-expiry.js

@@ -0,0 +1,22 @@
+"use strict";
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.sequelize.transaction(async (transaction) => {
+      await queryInterface.addColumn("attachments", "expiresAt", {
+        type: Sequelize.DATE,
+        allowNull: true,
+        transaction,
+      });
+      await queryInterface.addIndex("attachments", ["expiresAt"], {
+        transaction
+      });
+    });
+  },
+  down: async (queryInterface) => {
+    await queryInterface.sequelize.transaction(async (transaction) => {
+      await queryInterface.removeColumn("attachments", "expiresAt", { transaction });
+      await queryInterface.removeIndex("attachments", ["expiresAt"], { transaction });
+    });
+  },
+};

server/models/Attachment.ts

@@ -10,7 +10,12 @@ import {
   DataType,
   IsNumeric,
 } from "sequelize-typescript";
-import { publicS3Endpoint, deleteFromS3, getFileByKey } from "@server/utils/s3";
+import {
+  publicS3Endpoint,
+  deleteFromS3,
+  getFileByKey,
+  getSignedUrl,
+} from "@server/utils/s3";
 import Document from "./Document";
 import Team from "./Team";
 import User from "./User";
@@ -47,26 +52,59 @@ class Attachment extends IdModel {
   @Column
   lastAccessedAt: Date | null;
 
+  @Column
+  expiresAt: Date | null;
+
   // getters
 
+  /**
+   * Get the original uploaded file name.
+   */
   get name() {
     return path.parse(this.key).base;
   }
 
-  get redirectUrl() {
+  /**
-    return `/api/attachments.redirect?id=${this.id}`;
+   * Whether the attachment is private or not.
-  }
+   */
-
   get isPrivate() {
     return this.acl === "private";
   }
 
+  /**
+   * Get the contents of this attachment as a Buffer
+   */
   get buffer() {
     return getFileByKey(this.key);
   }
 
+  /**
+   * Get a url that can be used to download the attachment if the user has a valid session.
+   */
+  get url() {
+    return this.isPrivate ? this.redirectUrl : this.canonicalUrl;
+  }
+
+  /**
+   * Get a url that can be used to download a private attachment if the user has a valid session.
+   */
+  get redirectUrl() {
+    return `/api/attachments.redirect?id=${this.id}`;
+  }
+
+  /**
+   * Get a direct URL to the attachment in storage. Note that this will not work for private attachments,
+   * a signed URL must be used.
+   */
   get canonicalUrl() {
-    return `${publicS3Endpoint()}/${this.key}`;
+    return encodeURI(`${publicS3Endpoint()}/${this.key}`);
+  }
+
+  /**
+   * Get a signed URL with the default expirt to download the attachment from storage.
+   */
+  get signedUrl() {
+    return getSignedUrl(this.key);
   }
 
   // hooks

server/models/FileOperation.ts

@@ -73,7 +73,13 @@ class FileOperation extends IdModel {
 
   expire = async function () {
     this.state = "expired";
-    await deleteFromS3(this.key);
+    try {
+      await deleteFromS3(this.key);
+    } catch (err) {
+      if (err.retryable) {
+        throw err;
+      }
+    }
     await this.save();
   };
 

server/models/helpers/AttachmentHelper.ts

@@ -0,0 +1,76 @@
+import { addHours } from "date-fns";
+import { AttachmentPreset } from "@shared/types";
+import env from "@server/env";
+
+export default class AttachmentHelper {
+  /**
+   * Get the upload location for the given upload details
+   *
+   * @param acl The ACL to use
+   * @param id The ID of the attachment
+   * @param name The name of the attachment
+   * @param userId The ID of the user uploading the attachment
+   */
+  static getKey({
+    acl,
+    id,
+    name,
+    userId,
+  }: {
+    acl: string;
+    id: string;
+    name: string;
+    userId: string;
+  }) {
+    const bucket = acl === "public-read" ? "public" : "uploads";
+    const keyPrefix = `${bucket}/${userId}/${id}`;
+    return `${keyPrefix}/${name}`;
+  }
+
+  /**
+   * Get the ACL to use for a given attachment preset
+   *
+   * @param preset The preset to use
+   * @returns A valid S3 ACL
+   */
+  static presetToAcl(preset: AttachmentPreset) {
+    switch (preset) {
+      case AttachmentPreset.Avatar:
+        return "public-read";
+      default:
+        return env.AWS_S3_ACL;
+    }
+  }
+
+  /**
+   * Get the expiration time for a given attachment preset
+   *
+   * @param preset The preset to use
+   * @returns An expiration time
+   */
+  static presetToExpiry(preset: AttachmentPreset) {
+    switch (preset) {
+      case AttachmentPreset.Import:
+        return addHours(new Date(), 24);
+      default:
+        return undefined;
+    }
+  }
+
+  /**
+   * Get the maximum upload size for a given attachment preset
+   *
+   * @param preset The preset to use
+   * @returns The maximum upload size in bytes
+   */
+  static presetToMaxUploadSize(preset: AttachmentPreset) {
+    switch (preset) {
+      case AttachmentPreset.Avatar:
+        return Math.min(1024 * 1024 * 5, env.AWS_S3_UPLOAD_MAX_SIZE);
+      case AttachmentPreset.Import:
+        return env.MAXIMUM_IMPORT_SIZE;
+      default:
+        return env.AWS_S3_UPLOAD_MAX_SIZE;
+    }
+  }
+}

server/presenters/attachment.ts

@@ -0,0 +1,12 @@
+import { Attachment } from "@server/models";
+
+export default function present(attachment: Attachment) {
+  return {
+    documentId: attachment.documentId,
+    contentType: attachment.contentType,
+    name: attachment.name,
+    id: attachment.id,
+    url: attachment.url,
+    size: attachment.size,
+  };
+}

server/presenters/index.ts

@@ -1,4 +1,5 @@
 import presentApiKey from "./apiKey";
+import presentAttachment from "./attachment";
 import presentAuthenticationProvider from "./authenticationProvider";
 import presentAvailableTeam from "./availableTeam";
 import presentCollection from "./collection";
@@ -27,6 +28,7 @@ import presentWebhookSubscription from "./webhookSubscription";
 
 export {
   presentApiKey,
+  presentAttachment,
   presentAuthenticationProvider,
   presentAvailableTeam,
   presentCollection,

server/queues/tasks/CleanupExpiredAttachmentsTask.ts

@@ -0,0 +1,31 @@
+import { Op } from "sequelize";
+import Logger from "@server/logging/Logger";
+import { Attachment } from "@server/models";
+import BaseTask, { TaskPriority } from "./BaseTask";
+
+type Props = {
+  limit: number;
+};
+
+export default class CleanupExpiredAttachmentsTask extends BaseTask<Props> {
+  public async perform({ limit }: Props) {
+    Logger.info("task", `Deleting expired attachments…`);
+    const attachments = await Attachment.unscoped().findAll({
+      where: {
+        expiresAt: {
+          [Op.lt]: new Date(),
+        },
+      },
+      limit,
+    });
+    await Promise.all(attachments.map((attachment) => attachment.destroy()));
+    Logger.info("task", `Removed ${attachments.length} attachments`);
+  }
+
+  public get options() {
+    return {
+      attempts: 1,
+      priority: TaskPriority.Background,
+    };
+  }
+}

server/routes/api/attachments.test.ts

@@ -1,3 +1,4 @@
+import { AttachmentPreset } from "@shared/types";
 import Attachment from "@server/models/Attachment";
 import {
   buildUser,
@@ -34,6 +35,42 @@ describe("#attachments.create", () => {
       expect(res.status).toEqual(200);
     });
 
+    it("should allow upload using avatar preset", async () => {
+      const user = await buildUser();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.png",
+          contentType: "image/png",
+          size: 1000,
+          preset: AttachmentPreset.Avatar,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(200);
+
+      const body = await res.json();
+      const attachment = await Attachment.findByPk(body.data.attachment.id);
+      expect(attachment!.expiresAt).toBeNull();
+    });
+
+    it("should create expiring attachment using import preset", async () => {
+      const user = await buildUser();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.zip",
+          contentType: "application/zip",
+          size: 10000,
+          preset: AttachmentPreset.Import,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(200);
+
+      const body = await res.json();
+      const attachment = await Attachment.findByPk(body.data.attachment.id);
+      expect(attachment!.expiresAt).toBeTruthy();
+    });
+
     it("should not allow file upload for public attachments", async () => {
       const user = await buildUser();
       const res = await server.post("/api/attachments.create", {
@@ -47,6 +84,20 @@ describe("#attachments.create", () => {
       });
       expect(res.status).toEqual(400);
     });
+
+    it("should not allow file upload for avatar preset", async () => {
+      const user = await buildUser();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.pdf",
+          contentType: "application/pdf",
+          size: 1000,
+          preset: AttachmentPreset.Avatar,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(400);
+    });
   });
 
   describe("viewer", () => {
@@ -63,6 +114,20 @@ describe("#attachments.create", () => {
       });
       expect(res.status).toEqual(200);
     });
+
+    it("should allow upload using avatar preset", async () => {
+      const user = await buildViewer();
+      const res = await server.post("/api/attachments.create", {
+        body: {
+          name: "test.png",
+          contentType: "image/png",
+          size: 1000,
+          preset: AttachmentPreset.Avatar,
+          token: user.getJwtToken(),
+        },
+      });
+      expect(res.status).toEqual(200);
+    });
   });
 });
 

server/routes/api/attachments.ts

@@ -1,81 +1,87 @@
 import Router from "koa-router";
 import { v4 as uuidv4 } from "uuid";
+import { AttachmentPreset } from "@shared/types";
 import { bytesToHumanReadable } from "@shared/utils/files";
 import { AttachmentValidation } from "@shared/validations";
-import { sequelize } from "@server/database/sequelize";
 import { AuthorizationError, ValidationError } from "@server/errors";
 import auth from "@server/middlewares/authentication";
-import { Attachment, Document, Event } from "@server/models";
-import { authorize } from "@server/policies";
-import { ContextWithState } from "@server/types";
 import {
-  getPresignedPost,
+  transaction,
-  publicS3Endpoint,
+  TransactionContext,
-  getSignedUrl,
+} from "@server/middlewares/transaction";
-} from "@server/utils/s3";
+import { Attachment, Document, Event } from "@server/models";
+import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
+import { authorize } from "@server/policies";
+import { presentAttachment } from "@server/presenters";
+import { ContextWithState } from "@server/types";
+import { getPresignedPost, publicS3Endpoint } from "@server/utils/s3";
 import { assertIn, assertPresent, assertUuid } from "@server/validation";
 
 const router = new Router();
-const AWS_S3_ACL = process.env.AWS_S3_ACL || "private";
 
-router.post("attachments.create", auth(), async (ctx) => {
+router.post(
-  const {
+  "attachments.create",
-    name,
+  auth(),
-    documentId,
+  transaction(),
-    contentType = "application/octet-stream",
+  async (ctx: TransactionContext) => {
-    size,
+    const {
-    public: isPublic,
+      name,
-  } = ctx.request.body;
+      documentId,
-  assertPresent(name, "name is required");
+      contentType = "application/octet-stream",
-  assertPresent(size, "size is required");
+      size,
+      // 'public' is now deprecated and can be removed on December 1 2022.
+      public: isPublicDeprecated,
+      preset = isPublicDeprecated
+        ? AttachmentPreset.Avatar
+        : AttachmentPreset.DocumentAttachment,
+    } = ctx.request.body;
+    const { user, transaction } = ctx.state;
 
-  const { user } = ctx.state;
+    assertPresent(name, "name is required");
+    assertPresent(size, "size is required");
 
-  // Public attachments are only used for avatars, so this is loosely coupled –
+    // Public attachments are only used for avatars, so this is loosely coupled –
-  // all user types can upload an avatar so no additional authorization is needed.
+    // all user types can upload an avatar so no additional authorization is needed.
-  if (isPublic) {
+    if (preset === AttachmentPreset.Avatar) {
-    assertIn(contentType, AttachmentValidation.avatarContentTypes);
+      assertIn(contentType, AttachmentValidation.avatarContentTypes);
-  } else {
+    } else {
-    authorize(user, "createAttachment", user.team);
+      authorize(user, "createAttachment", user.team);
-  }
+    }
 
-  if (
+    const maxUploadSize = AttachmentHelper.presetToMaxUploadSize(preset);
-    process.env.AWS_S3_UPLOAD_MAX_SIZE &&
-    size > process.env.AWS_S3_UPLOAD_MAX_SIZE
-  ) {
-    throw ValidationError(
-      `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable(
-        parseInt(process.env.AWS_S3_UPLOAD_MAX_SIZE, 10)
-      )}`
-    );
-  }
 
-  const modelId = uuidv4();
+    if (size > maxUploadSize) {
-  const acl =
+      throw ValidationError(
-    isPublic === undefined ? AWS_S3_ACL : isPublic ? "public-read" : "private";
+        `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable(
-  const bucket = acl === "public-read" ? "public" : "uploads";
+          maxUploadSize
-  const keyPrefix = `${bucket}/${user.id}/${modelId}`;
+        )}`
-  const key = `${keyPrefix}/${name}`;
+      );
-  const presignedPost = await getPresignedPost(key, acl, contentType);
+    }
-  const endpoint = publicS3Endpoint();
-  const url = `${endpoint}/${keyPrefix}/${encodeURIComponent(name)}`;
 
-  if (documentId !== undefined) {
+    if (documentId !== undefined) {
-    assertUuid(documentId, "documentId must be a uuid");
+      assertUuid(documentId, "documentId must be a uuid");
-    const document = await Document.findByPk(documentId, {
+      const document = await Document.findByPk(documentId, {
+        userId: user.id,
+      });
+      authorize(user, "update", document);
+    }
+
+    const modelId = uuidv4();
+    const acl = AttachmentHelper.presetToAcl(preset);
+    const key = AttachmentHelper.getKey({
+      acl,
+      id: modelId,
+      name,
       userId: user.id,
     });
-    authorize(user, "update", document);
-  }
 
-  const attachment = await sequelize.transaction(async (transaction) => {
     const attachment = await Attachment.create(
       {
         id: modelId,
         key,
         acl,
         size,
-        url,
+        expiresAt: AttachmentHelper.presetToExpiry(preset),
         contentType,
         documentId,
         teamId: user.teamId,
@@ -97,29 +103,26 @@ router.post("attachments.create", auth(), async (ctx) => {
       { transaction }
     );
 
-    return attachment;
+    const presignedPost = await getPresignedPost(
-  });
+      key,
+      acl,
+      maxUploadSize,
+      contentType
+    );
 
-  ctx.body = {
+    ctx.body = {
-    data: {
+      data: {
-      maxUploadSize: process.env.AWS_S3_UPLOAD_MAX_SIZE,
+        uploadUrl: publicS3Endpoint(),
-      uploadUrl: endpoint,
+        form: {
-      form: {
+          "Cache-Control": "max-age=31557600",
-        "Cache-Control": "max-age=31557600",
+          "Content-Type": contentType,
-        "Content-Type": contentType,
+          ...presignedPost.fields,
-        ...presignedPost.fields,
+        },
+        attachment: presentAttachment(attachment),
       },
-      attachment: {
+    };
-        documentId,
+  }
-        contentType,
+);
-        name,
-        id: attachment.id,
-        url: isPublic ? url : attachment.redirectUrl,
-        size,
-      },
-    },
-  };
-});
 
 router.post("attachments.delete", auth(), async (ctx) => {
   const { id } = ctx.request.body;
@@ -168,8 +171,7 @@ const handleAttachmentsRedirect = async (ctx: ContextWithState) => {
   });
 
   if (attachment.isPrivate) {
-    const accessUrl = await getSignedUrl(attachment.key);
+    ctx.redirect(await attachment.signedUrl);
-    ctx.redirect(accessUrl);
   } else {
     ctx.redirect(attachment.canonicalUrl);
   }

server/routes/api/cron.ts

@@ -5,6 +5,7 @@ import env from "@server/env";
 import { AuthenticationError } from "@server/errors";
 import CleanupDeletedDocumentsTask from "@server/queues/tasks/CleanupDeletedDocumentsTask";
 import CleanupDeletedTeamsTask from "@server/queues/tasks/CleanupDeletedTeamsTask";
+import CleanupExpiredAttachmentsTask from "@server/queues/tasks/CleanupExpiredAttachmentsTask";
 import CleanupExpiredFileOperationsTask from "@server/queues/tasks/CleanupExpiredFileOperationsTask";
 import CleanupWebhookDeliveriesTask from "@server/queues/tasks/CleanupWebhookDeliveriesTask";
 import InviteReminderTask from "@server/queues/tasks/InviteReminderTask";
@@ -37,6 +38,8 @@ const cronHandler = async (ctx: Context) => {
 
   await CleanupExpiredFileOperationsTask.schedule({ limit });
 
+  await CleanupExpiredAttachmentsTask.schedule({ limit });
+
   await CleanupDeletedTeamsTask.schedule({ limit });
 
   await CleanupWebhookDeliveriesTask.schedule({ limit });

server/routes/api/documents.ts

@@ -1114,10 +1114,10 @@ router.post("documents.import", auth(), async (ctx) => {
     throw InvalidRequestError("Request must include a file parameter");
   }
 
-  if (env.MAXIMUM_IMPORT_SIZE && file.size > env.MAXIMUM_IMPORT_SIZE) {
+  if (file.size > env.AWS_S3_UPLOAD_MAX_SIZE) {
     throw InvalidRequestError(
       `The selected file was larger than the ${bytesToHumanReadable(
-        env.MAXIMUM_IMPORT_SIZE
+        env.AWS_S3_UPLOAD_MAX_SIZE
       )} maximum size`
     );
   }

server/utils/s3.ts

@@ -107,14 +107,13 @@ export const getSignature = (policy: string) => {
 export const getPresignedPost = (
   key: string,
   acl: string,
+  maxUploadSize: number,
   contentType = "image"
 ) => {
   const params = {
     Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME,
     Conditions: compact([
-      process.env.AWS_S3_UPLOAD_MAX_SIZE
+      ["content-length-range", 0, maxUploadSize],
-        ? ["content-length-range", 0, +process.env.AWS_S3_UPLOAD_MAX_SIZE]
-        : undefined,
       ["starts-with", "$Content-Type", contentType],
       ["starts-with", "$Cache-Control", ""],
     ]),

shared/types.ts

@@ -22,6 +22,12 @@ export type PublicEnv = {
   RELEASE: string | undefined;
 };
 
+export enum AttachmentPreset {
+  DocumentAttachment = "documentAttachment",
+  Import = "import",
+  Avatar = "avatar",
+}
+
 export enum IntegrationType {
   Post = "post",
   Command = "command",