feat: Allow data imports larger than the standard attachment size (#4449)

* feat: Allow data imports larger than the standard attachment size

* Use correct preset for data imports

* Cleanup of expired attachments

* lint

server/models/Attachment.ts

@@ -10,7 +10,12 @@ import {
   DataType,
   IsNumeric,
 } from "sequelize-typescript";
-import { publicS3Endpoint, deleteFromS3, getFileByKey } from "@server/utils/s3";
+import {
+  publicS3Endpoint,
+  deleteFromS3,
+  getFileByKey,
+  getSignedUrl,
+} from "@server/utils/s3";
 import Document from "./Document";
 import Team from "./Team";
 import User from "./User";
@@ -47,26 +52,59 @@ class Attachment extends IdModel {
   @Column
   lastAccessedAt: Date | null;
 
+  @Column
+  expiresAt: Date | null;
+
   // getters
 
+  /**
+   * Get the original uploaded file name.
+   */
   get name() {
     return path.parse(this.key).base;
   }
 
-  get redirectUrl() {
-    return `/api/attachments.redirect?id=${this.id}`;
-  }
-
+  /**
+   * Whether the attachment is private or not.
+   */
   get isPrivate() {
     return this.acl === "private";
   }
 
+  /**
+   * Get the contents of this attachment as a Buffer
+   */
   get buffer() {
     return getFileByKey(this.key);
   }
 
+  /**
+   * Get a url that can be used to download the attachment if the user has a valid session.
+   */
+  get url() {
+    return this.isPrivate ? this.redirectUrl : this.canonicalUrl;
+  }
+
+  /**
+   * Get a url that can be used to download a private attachment if the user has a valid session.
+   */
+  get redirectUrl() {
+    return `/api/attachments.redirect?id=${this.id}`;
+  }
+
+  /**
+   * Get a direct URL to the attachment in storage. Note that this will not work for private attachments,
+   * a signed URL must be used.
+   */
   get canonicalUrl() {
-    return `${publicS3Endpoint()}/${this.key}`;
+    return encodeURI(`${publicS3Endpoint()}/${this.key}`);
+  }
+
+  /**
+   * Get a signed URL with the default expirt to download the attachment from storage.
+   */
+  get signedUrl() {
+    return getSignedUrl(this.key);
   }
 
   // hooks

server/models/FileOperation.ts

@@ -73,7 +73,13 @@ class FileOperation extends IdModel {
 
   expire = async function () {
     this.state = "expired";
-    await deleteFromS3(this.key);
+    try {
+      await deleteFromS3(this.key);
+    } catch (err) {
+      if (err.retryable) {
+        throw err;
+      }
+    }
     await this.save();
   };
 

server/models/helpers/AttachmentHelper.ts

@@ -0,0 +1,76 @@
+import { addHours } from "date-fns";
+import { AttachmentPreset } from "@shared/types";
+import env from "@server/env";
+
+export default class AttachmentHelper {
+  /**
+   * Get the upload location for the given upload details
+   *
+   * @param acl The ACL to use
+   * @param id The ID of the attachment
+   * @param name The name of the attachment
+   * @param userId The ID of the user uploading the attachment
+   */
+  static getKey({
+    acl,
+    id,
+    name,
+    userId,
+  }: {
+    acl: string;
+    id: string;
+    name: string;
+    userId: string;
+  }) {
+    const bucket = acl === "public-read" ? "public" : "uploads";
+    const keyPrefix = `${bucket}/${userId}/${id}`;
+    return `${keyPrefix}/${name}`;
+  }
+
+  /**
+   * Get the ACL to use for a given attachment preset
+   *
+   * @param preset The preset to use
+   * @returns A valid S3 ACL
+   */
+  static presetToAcl(preset: AttachmentPreset) {
+    switch (preset) {
+      case AttachmentPreset.Avatar:
+        return "public-read";
+      default:
+        return env.AWS_S3_ACL;
+    }
+  }
+
+  /**
+   * Get the expiration time for a given attachment preset
+   *
+   * @param preset The preset to use
+   * @returns An expiration time
+   */
+  static presetToExpiry(preset: AttachmentPreset) {
+    switch (preset) {
+      case AttachmentPreset.Import:
+        return addHours(new Date(), 24);
+      default:
+        return undefined;
+    }
+  }
+
+  /**
+   * Get the maximum upload size for a given attachment preset
+   *
+   * @param preset The preset to use
+   * @returns The maximum upload size in bytes
+   */
+  static presetToMaxUploadSize(preset: AttachmentPreset) {
+    switch (preset) {
+      case AttachmentPreset.Avatar:
+        return Math.min(1024 * 1024 * 5, env.AWS_S3_UPLOAD_MAX_SIZE);
+      case AttachmentPreset.Import:
+        return env.MAXIMUM_IMPORT_SIZE;
+      default:
+        return env.AWS_S3_UPLOAD_MAX_SIZE;
+    }
+  }
+}