Local file storage (#5763)

Co-authored-by: Tom Moor <tom.moor@gmail.com>

server/models/Attachment.test.ts

@@ -0,0 +1,10 @@
+import { buildAttachment } from "@server/test/factories";
+import Attachment from "./Attachment";
+
+describe("#findByKey", () => {
+  it("should return the correct attachment given a key", async () => {
+    const attachment = await buildAttachment();
+    const found = await Attachment.findByKey(attachment.key);
+    expect(found?.id).toBe(attachment.id);
+  });
+});

server/models/Attachment.ts

@@ -1,4 +1,7 @@
+import { createReadStream } from "fs";
 import path from "path";
+import { File } from "formidable";
+import JWT from "jsonwebtoken";
 import { QueryTypes } from "sequelize";
 import {
   BeforeDestroy,
@@ -10,8 +13,13 @@ import {
   Table,
   DataType,
   IsNumeric,
+  BeforeUpdate,
 } from "sequelize-typescript";
+import env from "@server/env";
+import { AuthenticationError } from "@server/errors";
 import FileStorage from "@server/storage/files";
+import { getJWTPayload } from "@server/utils/jwt";
+import { ValidateKey } from "@server/validation";
 import Document from "./Document";
 import Team from "./Team";
 import User from "./User";
@@ -96,11 +104,11 @@ class Attachment extends IdModel {
   }
 
   /**
-   * Get a direct URL to the attachment in storage. Note that this will not work for private attachments,
-   * a signed URL must be used.
+   * Get a direct URL to the attachment in storage. Note that this will not work
+   * for private attachments, a signed URL must be used.
    */
   get canonicalUrl() {
-    return encodeURI(`${FileStorage.getPublicEndpoint()}/${this.key}`);
+    return encodeURI(FileStorage.getUrlForKey(this.key));
   }
 
   /**
@@ -110,8 +118,31 @@ class Attachment extends IdModel {
     return FileStorage.getSignedUrl(this.key);
   }
 
+  /**
+   * Store the given file in storage at the location specified by the attachment key.
+   * If the attachment already exists, it will be overwritten.
+   *
+   * @param file The file to store
+   * @returns A promise resolving to the attachment
+   */
+  async overwriteFile(file: File) {
+    return FileStorage.store({
+      body: createReadStream(file.filepath),
+      contentLength: file.size,
+      contentType: this.contentType,
+      key: this.key,
+      acl: this.acl,
+    });
+  }
+
   // hooks
 
+  @BeforeUpdate
+  static async sanitizeKey(model: Attachment) {
+    model.key = ValidateKey.sanitize(model.key);
+    return model;
+  }
+
   @BeforeDestroy
   static async deleteAttachmentFromS3(model: Attachment) {
     await FileStorage.deleteFile(model.key);
@@ -141,6 +172,42 @@ class Attachment extends IdModel {
     return parseInt(result?.[0]?.total ?? "0", 10);
   }
 
+  /**
+   * Find an attachment given a JWT signature.
+   *
+   * @param sign - The signature that uniquely identifies an attachment
+   * @returns A promise resolving to attachment corresponding to the signature
+   * @throws {AuthenticationError} Invalid signature if the signature verification fails
+   */
+  static async findBySignature(sign: string): Promise<Attachment> {
+    const payload = getJWTPayload(sign);
+
+    if (payload.type !== "attachment") {
+      throw AuthenticationError("Invalid signature");
+    }
+
+    try {
+      JWT.verify(sign, env.SECRET_KEY);
+    } catch (err) {
+      throw AuthenticationError("Invalid signature");
+    }
+
+    return this.findByKey(payload.key);
+  }
+
+  /**
+   * Find an attachment given a key
+   *
+   * @param key The key representing attachment file path
+   * @returns A promise resolving to attachment corresponding to the key
+   */
+  static async findByKey(key: string): Promise<Attachment> {
+    return this.findOne({
+      where: { key },
+      rejectOnEmpty: true,
+    });
+  }
+
   // associations
 
   @BelongsTo(() => Team, "teamId")

server/models/Team.ts

@@ -17,7 +17,6 @@ import {
   Is,
   DataType,
   IsUUID,
-  IsUrl,
   AllowNull,
   AfterUpdate,
 } from "sequelize-typescript";
@@ -40,6 +39,7 @@ import User from "./User";
 import ParanoidModel from "./base/ParanoidModel";
 import Fix from "./decorators/Fix";
 import IsFQDN from "./validators/IsFQDN";
+import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath";
 import Length from "./validators/Length";
 import NotContainsUrl from "./validators/NotContainsUrl";
 
@@ -97,7 +97,7 @@ class Team extends ParanoidModel {
   defaultCollectionId: string | null;
 
   @AllowNull
-  @IsUrl
+  @IsUrlOrRelativePath
   @Length({ max: 4096, msg: "avatarUrl must be 4096 characters or less" })
   @Column(DataType.STRING)
   get avatarUrl() {

server/models/User.ts

@@ -18,7 +18,6 @@ import {
   HasMany,
   Scopes,
   IsDate,
-  IsUrl,
   AllowNull,
   AfterUpdate,
 } from "sequelize-typescript";
@@ -52,6 +51,7 @@ import Encrypted, {
   getEncryptedColumn,
 } from "./decorators/Encrypted";
 import Fix from "./decorators/Fix";
+import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath";
 import Length from "./validators/Length";
 import NotContainsUrl from "./validators/NotContainsUrl";
 
@@ -182,7 +182,7 @@ class User extends ParanoidModel {
   language: string;
 
   @AllowNull
-  @IsUrl
+  @IsUrlOrRelativePath
   @Length({ max: 4096, msg: "avatarUrl must be less than 4096 characters" })
   @Column(DataType.STRING)
   get avatarUrl() {

server/models/helpers/AttachmentHelper.ts

@@ -70,7 +70,7 @@ export default class AttachmentHelper {
       case AttachmentPreset.Avatar:
       case AttachmentPreset.DocumentAttachment:
       default:
-        return env.AWS_S3_UPLOAD_MAX_SIZE;
+        return env.FILE_STORAGE_UPLOAD_MAX_SIZE;
     }
   }
 }

server/models/validators/IsUrlOrRelativePath.ts

@@ -0,0 +1,23 @@
+import { isURL } from "class-validator";
+import { addAttributeOptions } from "sequelize-typescript";
+
+/**
+ * A decorator that validates that a string is a url or relative path.
+ */
+export default function IsUrlOrRelativePath(target: any, propertyName: string) {
+  return addAttributeOptions(target, propertyName, {
+    validate: {
+      validUrlOrPath(value: string) {
+        if (
+          value &&
+          !isURL(value, {
+            require_host: false,
+            require_protocol: false,
+          })
+        ) {
+          throw new Error("Must be a URL or relative path");
+        }
+      },
+    },
+  });
+}