Local file storage (#5763)

Co-authored-by: Tom Moor <tom.moor@gmail.com>

server/commands/attachmentCreator.ts

@@ -48,7 +48,7 @@ export default async function attachmentCreator({
 
   if ("url" in rest) {
     const { url } = rest;
-    const res = await FileStorage.uploadFromUrl(url, key, acl);
+    const res = await FileStorage.storeFromUrl(url, key, acl);
 
     if (!res) {
       return;
@@ -69,7 +69,7 @@ export default async function attachmentCreator({
     );
   } else {
     const { buffer, type } = rest;
-    await FileStorage.upload({
+    await FileStorage.store({
       body: buffer,
       contentType: type,
       contentLength: buffer.length,

server/env.ts

@@ -545,12 +545,14 @@ export class Environment {
     this.toOptionalNumber(process.env.RATE_LIMITER_DURATION_WINDOW) ?? 60;
 
   /**
-   * Set max allowed upload size for file attachments.
+   * @deprecated Set max allowed upload size for file attachments.
    */
   @IsOptional()
   @IsNumber()
-  public AWS_S3_UPLOAD_MAX_SIZE =
-    this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? 100000000;
+  @Deprecated("Use FILE_STORAGE_UPLOAD_MAX_SIZE instead")
+  public AWS_S3_UPLOAD_MAX_SIZE = this.toOptionalNumber(
+    process.env.AWS_S3_UPLOAD_MAX_SIZE
+  );
 
   /**
    * Access key ID for AWS S3.
@@ -612,6 +614,28 @@ export class Environment {
   @IsOptional()
   public AWS_S3_ACL = process.env.AWS_S3_ACL ?? "private";
 
+  /**
+   * Which file storage system to use
+   */
+  @IsIn(["local", "s3"])
+  public FILE_STORAGE = this.toOptionalString(process.env.FILE_STORAGE) ?? "s3";
+
+  /**
+   * Set default root dir path for local file storage
+   */
+  public FILE_STORAGE_LOCAL_ROOT_DIR =
+    this.toOptionalString(process.env.FILE_STORAGE_LOCAL_ROOT_DIR) ??
+    "/var/lib/outline/data";
+
+  /**
+   * Set max allowed upload size for file attachments.
+   */
+  @IsNumber()
+  public FILE_STORAGE_UPLOAD_MAX_SIZE =
+    this.toOptionalNumber(process.env.FILE_STORAGE_UPLOAD_MAX_SIZE) ??
+    this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ??
+    100000000;
+
   /**
    * Because imports can be much larger than regular file attachments and are
    * deleted automatically we allow an optional separate limit on the size of
@@ -620,7 +644,7 @@ export class Environment {
   @IsNumber()
   public MAXIMUM_IMPORT_SIZE = Math.max(
     this.toOptionalNumber(process.env.MAXIMUM_IMPORT_SIZE) ?? 100000000,
-    this.AWS_S3_UPLOAD_MAX_SIZE
+    this.FILE_STORAGE_UPLOAD_MAX_SIZE
   );
 
   /**

server/migrations/20230920032853-add-key-index.js

@@ -0,0 +1,11 @@
+"use strict";
+
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.addIndex("attachments", ["key"]);
+  },
+
+  async down(queryInterface) {
+    await queryInterface.removeIndex("attachments", ["key"]);
+  },
+};

server/models/Attachment.test.ts

@@ -0,0 +1,10 @@
+import { buildAttachment } from "@server/test/factories";
+import Attachment from "./Attachment";
+
+describe("#findByKey", () => {
+  it("should return the correct attachment given a key", async () => {
+    const attachment = await buildAttachment();
+    const found = await Attachment.findByKey(attachment.key);
+    expect(found?.id).toBe(attachment.id);
+  });
+});

server/models/Attachment.ts

@@ -1,4 +1,7 @@
+import { createReadStream } from "fs";
 import path from "path";
+import { File } from "formidable";
+import JWT from "jsonwebtoken";
 import { QueryTypes } from "sequelize";
 import {
   BeforeDestroy,
@@ -10,8 +13,13 @@ import {
   Table,
   DataType,
   IsNumeric,
+  BeforeUpdate,
 } from "sequelize-typescript";
+import env from "@server/env";
+import { AuthenticationError } from "@server/errors";
 import FileStorage from "@server/storage/files";
+import { getJWTPayload } from "@server/utils/jwt";
+import { ValidateKey } from "@server/validation";
 import Document from "./Document";
 import Team from "./Team";
 import User from "./User";
@@ -96,11 +104,11 @@ class Attachment extends IdModel {
   }
 
   /**
-   * Get a direct URL to the attachment in storage. Note that this will not work for private attachments,
-   * a signed URL must be used.
+   * Get a direct URL to the attachment in storage. Note that this will not work
+   * for private attachments, a signed URL must be used.
    */
   get canonicalUrl() {
-    return encodeURI(`${FileStorage.getPublicEndpoint()}/${this.key}`);
+    return encodeURI(FileStorage.getUrlForKey(this.key));
   }
 
   /**
@@ -110,8 +118,31 @@ class Attachment extends IdModel {
     return FileStorage.getSignedUrl(this.key);
   }
 
+  /**
+   * Store the given file in storage at the location specified by the attachment key.
+   * If the attachment already exists, it will be overwritten.
+   *
+   * @param file The file to store
+   * @returns A promise resolving to the attachment
+   */
+  async overwriteFile(file: File) {
+    return FileStorage.store({
+      body: createReadStream(file.filepath),
+      contentLength: file.size,
+      contentType: this.contentType,
+      key: this.key,
+      acl: this.acl,
+    });
+  }
+
   // hooks
 
+  @BeforeUpdate
+  static async sanitizeKey(model: Attachment) {
+    model.key = ValidateKey.sanitize(model.key);
+    return model;
+  }
+
   @BeforeDestroy
   static async deleteAttachmentFromS3(model: Attachment) {
     await FileStorage.deleteFile(model.key);
@@ -141,6 +172,42 @@ class Attachment extends IdModel {
     return parseInt(result?.[0]?.total ?? "0", 10);
   }
 
+  /**
+   * Find an attachment given a JWT signature.
+   *
+   * @param sign - The signature that uniquely identifies an attachment
+   * @returns A promise resolving to attachment corresponding to the signature
+   * @throws {AuthenticationError} Invalid signature if the signature verification fails
+   */
+  static async findBySignature(sign: string): Promise<Attachment> {
+    const payload = getJWTPayload(sign);
+
+    if (payload.type !== "attachment") {
+      throw AuthenticationError("Invalid signature");
+    }
+
+    try {
+      JWT.verify(sign, env.SECRET_KEY);
+    } catch (err) {
+      throw AuthenticationError("Invalid signature");
+    }
+
+    return this.findByKey(payload.key);
+  }
+
+  /**
+   * Find an attachment given a key
+   *
+   * @param key The key representing attachment file path
+   * @returns A promise resolving to attachment corresponding to the key
+   */
+  static async findByKey(key: string): Promise<Attachment> {
+    return this.findOne({
+      where: { key },
+      rejectOnEmpty: true,
+    });
+  }
+
   // associations
 
   @BelongsTo(() => Team, "teamId")

server/models/Team.ts

@@ -17,7 +17,6 @@ import {
   Is,
   DataType,
   IsUUID,
-  IsUrl,
   AllowNull,
   AfterUpdate,
 } from "sequelize-typescript";
@@ -40,6 +39,7 @@ import User from "./User";
 import ParanoidModel from "./base/ParanoidModel";
 import Fix from "./decorators/Fix";
 import IsFQDN from "./validators/IsFQDN";
+import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath";
 import Length from "./validators/Length";
 import NotContainsUrl from "./validators/NotContainsUrl";
 
@@ -97,7 +97,7 @@ class Team extends ParanoidModel {
   defaultCollectionId: string | null;
 
   @AllowNull
-  @IsUrl
+  @IsUrlOrRelativePath
   @Length({ max: 4096, msg: "avatarUrl must be 4096 characters or less" })
   @Column(DataType.STRING)
   get avatarUrl() {

server/models/User.ts

@@ -18,7 +18,6 @@ import {
   HasMany,
   Scopes,
   IsDate,
-  IsUrl,
   AllowNull,
   AfterUpdate,
 } from "sequelize-typescript";
@@ -52,6 +51,7 @@ import Encrypted, {
   getEncryptedColumn,
 } from "./decorators/Encrypted";
 import Fix from "./decorators/Fix";
+import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath";
 import Length from "./validators/Length";
 import NotContainsUrl from "./validators/NotContainsUrl";
 
@@ -182,7 +182,7 @@ class User extends ParanoidModel {
   language: string;
 
   @AllowNull
-  @IsUrl
+  @IsUrlOrRelativePath
   @Length({ max: 4096, msg: "avatarUrl must be less than 4096 characters" })
   @Column(DataType.STRING)
   get avatarUrl() {

server/models/helpers/AttachmentHelper.ts

@@ -70,7 +70,7 @@ export default class AttachmentHelper {
       case AttachmentPreset.Avatar:
       case AttachmentPreset.DocumentAttachment:
       default:
-        return env.AWS_S3_UPLOAD_MAX_SIZE;
+        return env.FILE_STORAGE_UPLOAD_MAX_SIZE;
     }
   }
 }

server/models/validators/IsUrlOrRelativePath.ts

@@ -0,0 +1,23 @@
+import { isURL } from "class-validator";
+import { addAttributeOptions } from "sequelize-typescript";
+
+/**
+ * A decorator that validates that a string is a url or relative path.
+ */
+export default function IsUrlOrRelativePath(target: any, propertyName: string) {
+  return addAttributeOptions(target, propertyName, {
+    validate: {
+      validUrlOrPath(value: string) {
+        if (
+          value &&
+          !isURL(value, {
+            require_host: false,
+            require_protocol: false,
+          })
+        ) {
+          throw new Error("Must be a URL or relative path");
+        }
+      },
+    },
+  });
+}

server/policies/attachment.ts

@@ -9,7 +9,7 @@ allow(User, "createAttachment", Team, (user, team) => {
 });
 
 allow(User, "read", Attachment, (actor, attachment) => {
-  if (!attachment || attachment.teamId !== actor.teamId) {
+  if (!attachment || !actor || attachment.teamId !== actor.teamId) {
     return false;
   }
   if (actor.isAdmin) {

server/queues/tasks/ExportTask.ts

@@ -90,7 +90,7 @@ export default abstract class ExportTask extends BaseTask<Props> {
       });
 
       const stat = await fs.promises.stat(filePath);
-      const url = await FileStorage.upload({
+      const url = await FileStorage.store({
         body: fs.createReadStream(filePath),
         contentLength: stat.size,
         contentType: "application/zip",

server/queues/tasks/UploadTeamAvatarTask.ts

@@ -20,14 +20,14 @@ export default class UploadTeamAvatarTask extends BaseTask<Props> {
       rejectOnEmpty: true,
     });
 
-    const res = await FileStorage.uploadFromUrl(
+    const res = await FileStorage.storeFromUrl(
       props.avatarUrl,
       `avatars/${team.id}/${uuidv4()}`,
       "public-read"
     );
 
     if (res?.url) {
-      await team.update({ avatarUrl: res?.url });
+      await team.update({ avatarUrl: res.url });
     }
   }
 

server/queues/tasks/UploadUserAvatarTask.ts

@@ -20,14 +20,14 @@ export default class UploadUserAvatarTask extends BaseTask<Props> {
       rejectOnEmpty: true,
     });
 
-    const res = await FileStorage.uploadFromUrl(
+    const res = await FileStorage.storeFromUrl(
       props.avatarUrl,
       `avatars/${user.id}/${uuidv4()}`,
       "public-read"
     );
 
     if (res?.url) {
-      await user.update({ avatarUrl: res?.url });
+      await user.update({ avatarUrl: res.url });
     }
   }
 

server/routes/api/attachments/attachments.ts

@@ -99,7 +99,7 @@ router.post(
 
     ctx.body = {
       data: {
-        uploadUrl: FileStorage.getPublicEndpoint(),
+        uploadUrl: FileStorage.getUploadUrl(),
         form: {
           "Cache-Control": "max-age=31557600",
           "Content-Type": contentType,

server/routes/auth/index.test.ts

@@ -13,7 +13,8 @@ describe("auth/redirect", () => {
       }
     );
     expect(res.status).toEqual(302);
-    expect(res.headers.get("location").endsWith("/home")).toBeTruthy();
+    expect(res.headers.get("location")).not.toBeNull();
+    expect(res.headers.get("location")!.endsWith("/home")).toBeTruthy();
   });
 
   it("should redirect to first collection", async () => {
@@ -28,6 +29,7 @@ describe("auth/redirect", () => {
       }
     );
     expect(res.status).toEqual(302);
-    expect(res.headers.get("location").endsWith(collection.url)).toBeTruthy();
+    expect(res.headers.get("location")).not.toBeNull();
+    expect(res.headers.get("location")!.endsWith(collection.url)).toBeTruthy();
   });
 });

server/storage/files/BaseStorage.ts

@@ -1,3 +1,4 @@
+import { Blob } from "buffer";
 import { Readable } from "stream";
 import { PresignedPost } from "aws-sdk/clients/s3";
 import env from "@server/env";
@@ -5,6 +6,9 @@ import Logger from "@server/logging/Logger";
 import fetch from "@server/utils/fetch";
 
 export default abstract class BaseStorage {
+  /** The default number of seconds until a signed URL expires. */
+  public static defaultSignedUrlExpires = 60;
+
   /**
    * Returns a presigned post for uploading files to the storage provider.
    *
@@ -19,7 +23,7 @@ export default abstract class BaseStorage {
     acl: string,
     maxUploadSize: number,
     contentType: string
-  ): Promise<PresignedPost>;
+  ): Promise<Partial<PresignedPost>>;
 
   /**
    * Returns a stream for reading a file from the storage provider.
@@ -29,19 +33,20 @@ export default abstract class BaseStorage {
   public abstract getFileStream(key: string): NodeJS.ReadableStream | null;
 
   /**
-   * Returns a buffer of a file from the storage provider.
-   *
-   * @param key The path to the file
-   */
-  public abstract getFileBuffer(key: string): Promise<Blob>;
-
-  /**
-   * Returns the public endpoint for the storage provider.
+   * Returns the upload URL for the storage provider.
    *
    * @param isServerUpload Whether the upload is happening on the server or not
-   * @returns The public endpoint as a string
+   * @returns {string} The upload URL
    */
-  public abstract getPublicEndpoint(isServerUpload?: boolean): string;
+  public abstract getUploadUrl(isServerUpload?: boolean): string;
+
+  /**
+   * Returns the download URL for a given file.
+   *
+   * @param key The path to the file
+   * @returns {string} The download URL for the file
+   */
+  public abstract getUrlForKey(key: string): string;
 
   /**
    * Returns a signed URL for a file from the storage provider.
@@ -55,7 +60,7 @@ export default abstract class BaseStorage {
   ): Promise<string>;
 
   /**
-   * Upload a file to the storage provider.
+   * Store a file in the storage provider.
    *
    * @param body The file body
    * @param contentLength The content length of the file
@@ -64,7 +69,7 @@ export default abstract class BaseStorage {
    * @param acl The ACL to use
    * @returns The URL of the file
    */
-  public abstract upload({
+  public abstract store({
     body,
     contentLength,
     contentType,
@@ -72,12 +77,35 @@ export default abstract class BaseStorage {
     acl,
   }: {
     body: Buffer | Uint8Array | Blob | string | Readable;
-    contentLength: number;
-    contentType: string;
+    contentLength?: number;
+    contentType?: string;
     key: string;
-    acl: string;
+    acl?: string;
   }): Promise<string | undefined>;
 
+  /**
+   * Returns a buffer of a file from the storage provider.
+   *
+   * @param key The path to the file
+   */
+  public async getFileBuffer(key: string) {
+    const stream = this.getFileStream(key);
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      if (!stream) {
+        return reject(new Error("No stream available"));
+      }
+
+      stream.on("data", (d) => {
+        chunks.push(d);
+      });
+      stream.once("end", () => {
+        resolve(Buffer.concat(chunks));
+      });
+      stream.once("error", reject);
+    });
+  }
+
   /**
    * Upload a file to the storage provider directly from a remote or base64 encoded URL.
    *
@@ -86,7 +114,7 @@ export default abstract class BaseStorage {
    * @param acl The ACL to use
    * @returns A promise that resolves when the file is uploaded
    */
-  public async uploadFromUrl(
+  public async storeFromUrl(
     url: string,
     key: string,
     acl: string
@@ -98,7 +126,7 @@ export default abstract class BaseStorage {
       }
     | undefined
   > {
-    const endpoint = this.getPublicEndpoint(true);
+    const endpoint = this.getUploadUrl(true);
     if (url.startsWith("/api") || url.startsWith(endpoint)) {
       return;
     }
@@ -115,7 +143,7 @@ export default abstract class BaseStorage {
         const res = await fetch(url, {
           follow: 3,
           redirect: "follow",
-          size: env.AWS_S3_UPLOAD_MAX_SIZE,
+          size: env.FILE_STORAGE_UPLOAD_MAX_SIZE,
           timeout: 10000,
         });
 
@@ -143,7 +171,7 @@ export default abstract class BaseStorage {
     }
 
     try {
-      const result = await this.upload({
+      const result = await this.store({
         body: buffer,
         contentLength,
         contentType,

server/storage/files/LocalStorage.ts

@@ -0,0 +1,120 @@
+import { Blob } from "buffer";
+import {
+  ReadStream,
+  closeSync,
+  createReadStream,
+  createWriteStream,
+  existsSync,
+  openSync,
+} from "fs";
+import { mkdir, unlink } from "fs/promises";
+import path from "path";
+import { Readable } from "stream";
+import invariant from "invariant";
+import JWT from "jsonwebtoken";
+import env from "@server/env";
+import Logger from "@server/logging/Logger";
+import BaseStorage from "./BaseStorage";
+
+export default class LocalStorage extends BaseStorage {
+  public async getPresignedPost(
+    key: string,
+    acl: string,
+    maxUploadSize: number,
+    contentType = "image"
+  ) {
+    return Promise.resolve({
+      url: this.getUrlForKey(key),
+      fields: {
+        key,
+        acl,
+        maxUploadSize,
+        contentType,
+      },
+    } as any);
+  }
+
+  public getUploadUrl() {
+    return "/api/files.create";
+  }
+
+  public getUrlForKey(key: string): string {
+    return `/api/files.get?key=${key}`;
+  }
+
+  public store = async ({
+    body,
+    key,
+  }: {
+    body: string | ReadStream | Buffer | Uint8Array | Blob;
+    contentLength?: number;
+    contentType?: string;
+    key: string;
+    acl?: string;
+  }) => {
+    const subdir = key.split("/").slice(0, -1).join("/");
+    if (!existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir))) {
+      await mkdir(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir), {
+        recursive: true,
+      });
+    }
+
+    let src: NodeJS.ReadableStream;
+    if (body instanceof ReadStream) {
+      src = body;
+    } else if (body instanceof Blob) {
+      src = Readable.from(Buffer.from(await body.arrayBuffer()));
+    } else {
+      src = Readable.from(body);
+    }
+
+    const destPath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key);
+    closeSync(openSync(destPath, "w"));
+    const dest = createWriteStream(destPath);
+    src.pipe(dest);
+
+    return new Promise<string>((resolve, reject) => {
+      src.once("end", () => resolve(this.getUrlForKey(key)));
+      src.once("err", (err) => {
+        dest.end();
+        reject(err);
+      });
+    });
+  };
+
+  public async deleteFile(key: string) {
+    const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key);
+    try {
+      await unlink(filePath);
+    } catch (err) {
+      Logger.warn(`Couldn't delete ${filePath}`, err);
+    }
+  }
+
+  public getSignedUrl = async (
+    key: string,
+    expiresIn = LocalStorage.defaultSignedUrlExpires
+  ) => {
+    const sig = JWT.sign(
+      {
+        key,
+        type: "attachment",
+      },
+      env.SECRET_KEY,
+      {
+        expiresIn,
+      }
+    );
+    return Promise.resolve(`/api/files.get?sig=${sig}`);
+  };
+
+  public getFileStream(key: string) {
+    invariant(
+      env.FILE_STORAGE_LOCAL_ROOT_DIR,
+      "FILE_STORAGE_LOCAL_ROOT_DIR is required"
+    );
+
+    const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key);
+    return createReadStream(filePath);
+  }
+}

server/storage/files/S3Storage.ts

@@ -47,7 +47,7 @@ export default class S3Storage extends BaseStorage {
     );
   }
 
-  public getPublicEndpoint(isServerUpload?: boolean) {
+  private getPublicEndpoint(isServerUpload?: boolean) {
     if (env.AWS_S3_ACCELERATE_URL) {
       return env.AWS_S3_ACCELERATE_URL;
     }
@@ -78,7 +78,15 @@ export default class S3Storage extends BaseStorage {
     }`;
   }
 
-  public upload = async ({
+  public getUploadUrl(isServerUpload?: boolean) {
+    return this.getPublicEndpoint(isServerUpload);
+  }
+
+  public getUrlForKey(key: string): string {
+    return `${this.getPublicEndpoint()}/${key}`;
+  }
+
+  public store = async ({
     body,
     contentLength,
     contentType,
@@ -86,10 +94,10 @@ export default class S3Storage extends BaseStorage {
     acl,
   }: {
     body: S3.Body;
-    contentLength: number;
-    contentType: string;
+    contentLength?: number;
+    contentType?: string;
     key: string;
-    acl: string;
+    acl?: string;
   }) => {
     invariant(
       env.AWS_S3_UPLOAD_BUCKET_NAME,
@@ -125,7 +133,10 @@ export default class S3Storage extends BaseStorage {
       .promise();
   }
 
-  public getSignedUrl = async (key: string, expiresIn = 60) => {
+  public getSignedUrl = async (
+    key: string,
+    expiresIn = S3Storage.defaultSignedUrlExpires
+  ) => {
     const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
     const params = {
       Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
@@ -170,26 +181,6 @@ export default class S3Storage extends BaseStorage {
     return null;
   }
 
-  public async getFileBuffer(key: string) {
-    invariant(
-      env.AWS_S3_UPLOAD_BUCKET_NAME,
-      "AWS_S3_UPLOAD_BUCKET_NAME is required"
-    );
-
-    const response = await this.client
-      .getObject({
-        Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
-        Key: key,
-      })
-      .promise();
-
-    if (response.Body) {
-      return response.Body as Blob;
-    }
-
-    throw new Error("Error getting file buffer from S3");
-  }
-
   private client: AWS.S3;
 
   private getEndpoint() {

server/storage/files/__mocks__/index.ts

@@ -1,7 +1,9 @@
 export default {
   upload: jest.fn().mockReturnValue("/endpoint/key"),
 
-  getPublicEndpoint: jest.fn().mockReturnValue("http://mock"),
+  getUploadUrl: jest.fn().mockReturnValue("http://mock/create"),
+
+  getUrlForKey: jest.fn().mockReturnValue("http://mock/get"),
 
   getSignedUrl: jest.fn().mockReturnValue("http://s3mock"),
 

server/storage/files/index.ts

@@ -1,3 +1,8 @@
+import env from "@server/env";
+import LocalStorage from "./LocalStorage";
 import S3Storage from "./S3Storage";
 
-export default new S3Storage();
+const storage =
+  env.FILE_STORAGE === "local" ? new LocalStorage() : new S3Storage();
+
+export default storage;

server/test/TestServer.ts

@@ -0,0 +1,84 @@
+import http from "http";
+import { AddressInfo } from "net";
+import Koa from "koa";
+// eslint-disable-next-line no-restricted-imports
+import nodeFetch from "node-fetch";
+
+class TestServer {
+  private server: http.Server;
+  private listener?: Promise<void> | null;
+
+  constructor(app: Koa) {
+    this.server = http.createServer(app.callback() as any);
+  }
+
+  get address(): string {
+    const { port } = this.server.address() as AddressInfo;
+    return `http://localhost:${port}`;
+  }
+
+  listen() {
+    if (!this.listener) {
+      this.listener = new Promise((resolve, reject) => {
+        this.server
+          .listen(0, () => resolve())
+          .on("error", (err) => reject(err));
+      });
+    }
+
+    return this.listener;
+  }
+
+  fetch(path: string, opts: any) {
+    return this.listen().then(() => {
+      const url = `${this.address}${path}`;
+      const options = Object.assign({ headers: {} }, opts);
+      const contentType =
+        options.headers["Content-Type"] ?? options.headers["content-type"];
+      // automatic JSON encoding
+      if (!contentType && typeof options.body === "object") {
+        options.headers["Content-Type"] = "application/json";
+        options.body = JSON.stringify(options.body);
+      }
+
+      return nodeFetch(url, options);
+    });
+  }
+
+  close() {
+    this.listener = null;
+    return new Promise<void>((resolve, reject) => {
+      this.server.close((err) => (err ? reject(err) : resolve()));
+    });
+  }
+
+  delete(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "DELETE" });
+  }
+
+  get(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "GET" });
+  }
+
+  head(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "HEAD" });
+  }
+
+  options(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "OPTIONS" });
+  }
+
+  patch(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "PATCH" });
+  }
+
+  post(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "POST" });
+  }
+
+  put(path: string, options?: any) {
+    return this.fetch(path, { ...options, method: "PUT" });
+  }
+}
+
+export default TestServer;

server/test/env.ts

@@ -18,6 +18,8 @@ env.OIDC_USERINFO_URI = "http://localhost/userinfo";
 
 env.RATE_LIMITER_ENABLED = false;
 
+env.FILE_STORAGE = "local";
+env.FILE_STORAGE_LOCAL_ROOT_DIR = "/tmp";
 env.IFRAMELY_API_KEY = "123";
 
 if (process.env.DATABASE_URL_TEST) {

server/test/factories.ts

@@ -33,6 +33,7 @@ import {
   SearchQuery,
   Pin,
 } from "@server/models";
+import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
 
 export async function buildApiKey(overrides: Partial<ApiKey> = {}) {
   if (!overrides.userId) {
@@ -420,7 +421,10 @@ export async function buildFileOperation(
   });
 }
 
-export async function buildAttachment(overrides: Partial<Attachment> = {}) {
+export async function buildAttachment(
+  overrides: Partial<Attachment> = {},
+  fileName?: string
+) {
   if (!overrides.teamId) {
     const team = await buildTeam();
     overrides.teamId = team.id;
@@ -441,11 +445,14 @@ export async function buildAttachment(overrides: Partial<Attachment> = {}) {
     overrides.documentId = document.id;
   }
 
+  const id = uuidv4();
+  const acl = overrides.acl || "public-read";
+  const name = fileName || faker.system.fileName();
   return Attachment.create({
-    key: `uploads/key/to/${faker.system.fileName}.png`,
+    key: AttachmentHelper.getKey({ acl, id, name, userId: overrides.userId }),
     contentType: "image/png",
     size: 100,
-    acl: "public-read",
+    acl,
     createdAt: new Date("2018-01-02T00:00:00.000Z"),
     updatedAt: new Date("2018-01-02T00:00:00.000Z"),
     ...overrides,

server/test/support.ts

@@ -1,22 +1,22 @@
 import { faker } from "@faker-js/faker";
-import TestServer from "fetch-test-server";
 import sharedEnv from "@shared/env";
 import env from "@server/env";
 import onerror from "@server/onerror";
 import webService from "@server/services/web";
 import { sequelize } from "@server/storage/database";
+import TestServer from "./TestServer";
 
 export function getTestServer() {
   const app = webService();
   onerror(app);
-  const server = new TestServer(app.callback());
+  const server = new TestServer(app);
 
-  server.disconnect = async () => {
+  const disconnect = async () => {
     await sequelize.close();
-    server.close();
+    return server.close();
   };
 
-  afterAll(server.disconnect);
+  afterAll(disconnect);
 
   return server;
 }

server/typings/index.d.ts

@@ -6,8 +6,6 @@ declare module "formidable/lib/file";
 
 declare module "oy-vey";
 
-declare module "fetch-test-server";
-
 declare module "dotenv";
 
 declare module "email-providers" {

server/utils/jwt.ts

@@ -4,7 +4,7 @@ import JWT from "jsonwebtoken";
 import { Team, User } from "@server/models";
 import { AuthenticationError } from "../errors";
 
-function getJWTPayload(token: string) {
+export function getJWTPayload(token: string) {
   let payload;
 
   try {

server/validation.test.ts

@@ -0,0 +1,41 @@
+import { v4 as uuidv4 } from "uuid";
+import { ValidateKey } from "./validation";
+
+describe("#ValidateKey.isValid", () => {
+  it("should return false if number of key components are not equal to 4", () => {
+    expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}`)).toBe(false);
+    expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}/foo/bar`)).toBe(
+      false
+    );
+  });
+
+  it("should return false if the first key component is neither 'public' nor 'uploads' ", () => {
+    expect(ValidateKey.isValid(`foo/${uuidv4()}/${uuidv4()}/bar.png`)).toBe(
+      false
+    );
+  });
+
+  it("should return false if second and third key components are not UUID", () => {
+    expect(ValidateKey.isValid(`uploads/foo/${uuidv4()}/bar.png`)).toBe(false);
+    expect(ValidateKey.isValid(`uploads/${uuidv4()}/foo/bar.png`)).toBe(false);
+  });
+
+  it("should return true successfully validating key", () => {
+    expect(ValidateKey.isValid(`public/${uuidv4()}/${uuidv4()}/foo.png`)).toBe(
+      true
+    );
+    expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}/foo.png`)).toBe(
+      true
+    );
+  });
+});
+
+describe("#ValidateKey.sanitize", () => {
+  it("should sanitize malicious looking keys", () => {
+    const uuid1 = uuidv4();
+    const uuid2 = uuidv4();
+    expect(
+      ValidateKey.sanitize(`public/${uuid1}/${uuid2}/~\.\u0000\malicious_key`)
+    ).toEqual(`public/${uuid1}/${uuid2}/~.malicious_key`);
+  });
+});

server/validation.ts

@@ -1,6 +1,8 @@
 import isArrayLike from "lodash/isArrayLike";
+import sanitize from "sanitize-filename";
 import { Primitive } from "utility-types";
 import validator from "validator";
+import isIn from "validator/lib/isIn";
 import isUUID from "validator/lib/isUUID";
 import parseMentionUrl from "@shared/utils/parseMentionUrl";
 import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
@@ -170,6 +172,30 @@ export const assertCollectionPermission = (
   assertIn(value, [...Object.values(CollectionPermission), null], message);
 };
 
+export class ValidateKey {
+  public static isValid = (key: string) => {
+    const parts = key.split("/").slice(0, -1);
+    return (
+      parts.length === 3 &&
+      isIn(parts[0], ["uploads", "public"]) &&
+      isUUID(parts[1]) &&
+      isUUID(parts[2])
+    );
+  };
+
+  public static sanitize = (key: string) => {
+    const [filename] = key.split("/").slice(-1);
+    return key
+      .split("/")
+      .slice(0, -1)
+      .join("/")
+      .concat(`/${sanitize(filename)}`);
+  };
+
+  public static message =
+    "Must be of the form uploads/<uuid>/<uuid>/<name> or public/<uuid>/<uuid>/<name>";
+}
+
 export class ValidateDocumentId {
   /**
    * Checks if documentId is valid. A valid documentId is either