Local file storage (#5763)

Co-authored-by: Tom Moor <tom.moor@gmail.com>

server/storage/files/BaseStorage.ts

@@ -1,3 +1,4 @@
+import { Blob } from "buffer";
 import { Readable } from "stream";
 import { PresignedPost } from "aws-sdk/clients/s3";
 import env from "@server/env";
@@ -5,6 +6,9 @@ import Logger from "@server/logging/Logger";
 import fetch from "@server/utils/fetch";
 
 export default abstract class BaseStorage {
+  /** The default number of seconds until a signed URL expires. */
+  public static defaultSignedUrlExpires = 60;
+
   /**
    * Returns a presigned post for uploading files to the storage provider.
    *
@@ -19,7 +23,7 @@ export default abstract class BaseStorage {
     acl: string,
     maxUploadSize: number,
     contentType: string
-  ): Promise<PresignedPost>;
+  ): Promise<Partial<PresignedPost>>;
 
   /**
    * Returns a stream for reading a file from the storage provider.
@@ -29,19 +33,20 @@ export default abstract class BaseStorage {
   public abstract getFileStream(key: string): NodeJS.ReadableStream | null;
 
   /**
-   * Returns a buffer of a file from the storage provider.
-   *
-   * @param key The path to the file
-   */
-  public abstract getFileBuffer(key: string): Promise<Blob>;
-
-  /**
-   * Returns the public endpoint for the storage provider.
+   * Returns the upload URL for the storage provider.
    *
    * @param isServerUpload Whether the upload is happening on the server or not
-   * @returns The public endpoint as a string
+   * @returns {string} The upload URL
    */
-  public abstract getPublicEndpoint(isServerUpload?: boolean): string;
+  public abstract getUploadUrl(isServerUpload?: boolean): string;
+
+  /**
+   * Returns the download URL for a given file.
+   *
+   * @param key The path to the file
+   * @returns {string} The download URL for the file
+   */
+  public abstract getUrlForKey(key: string): string;
 
   /**
    * Returns a signed URL for a file from the storage provider.
@@ -55,7 +60,7 @@ export default abstract class BaseStorage {
   ): Promise<string>;
 
   /**
-   * Upload a file to the storage provider.
+   * Store a file in the storage provider.
    *
    * @param body The file body
    * @param contentLength The content length of the file
@@ -64,7 +69,7 @@ export default abstract class BaseStorage {
    * @param acl The ACL to use
    * @returns The URL of the file
    */
-  public abstract upload({
+  public abstract store({
     body,
     contentLength,
     contentType,
@@ -72,12 +77,35 @@ export default abstract class BaseStorage {
     acl,
   }: {
     body: Buffer | Uint8Array | Blob | string | Readable;
-    contentLength: number;
-    contentType: string;
+    contentLength?: number;
+    contentType?: string;
     key: string;
-    acl: string;
+    acl?: string;
   }): Promise<string | undefined>;
 
+  /**
+   * Returns a buffer of a file from the storage provider.
+   *
+   * @param key The path to the file
+   */
+  public async getFileBuffer(key: string) {
+    const stream = this.getFileStream(key);
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      if (!stream) {
+        return reject(new Error("No stream available"));
+      }
+
+      stream.on("data", (d) => {
+        chunks.push(d);
+      });
+      stream.once("end", () => {
+        resolve(Buffer.concat(chunks));
+      });
+      stream.once("error", reject);
+    });
+  }
+
   /**
    * Upload a file to the storage provider directly from a remote or base64 encoded URL.
    *
@@ -86,7 +114,7 @@ export default abstract class BaseStorage {
    * @param acl The ACL to use
    * @returns A promise that resolves when the file is uploaded
    */
-  public async uploadFromUrl(
+  public async storeFromUrl(
     url: string,
     key: string,
     acl: string
@@ -98,7 +126,7 @@ export default abstract class BaseStorage {
       }
     | undefined
   > {
-    const endpoint = this.getPublicEndpoint(true);
+    const endpoint = this.getUploadUrl(true);
     if (url.startsWith("/api") || url.startsWith(endpoint)) {
       return;
     }
@@ -115,7 +143,7 @@ export default abstract class BaseStorage {
         const res = await fetch(url, {
           follow: 3,
           redirect: "follow",
-          size: env.AWS_S3_UPLOAD_MAX_SIZE,
+          size: env.FILE_STORAGE_UPLOAD_MAX_SIZE,
           timeout: 10000,
         });
 
@@ -143,7 +171,7 @@ export default abstract class BaseStorage {
     }
 
     try {
-      const result = await this.upload({
+      const result = await this.store({
         body: buffer,
         contentLength,
         contentType,

server/storage/files/LocalStorage.ts

@@ -0,0 +1,120 @@
+import { Blob } from "buffer";
+import {
+  ReadStream,
+  closeSync,
+  createReadStream,
+  createWriteStream,
+  existsSync,
+  openSync,
+} from "fs";
+import { mkdir, unlink } from "fs/promises";
+import path from "path";
+import { Readable } from "stream";
+import invariant from "invariant";
+import JWT from "jsonwebtoken";
+import env from "@server/env";
+import Logger from "@server/logging/Logger";
+import BaseStorage from "./BaseStorage";
+
+export default class LocalStorage extends BaseStorage {
+  public async getPresignedPost(
+    key: string,
+    acl: string,
+    maxUploadSize: number,
+    contentType = "image"
+  ) {
+    return Promise.resolve({
+      url: this.getUrlForKey(key),
+      fields: {
+        key,
+        acl,
+        maxUploadSize,
+        contentType,
+      },
+    } as any);
+  }
+
+  public getUploadUrl() {
+    return "/api/files.create";
+  }
+
+  public getUrlForKey(key: string): string {
+    return `/api/files.get?key=${key}`;
+  }
+
+  public store = async ({
+    body,
+    key,
+  }: {
+    body: string | ReadStream | Buffer | Uint8Array | Blob;
+    contentLength?: number;
+    contentType?: string;
+    key: string;
+    acl?: string;
+  }) => {
+    const subdir = key.split("/").slice(0, -1).join("/");
+    if (!existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir))) {
+      await mkdir(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir), {
+        recursive: true,
+      });
+    }
+
+    let src: NodeJS.ReadableStream;
+    if (body instanceof ReadStream) {
+      src = body;
+    } else if (body instanceof Blob) {
+      src = Readable.from(Buffer.from(await body.arrayBuffer()));
+    } else {
+      src = Readable.from(body);
+    }
+
+    const destPath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key);
+    closeSync(openSync(destPath, "w"));
+    const dest = createWriteStream(destPath);
+    src.pipe(dest);
+
+    return new Promise<string>((resolve, reject) => {
+      src.once("end", () => resolve(this.getUrlForKey(key)));
+      src.once("err", (err) => {
+        dest.end();
+        reject(err);
+      });
+    });
+  };
+
+  public async deleteFile(key: string) {
+    const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key);
+    try {
+      await unlink(filePath);
+    } catch (err) {
+      Logger.warn(`Couldn't delete ${filePath}`, err);
+    }
+  }
+
+  public getSignedUrl = async (
+    key: string,
+    expiresIn = LocalStorage.defaultSignedUrlExpires
+  ) => {
+    const sig = JWT.sign(
+      {
+        key,
+        type: "attachment",
+      },
+      env.SECRET_KEY,
+      {
+        expiresIn,
+      }
+    );
+    return Promise.resolve(`/api/files.get?sig=${sig}`);
+  };
+
+  public getFileStream(key: string) {
+    invariant(
+      env.FILE_STORAGE_LOCAL_ROOT_DIR,
+      "FILE_STORAGE_LOCAL_ROOT_DIR is required"
+    );
+
+    const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key);
+    return createReadStream(filePath);
+  }
+}

server/storage/files/S3Storage.ts

@@ -47,7 +47,7 @@ export default class S3Storage extends BaseStorage {
     );
   }
 
-  public getPublicEndpoint(isServerUpload?: boolean) {
+  private getPublicEndpoint(isServerUpload?: boolean) {
     if (env.AWS_S3_ACCELERATE_URL) {
       return env.AWS_S3_ACCELERATE_URL;
     }
@@ -78,7 +78,15 @@ export default class S3Storage extends BaseStorage {
     }`;
   }
 
-  public upload = async ({
+  public getUploadUrl(isServerUpload?: boolean) {
+    return this.getPublicEndpoint(isServerUpload);
+  }
+
+  public getUrlForKey(key: string): string {
+    return `${this.getPublicEndpoint()}/${key}`;
+  }
+
+  public store = async ({
     body,
     contentLength,
     contentType,
@@ -86,10 +94,10 @@ export default class S3Storage extends BaseStorage {
     acl,
   }: {
     body: S3.Body;
-    contentLength: number;
-    contentType: string;
+    contentLength?: number;
+    contentType?: string;
     key: string;
-    acl: string;
+    acl?: string;
   }) => {
     invariant(
       env.AWS_S3_UPLOAD_BUCKET_NAME,
@@ -125,7 +133,10 @@ export default class S3Storage extends BaseStorage {
       .promise();
   }
 
-  public getSignedUrl = async (key: string, expiresIn = 60) => {
+  public getSignedUrl = async (
+    key: string,
+    expiresIn = S3Storage.defaultSignedUrlExpires
+  ) => {
     const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
     const params = {
       Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
@@ -170,26 +181,6 @@ export default class S3Storage extends BaseStorage {
     return null;
   }
 
-  public async getFileBuffer(key: string) {
-    invariant(
-      env.AWS_S3_UPLOAD_BUCKET_NAME,
-      "AWS_S3_UPLOAD_BUCKET_NAME is required"
-    );
-
-    const response = await this.client
-      .getObject({
-        Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
-        Key: key,
-      })
-      .promise();
-
-    if (response.Body) {
-      return response.Body as Blob;
-    }
-
-    throw new Error("Error getting file buffer from S3");
-  }
-
   private client: AWS.S3;
 
   private getEndpoint() {

server/storage/files/__mocks__/index.ts

@@ -1,7 +1,9 @@
 export default {
   upload: jest.fn().mockReturnValue("/endpoint/key"),
 
-  getPublicEndpoint: jest.fn().mockReturnValue("http://mock"),
+  getUploadUrl: jest.fn().mockReturnValue("http://mock/create"),
+
+  getUrlForKey: jest.fn().mockReturnValue("http://mock/get"),
 
   getSignedUrl: jest.fn().mockReturnValue("http://s3mock"),
 

server/storage/files/index.ts

@@ -1,3 +1,8 @@
+import env from "@server/env";
+import LocalStorage from "./LocalStorage";
 import S3Storage from "./S3Storage";
 
-export default new S3Storage();
+const storage =
+  env.FILE_STORAGE === "local" ? new LocalStorage() : new S3Storage();
+
+export default storage;