Local file storage (#5763)

Co-authored-by: Tom Moor <tom.moor@gmail.com>

plugins/oidc/server/auth/oidc.test.ts

@@ -7,7 +7,8 @@ describe("oidc", () => {
     const res = await server.get("/auth/oidc?myParam=someParam", {
       redirect: "manual",
     });
-    const redirectLocation = new URL(res.headers.get("location"));
+    expect(res.headers.get("location")).not.toBeNull();
+    const redirectLocation = new URL(res.headers.get("location")!);
     expect(res.status).toEqual(302);
     expect(redirectLocation.searchParams.get("myParam")).toEqual("someParam");
   });

plugins/storage/plugin.json

@@ -0,0 +1,8 @@
+{
+  "name": "Storage",
+  "description": "Plugin for storing files on the local file system",
+  "requiredEnvVars": [
+    "FILE_STORAGE_UPLOAD_MAX_SIZE",
+    "FILE_STORAGE_LOCAL_ROOT_DIR"
+  ]
+}

plugins/storage/server/.babelrc

@@ -0,0 +1,3 @@
+{
+  "extends": "../../../server/.babelrc"
+}

plugins/storage/server/api/files.test.ts

@@ -0,0 +1,150 @@
+import { existsSync } from "fs";
+import { readFile } from "fs/promises";
+import path from "path";
+import FormData from "form-data";
+import env from "@server/env";
+import "@server/test/env";
+import { buildAttachment, buildUser } from "@server/test/factories";
+import { getTestServer } from "@server/test/support";
+
+const server = getTestServer();
+
+describe("#files.create", () => {
+  it("should fail with status 400 bad request if key is invalid", async () => {
+    const user = await buildUser();
+    const res = await server.post("/api/files.create", {
+      body: {
+        token: user.getJwtToken(),
+        key: "public/foo/bar/baz.png",
+      },
+    });
+    const body = await res.json();
+    expect(res.status).toEqual(400);
+    expect(body.message).toEqual(
+      "key: Must be of the form uploads/<uuid>/<uuid>/<name> or public/<uuid>/<uuid>/<name>"
+    );
+  });
+
+  it("should succeed with status 200 ok and create a file", async () => {
+    const user = await buildUser();
+    const fileName = "images.docx";
+    const attachment = await buildAttachment(
+      {
+        teamId: user.teamId,
+        userId: user.id,
+        contentType:
+          "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      },
+      fileName
+    );
+
+    const content = await readFile(
+      path.resolve(__dirname, "..", "test", "fixtures", fileName)
+    );
+    const form = new FormData();
+    form.append("key", attachment.key);
+    form.append("file", content, fileName);
+    form.append("token", user.getJwtToken());
+
+    const res = await server.post(`/api/files.create`, {
+      headers: form.getHeaders(),
+      body: form,
+    });
+
+    const body = await res.json();
+    expect(res.status).toEqual(200);
+    expect(body.success).toEqual(true);
+    expect(
+      existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, attachment.key))
+    ).toBe(true);
+  });
+});
+
+describe("#files.get", () => {
+  it("should fail with status 400 bad request if key is invalid", async () => {
+    const res = await server.get(`/api/files.get?key=public/foo/bar/baz.png`);
+    const body = await res.json();
+    expect(res.status).toEqual(400);
+    expect(body.message).toEqual(
+      "key: Must be of the form uploads/<uuid>/<uuid>/<name> or public/<uuid>/<uuid>/<name>"
+    );
+  });
+
+  it("should fail with status 400 bad request if none of key or sig is supplied", async () => {
+    const res = await server.get("/api/files.get");
+    const body = await res.json();
+    expect(res.status).toEqual(400);
+    expect(body.message).toEqual("query: One of key or sig is required");
+  });
+
+  it("should succeed with status 200 ok when file is requested using key", async () => {
+    const user = await buildUser();
+    const fileName = "images.docx";
+
+    const attachment = await buildAttachment(
+      {
+        teamId: user.teamId,
+        userId: user.id,
+        contentType:
+          "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      },
+      fileName
+    );
+
+    const content = await readFile(
+      path.resolve(__dirname, "..", "test", "fixtures", fileName)
+    );
+    const form = new FormData();
+    form.append("key", attachment.key);
+    form.append("file", content, fileName);
+    form.append("token", user.getJwtToken());
+
+    await server.post(`/api/files.create`, {
+      headers: form.getHeaders(),
+      body: form,
+    });
+
+    const res = await server.get(attachment.canonicalUrl);
+    expect(res.status).toEqual(200);
+    expect(res.headers.get("Content-Type")).toEqual(attachment.contentType);
+    expect(res.headers.get("Content-Disposition")).toEqual(
+      'attachment; filename="images.docx"'
+    );
+  });
+
+  it("should succeed with status 200 ok when private file is requested using signature", async () => {
+    const user = await buildUser();
+    const fileName = "images.docx";
+
+    const attachment = await buildAttachment(
+      {
+        teamId: user.teamId,
+        userId: user.id,
+        contentType:
+          "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        acl: "private",
+      },
+      fileName
+    );
+
+    const content = await readFile(
+      path.resolve(__dirname, "..", "test", "fixtures", fileName)
+    );
+    const form = new FormData();
+    form.append("key", attachment.key);
+    form.append("file", content, fileName);
+    form.append("token", user.getJwtToken());
+
+    await server.post(`/api/files.create`, {
+      headers: form.getHeaders(),
+      body: form,
+    });
+
+    const res = await server.get(await attachment.signedUrl);
+    expect(res.status).toEqual(200);
+    expect(res.headers.get("Content-Type")).toEqual(attachment.contentType);
+    expect(res.headers.get("Content-Disposition")).toEqual(
+      'attachment; filename="images.docx"'
+    );
+  });
+});

plugins/storage/server/api/files.ts

@@ -0,0 +1,71 @@
+import Router from "koa-router";
+import env from "@server/env";
+import { ValidationError } from "@server/errors";
+import auth from "@server/middlewares/authentication";
+import multipart from "@server/middlewares/multipart";
+import { rateLimiter } from "@server/middlewares/rateLimiter";
+import validate from "@server/middlewares/validate";
+import { Attachment } from "@server/models";
+import { authorize } from "@server/policies";
+import { APIContext } from "@server/types";
+import { RateLimiterStrategy } from "@server/utils/RateLimiter";
+import { createRootDirForLocalStorage } from "../utils";
+import * as T from "./schema";
+
+createRootDirForLocalStorage();
+
+const router = new Router();
+
+router.post(
+  "files.create",
+  rateLimiter(RateLimiterStrategy.TenPerMinute),
+  auth(),
+  validate(T.FilesCreateSchema),
+  multipart({ maximumFileSize: env.FILE_STORAGE_UPLOAD_MAX_SIZE }),
+  async (ctx: APIContext<T.FilesCreateReq>) => {
+    const actor = ctx.state.auth.user;
+    const { key } = ctx.input.body;
+    const file = ctx.input.file;
+
+    const attachment = await Attachment.findByKey(key);
+
+    if (attachment.isPrivate) {
+      authorize(actor, "createAttachment", actor.team);
+    }
+
+    await attachment.overwriteFile(file);
+
+    ctx.body = {
+      success: true,
+    };
+  }
+);
+
+router.get(
+  "files.get",
+  auth({ optional: true }),
+  validate(T.FilesGetSchema),
+  async (ctx: APIContext<T.FilesGetReq>) => {
+    const { key, sig } = ctx.input.query;
+    const actor = ctx.state.auth.user;
+    let attachment: Attachment | null;
+
+    if (key) {
+      attachment = await Attachment.findByKey(key);
+
+      if (attachment.isPrivate) {
+        authorize(actor, "read", attachment);
+      }
+    } else if (sig) {
+      attachment = await Attachment.findBySignature(sig);
+    } else {
+      throw ValidationError("Must provide either key or signature");
+    }
+
+    ctx.set("Content-Type", attachment.contentType);
+    ctx.attachment(attachment.name);
+    ctx.body = attachment.stream;
+  }
+);
+
+export default router;

plugins/storage/server/api/schema.ts

@@ -0,0 +1,33 @@
+import formidable from "formidable";
+import isEmpty from "lodash/isEmpty";
+import { z } from "zod";
+import { ValidateKey } from "@server/validation";
+
+export const FilesCreateSchema = z.object({
+  body: z.object({
+    key: z
+      .string()
+      .refine(ValidateKey.isValid, { message: ValidateKey.message })
+      .transform(ValidateKey.sanitize),
+  }),
+  file: z.custom<formidable.File>(),
+});
+
+export type FilesCreateReq = z.infer<typeof FilesCreateSchema>;
+
+export const FilesGetSchema = z.object({
+  query: z
+    .object({
+      key: z
+        .string()
+        .refine(ValidateKey.isValid, { message: ValidateKey.message })
+        .optional()
+        .transform((val) => (val ? ValidateKey.sanitize(val) : undefined)),
+      sig: z.string().optional(),
+    })
+    .refine((obj) => !(isEmpty(obj.key) && isEmpty(obj.sig)), {
+      message: "One of key or sig is required",
+    }),
+});
+
+export type FilesGetReq = z.infer<typeof FilesGetSchema>;

plugins/storage/server/utils.ts

@@ -0,0 +1,20 @@
+import { existsSync, mkdirSync } from "fs";
+import env from "@server/env";
+import Logger from "@server/logging/Logger";
+
+export const createRootDirForLocalStorage = () => {
+  if (env.FILE_STORAGE === "local") {
+    const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR;
+    try {
+      if (!existsSync(rootDir)) {
+        mkdirSync(rootDir, { recursive: true });
+        Logger.debug("utils", `Created ${rootDir} for local storage`);
+      }
+    } catch (err) {
+      Logger.fatal(
+        "Couldn't create root dir for local storage of attachments",
+        err
+      );
+    }
+  }
+};