From 67b1fe55146efb5b4ee6ffa89dd1fbd60b3709dc Mon Sep 17 00:00:00 2001 From: Apoorv Mishra Date: Thu, 21 Sep 2023 03:42:03 +0530 Subject: [PATCH] Local file storage (#5763) Co-authored-by: Tom Moor --- .circleci/config.yml | 7 +- .env.sample | 12 +- app.json | 10 +- package.json | 7 +- plugins/oidc/server/auth/oidc.test.ts | 3 +- plugins/storage/plugin.json | 8 + plugins/storage/server/.babelrc | 3 + plugins/storage/server/api/files.test.ts | 150 ++++++++++++++++++ plugins/storage/server/api/files.ts | 71 +++++++++ plugins/storage/server/api/schema.ts | 33 ++++ .../storage/server/test/fixtures/images.docx | Bin 0 -> 9258 bytes plugins/storage/server/utils.ts | 20 +++ server/commands/attachmentCreator.ts | 4 +- server/env.ts | 32 +++- .../20230920032853-add-key-index.js | 11 ++ server/models/Attachment.test.ts | 10 ++ server/models/Attachment.ts | 73 ++++++++- server/models/Team.ts | 4 +- server/models/User.ts | 4 +- server/models/helpers/AttachmentHelper.ts | 2 +- .../models/validators/IsUrlOrRelativePath.ts | 23 +++ server/policies/attachment.ts | 2 +- server/queues/tasks/ExportTask.ts | 2 +- server/queues/tasks/UploadTeamAvatarTask.ts | 4 +- server/queues/tasks/UploadUserAvatarTask.ts | 4 +- server/routes/api/attachments/attachments.ts | 2 +- server/routes/auth/index.test.ts | 6 +- server/storage/files/BaseStorage.ts | 68 +++++--- server/storage/files/LocalStorage.ts | 120 ++++++++++++++ server/storage/files/S3Storage.ts | 43 ++--- server/storage/files/__mocks__/index.ts | 4 +- server/storage/files/index.ts | 7 +- server/test/TestServer.ts | 84 ++++++++++ server/test/env.ts | 2 + server/test/factories.ts | 13 +- server/test/support.ts | 10 +- server/typings/index.d.ts | 2 - server/utils/jwt.ts | 2 +- server/validation.test.ts | 41 +++++ server/validation.ts | 26 +++ yarn.lock | 103 ++++++------ 41 files changed, 893 insertions(+), 139 deletions(-) create mode 100644 plugins/storage/plugin.json create mode 100644 plugins/storage/server/.babelrc create mode 100644 plugins/storage/server/api/files.test.ts create mode 100644 plugins/storage/server/api/files.ts create mode 100644 plugins/storage/server/api/schema.ts create mode 100644 plugins/storage/server/test/fixtures/images.docx create mode 100644 plugins/storage/server/utils.ts create mode 100644 server/migrations/20230920032853-add-key-index.js create mode 100644 server/models/Attachment.test.ts create mode 100644 server/models/validators/IsUrlOrRelativePath.ts create mode 100644 server/storage/files/LocalStorage.ts create mode 100644 server/test/TestServer.ts create mode 100644 server/validation.test.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fb8ee280..a95687b8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,7 +145,12 @@ jobs: command: docker push $BASE_IMAGE_NAME:latest - run: name: Build and push Docker image - command: docker buildx build -t $IMAGE_NAME:latest -t $IMAGE_NAME:${CIRCLE_TAG/v/''} --platform linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x --push . + command: | + if [ "$CIRCLE_BRANCH" == "main" ]; then + docker buildx build -t $IMAGE_NAME:latest -t $IMAGE_NAME:${CIRCLE_TAG/v/''} --platform linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x --push . + else + docker buildx build -t $IMAGE_NAME:${CIRCLE_TAG/v/''} --platform linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x --push . + fi workflows: version: 2 diff --git a/.env.sample b/.env.sample index 9b199b806..3e8cadbe4 100644 --- a/.env.sample +++ b/.env.sample @@ -51,10 +51,20 @@ AWS_REGION=xx-xxxx-x AWS_S3_ACCELERATE_URL= AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569 AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here -AWS_S3_UPLOAD_MAX_SIZE=26214400 AWS_S3_FORCE_PATH_STYLE=true AWS_S3_ACL=private +# Specify what storage system to use. Possible value is one of "s3" or "local". +# For "local", the avatar images and document attachments will be saved on local disk. +FILE_STORAGE=local + +# If "local" is configured for FILE_STORAGE above, then this sets the parent directory under +# which all attachments/images go. Make sure that the process has permissions to create +# this path and also to write files to it. +FILE_STORAGE_LOCAL_ROOT_DIR=/var/lib/outline/data + +# Maximum allowed size for the uploaded attachment. +FILE_STORAGE_UPLOAD_MAX_SIZE=26214400 # –––––––––––––– AUTHENTICATION –––––––––––––– diff --git a/app.json b/app.json index f5eece7b3..9169adf16 100644 --- a/app.json +++ b/app.json @@ -128,11 +128,6 @@ "description": "Live web link to your bucket. For CNAMEs, https://yourbucket.example.com", "required": false }, - "AWS_S3_UPLOAD_MAX_SIZE": { - "description": "Maximum file upload size in bytes", - "value": "26214400", - "required": false - }, "AWS_S3_FORCE_PATH_STYLE": { "description": "Use path-style URL's for connecting to S3 instead of subdomain. This is useful for S3-compatible storage.", "value": "true", @@ -148,6 +143,11 @@ "description": "S3 canned ACL for document attachments", "required": false }, + "FILE_STORAGE_UPLOAD_MAX_SIZE": { + "description": "Maximum file upload size in bytes", + "value": "26214400", + "required": false + }, "SMTP_HOST": { "description": "smtp.example.com (optional)", "required": false diff --git a/package.json b/package.json index ca33fc4e4..8bac49f74 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,8 @@ "@sentry/tracing": "^7.51.2", "@tippyjs/react": "^4.2.6", "@tommoor/remove-markdown": "^0.3.2", + "@types/form-data": "^2.5.0", + "@types/sanitize-filename": "^1.6.3", "@vitejs/plugin-react": "^3.1.0", "addressparser": "^1.0.1", "autotrack": "^2.4.1", @@ -103,6 +105,7 @@ "fetch-retry": "^5.0.5", "fetch-with-proxy": "^3.0.1", "focus-visible": "^5.2.0", + "form-data": "^4.0.0", "fractional-index": "^1.0.0", "framer-motion": "^4.1.17", "fs-extra": "^11.1.1", @@ -193,6 +196,7 @@ "reflect-metadata": "^0.1.13", "refractor": "^3.6.0", "request-filtering-agent": "^1.1.2", + "sanitize-filename": "^1.6.3", "semver": "^7.5.2", "sequelize": "^6.32.1", "sequelize-cli": "^6.6.1", @@ -315,7 +319,6 @@ "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react": "^7.20.0", "eslint-plugin-react-hooks": "^4.6.0", - "fetch-test-server": "^1.1.0", "husky": "^8.0.2", "i18next-parser": "^7.9.0", "jest-cli": "^29.6.4", @@ -343,5 +346,5 @@ "qs": "6.9.7", "rollup": "^3.14.0" }, - "version": "0.71.0" + "version": "0.72.0-1" } diff --git a/plugins/oidc/server/auth/oidc.test.ts b/plugins/oidc/server/auth/oidc.test.ts index 0e0013dc1..72e37f374 100644 --- a/plugins/oidc/server/auth/oidc.test.ts +++ b/plugins/oidc/server/auth/oidc.test.ts @@ -7,7 +7,8 @@ describe("oidc", () => { const res = await server.get("/auth/oidc?myParam=someParam", { redirect: "manual", }); - const redirectLocation = new URL(res.headers.get("location")); + expect(res.headers.get("location")).not.toBeNull(); + const redirectLocation = new URL(res.headers.get("location")!); expect(res.status).toEqual(302); expect(redirectLocation.searchParams.get("myParam")).toEqual("someParam"); }); diff --git a/plugins/storage/plugin.json b/plugins/storage/plugin.json new file mode 100644 index 000000000..b7274ca29 --- /dev/null +++ b/plugins/storage/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "Storage", + "description": "Plugin for storing files on the local file system", + "requiredEnvVars": [ + "FILE_STORAGE_UPLOAD_MAX_SIZE", + "FILE_STORAGE_LOCAL_ROOT_DIR" + ] +} diff --git a/plugins/storage/server/.babelrc b/plugins/storage/server/.babelrc new file mode 100644 index 000000000..2bc0ef3ed --- /dev/null +++ b/plugins/storage/server/.babelrc @@ -0,0 +1,3 @@ +{ + "extends": "../../../server/.babelrc" +} diff --git a/plugins/storage/server/api/files.test.ts b/plugins/storage/server/api/files.test.ts new file mode 100644 index 000000000..6683061d0 --- /dev/null +++ b/plugins/storage/server/api/files.test.ts @@ -0,0 +1,150 @@ +import { existsSync } from "fs"; +import { readFile } from "fs/promises"; +import path from "path"; +import FormData from "form-data"; +import env from "@server/env"; +import "@server/test/env"; +import { buildAttachment, buildUser } from "@server/test/factories"; +import { getTestServer } from "@server/test/support"; + +const server = getTestServer(); + +describe("#files.create", () => { + it("should fail with status 400 bad request if key is invalid", async () => { + const user = await buildUser(); + const res = await server.post("/api/files.create", { + body: { + token: user.getJwtToken(), + key: "public/foo/bar/baz.png", + }, + }); + const body = await res.json(); + expect(res.status).toEqual(400); + expect(body.message).toEqual( + "key: Must be of the form uploads/// or public///" + ); + }); + + it("should succeed with status 200 ok and create a file", async () => { + const user = await buildUser(); + const fileName = "images.docx"; + const attachment = await buildAttachment( + { + teamId: user.teamId, + userId: user.id, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }, + fileName + ); + + const content = await readFile( + path.resolve(__dirname, "..", "test", "fixtures", fileName) + ); + const form = new FormData(); + form.append("key", attachment.key); + form.append("file", content, fileName); + form.append("token", user.getJwtToken()); + + const res = await server.post(`/api/files.create`, { + headers: form.getHeaders(), + body: form, + }); + + const body = await res.json(); + expect(res.status).toEqual(200); + expect(body.success).toEqual(true); + expect( + existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, attachment.key)) + ).toBe(true); + }); +}); + +describe("#files.get", () => { + it("should fail with status 400 bad request if key is invalid", async () => { + const res = await server.get(`/api/files.get?key=public/foo/bar/baz.png`); + const body = await res.json(); + expect(res.status).toEqual(400); + expect(body.message).toEqual( + "key: Must be of the form uploads/// or public///" + ); + }); + + it("should fail with status 400 bad request if none of key or sig is supplied", async () => { + const res = await server.get("/api/files.get"); + const body = await res.json(); + expect(res.status).toEqual(400); + expect(body.message).toEqual("query: One of key or sig is required"); + }); + + it("should succeed with status 200 ok when file is requested using key", async () => { + const user = await buildUser(); + const fileName = "images.docx"; + + const attachment = await buildAttachment( + { + teamId: user.teamId, + userId: user.id, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }, + fileName + ); + + const content = await readFile( + path.resolve(__dirname, "..", "test", "fixtures", fileName) + ); + const form = new FormData(); + form.append("key", attachment.key); + form.append("file", content, fileName); + form.append("token", user.getJwtToken()); + + await server.post(`/api/files.create`, { + headers: form.getHeaders(), + body: form, + }); + + const res = await server.get(attachment.canonicalUrl); + expect(res.status).toEqual(200); + expect(res.headers.get("Content-Type")).toEqual(attachment.contentType); + expect(res.headers.get("Content-Disposition")).toEqual( + 'attachment; filename="images.docx"' + ); + }); + + it("should succeed with status 200 ok when private file is requested using signature", async () => { + const user = await buildUser(); + const fileName = "images.docx"; + + const attachment = await buildAttachment( + { + teamId: user.teamId, + userId: user.id, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + acl: "private", + }, + fileName + ); + + const content = await readFile( + path.resolve(__dirname, "..", "test", "fixtures", fileName) + ); + const form = new FormData(); + form.append("key", attachment.key); + form.append("file", content, fileName); + form.append("token", user.getJwtToken()); + + await server.post(`/api/files.create`, { + headers: form.getHeaders(), + body: form, + }); + + const res = await server.get(await attachment.signedUrl); + expect(res.status).toEqual(200); + expect(res.headers.get("Content-Type")).toEqual(attachment.contentType); + expect(res.headers.get("Content-Disposition")).toEqual( + 'attachment; filename="images.docx"' + ); + }); +}); diff --git a/plugins/storage/server/api/files.ts b/plugins/storage/server/api/files.ts new file mode 100644 index 000000000..ef39a4ff1 --- /dev/null +++ b/plugins/storage/server/api/files.ts @@ -0,0 +1,71 @@ +import Router from "koa-router"; +import env from "@server/env"; +import { ValidationError } from "@server/errors"; +import auth from "@server/middlewares/authentication"; +import multipart from "@server/middlewares/multipart"; +import { rateLimiter } from "@server/middlewares/rateLimiter"; +import validate from "@server/middlewares/validate"; +import { Attachment } from "@server/models"; +import { authorize } from "@server/policies"; +import { APIContext } from "@server/types"; +import { RateLimiterStrategy } from "@server/utils/RateLimiter"; +import { createRootDirForLocalStorage } from "../utils"; +import * as T from "./schema"; + +createRootDirForLocalStorage(); + +const router = new Router(); + +router.post( + "files.create", + rateLimiter(RateLimiterStrategy.TenPerMinute), + auth(), + validate(T.FilesCreateSchema), + multipart({ maximumFileSize: env.FILE_STORAGE_UPLOAD_MAX_SIZE }), + async (ctx: APIContext) => { + const actor = ctx.state.auth.user; + const { key } = ctx.input.body; + const file = ctx.input.file; + + const attachment = await Attachment.findByKey(key); + + if (attachment.isPrivate) { + authorize(actor, "createAttachment", actor.team); + } + + await attachment.overwriteFile(file); + + ctx.body = { + success: true, + }; + } +); + +router.get( + "files.get", + auth({ optional: true }), + validate(T.FilesGetSchema), + async (ctx: APIContext) => { + const { key, sig } = ctx.input.query; + const actor = ctx.state.auth.user; + let attachment: Attachment | null; + + if (key) { + attachment = await Attachment.findByKey(key); + + if (attachment.isPrivate) { + authorize(actor, "read", attachment); + } + } else if (sig) { + attachment = await Attachment.findBySignature(sig); + } else { + throw ValidationError("Must provide either key or signature"); + } + + ctx.set("Content-Type", attachment.contentType); + ctx.attachment(attachment.name); + ctx.body = attachment.stream; + } +); + +export default router; diff --git a/plugins/storage/server/api/schema.ts b/plugins/storage/server/api/schema.ts new file mode 100644 index 000000000..2dea96d68 --- /dev/null +++ b/plugins/storage/server/api/schema.ts @@ -0,0 +1,33 @@ +import formidable from "formidable"; +import isEmpty from "lodash/isEmpty"; +import { z } from "zod"; +import { ValidateKey } from "@server/validation"; + +export const FilesCreateSchema = z.object({ + body: z.object({ + key: z + .string() + .refine(ValidateKey.isValid, { message: ValidateKey.message }) + .transform(ValidateKey.sanitize), + }), + file: z.custom(), +}); + +export type FilesCreateReq = z.infer; + +export const FilesGetSchema = z.object({ + query: z + .object({ + key: z + .string() + .refine(ValidateKey.isValid, { message: ValidateKey.message }) + .optional() + .transform((val) => (val ? ValidateKey.sanitize(val) : undefined)), + sig: z.string().optional(), + }) + .refine((obj) => !(isEmpty(obj.key) && isEmpty(obj.sig)), { + message: "One of key or sig is required", + }), +}); + +export type FilesGetReq = z.infer; diff --git a/plugins/storage/server/test/fixtures/images.docx b/plugins/storage/server/test/fixtures/images.docx new file mode 100644 index 0000000000000000000000000000000000000000..25a9156a8dfb80c627b6a1d6bc2afdcb5684087d GIT binary patch literal 9258 zcma)i1#p}@vbC8hW~P|gj+rrLW{R0T=9rn8nHghdW@d<)ImVdTeopr7-t7JV-FiJ$ zQ}az#OP^XgYU!MolLQ4r1%iTt0wT93kq7$2kl%khS=$-XTYUxS8`+sznb0`{EDPeK zWqKKq0%W`Jh|gC@&@4sLEG zhqL!h!4Kp6MuWk$J`MTPDO^5W)HoCSd{r`0^4+K8u#;>^YyBE`C~?a1=;0%3nnsZr zJHrv@pJPO-I!D)Sa~j5xojXBa4TaV(WP}bEpCfY;fO>{mk$V{9fGH*(!1)6SmK4)B zU^RmTdbe4(?dcjOeS1a1v5Ts|o{KARsNfv9jy9m0M|!;AL4Qf;s1re+dgHm>1o2M@RK?~V1KS}8fN=dhNjXYI(3=Pq zx9uP{@j;Y`_J3cV2thFv_elJG-D(WUuJcfDq#8DT9fyuWDp?2eKa|JIa2?-^R zBIIQX3~T4ppU~1NFD;QJA0?(gNfhs`L~RV#JpnpX5ggxcwf-mZF2{hNG0RT73#8)se25y zbNMRy5;vxLIHSRuV(WxW%%^A1Mh#Tej?(XHU}d8lkeY-e>|4Z;!*Hm&D&h-gaHx&I z*u7Z35_jQhw)aVx+MIN;+aHj~&eCEtmI{~0^arcK1V+@9+IfiOl!oeYzWPIbB0opE z{ANR4?2r^5lC!#yoexNE@F6=Za4|q3EQ-S4CUO3Ge4TFD$A5rBdmmk_7-*WuP(NQb z>1Ru9{39IVHkeXh26}-*h!i9_o#Dl1)w^(&I?K&+861s(&<3zVBeM1%4 z>G}%LW?Wl%AGn#PWl`wU2I+&tKnHZs16J`fCia|AmI3}#iAls2dT+y z;(GXK2e9jlP**p!rK@Sha({Y?+059YasP2I(VGdf|YI~2~dHffLe&}aZuW`lr z>wUe^{#nn(O4U&N@A8+_%Nybbd_Ah>cZk2%P-JayJ~#*v&=JIcTSNcCy0NvDgOZ-U zrP1$26#pmI^P!MJQ-!?{6Lp*{dCVHZlxB~9nNAngHSyci^56sA`q*uEe_qY`%Ft_;lYex^%+rXCu{X=sm{VOr4_gU&V*oDo_v^+ znjl>|tPVkBzfY3{8Qf2AOjJdU`yWeAxdN%c_F@Cjra!+xoLiad6x)8>p-E4j8LGaf z*?=~EYlTf_oIH1>CNH@6M^fs@A&Ry~YJ<4s;U6cW*wvI@^;_yj+RsfzJwQ<)$|0UI z5L;6xZl*B`Le%4tm4S$IA$`zZfmR`AY zra#69{A=RMg8)D-@8H+?ui!`g7x*1qERB9c|9j%8)iMKWV5RPz_hqNF7>JO(nR5LR zYspbzeXF;hL9p^*gC*ohh35ntNsa_4GTt)~!Bg9y=QHQT_i;@rQ41QhElaBidOQn% ztZgE_vgVtEy)Av?hFn!7eHNw%DR9PA&6s)ODnI}+u3yV8 z$fw!nT_y-Xs-4jwp$>|zk24K_9C(hfHDJNEP=%=XK!sRcPE4y0OuXV=p@MHcj6*_y%t2*3F*%jPrjVFesRSKS`BR2GWsRrxhvVeL9oAggdbyqf%g<6* zf;exU5DTf~G>XeZg_&|ywRC$xAw-$+?LVT(u$m%}}6_94&krf6wc?B;|K;z2tl5Hk}EbIeg+BrleMaz}X z6h4!N8BdRR;P)b{ z%l6He9t8t?#Yk=Pj0=#oE9=B9u8Ai%&84Q>%=5bTq(Mh_RQqHw6I`{*B-&x7 zPeCzF1ivj;v#HvmV&V;%oA9n+j#|mPvKZG;ECX7oe3Q zIzHuoc>MXgFWr-V@U=UkL@yJ!o75V^YxVr9CpaT;`IBI=c11sTNn4{!DyO{9{@J`* zvA8<++-h9#ijw_vLoHj6>1g|^80iaRh74I{9Kyg@9NxSOq-|aS|BloTKS`?*1(?Is zbx`^c9PLEy5kMcRfILE>V?tep=Gze>Q_^XeT{jJca{cV~?w;Ha zo465ld3O9`WEiLn{BZb5`8N2qlfw4uhzvfsQQm=zji{~E8n7%fhTj4dlSq)2vvwgk z@Ux7&D~k_GIukIe?>~UJJm4Ytw?POh;d&ruR|v>`cK6$+|H9@@XkXnPJP{QQGu$|I zBc^aCM4cv$KfNc}PaAH!k|p%yZEPaiGFc(OwdAu@MgW?83k1sv!dA#0IA7x-LvC7~ zNgj6==O830Ckv@68#FIT@gVbH!c5JLGiKbpH^hhDYdTM6po)pDrPtCVEBa6%oGE_K ztrfJ$`XxrZ7gP#a$RYI>W+-FNcLFGE69`8P9WlWnyd&MkCKLV+Ts}&Cym1pGFiMPZ zZzDT-R!3uOqZHNY&4qGi%5f0!D}c^|)IvAF;T4ZZv}Q@0A8Na|`TmEN7NR~ZC?r8L z8N!tzXRF&MSmSMW7_&MmNu<|v(^hEe?&TZZ#n(qGz));cEAfniS&jd9EtE-32$ROd zvoAGU9C#@WvD^8ryP36kBUw%I=y!9ljS44o;}W5AuY|#q(n+(s59zLXn?0tQiQdPz z!?Ru3>CXr}cbWXw@pb0~3VKXAlMImUz%T&>VGFh$2>q$0cYax|0WmQapx^ApQJdmLy$NE2htM9~5-LI^ZUB z?rVu&-d3HLoqe6>Z>O0NjP?_RVvVn;Tn&52tvc+dOA$dNx|sMI7DJO`$(>8#OF>Gc z7a{r^$S-hyNZV!k@(#`bxc>%D+<&G$9XlgSd-^{iO84uZ@;GVp-?x-0uHdDG&{@`a zWR5h2p@t?%xsz&I{TD)M36CcP=maH6jwdqDErg&)R+#GS8i4RwL~x=f$$ zOqx&e9m&|jhn)wrOUwxm^Hm4TAP%;g)`7*<*-4Ghqq}{fIXlc{*~7FPXAA|2!EHmj zEe@`p@yhdg^$*QzK=Q`oEGDD zcBwzSj6%@~Ar32M{M7VRT6$KUJ5sUSY-UL|k+tbg(vBQaJ+q8oF4$QZ9`{VikmM7LvFqm&-pR?vAs( z;|<)~q8Eg=2nf>C^g&-Ncw%1Sw8WdLtm?#1q)eO0lWs&PM`RU$c|C^|f4EHO4bJf( zCfT!Hv7?)(5!R1-Vn$e=KNWZr92Hd9S{w`?%m+vZ*HwJ;y6-k>D-=Zj0Crui)eE3j zqEfE~{OD#%#BdSRcV^ut?tv*JuJZ6f^tSFhaV>Aq7fl{O548oew2t^Z_t7-Erf$Ay z%nhzXpL7;HDny$;u2|7f&1+KJYggDYh5sqJo;G^t$`6=B30+@}%6Y26d}**|fqsSb zgQBDf(h`-Nn`p% zd=bP*P*__lOXK{vZI;&v@MrU)SzA@=`vJHOt*%CM>hTCy@Qw7XHIWxCoV7Cv@bihE z)Ufxj1>X~l(XxC5Kku}IJ77;TF_9YfEwOxT1OGYC3~WrHIa-m}fK!?}OH}Z^+ zr{!&Q+7D9n&Wlw}yVo{bvHkuvtdqba^wsrpEi6bURC z;`-vsn)mH0CR9LSJddW1fbeyO5mpHCm7j2YZvCJgmq z8p>D|@B)!C;`)av5UH-R9XGT9Cm}|aIiNQD&uE+I2j6-FrVEctXx3nN*co6k4DcHL zE;_^=o#;B+Z9$r)q-k1-N{XAu33u5E8RaQF2JWC3qt^=RwijrRpy?jOa#~TpGO)B5 z@&}DbIYfL$N(vDd2zKgc({z#x`h@Z%U85r+sH|(OFRs=fp?aaVHB`(PnH<^Xqim{R z=S18V+FRf&2I+4*cNG#(ma3_*Ab-bMhhV?hxJXh;9FXfqwhKSj9J#vdwuU4cB5#5n z#CiL%8Im$|npO2PsdK&H3`ZaEpk%K{5c1cCY^f?IsQbh^J)?LR%+ydu7QPy-{+6hlxe?TF8T%n2=$XV{kEp^I z7Hncb%;pAKdCMvo1!>0+8AQDbfkdLozbv3W-pP@#K-a!uUxq%xd%!&G>A_B@(=ySr zMxF4hKtz1ESpW-NdnN-pWTryuPHI8(phPW;wcr|IdWVFK%2Bha80uJX)+3Jz1N~1sYKzWd7 z!(}rD*zJbBlGUy_3)_ipWqTD47N?YMv=1TC<|v~KMI-pqIt6{7UXHCk-Sj?S>Y9=j z&E`(qy$aEuJWRz-2RoY&k63DO?-+>mSsfB&6XAs6!MU%;N8b;Kx=^=Zm@CX~?rSX1q#=RiI zgsj*lalEpG#REXC=D~v|QwwadePkSVd^VF|;#nrWndmb^Z@Q}FZ0X32_Y*Iem+H<1 zEW|lSIbNrk9V|mL)t0i%>lk%T1Nr=lo6CH{ipKW#V*AV5QbqT|*+oSIhUDD*8PSsE zRulzq8)OaJmg^_uhfpcqM8pOW&SMZQktA$PD4=gZ#LNT98u_R*YRl& zGIWIX?If1zQfiwfX=379d?`)&$?F#q+b8HI`d!Ac_U`z5H6n?b`WMGdC#WN7+Qwt+ zvn?m6KCa+NCA~Y^D$XOUr1%_4AWlVI+5)f3wei|-kiQZukw&gR%{!&Cg7<$)Hvl6; zGd+4UfS!pFBb|+v$!X#PU;GnBR~DAR@XQ>NLNc)?5gW-$G7U=zdM=H@4;8Hig->uj za2hmA;tTag;zW`Zj0R*q?$l%n1vCZ(shYE6HscHt=Tfh}=jR&)yptT~R~!c%lUI-F z_FuB*YAeW54Zj-#1Ih7cIItrE5%RYg<_q^64D6HeN99!2=c##lGTDdND z#-DH3tB3I}yQj}Eon83!to3MaG+XGHa_pVblOdJll_R56lx}0r+ z31xaXL_!3z^h67`w3k$8t3Xx{#7SMH-G)z6Jr~xczY6NqcbuBt?$zNB8~aDY74OZG zGk!|Ft57AzP5@GD8fi2~kD4!AAQZXBR*&ykHe}<&t^|jp=TiXi@c3~6HQ|7fYxQX} zH|MawU@(*yF=bpBhi*a6`pe-)46hkVqi$X>HW>Qm|!*KjqpRJt@x91MASe`9Yw_UcBY}JSXn2C&=&5uW#BKP!( z7N1U)3jQn6=b^z7853qx{IQGb~?k8AI*8T?V(ND@1uL_rtjrvW*@Q{^ZctM-4+8~UAmK8-wFI9qx%;@ zNe74cdTO6G911ZzMjWQfY>rVoH!ddU?7?;);A1=+Wph1FI+Uc^zZOdGj8`lfspr}W zBe-e-I+ntg=y>?KmZ!&MExSF{{Y~HxZZ*LJL|97O{4@yQ@G(SGi(sp}@2f@RPJ1?MFlg!OBv-bR-YZ8W^?(5h@mkX z?+vLpS1shfAjKTA3+p&*RJv7Z?1&l@<~~uej)%HF zgi2#dZmw2h)K|`fl0AmTG}jB%%(`o9q@8Yrh$1MiF@5|VvO2JqRu%HS>jWEMY{}d^ zIdYkhSD`fK;iZF?$nWRl@3Pw-IBRz-v@da}p#Z$F$8p1hC!^@D5Rl7MIMWk~vNj{< zEorLM&SGIE2w*fp?`N3b9Y@4ep(NAHJ5C8Bz5d8O3@St6Qf-1o=fg^NfEBHs$B5j_ zG_5#IGo*zc5e8j*8al1Z zAxnP|b@W;ka~J!WZomiE)X+mou+ljHe#{33g69oak%Jdyi3U$mHr{U#268!w(`t6p z#8FoKtZHd)+Vf~Q?qHYx9zX#e%30h+{(P=3ftF;f_jF?mNGfi8+tC;}&`L~pym%u#8I8yQB8M(a*K zF*Z{R;_L~2M2^yEAZu9n3`mS`LN%VbN39?g2e5)sCPO@3Fp5^XLA$AsP4w5}a4&o` z@y+>|{l1^81h2@QNMxO$PUVFn2T(t-v-4gvxpg{rz@1Apy&>9kaJ@r%wSe`V%!7lV+C=}3S3 z5gxLq?>w=bYWvN`wkd-j8!qUX6Ak-inw;3jAK@uLgtwwEC1AwwdbvRbgi;UI z>$@L?%KgDSUBs6as?aqeDtOK3BT7Aw$0dA}p#ChrS;-%b!lKKDrV{+1B4xL4R@}TM zFszvxdBLSz+HI2XW7Z?W7IQrI22C{s=l8I7CSt?vVfY0P9 z4bFraw^Ydy_d3A-aB_f*sLyfHurxdr+G&&BZk}?0Ku;e&%AP{_>~X5@tAK(v=%o=L z@x=68`KS@nq|ON6-@SrAp9Uixj!S4~&G~~0tpTQBmgRA3;XCrjtWc?T*C2D{SUka0 z>`URDuRm@B64ZC4taucegHrnA>$H>sm33k|F?oJgPs3d&gN2$qTvR2q%J?csht%W9 z{?{4Kru?A|H=Wzm-N+O~O7&hAGlzZ9W%VQ6>Wv-v3>+cKx~Z#%Q-Tr~CFiDy5l4YDi z|5W7(!%)YjuLu^1?R1u4&lOps^XHG|So&W)RrPLXBhOP#U~>v{c=xE}mYCQg`%`gj z??c)4kufaYsTkbEH`FtfUt_gB(x`36)%y^~nWBhF*Y#1!yPx~6#bWmEtnIPO^44Re z$cEx5sR*7pru?iWnk>!9yQ2Ztjfh6X84^+ie$zw}cL{!RPh z6GR0Ea%$uPP^t>^ORZP*7^zMgj1vk?ls8a?Ig4l^^{ZDkd{-wq*Kgc$?%ga`j90Ip z!tRX8SzvkjQc#!TTIx8=cQZ@~R8qoF$`2-LNY)}Uxkb(p&5#U{`7|)v?)y5}G;>p` zxmO`48T0z-MP=iMKXe{TY8M%;sh6+u^J;0+jtbeZ%H7fu{~;7w%kLzD3l^JUJk$7yaCFV30Vif_viq#2Yq<1gU-U&D1VfW_7 z!bLr)=0}XzUH>ninYCq?52Gu*K31pN^-!NP9bvzr zZl!&u{?@*%fltZj@lEI&Msa&VS2`s+Shi@6E%+KF5nyx{Z&J@)ZiEZQiHQqxSSP=O zbF0z4ylLe71s|;kaEkpgi(csZvKXmdU!ZlA&|{R(4#{g7bo4xb$KEkAe(g=~;&zog2)tl(YN{73iy5-$Jl{3jLs@9+Kw^}o)462E_U z|C9Ut#TWl&>F?a~@4o*7d;Is7{^S;aaixFR5&XX~r+;_+^OpFFdi=}g5dQZS{!T>x zy@@|@`R`2#y+8gQ#osTNe{bT?_4NzDe;F#_{};}3lHl)GTOc5q_fNok?LZXKuc!Y3 DOD}@! literal 0 HcmV?d00001 diff --git a/plugins/storage/server/utils.ts b/plugins/storage/server/utils.ts new file mode 100644 index 000000000..c3c88fba5 --- /dev/null +++ b/plugins/storage/server/utils.ts @@ -0,0 +1,20 @@ +import { existsSync, mkdirSync } from "fs"; +import env from "@server/env"; +import Logger from "@server/logging/Logger"; + +export const createRootDirForLocalStorage = () => { + if (env.FILE_STORAGE === "local") { + const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR; + try { + if (!existsSync(rootDir)) { + mkdirSync(rootDir, { recursive: true }); + Logger.debug("utils", `Created ${rootDir} for local storage`); + } + } catch (err) { + Logger.fatal( + "Couldn't create root dir for local storage of attachments", + err + ); + } + } +}; diff --git a/server/commands/attachmentCreator.ts b/server/commands/attachmentCreator.ts index c035ebc2f..cb0e748c2 100644 --- a/server/commands/attachmentCreator.ts +++ b/server/commands/attachmentCreator.ts @@ -48,7 +48,7 @@ export default async function attachmentCreator({ if ("url" in rest) { const { url } = rest; - const res = await FileStorage.uploadFromUrl(url, key, acl); + const res = await FileStorage.storeFromUrl(url, key, acl); if (!res) { return; @@ -69,7 +69,7 @@ export default async function attachmentCreator({ ); } else { const { buffer, type } = rest; - await FileStorage.upload({ + await FileStorage.store({ body: buffer, contentType: type, contentLength: buffer.length, diff --git a/server/env.ts b/server/env.ts index 35459e035..4708e769e 100644 --- a/server/env.ts +++ b/server/env.ts @@ -545,12 +545,14 @@ export class Environment { this.toOptionalNumber(process.env.RATE_LIMITER_DURATION_WINDOW) ?? 60; /** - * Set max allowed upload size for file attachments. + * @deprecated Set max allowed upload size for file attachments. */ @IsOptional() @IsNumber() - public AWS_S3_UPLOAD_MAX_SIZE = - this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? 100000000; + @Deprecated("Use FILE_STORAGE_UPLOAD_MAX_SIZE instead") + public AWS_S3_UPLOAD_MAX_SIZE = this.toOptionalNumber( + process.env.AWS_S3_UPLOAD_MAX_SIZE + ); /** * Access key ID for AWS S3. @@ -612,6 +614,28 @@ export class Environment { @IsOptional() public AWS_S3_ACL = process.env.AWS_S3_ACL ?? "private"; + /** + * Which file storage system to use + */ + @IsIn(["local", "s3"]) + public FILE_STORAGE = this.toOptionalString(process.env.FILE_STORAGE) ?? "s3"; + + /** + * Set default root dir path for local file storage + */ + public FILE_STORAGE_LOCAL_ROOT_DIR = + this.toOptionalString(process.env.FILE_STORAGE_LOCAL_ROOT_DIR) ?? + "/var/lib/outline/data"; + + /** + * Set max allowed upload size for file attachments. + */ + @IsNumber() + public FILE_STORAGE_UPLOAD_MAX_SIZE = + this.toOptionalNumber(process.env.FILE_STORAGE_UPLOAD_MAX_SIZE) ?? + this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? + 100000000; + /** * Because imports can be much larger than regular file attachments and are * deleted automatically we allow an optional separate limit on the size of @@ -620,7 +644,7 @@ export class Environment { @IsNumber() public MAXIMUM_IMPORT_SIZE = Math.max( this.toOptionalNumber(process.env.MAXIMUM_IMPORT_SIZE) ?? 100000000, - this.AWS_S3_UPLOAD_MAX_SIZE + this.FILE_STORAGE_UPLOAD_MAX_SIZE ); /** diff --git a/server/migrations/20230920032853-add-key-index.js b/server/migrations/20230920032853-add-key-index.js new file mode 100644 index 000000000..50c11a400 --- /dev/null +++ b/server/migrations/20230920032853-add-key-index.js @@ -0,0 +1,11 @@ +"use strict"; + +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addIndex("attachments", ["key"]); + }, + + async down(queryInterface) { + await queryInterface.removeIndex("attachments", ["key"]); + }, +}; diff --git a/server/models/Attachment.test.ts b/server/models/Attachment.test.ts new file mode 100644 index 000000000..b220e705f --- /dev/null +++ b/server/models/Attachment.test.ts @@ -0,0 +1,10 @@ +import { buildAttachment } from "@server/test/factories"; +import Attachment from "./Attachment"; + +describe("#findByKey", () => { + it("should return the correct attachment given a key", async () => { + const attachment = await buildAttachment(); + const found = await Attachment.findByKey(attachment.key); + expect(found?.id).toBe(attachment.id); + }); +}); diff --git a/server/models/Attachment.ts b/server/models/Attachment.ts index 0fe297e13..49f1c2a2b 100644 --- a/server/models/Attachment.ts +++ b/server/models/Attachment.ts @@ -1,4 +1,7 @@ +import { createReadStream } from "fs"; import path from "path"; +import { File } from "formidable"; +import JWT from "jsonwebtoken"; import { QueryTypes } from "sequelize"; import { BeforeDestroy, @@ -10,8 +13,13 @@ import { Table, DataType, IsNumeric, + BeforeUpdate, } from "sequelize-typescript"; +import env from "@server/env"; +import { AuthenticationError } from "@server/errors"; import FileStorage from "@server/storage/files"; +import { getJWTPayload } from "@server/utils/jwt"; +import { ValidateKey } from "@server/validation"; import Document from "./Document"; import Team from "./Team"; import User from "./User"; @@ -96,11 +104,11 @@ class Attachment extends IdModel { } /** - * Get a direct URL to the attachment in storage. Note that this will not work for private attachments, - * a signed URL must be used. + * Get a direct URL to the attachment in storage. Note that this will not work + * for private attachments, a signed URL must be used. */ get canonicalUrl() { - return encodeURI(`${FileStorage.getPublicEndpoint()}/${this.key}`); + return encodeURI(FileStorage.getUrlForKey(this.key)); } /** @@ -110,8 +118,31 @@ class Attachment extends IdModel { return FileStorage.getSignedUrl(this.key); } + /** + * Store the given file in storage at the location specified by the attachment key. + * If the attachment already exists, it will be overwritten. + * + * @param file The file to store + * @returns A promise resolving to the attachment + */ + async overwriteFile(file: File) { + return FileStorage.store({ + body: createReadStream(file.filepath), + contentLength: file.size, + contentType: this.contentType, + key: this.key, + acl: this.acl, + }); + } + // hooks + @BeforeUpdate + static async sanitizeKey(model: Attachment) { + model.key = ValidateKey.sanitize(model.key); + return model; + } + @BeforeDestroy static async deleteAttachmentFromS3(model: Attachment) { await FileStorage.deleteFile(model.key); @@ -141,6 +172,42 @@ class Attachment extends IdModel { return parseInt(result?.[0]?.total ?? "0", 10); } + /** + * Find an attachment given a JWT signature. + * + * @param sign - The signature that uniquely identifies an attachment + * @returns A promise resolving to attachment corresponding to the signature + * @throws {AuthenticationError} Invalid signature if the signature verification fails + */ + static async findBySignature(sign: string): Promise { + const payload = getJWTPayload(sign); + + if (payload.type !== "attachment") { + throw AuthenticationError("Invalid signature"); + } + + try { + JWT.verify(sign, env.SECRET_KEY); + } catch (err) { + throw AuthenticationError("Invalid signature"); + } + + return this.findByKey(payload.key); + } + + /** + * Find an attachment given a key + * + * @param key The key representing attachment file path + * @returns A promise resolving to attachment corresponding to the key + */ + static async findByKey(key: string): Promise { + return this.findOne({ + where: { key }, + rejectOnEmpty: true, + }); + } + // associations @BelongsTo(() => Team, "teamId") diff --git a/server/models/Team.ts b/server/models/Team.ts index 392604b48..8f91ccb4e 100644 --- a/server/models/Team.ts +++ b/server/models/Team.ts @@ -17,7 +17,6 @@ import { Is, DataType, IsUUID, - IsUrl, AllowNull, AfterUpdate, } from "sequelize-typescript"; @@ -40,6 +39,7 @@ import User from "./User"; import ParanoidModel from "./base/ParanoidModel"; import Fix from "./decorators/Fix"; import IsFQDN from "./validators/IsFQDN"; +import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath"; import Length from "./validators/Length"; import NotContainsUrl from "./validators/NotContainsUrl"; @@ -97,7 +97,7 @@ class Team extends ParanoidModel { defaultCollectionId: string | null; @AllowNull - @IsUrl + @IsUrlOrRelativePath @Length({ max: 4096, msg: "avatarUrl must be 4096 characters or less" }) @Column(DataType.STRING) get avatarUrl() { diff --git a/server/models/User.ts b/server/models/User.ts index 020e4058b..d8afe4370 100644 --- a/server/models/User.ts +++ b/server/models/User.ts @@ -18,7 +18,6 @@ import { HasMany, Scopes, IsDate, - IsUrl, AllowNull, AfterUpdate, } from "sequelize-typescript"; @@ -52,6 +51,7 @@ import Encrypted, { getEncryptedColumn, } from "./decorators/Encrypted"; import Fix from "./decorators/Fix"; +import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath"; import Length from "./validators/Length"; import NotContainsUrl from "./validators/NotContainsUrl"; @@ -182,7 +182,7 @@ class User extends ParanoidModel { language: string; @AllowNull - @IsUrl + @IsUrlOrRelativePath @Length({ max: 4096, msg: "avatarUrl must be less than 4096 characters" }) @Column(DataType.STRING) get avatarUrl() { diff --git a/server/models/helpers/AttachmentHelper.ts b/server/models/helpers/AttachmentHelper.ts index 6933ff07d..e99c6db32 100644 --- a/server/models/helpers/AttachmentHelper.ts +++ b/server/models/helpers/AttachmentHelper.ts @@ -70,7 +70,7 @@ export default class AttachmentHelper { case AttachmentPreset.Avatar: case AttachmentPreset.DocumentAttachment: default: - return env.AWS_S3_UPLOAD_MAX_SIZE; + return env.FILE_STORAGE_UPLOAD_MAX_SIZE; } } } diff --git a/server/models/validators/IsUrlOrRelativePath.ts b/server/models/validators/IsUrlOrRelativePath.ts new file mode 100644 index 000000000..8cec74aef --- /dev/null +++ b/server/models/validators/IsUrlOrRelativePath.ts @@ -0,0 +1,23 @@ +import { isURL } from "class-validator"; +import { addAttributeOptions } from "sequelize-typescript"; + +/** + * A decorator that validates that a string is a url or relative path. + */ +export default function IsUrlOrRelativePath(target: any, propertyName: string) { + return addAttributeOptions(target, propertyName, { + validate: { + validUrlOrPath(value: string) { + if ( + value && + !isURL(value, { + require_host: false, + require_protocol: false, + }) + ) { + throw new Error("Must be a URL or relative path"); + } + }, + }, + }); +} diff --git a/server/policies/attachment.ts b/server/policies/attachment.ts index 0cf2f82d1..6c486ed1f 100644 --- a/server/policies/attachment.ts +++ b/server/policies/attachment.ts @@ -9,7 +9,7 @@ allow(User, "createAttachment", Team, (user, team) => { }); allow(User, "read", Attachment, (actor, attachment) => { - if (!attachment || attachment.teamId !== actor.teamId) { + if (!attachment || !actor || attachment.teamId !== actor.teamId) { return false; } if (actor.isAdmin) { diff --git a/server/queues/tasks/ExportTask.ts b/server/queues/tasks/ExportTask.ts index 65ee136ca..b2d3996de 100644 --- a/server/queues/tasks/ExportTask.ts +++ b/server/queues/tasks/ExportTask.ts @@ -90,7 +90,7 @@ export default abstract class ExportTask extends BaseTask { }); const stat = await fs.promises.stat(filePath); - const url = await FileStorage.upload({ + const url = await FileStorage.store({ body: fs.createReadStream(filePath), contentLength: stat.size, contentType: "application/zip", diff --git a/server/queues/tasks/UploadTeamAvatarTask.ts b/server/queues/tasks/UploadTeamAvatarTask.ts index 657c71e4a..3301f4f9a 100644 --- a/server/queues/tasks/UploadTeamAvatarTask.ts +++ b/server/queues/tasks/UploadTeamAvatarTask.ts @@ -20,14 +20,14 @@ export default class UploadTeamAvatarTask extends BaseTask { rejectOnEmpty: true, }); - const res = await FileStorage.uploadFromUrl( + const res = await FileStorage.storeFromUrl( props.avatarUrl, `avatars/${team.id}/${uuidv4()}`, "public-read" ); if (res?.url) { - await team.update({ avatarUrl: res?.url }); + await team.update({ avatarUrl: res.url }); } } diff --git a/server/queues/tasks/UploadUserAvatarTask.ts b/server/queues/tasks/UploadUserAvatarTask.ts index 35744c2b3..541ca1c38 100644 --- a/server/queues/tasks/UploadUserAvatarTask.ts +++ b/server/queues/tasks/UploadUserAvatarTask.ts @@ -20,14 +20,14 @@ export default class UploadUserAvatarTask extends BaseTask { rejectOnEmpty: true, }); - const res = await FileStorage.uploadFromUrl( + const res = await FileStorage.storeFromUrl( props.avatarUrl, `avatars/${user.id}/${uuidv4()}`, "public-read" ); if (res?.url) { - await user.update({ avatarUrl: res?.url }); + await user.update({ avatarUrl: res.url }); } } diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts index d00a84ce4..8552ad6a4 100644 --- a/server/routes/api/attachments/attachments.ts +++ b/server/routes/api/attachments/attachments.ts @@ -99,7 +99,7 @@ router.post( ctx.body = { data: { - uploadUrl: FileStorage.getPublicEndpoint(), + uploadUrl: FileStorage.getUploadUrl(), form: { "Cache-Control": "max-age=31557600", "Content-Type": contentType, diff --git a/server/routes/auth/index.test.ts b/server/routes/auth/index.test.ts index fef9f124b..f0aef1adf 100644 --- a/server/routes/auth/index.test.ts +++ b/server/routes/auth/index.test.ts @@ -13,7 +13,8 @@ describe("auth/redirect", () => { } ); expect(res.status).toEqual(302); - expect(res.headers.get("location").endsWith("/home")).toBeTruthy(); + expect(res.headers.get("location")).not.toBeNull(); + expect(res.headers.get("location")!.endsWith("/home")).toBeTruthy(); }); it("should redirect to first collection", async () => { @@ -28,6 +29,7 @@ describe("auth/redirect", () => { } ); expect(res.status).toEqual(302); - expect(res.headers.get("location").endsWith(collection.url)).toBeTruthy(); + expect(res.headers.get("location")).not.toBeNull(); + expect(res.headers.get("location")!.endsWith(collection.url)).toBeTruthy(); }); }); diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts index 5e7c85229..accd21279 100644 --- a/server/storage/files/BaseStorage.ts +++ b/server/storage/files/BaseStorage.ts @@ -1,3 +1,4 @@ +import { Blob } from "buffer"; import { Readable } from "stream"; import { PresignedPost } from "aws-sdk/clients/s3"; import env from "@server/env"; @@ -5,6 +6,9 @@ import Logger from "@server/logging/Logger"; import fetch from "@server/utils/fetch"; export default abstract class BaseStorage { + /** The default number of seconds until a signed URL expires. */ + public static defaultSignedUrlExpires = 60; + /** * Returns a presigned post for uploading files to the storage provider. * @@ -19,7 +23,7 @@ export default abstract class BaseStorage { acl: string, maxUploadSize: number, contentType: string - ): Promise; + ): Promise>; /** * Returns a stream for reading a file from the storage provider. @@ -29,19 +33,20 @@ export default abstract class BaseStorage { public abstract getFileStream(key: string): NodeJS.ReadableStream | null; /** - * Returns a buffer of a file from the storage provider. - * - * @param key The path to the file - */ - public abstract getFileBuffer(key: string): Promise; - - /** - * Returns the public endpoint for the storage provider. + * Returns the upload URL for the storage provider. * * @param isServerUpload Whether the upload is happening on the server or not - * @returns The public endpoint as a string + * @returns {string} The upload URL */ - public abstract getPublicEndpoint(isServerUpload?: boolean): string; + public abstract getUploadUrl(isServerUpload?: boolean): string; + + /** + * Returns the download URL for a given file. + * + * @param key The path to the file + * @returns {string} The download URL for the file + */ + public abstract getUrlForKey(key: string): string; /** * Returns a signed URL for a file from the storage provider. @@ -55,7 +60,7 @@ export default abstract class BaseStorage { ): Promise; /** - * Upload a file to the storage provider. + * Store a file in the storage provider. * * @param body The file body * @param contentLength The content length of the file @@ -64,7 +69,7 @@ export default abstract class BaseStorage { * @param acl The ACL to use * @returns The URL of the file */ - public abstract upload({ + public abstract store({ body, contentLength, contentType, @@ -72,12 +77,35 @@ export default abstract class BaseStorage { acl, }: { body: Buffer | Uint8Array | Blob | string | Readable; - contentLength: number; - contentType: string; + contentLength?: number; + contentType?: string; key: string; - acl: string; + acl?: string; }): Promise; + /** + * Returns a buffer of a file from the storage provider. + * + * @param key The path to the file + */ + public async getFileBuffer(key: string) { + const stream = this.getFileStream(key); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + if (!stream) { + return reject(new Error("No stream available")); + } + + stream.on("data", (d) => { + chunks.push(d); + }); + stream.once("end", () => { + resolve(Buffer.concat(chunks)); + }); + stream.once("error", reject); + }); + } + /** * Upload a file to the storage provider directly from a remote or base64 encoded URL. * @@ -86,7 +114,7 @@ export default abstract class BaseStorage { * @param acl The ACL to use * @returns A promise that resolves when the file is uploaded */ - public async uploadFromUrl( + public async storeFromUrl( url: string, key: string, acl: string @@ -98,7 +126,7 @@ export default abstract class BaseStorage { } | undefined > { - const endpoint = this.getPublicEndpoint(true); + const endpoint = this.getUploadUrl(true); if (url.startsWith("/api") || url.startsWith(endpoint)) { return; } @@ -115,7 +143,7 @@ export default abstract class BaseStorage { const res = await fetch(url, { follow: 3, redirect: "follow", - size: env.AWS_S3_UPLOAD_MAX_SIZE, + size: env.FILE_STORAGE_UPLOAD_MAX_SIZE, timeout: 10000, }); @@ -143,7 +171,7 @@ export default abstract class BaseStorage { } try { - const result = await this.upload({ + const result = await this.store({ body: buffer, contentLength, contentType, diff --git a/server/storage/files/LocalStorage.ts b/server/storage/files/LocalStorage.ts new file mode 100644 index 000000000..e0297ad6f --- /dev/null +++ b/server/storage/files/LocalStorage.ts @@ -0,0 +1,120 @@ +import { Blob } from "buffer"; +import { + ReadStream, + closeSync, + createReadStream, + createWriteStream, + existsSync, + openSync, +} from "fs"; +import { mkdir, unlink } from "fs/promises"; +import path from "path"; +import { Readable } from "stream"; +import invariant from "invariant"; +import JWT from "jsonwebtoken"; +import env from "@server/env"; +import Logger from "@server/logging/Logger"; +import BaseStorage from "./BaseStorage"; + +export default class LocalStorage extends BaseStorage { + public async getPresignedPost( + key: string, + acl: string, + maxUploadSize: number, + contentType = "image" + ) { + return Promise.resolve({ + url: this.getUrlForKey(key), + fields: { + key, + acl, + maxUploadSize, + contentType, + }, + } as any); + } + + public getUploadUrl() { + return "/api/files.create"; + } + + public getUrlForKey(key: string): string { + return `/api/files.get?key=${key}`; + } + + public store = async ({ + body, + key, + }: { + body: string | ReadStream | Buffer | Uint8Array | Blob; + contentLength?: number; + contentType?: string; + key: string; + acl?: string; + }) => { + const subdir = key.split("/").slice(0, -1).join("/"); + if (!existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir))) { + await mkdir(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir), { + recursive: true, + }); + } + + let src: NodeJS.ReadableStream; + if (body instanceof ReadStream) { + src = body; + } else if (body instanceof Blob) { + src = Readable.from(Buffer.from(await body.arrayBuffer())); + } else { + src = Readable.from(body); + } + + const destPath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key); + closeSync(openSync(destPath, "w")); + const dest = createWriteStream(destPath); + src.pipe(dest); + + return new Promise((resolve, reject) => { + src.once("end", () => resolve(this.getUrlForKey(key))); + src.once("err", (err) => { + dest.end(); + reject(err); + }); + }); + }; + + public async deleteFile(key: string) { + const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key); + try { + await unlink(filePath); + } catch (err) { + Logger.warn(`Couldn't delete ${filePath}`, err); + } + } + + public getSignedUrl = async ( + key: string, + expiresIn = LocalStorage.defaultSignedUrlExpires + ) => { + const sig = JWT.sign( + { + key, + type: "attachment", + }, + env.SECRET_KEY, + { + expiresIn, + } + ); + return Promise.resolve(`/api/files.get?sig=${sig}`); + }; + + public getFileStream(key: string) { + invariant( + env.FILE_STORAGE_LOCAL_ROOT_DIR, + "FILE_STORAGE_LOCAL_ROOT_DIR is required" + ); + + const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key); + return createReadStream(filePath); + } +} diff --git a/server/storage/files/S3Storage.ts b/server/storage/files/S3Storage.ts index ea2b6cf39..d1ba71386 100644 --- a/server/storage/files/S3Storage.ts +++ b/server/storage/files/S3Storage.ts @@ -47,7 +47,7 @@ export default class S3Storage extends BaseStorage { ); } - public getPublicEndpoint(isServerUpload?: boolean) { + private getPublicEndpoint(isServerUpload?: boolean) { if (env.AWS_S3_ACCELERATE_URL) { return env.AWS_S3_ACCELERATE_URL; } @@ -78,7 +78,15 @@ export default class S3Storage extends BaseStorage { }`; } - public upload = async ({ + public getUploadUrl(isServerUpload?: boolean) { + return this.getPublicEndpoint(isServerUpload); + } + + public getUrlForKey(key: string): string { + return `${this.getPublicEndpoint()}/${key}`; + } + + public store = async ({ body, contentLength, contentType, @@ -86,10 +94,10 @@ export default class S3Storage extends BaseStorage { acl, }: { body: S3.Body; - contentLength: number; - contentType: string; + contentLength?: number; + contentType?: string; key: string; - acl: string; + acl?: string; }) => { invariant( env.AWS_S3_UPLOAD_BUCKET_NAME, @@ -125,7 +133,10 @@ export default class S3Storage extends BaseStorage { .promise(); } - public getSignedUrl = async (key: string, expiresIn = 60) => { + public getSignedUrl = async ( + key: string, + expiresIn = S3Storage.defaultSignedUrlExpires + ) => { const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/); const params = { Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME, @@ -170,26 +181,6 @@ export default class S3Storage extends BaseStorage { return null; } - public async getFileBuffer(key: string) { - invariant( - env.AWS_S3_UPLOAD_BUCKET_NAME, - "AWS_S3_UPLOAD_BUCKET_NAME is required" - ); - - const response = await this.client - .getObject({ - Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME, - Key: key, - }) - .promise(); - - if (response.Body) { - return response.Body as Blob; - } - - throw new Error("Error getting file buffer from S3"); - } - private client: AWS.S3; private getEndpoint() { diff --git a/server/storage/files/__mocks__/index.ts b/server/storage/files/__mocks__/index.ts index 07ffc8db2..0c8716df8 100644 --- a/server/storage/files/__mocks__/index.ts +++ b/server/storage/files/__mocks__/index.ts @@ -1,7 +1,9 @@ export default { upload: jest.fn().mockReturnValue("/endpoint/key"), - getPublicEndpoint: jest.fn().mockReturnValue("http://mock"), + getUploadUrl: jest.fn().mockReturnValue("http://mock/create"), + + getUrlForKey: jest.fn().mockReturnValue("http://mock/get"), getSignedUrl: jest.fn().mockReturnValue("http://s3mock"), diff --git a/server/storage/files/index.ts b/server/storage/files/index.ts index c87b0b50c..08e11a8ac 100644 --- a/server/storage/files/index.ts +++ b/server/storage/files/index.ts @@ -1,3 +1,8 @@ +import env from "@server/env"; +import LocalStorage from "./LocalStorage"; import S3Storage from "./S3Storage"; -export default new S3Storage(); +const storage = + env.FILE_STORAGE === "local" ? new LocalStorage() : new S3Storage(); + +export default storage; diff --git a/server/test/TestServer.ts b/server/test/TestServer.ts new file mode 100644 index 000000000..1a2df84b2 --- /dev/null +++ b/server/test/TestServer.ts @@ -0,0 +1,84 @@ +import http from "http"; +import { AddressInfo } from "net"; +import Koa from "koa"; +// eslint-disable-next-line no-restricted-imports +import nodeFetch from "node-fetch"; + +class TestServer { + private server: http.Server; + private listener?: Promise | null; + + constructor(app: Koa) { + this.server = http.createServer(app.callback() as any); + } + + get address(): string { + const { port } = this.server.address() as AddressInfo; + return `http://localhost:${port}`; + } + + listen() { + if (!this.listener) { + this.listener = new Promise((resolve, reject) => { + this.server + .listen(0, () => resolve()) + .on("error", (err) => reject(err)); + }); + } + + return this.listener; + } + + fetch(path: string, opts: any) { + return this.listen().then(() => { + const url = `${this.address}${path}`; + const options = Object.assign({ headers: {} }, opts); + const contentType = + options.headers["Content-Type"] ?? options.headers["content-type"]; + // automatic JSON encoding + if (!contentType && typeof options.body === "object") { + options.headers["Content-Type"] = "application/json"; + options.body = JSON.stringify(options.body); + } + + return nodeFetch(url, options); + }); + } + + close() { + this.listener = null; + return new Promise((resolve, reject) => { + this.server.close((err) => (err ? reject(err) : resolve())); + }); + } + + delete(path: string, options?: any) { + return this.fetch(path, { ...options, method: "DELETE" }); + } + + get(path: string, options?: any) { + return this.fetch(path, { ...options, method: "GET" }); + } + + head(path: string, options?: any) { + return this.fetch(path, { ...options, method: "HEAD" }); + } + + options(path: string, options?: any) { + return this.fetch(path, { ...options, method: "OPTIONS" }); + } + + patch(path: string, options?: any) { + return this.fetch(path, { ...options, method: "PATCH" }); + } + + post(path: string, options?: any) { + return this.fetch(path, { ...options, method: "POST" }); + } + + put(path: string, options?: any) { + return this.fetch(path, { ...options, method: "PUT" }); + } +} + +export default TestServer; diff --git a/server/test/env.ts b/server/test/env.ts index d04695d9b..d5fab9ce7 100644 --- a/server/test/env.ts +++ b/server/test/env.ts @@ -18,6 +18,8 @@ env.OIDC_USERINFO_URI = "http://localhost/userinfo"; env.RATE_LIMITER_ENABLED = false; +env.FILE_STORAGE = "local"; +env.FILE_STORAGE_LOCAL_ROOT_DIR = "/tmp"; env.IFRAMELY_API_KEY = "123"; if (process.env.DATABASE_URL_TEST) { diff --git a/server/test/factories.ts b/server/test/factories.ts index 0f1e5597c..8a67f5d27 100644 --- a/server/test/factories.ts +++ b/server/test/factories.ts @@ -33,6 +33,7 @@ import { SearchQuery, Pin, } from "@server/models"; +import AttachmentHelper from "@server/models/helpers/AttachmentHelper"; export async function buildApiKey(overrides: Partial = {}) { if (!overrides.userId) { @@ -420,7 +421,10 @@ export async function buildFileOperation( }); } -export async function buildAttachment(overrides: Partial = {}) { +export async function buildAttachment( + overrides: Partial = {}, + fileName?: string +) { if (!overrides.teamId) { const team = await buildTeam(); overrides.teamId = team.id; @@ -441,11 +445,14 @@ export async function buildAttachment(overrides: Partial = {}) { overrides.documentId = document.id; } + const id = uuidv4(); + const acl = overrides.acl || "public-read"; + const name = fileName || faker.system.fileName(); return Attachment.create({ - key: `uploads/key/to/${faker.system.fileName}.png`, + key: AttachmentHelper.getKey({ acl, id, name, userId: overrides.userId }), contentType: "image/png", size: 100, - acl: "public-read", + acl, createdAt: new Date("2018-01-02T00:00:00.000Z"), updatedAt: new Date("2018-01-02T00:00:00.000Z"), ...overrides, diff --git a/server/test/support.ts b/server/test/support.ts index 14af26807..458a35931 100644 --- a/server/test/support.ts +++ b/server/test/support.ts @@ -1,22 +1,22 @@ import { faker } from "@faker-js/faker"; -import TestServer from "fetch-test-server"; import sharedEnv from "@shared/env"; import env from "@server/env"; import onerror from "@server/onerror"; import webService from "@server/services/web"; import { sequelize } from "@server/storage/database"; +import TestServer from "./TestServer"; export function getTestServer() { const app = webService(); onerror(app); - const server = new TestServer(app.callback()); + const server = new TestServer(app); - server.disconnect = async () => { + const disconnect = async () => { await sequelize.close(); - server.close(); + return server.close(); }; - afterAll(server.disconnect); + afterAll(disconnect); return server; } diff --git a/server/typings/index.d.ts b/server/typings/index.d.ts index a6fad124b..c1bdffcb8 100644 --- a/server/typings/index.d.ts +++ b/server/typings/index.d.ts @@ -6,8 +6,6 @@ declare module "formidable/lib/file"; declare module "oy-vey"; -declare module "fetch-test-server"; - declare module "dotenv"; declare module "email-providers" { diff --git a/server/utils/jwt.ts b/server/utils/jwt.ts index e3ccdb949..64f201a4e 100644 --- a/server/utils/jwt.ts +++ b/server/utils/jwt.ts @@ -4,7 +4,7 @@ import JWT from "jsonwebtoken"; import { Team, User } from "@server/models"; import { AuthenticationError } from "../errors"; -function getJWTPayload(token: string) { +export function getJWTPayload(token: string) { let payload; try { diff --git a/server/validation.test.ts b/server/validation.test.ts new file mode 100644 index 000000000..826b2e652 --- /dev/null +++ b/server/validation.test.ts @@ -0,0 +1,41 @@ +import { v4 as uuidv4 } from "uuid"; +import { ValidateKey } from "./validation"; + +describe("#ValidateKey.isValid", () => { + it("should return false if number of key components are not equal to 4", () => { + expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}`)).toBe(false); + expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}/foo/bar`)).toBe( + false + ); + }); + + it("should return false if the first key component is neither 'public' nor 'uploads' ", () => { + expect(ValidateKey.isValid(`foo/${uuidv4()}/${uuidv4()}/bar.png`)).toBe( + false + ); + }); + + it("should return false if second and third key components are not UUID", () => { + expect(ValidateKey.isValid(`uploads/foo/${uuidv4()}/bar.png`)).toBe(false); + expect(ValidateKey.isValid(`uploads/${uuidv4()}/foo/bar.png`)).toBe(false); + }); + + it("should return true successfully validating key", () => { + expect(ValidateKey.isValid(`public/${uuidv4()}/${uuidv4()}/foo.png`)).toBe( + true + ); + expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}/foo.png`)).toBe( + true + ); + }); +}); + +describe("#ValidateKey.sanitize", () => { + it("should sanitize malicious looking keys", () => { + const uuid1 = uuidv4(); + const uuid2 = uuidv4(); + expect( + ValidateKey.sanitize(`public/${uuid1}/${uuid2}/~\.\u0000\malicious_key`) + ).toEqual(`public/${uuid1}/${uuid2}/~.malicious_key`); + }); +}); diff --git a/server/validation.ts b/server/validation.ts index a05f09e7e..c8eaa7b05 100644 --- a/server/validation.ts +++ b/server/validation.ts @@ -1,6 +1,8 @@ import isArrayLike from "lodash/isArrayLike"; +import sanitize from "sanitize-filename"; import { Primitive } from "utility-types"; import validator from "validator"; +import isIn from "validator/lib/isIn"; import isUUID from "validator/lib/isUUID"; import parseMentionUrl from "@shared/utils/parseMentionUrl"; import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers"; @@ -170,6 +172,30 @@ export const assertCollectionPermission = ( assertIn(value, [...Object.values(CollectionPermission), null], message); }; +export class ValidateKey { + public static isValid = (key: string) => { + const parts = key.split("/").slice(0, -1); + return ( + parts.length === 3 && + isIn(parts[0], ["uploads", "public"]) && + isUUID(parts[1]) && + isUUID(parts[2]) + ); + }; + + public static sanitize = (key: string) => { + const [filename] = key.split("/").slice(-1); + return key + .split("/") + .slice(0, -1) + .join("/") + .concat(`/${sanitize(filename)}`); + }; + + public static message = + "Must be of the form uploads/// or public///"; +} + export class ValidateDocumentId { /** * Checks if documentId is valid. A valid documentId is either diff --git a/yarn.lock b/yarn.lock index 14a642ca7..243498280 100644 --- a/yarn.lock +++ b/yarn.lock @@ -191,14 +191,7 @@ dependencies: "@babel/types" "^7.21.0" -"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.14.5", "@babel/helper-module-imports@^7.18.6", "@babel/helper-module-imports@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz#1a8f4c9f4027d23f520bd76b364d44434a72660c" - integrity sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg== - dependencies: - "@babel/types" "^7.22.5" - -"@babel/helper-module-imports@^7.22.15": +"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.14.5", "@babel/helper-module-imports@^7.18.6", "@babel/helper-module-imports@^7.22.15", "@babel/helper-module-imports@^7.22.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz#16146307acdc40cc00c3b2c647713076464bdbf0" integrity sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w== @@ -279,12 +272,7 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== -"@babel/helper-validator-identifier@^7.19.1", "@babel/helper-validator-identifier@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" - integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ== - -"@babel/helper-validator-identifier@^7.22.19": +"@babel/helper-validator-identifier@^7.19.1", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.5": version "7.22.20" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== @@ -1095,16 +1083,7 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.22.5", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe" - integrity sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA== - dependencies: - "@babel/helper-string-parser" "^7.22.5" - "@babel/helper-validator-identifier" "^7.22.5" - to-fast-properties "^2.0.0" - -"@babel/types@^7.22.15": +"@babel/types@^7.0.0", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0": version "7.22.19" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.19.tgz#7425343253556916e440e662bb221a93ddb75684" integrity sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg== @@ -2921,6 +2900,13 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/form-data@^2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@types/form-data/-/form-data-2.5.0.tgz#5025f7433016f923348434c40006d9a797c1b0e8" + integrity sha512-23/wYiuckYYtFpL+4RPWiWmRQH2BjFuqCUi2+N3amB1a1Drv+i/byTrGvlLwRVLFNAZbwpbQ7JvTK+VCAPMbcg== + dependencies: + form-data "*" + "@types/formidable@^2.0.5", "@types/formidable@^2.0.6": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-2.0.6.tgz#811ed3cd8a8a7675e02420b3f861c317e055376a" @@ -3241,7 +3227,12 @@ "@types/node" "*" form-data "^4.0.0" -"@types/node@*", "@types/node@18.0.6", "@types/node@>=10.0.0", "@types/node@>=13.7.0": +"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=13.7.0": + version "20.5.9" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.9.tgz#a70ec9d8fa0180a314c3ede0e20ea56ff71aed9a" + integrity sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ== + +"@types/node@18.0.6": version "18.0.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.6.tgz#0ba49ac517ad69abe7a1508bc9b3a5483df9d5d7" integrity sha512-/xUq6H2aQm261exT6iZTMifUySEt4GR5KX8eYyY+C4MSNPqSh9oNIP7tz2GLKTlFaiBbgZNxffoR3CVRG+cljw== @@ -3437,6 +3428,13 @@ dependencies: "@types/node" "*" +"@types/sanitize-filename@^1.6.3": + version "1.6.3" + resolved "https://registry.yarnpkg.com/@types/sanitize-filename/-/sanitize-filename-1.6.3.tgz#182ebd5658fbd3fe36bcb771daad8b2623371705" + integrity sha512-1dAV8Va7KsiXNAstV2JmF4CRVG3Fsyl+VnBw87C9cCMccekpOqJBezS7MUnHYPChNAFee1WakwBXdfn7QJxzVg== + dependencies: + sanitize-filename "*" + "@types/scheduler@*": version "0.16.2" resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" @@ -3992,7 +3990,7 @@ async@^3.2.3: asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== at-least-node@^1.0.0: version "1.0.0" @@ -5689,7 +5687,7 @@ delay@^5.0.0: delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== delegates@^1.0.0: version "1.0.0" @@ -6807,14 +6805,6 @@ fetch-retry@^5.0.5: resolved "https://registry.yarnpkg.com/fetch-retry/-/fetch-retry-5.0.5.tgz#61079b816b6651d88a022ebd45d51d83aa72b521" integrity sha512-q9SvpKH5Ka6h7X2C6r1sP31pQoeDb3o6/R9cg21ahfPAqbIOkW9tus1dXfwYb6G6dOI4F7nVS4Q+LSssBGIz0A== -fetch-test-server@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/fetch-test-server/-/fetch-test-server-1.2.0.tgz#65f23af1d030c293249a49bbd1b51e45fc68eb69" - integrity sha512-KjxYDGGfVC/paLya7UN+AFxb3wt0Mj79eOBjlpRdn9B1o0uo3vJCC9VGVTd17Q5kiBx+HvglP/BzBi8BZs18sA== - dependencies: - debug "^3.1.0" - node-fetch "^2.1.2" - fetch-with-proxy@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/fetch-with-proxy/-/fetch-with-proxy-3.0.1.tgz#29ed6d0e2550ef999d40b18de2ba476af4b7dee4" @@ -6944,19 +6934,19 @@ for-each@^0.3.3: dependencies: is-callable "^1.1.3" -form-data@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" - integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== +form-data@*, form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== +form-data@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" + integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" @@ -9877,7 +9867,7 @@ node-addon-api@^6.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== -node-fetch@2.6.7, node-fetch@2.7.0, node-fetch@^2.1.2, node-fetch@^2.6.1, node-fetch@^2.6.12: +node-fetch@2.6.7, node-fetch@2.7.0, node-fetch@^2.6.1, node-fetch@^2.6.12: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -11793,6 +11783,13 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sanitize-filename@*, sanitize-filename@^1.6.3: + version "1.6.3" + resolved "https://registry.yarnpkg.com/sanitize-filename/-/sanitize-filename-1.6.3.tgz#755ebd752045931977e30b2025d340d7c9090378" + integrity sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg== + dependencies: + truncate-utf8-bytes "^1.0.0" + sanitizer@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/sanitizer/-/sanitizer-0.1.3.tgz#d4f0af7475d9a7baf2a9e5a611718baa178a39e1" @@ -12839,7 +12836,7 @@ tr46@^4.1.1: tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== tree-kill@^1.2.2: version "1.2.2" @@ -12851,6 +12848,13 @@ triple-beam@^1.3.0: resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== +truncate-utf8-bytes@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b" + integrity sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ== + dependencies: + utf8-byte-length "^1.0.1" + ts-dedent@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.2.0.tgz#39e4bd297cd036292ae2394eb3412be63f563bb5" @@ -13194,6 +13198,11 @@ use-sidecar@^1.1.2: detect-node-es "^1.1.0" tslib "^2.0.0" +utf8-byte-length@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz#f45f150c4c66eee968186505ab93fcbb8ad6bf61" + integrity sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA== + utf8@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96" @@ -13421,7 +13430,7 @@ walker@^1.0.8: webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== webidl-conversions@^4.0.2: version "4.0.2" @@ -13477,7 +13486,7 @@ whatwg-url@^12.0.0, whatwg-url@^12.0.1: whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0"