diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fb8ee280..a95687b8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,7 +145,12 @@ jobs: command: docker push $BASE_IMAGE_NAME:latest - run: name: Build and push Docker image - command: docker buildx build -t $IMAGE_NAME:latest -t $IMAGE_NAME:${CIRCLE_TAG/v/''} --platform linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x --push . + command: | + if [ "$CIRCLE_BRANCH" == "main" ]; then + docker buildx build -t $IMAGE_NAME:latest -t $IMAGE_NAME:${CIRCLE_TAG/v/''} --platform linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x --push . + else + docker buildx build -t $IMAGE_NAME:${CIRCLE_TAG/v/''} --platform linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x --push . + fi workflows: version: 2 diff --git a/.env.sample b/.env.sample index 9b199b806..3e8cadbe4 100644 --- a/.env.sample +++ b/.env.sample @@ -51,10 +51,20 @@ AWS_REGION=xx-xxxx-x AWS_S3_ACCELERATE_URL= AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569 AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here -AWS_S3_UPLOAD_MAX_SIZE=26214400 AWS_S3_FORCE_PATH_STYLE=true AWS_S3_ACL=private +# Specify what storage system to use. Possible value is one of "s3" or "local". +# For "local", the avatar images and document attachments will be saved on local disk. +FILE_STORAGE=local + +# If "local" is configured for FILE_STORAGE above, then this sets the parent directory under +# which all attachments/images go. Make sure that the process has permissions to create +# this path and also to write files to it. +FILE_STORAGE_LOCAL_ROOT_DIR=/var/lib/outline/data + +# Maximum allowed size for the uploaded attachment. +FILE_STORAGE_UPLOAD_MAX_SIZE=26214400 # –––––––––––––– AUTHENTICATION –––––––––––––– diff --git a/app.json b/app.json index f5eece7b3..9169adf16 100644 --- a/app.json +++ b/app.json @@ -128,11 +128,6 @@ "description": "Live web link to your bucket. For CNAMEs, https://yourbucket.example.com", "required": false }, - "AWS_S3_UPLOAD_MAX_SIZE": { - "description": "Maximum file upload size in bytes", - "value": "26214400", - "required": false - }, "AWS_S3_FORCE_PATH_STYLE": { "description": "Use path-style URL's for connecting to S3 instead of subdomain. This is useful for S3-compatible storage.", "value": "true", @@ -148,6 +143,11 @@ "description": "S3 canned ACL for document attachments", "required": false }, + "FILE_STORAGE_UPLOAD_MAX_SIZE": { + "description": "Maximum file upload size in bytes", + "value": "26214400", + "required": false + }, "SMTP_HOST": { "description": "smtp.example.com (optional)", "required": false diff --git a/package.json b/package.json index ca33fc4e4..8bac49f74 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,8 @@ "@sentry/tracing": "^7.51.2", "@tippyjs/react": "^4.2.6", "@tommoor/remove-markdown": "^0.3.2", + "@types/form-data": "^2.5.0", + "@types/sanitize-filename": "^1.6.3", "@vitejs/plugin-react": "^3.1.0", "addressparser": "^1.0.1", "autotrack": "^2.4.1", @@ -103,6 +105,7 @@ "fetch-retry": "^5.0.5", "fetch-with-proxy": "^3.0.1", "focus-visible": "^5.2.0", + "form-data": "^4.0.0", "fractional-index": "^1.0.0", "framer-motion": "^4.1.17", "fs-extra": "^11.1.1", @@ -193,6 +196,7 @@ "reflect-metadata": "^0.1.13", "refractor": "^3.6.0", "request-filtering-agent": "^1.1.2", + "sanitize-filename": "^1.6.3", "semver": "^7.5.2", "sequelize": "^6.32.1", "sequelize-cli": "^6.6.1", @@ -315,7 +319,6 @@ "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react": "^7.20.0", "eslint-plugin-react-hooks": "^4.6.0", - "fetch-test-server": "^1.1.0", "husky": "^8.0.2", "i18next-parser": "^7.9.0", "jest-cli": "^29.6.4", @@ -343,5 +346,5 @@ "qs": "6.9.7", "rollup": "^3.14.0" }, - "version": "0.71.0" + "version": "0.72.0-1" } diff --git a/plugins/oidc/server/auth/oidc.test.ts b/plugins/oidc/server/auth/oidc.test.ts index 0e0013dc1..72e37f374 100644 --- a/plugins/oidc/server/auth/oidc.test.ts +++ b/plugins/oidc/server/auth/oidc.test.ts @@ -7,7 +7,8 @@ describe("oidc", () => { const res = await server.get("/auth/oidc?myParam=someParam", { redirect: "manual", }); - const redirectLocation = new URL(res.headers.get("location")); + expect(res.headers.get("location")).not.toBeNull(); + const redirectLocation = new URL(res.headers.get("location")!); expect(res.status).toEqual(302); expect(redirectLocation.searchParams.get("myParam")).toEqual("someParam"); }); diff --git a/plugins/storage/plugin.json b/plugins/storage/plugin.json new file mode 100644 index 000000000..b7274ca29 --- /dev/null +++ b/plugins/storage/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "Storage", + "description": "Plugin for storing files on the local file system", + "requiredEnvVars": [ + "FILE_STORAGE_UPLOAD_MAX_SIZE", + "FILE_STORAGE_LOCAL_ROOT_DIR" + ] +} diff --git a/plugins/storage/server/.babelrc b/plugins/storage/server/.babelrc new file mode 100644 index 000000000..2bc0ef3ed --- /dev/null +++ b/plugins/storage/server/.babelrc @@ -0,0 +1,3 @@ +{ + "extends": "../../../server/.babelrc" +} diff --git a/plugins/storage/server/api/files.test.ts b/plugins/storage/server/api/files.test.ts new file mode 100644 index 000000000..6683061d0 --- /dev/null +++ b/plugins/storage/server/api/files.test.ts @@ -0,0 +1,150 @@ +import { existsSync } from "fs"; +import { readFile } from "fs/promises"; +import path from "path"; +import FormData from "form-data"; +import env from "@server/env"; +import "@server/test/env"; +import { buildAttachment, buildUser } from "@server/test/factories"; +import { getTestServer } from "@server/test/support"; + +const server = getTestServer(); + +describe("#files.create", () => { + it("should fail with status 400 bad request if key is invalid", async () => { + const user = await buildUser(); + const res = await server.post("/api/files.create", { + body: { + token: user.getJwtToken(), + key: "public/foo/bar/baz.png", + }, + }); + const body = await res.json(); + expect(res.status).toEqual(400); + expect(body.message).toEqual( + "key: Must be of the form uploads/// or public///" + ); + }); + + it("should succeed with status 200 ok and create a file", async () => { + const user = await buildUser(); + const fileName = "images.docx"; + const attachment = await buildAttachment( + { + teamId: user.teamId, + userId: user.id, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }, + fileName + ); + + const content = await readFile( + path.resolve(__dirname, "..", "test", "fixtures", fileName) + ); + const form = new FormData(); + form.append("key", attachment.key); + form.append("file", content, fileName); + form.append("token", user.getJwtToken()); + + const res = await server.post(`/api/files.create`, { + headers: form.getHeaders(), + body: form, + }); + + const body = await res.json(); + expect(res.status).toEqual(200); + expect(body.success).toEqual(true); + expect( + existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, attachment.key)) + ).toBe(true); + }); +}); + +describe("#files.get", () => { + it("should fail with status 400 bad request if key is invalid", async () => { + const res = await server.get(`/api/files.get?key=public/foo/bar/baz.png`); + const body = await res.json(); + expect(res.status).toEqual(400); + expect(body.message).toEqual( + "key: Must be of the form uploads/// or public///" + ); + }); + + it("should fail with status 400 bad request if none of key or sig is supplied", async () => { + const res = await server.get("/api/files.get"); + const body = await res.json(); + expect(res.status).toEqual(400); + expect(body.message).toEqual("query: One of key or sig is required"); + }); + + it("should succeed with status 200 ok when file is requested using key", async () => { + const user = await buildUser(); + const fileName = "images.docx"; + + const attachment = await buildAttachment( + { + teamId: user.teamId, + userId: user.id, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }, + fileName + ); + + const content = await readFile( + path.resolve(__dirname, "..", "test", "fixtures", fileName) + ); + const form = new FormData(); + form.append("key", attachment.key); + form.append("file", content, fileName); + form.append("token", user.getJwtToken()); + + await server.post(`/api/files.create`, { + headers: form.getHeaders(), + body: form, + }); + + const res = await server.get(attachment.canonicalUrl); + expect(res.status).toEqual(200); + expect(res.headers.get("Content-Type")).toEqual(attachment.contentType); + expect(res.headers.get("Content-Disposition")).toEqual( + 'attachment; filename="images.docx"' + ); + }); + + it("should succeed with status 200 ok when private file is requested using signature", async () => { + const user = await buildUser(); + const fileName = "images.docx"; + + const attachment = await buildAttachment( + { + teamId: user.teamId, + userId: user.id, + contentType: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + acl: "private", + }, + fileName + ); + + const content = await readFile( + path.resolve(__dirname, "..", "test", "fixtures", fileName) + ); + const form = new FormData(); + form.append("key", attachment.key); + form.append("file", content, fileName); + form.append("token", user.getJwtToken()); + + await server.post(`/api/files.create`, { + headers: form.getHeaders(), + body: form, + }); + + const res = await server.get(await attachment.signedUrl); + expect(res.status).toEqual(200); + expect(res.headers.get("Content-Type")).toEqual(attachment.contentType); + expect(res.headers.get("Content-Disposition")).toEqual( + 'attachment; filename="images.docx"' + ); + }); +}); diff --git a/plugins/storage/server/api/files.ts b/plugins/storage/server/api/files.ts new file mode 100644 index 000000000..ef39a4ff1 --- /dev/null +++ b/plugins/storage/server/api/files.ts @@ -0,0 +1,71 @@ +import Router from "koa-router"; +import env from "@server/env"; +import { ValidationError } from "@server/errors"; +import auth from "@server/middlewares/authentication"; +import multipart from "@server/middlewares/multipart"; +import { rateLimiter } from "@server/middlewares/rateLimiter"; +import validate from "@server/middlewares/validate"; +import { Attachment } from "@server/models"; +import { authorize } from "@server/policies"; +import { APIContext } from "@server/types"; +import { RateLimiterStrategy } from "@server/utils/RateLimiter"; +import { createRootDirForLocalStorage } from "../utils"; +import * as T from "./schema"; + +createRootDirForLocalStorage(); + +const router = new Router(); + +router.post( + "files.create", + rateLimiter(RateLimiterStrategy.TenPerMinute), + auth(), + validate(T.FilesCreateSchema), + multipart({ maximumFileSize: env.FILE_STORAGE_UPLOAD_MAX_SIZE }), + async (ctx: APIContext) => { + const actor = ctx.state.auth.user; + const { key } = ctx.input.body; + const file = ctx.input.file; + + const attachment = await Attachment.findByKey(key); + + if (attachment.isPrivate) { + authorize(actor, "createAttachment", actor.team); + } + + await attachment.overwriteFile(file); + + ctx.body = { + success: true, + }; + } +); + +router.get( + "files.get", + auth({ optional: true }), + validate(T.FilesGetSchema), + async (ctx: APIContext) => { + const { key, sig } = ctx.input.query; + const actor = ctx.state.auth.user; + let attachment: Attachment | null; + + if (key) { + attachment = await Attachment.findByKey(key); + + if (attachment.isPrivate) { + authorize(actor, "read", attachment); + } + } else if (sig) { + attachment = await Attachment.findBySignature(sig); + } else { + throw ValidationError("Must provide either key or signature"); + } + + ctx.set("Content-Type", attachment.contentType); + ctx.attachment(attachment.name); + ctx.body = attachment.stream; + } +); + +export default router; diff --git a/plugins/storage/server/api/schema.ts b/plugins/storage/server/api/schema.ts new file mode 100644 index 000000000..2dea96d68 --- /dev/null +++ b/plugins/storage/server/api/schema.ts @@ -0,0 +1,33 @@ +import formidable from "formidable"; +import isEmpty from "lodash/isEmpty"; +import { z } from "zod"; +import { ValidateKey } from "@server/validation"; + +export const FilesCreateSchema = z.object({ + body: z.object({ + key: z + .string() + .refine(ValidateKey.isValid, { message: ValidateKey.message }) + .transform(ValidateKey.sanitize), + }), + file: z.custom(), +}); + +export type FilesCreateReq = z.infer; + +export const FilesGetSchema = z.object({ + query: z + .object({ + key: z + .string() + .refine(ValidateKey.isValid, { message: ValidateKey.message }) + .optional() + .transform((val) => (val ? ValidateKey.sanitize(val) : undefined)), + sig: z.string().optional(), + }) + .refine((obj) => !(isEmpty(obj.key) && isEmpty(obj.sig)), { + message: "One of key or sig is required", + }), +}); + +export type FilesGetReq = z.infer; diff --git a/plugins/storage/server/test/fixtures/images.docx b/plugins/storage/server/test/fixtures/images.docx new file mode 100644 index 000000000..25a9156a8 Binary files /dev/null and b/plugins/storage/server/test/fixtures/images.docx differ diff --git a/plugins/storage/server/utils.ts b/plugins/storage/server/utils.ts new file mode 100644 index 000000000..c3c88fba5 --- /dev/null +++ b/plugins/storage/server/utils.ts @@ -0,0 +1,20 @@ +import { existsSync, mkdirSync } from "fs"; +import env from "@server/env"; +import Logger from "@server/logging/Logger"; + +export const createRootDirForLocalStorage = () => { + if (env.FILE_STORAGE === "local") { + const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR; + try { + if (!existsSync(rootDir)) { + mkdirSync(rootDir, { recursive: true }); + Logger.debug("utils", `Created ${rootDir} for local storage`); + } + } catch (err) { + Logger.fatal( + "Couldn't create root dir for local storage of attachments", + err + ); + } + } +}; diff --git a/server/commands/attachmentCreator.ts b/server/commands/attachmentCreator.ts index c035ebc2f..cb0e748c2 100644 --- a/server/commands/attachmentCreator.ts +++ b/server/commands/attachmentCreator.ts @@ -48,7 +48,7 @@ export default async function attachmentCreator({ if ("url" in rest) { const { url } = rest; - const res = await FileStorage.uploadFromUrl(url, key, acl); + const res = await FileStorage.storeFromUrl(url, key, acl); if (!res) { return; @@ -69,7 +69,7 @@ export default async function attachmentCreator({ ); } else { const { buffer, type } = rest; - await FileStorage.upload({ + await FileStorage.store({ body: buffer, contentType: type, contentLength: buffer.length, diff --git a/server/env.ts b/server/env.ts index 35459e035..4708e769e 100644 --- a/server/env.ts +++ b/server/env.ts @@ -545,12 +545,14 @@ export class Environment { this.toOptionalNumber(process.env.RATE_LIMITER_DURATION_WINDOW) ?? 60; /** - * Set max allowed upload size for file attachments. + * @deprecated Set max allowed upload size for file attachments. */ @IsOptional() @IsNumber() - public AWS_S3_UPLOAD_MAX_SIZE = - this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? 100000000; + @Deprecated("Use FILE_STORAGE_UPLOAD_MAX_SIZE instead") + public AWS_S3_UPLOAD_MAX_SIZE = this.toOptionalNumber( + process.env.AWS_S3_UPLOAD_MAX_SIZE + ); /** * Access key ID for AWS S3. @@ -612,6 +614,28 @@ export class Environment { @IsOptional() public AWS_S3_ACL = process.env.AWS_S3_ACL ?? "private"; + /** + * Which file storage system to use + */ + @IsIn(["local", "s3"]) + public FILE_STORAGE = this.toOptionalString(process.env.FILE_STORAGE) ?? "s3"; + + /** + * Set default root dir path for local file storage + */ + public FILE_STORAGE_LOCAL_ROOT_DIR = + this.toOptionalString(process.env.FILE_STORAGE_LOCAL_ROOT_DIR) ?? + "/var/lib/outline/data"; + + /** + * Set max allowed upload size for file attachments. + */ + @IsNumber() + public FILE_STORAGE_UPLOAD_MAX_SIZE = + this.toOptionalNumber(process.env.FILE_STORAGE_UPLOAD_MAX_SIZE) ?? + this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? + 100000000; + /** * Because imports can be much larger than regular file attachments and are * deleted automatically we allow an optional separate limit on the size of @@ -620,7 +644,7 @@ export class Environment { @IsNumber() public MAXIMUM_IMPORT_SIZE = Math.max( this.toOptionalNumber(process.env.MAXIMUM_IMPORT_SIZE) ?? 100000000, - this.AWS_S3_UPLOAD_MAX_SIZE + this.FILE_STORAGE_UPLOAD_MAX_SIZE ); /** diff --git a/server/migrations/20230920032853-add-key-index.js b/server/migrations/20230920032853-add-key-index.js new file mode 100644 index 000000000..50c11a400 --- /dev/null +++ b/server/migrations/20230920032853-add-key-index.js @@ -0,0 +1,11 @@ +"use strict"; + +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addIndex("attachments", ["key"]); + }, + + async down(queryInterface) { + await queryInterface.removeIndex("attachments", ["key"]); + }, +}; diff --git a/server/models/Attachment.test.ts b/server/models/Attachment.test.ts new file mode 100644 index 000000000..b220e705f --- /dev/null +++ b/server/models/Attachment.test.ts @@ -0,0 +1,10 @@ +import { buildAttachment } from "@server/test/factories"; +import Attachment from "./Attachment"; + +describe("#findByKey", () => { + it("should return the correct attachment given a key", async () => { + const attachment = await buildAttachment(); + const found = await Attachment.findByKey(attachment.key); + expect(found?.id).toBe(attachment.id); + }); +}); diff --git a/server/models/Attachment.ts b/server/models/Attachment.ts index 0fe297e13..49f1c2a2b 100644 --- a/server/models/Attachment.ts +++ b/server/models/Attachment.ts @@ -1,4 +1,7 @@ +import { createReadStream } from "fs"; import path from "path"; +import { File } from "formidable"; +import JWT from "jsonwebtoken"; import { QueryTypes } from "sequelize"; import { BeforeDestroy, @@ -10,8 +13,13 @@ import { Table, DataType, IsNumeric, + BeforeUpdate, } from "sequelize-typescript"; +import env from "@server/env"; +import { AuthenticationError } from "@server/errors"; import FileStorage from "@server/storage/files"; +import { getJWTPayload } from "@server/utils/jwt"; +import { ValidateKey } from "@server/validation"; import Document from "./Document"; import Team from "./Team"; import User from "./User"; @@ -96,11 +104,11 @@ class Attachment extends IdModel { } /** - * Get a direct URL to the attachment in storage. Note that this will not work for private attachments, - * a signed URL must be used. + * Get a direct URL to the attachment in storage. Note that this will not work + * for private attachments, a signed URL must be used. */ get canonicalUrl() { - return encodeURI(`${FileStorage.getPublicEndpoint()}/${this.key}`); + return encodeURI(FileStorage.getUrlForKey(this.key)); } /** @@ -110,8 +118,31 @@ class Attachment extends IdModel { return FileStorage.getSignedUrl(this.key); } + /** + * Store the given file in storage at the location specified by the attachment key. + * If the attachment already exists, it will be overwritten. + * + * @param file The file to store + * @returns A promise resolving to the attachment + */ + async overwriteFile(file: File) { + return FileStorage.store({ + body: createReadStream(file.filepath), + contentLength: file.size, + contentType: this.contentType, + key: this.key, + acl: this.acl, + }); + } + // hooks + @BeforeUpdate + static async sanitizeKey(model: Attachment) { + model.key = ValidateKey.sanitize(model.key); + return model; + } + @BeforeDestroy static async deleteAttachmentFromS3(model: Attachment) { await FileStorage.deleteFile(model.key); @@ -141,6 +172,42 @@ class Attachment extends IdModel { return parseInt(result?.[0]?.total ?? "0", 10); } + /** + * Find an attachment given a JWT signature. + * + * @param sign - The signature that uniquely identifies an attachment + * @returns A promise resolving to attachment corresponding to the signature + * @throws {AuthenticationError} Invalid signature if the signature verification fails + */ + static async findBySignature(sign: string): Promise { + const payload = getJWTPayload(sign); + + if (payload.type !== "attachment") { + throw AuthenticationError("Invalid signature"); + } + + try { + JWT.verify(sign, env.SECRET_KEY); + } catch (err) { + throw AuthenticationError("Invalid signature"); + } + + return this.findByKey(payload.key); + } + + /** + * Find an attachment given a key + * + * @param key The key representing attachment file path + * @returns A promise resolving to attachment corresponding to the key + */ + static async findByKey(key: string): Promise { + return this.findOne({ + where: { key }, + rejectOnEmpty: true, + }); + } + // associations @BelongsTo(() => Team, "teamId") diff --git a/server/models/Team.ts b/server/models/Team.ts index 392604b48..8f91ccb4e 100644 --- a/server/models/Team.ts +++ b/server/models/Team.ts @@ -17,7 +17,6 @@ import { Is, DataType, IsUUID, - IsUrl, AllowNull, AfterUpdate, } from "sequelize-typescript"; @@ -40,6 +39,7 @@ import User from "./User"; import ParanoidModel from "./base/ParanoidModel"; import Fix from "./decorators/Fix"; import IsFQDN from "./validators/IsFQDN"; +import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath"; import Length from "./validators/Length"; import NotContainsUrl from "./validators/NotContainsUrl"; @@ -97,7 +97,7 @@ class Team extends ParanoidModel { defaultCollectionId: string | null; @AllowNull - @IsUrl + @IsUrlOrRelativePath @Length({ max: 4096, msg: "avatarUrl must be 4096 characters or less" }) @Column(DataType.STRING) get avatarUrl() { diff --git a/server/models/User.ts b/server/models/User.ts index 020e4058b..d8afe4370 100644 --- a/server/models/User.ts +++ b/server/models/User.ts @@ -18,7 +18,6 @@ import { HasMany, Scopes, IsDate, - IsUrl, AllowNull, AfterUpdate, } from "sequelize-typescript"; @@ -52,6 +51,7 @@ import Encrypted, { getEncryptedColumn, } from "./decorators/Encrypted"; import Fix from "./decorators/Fix"; +import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath"; import Length from "./validators/Length"; import NotContainsUrl from "./validators/NotContainsUrl"; @@ -182,7 +182,7 @@ class User extends ParanoidModel { language: string; @AllowNull - @IsUrl + @IsUrlOrRelativePath @Length({ max: 4096, msg: "avatarUrl must be less than 4096 characters" }) @Column(DataType.STRING) get avatarUrl() { diff --git a/server/models/helpers/AttachmentHelper.ts b/server/models/helpers/AttachmentHelper.ts index 6933ff07d..e99c6db32 100644 --- a/server/models/helpers/AttachmentHelper.ts +++ b/server/models/helpers/AttachmentHelper.ts @@ -70,7 +70,7 @@ export default class AttachmentHelper { case AttachmentPreset.Avatar: case AttachmentPreset.DocumentAttachment: default: - return env.AWS_S3_UPLOAD_MAX_SIZE; + return env.FILE_STORAGE_UPLOAD_MAX_SIZE; } } } diff --git a/server/models/validators/IsUrlOrRelativePath.ts b/server/models/validators/IsUrlOrRelativePath.ts new file mode 100644 index 000000000..8cec74aef --- /dev/null +++ b/server/models/validators/IsUrlOrRelativePath.ts @@ -0,0 +1,23 @@ +import { isURL } from "class-validator"; +import { addAttributeOptions } from "sequelize-typescript"; + +/** + * A decorator that validates that a string is a url or relative path. + */ +export default function IsUrlOrRelativePath(target: any, propertyName: string) { + return addAttributeOptions(target, propertyName, { + validate: { + validUrlOrPath(value: string) { + if ( + value && + !isURL(value, { + require_host: false, + require_protocol: false, + }) + ) { + throw new Error("Must be a URL or relative path"); + } + }, + }, + }); +} diff --git a/server/policies/attachment.ts b/server/policies/attachment.ts index 0cf2f82d1..6c486ed1f 100644 --- a/server/policies/attachment.ts +++ b/server/policies/attachment.ts @@ -9,7 +9,7 @@ allow(User, "createAttachment", Team, (user, team) => { }); allow(User, "read", Attachment, (actor, attachment) => { - if (!attachment || attachment.teamId !== actor.teamId) { + if (!attachment || !actor || attachment.teamId !== actor.teamId) { return false; } if (actor.isAdmin) { diff --git a/server/queues/tasks/ExportTask.ts b/server/queues/tasks/ExportTask.ts index 65ee136ca..b2d3996de 100644 --- a/server/queues/tasks/ExportTask.ts +++ b/server/queues/tasks/ExportTask.ts @@ -90,7 +90,7 @@ export default abstract class ExportTask extends BaseTask { }); const stat = await fs.promises.stat(filePath); - const url = await FileStorage.upload({ + const url = await FileStorage.store({ body: fs.createReadStream(filePath), contentLength: stat.size, contentType: "application/zip", diff --git a/server/queues/tasks/UploadTeamAvatarTask.ts b/server/queues/tasks/UploadTeamAvatarTask.ts index 657c71e4a..3301f4f9a 100644 --- a/server/queues/tasks/UploadTeamAvatarTask.ts +++ b/server/queues/tasks/UploadTeamAvatarTask.ts @@ -20,14 +20,14 @@ export default class UploadTeamAvatarTask extends BaseTask { rejectOnEmpty: true, }); - const res = await FileStorage.uploadFromUrl( + const res = await FileStorage.storeFromUrl( props.avatarUrl, `avatars/${team.id}/${uuidv4()}`, "public-read" ); if (res?.url) { - await team.update({ avatarUrl: res?.url }); + await team.update({ avatarUrl: res.url }); } } diff --git a/server/queues/tasks/UploadUserAvatarTask.ts b/server/queues/tasks/UploadUserAvatarTask.ts index 35744c2b3..541ca1c38 100644 --- a/server/queues/tasks/UploadUserAvatarTask.ts +++ b/server/queues/tasks/UploadUserAvatarTask.ts @@ -20,14 +20,14 @@ export default class UploadUserAvatarTask extends BaseTask { rejectOnEmpty: true, }); - const res = await FileStorage.uploadFromUrl( + const res = await FileStorage.storeFromUrl( props.avatarUrl, `avatars/${user.id}/${uuidv4()}`, "public-read" ); if (res?.url) { - await user.update({ avatarUrl: res?.url }); + await user.update({ avatarUrl: res.url }); } } diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts index d00a84ce4..8552ad6a4 100644 --- a/server/routes/api/attachments/attachments.ts +++ b/server/routes/api/attachments/attachments.ts @@ -99,7 +99,7 @@ router.post( ctx.body = { data: { - uploadUrl: FileStorage.getPublicEndpoint(), + uploadUrl: FileStorage.getUploadUrl(), form: { "Cache-Control": "max-age=31557600", "Content-Type": contentType, diff --git a/server/routes/auth/index.test.ts b/server/routes/auth/index.test.ts index fef9f124b..f0aef1adf 100644 --- a/server/routes/auth/index.test.ts +++ b/server/routes/auth/index.test.ts @@ -13,7 +13,8 @@ describe("auth/redirect", () => { } ); expect(res.status).toEqual(302); - expect(res.headers.get("location").endsWith("/home")).toBeTruthy(); + expect(res.headers.get("location")).not.toBeNull(); + expect(res.headers.get("location")!.endsWith("/home")).toBeTruthy(); }); it("should redirect to first collection", async () => { @@ -28,6 +29,7 @@ describe("auth/redirect", () => { } ); expect(res.status).toEqual(302); - expect(res.headers.get("location").endsWith(collection.url)).toBeTruthy(); + expect(res.headers.get("location")).not.toBeNull(); + expect(res.headers.get("location")!.endsWith(collection.url)).toBeTruthy(); }); }); diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts index 5e7c85229..accd21279 100644 --- a/server/storage/files/BaseStorage.ts +++ b/server/storage/files/BaseStorage.ts @@ -1,3 +1,4 @@ +import { Blob } from "buffer"; import { Readable } from "stream"; import { PresignedPost } from "aws-sdk/clients/s3"; import env from "@server/env"; @@ -5,6 +6,9 @@ import Logger from "@server/logging/Logger"; import fetch from "@server/utils/fetch"; export default abstract class BaseStorage { + /** The default number of seconds until a signed URL expires. */ + public static defaultSignedUrlExpires = 60; + /** * Returns a presigned post for uploading files to the storage provider. * @@ -19,7 +23,7 @@ export default abstract class BaseStorage { acl: string, maxUploadSize: number, contentType: string - ): Promise; + ): Promise>; /** * Returns a stream for reading a file from the storage provider. @@ -29,19 +33,20 @@ export default abstract class BaseStorage { public abstract getFileStream(key: string): NodeJS.ReadableStream | null; /** - * Returns a buffer of a file from the storage provider. - * - * @param key The path to the file - */ - public abstract getFileBuffer(key: string): Promise; - - /** - * Returns the public endpoint for the storage provider. + * Returns the upload URL for the storage provider. * * @param isServerUpload Whether the upload is happening on the server or not - * @returns The public endpoint as a string + * @returns {string} The upload URL */ - public abstract getPublicEndpoint(isServerUpload?: boolean): string; + public abstract getUploadUrl(isServerUpload?: boolean): string; + + /** + * Returns the download URL for a given file. + * + * @param key The path to the file + * @returns {string} The download URL for the file + */ + public abstract getUrlForKey(key: string): string; /** * Returns a signed URL for a file from the storage provider. @@ -55,7 +60,7 @@ export default abstract class BaseStorage { ): Promise; /** - * Upload a file to the storage provider. + * Store a file in the storage provider. * * @param body The file body * @param contentLength The content length of the file @@ -64,7 +69,7 @@ export default abstract class BaseStorage { * @param acl The ACL to use * @returns The URL of the file */ - public abstract upload({ + public abstract store({ body, contentLength, contentType, @@ -72,12 +77,35 @@ export default abstract class BaseStorage { acl, }: { body: Buffer | Uint8Array | Blob | string | Readable; - contentLength: number; - contentType: string; + contentLength?: number; + contentType?: string; key: string; - acl: string; + acl?: string; }): Promise; + /** + * Returns a buffer of a file from the storage provider. + * + * @param key The path to the file + */ + public async getFileBuffer(key: string) { + const stream = this.getFileStream(key); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + if (!stream) { + return reject(new Error("No stream available")); + } + + stream.on("data", (d) => { + chunks.push(d); + }); + stream.once("end", () => { + resolve(Buffer.concat(chunks)); + }); + stream.once("error", reject); + }); + } + /** * Upload a file to the storage provider directly from a remote or base64 encoded URL. * @@ -86,7 +114,7 @@ export default abstract class BaseStorage { * @param acl The ACL to use * @returns A promise that resolves when the file is uploaded */ - public async uploadFromUrl( + public async storeFromUrl( url: string, key: string, acl: string @@ -98,7 +126,7 @@ export default abstract class BaseStorage { } | undefined > { - const endpoint = this.getPublicEndpoint(true); + const endpoint = this.getUploadUrl(true); if (url.startsWith("/api") || url.startsWith(endpoint)) { return; } @@ -115,7 +143,7 @@ export default abstract class BaseStorage { const res = await fetch(url, { follow: 3, redirect: "follow", - size: env.AWS_S3_UPLOAD_MAX_SIZE, + size: env.FILE_STORAGE_UPLOAD_MAX_SIZE, timeout: 10000, }); @@ -143,7 +171,7 @@ export default abstract class BaseStorage { } try { - const result = await this.upload({ + const result = await this.store({ body: buffer, contentLength, contentType, diff --git a/server/storage/files/LocalStorage.ts b/server/storage/files/LocalStorage.ts new file mode 100644 index 000000000..e0297ad6f --- /dev/null +++ b/server/storage/files/LocalStorage.ts @@ -0,0 +1,120 @@ +import { Blob } from "buffer"; +import { + ReadStream, + closeSync, + createReadStream, + createWriteStream, + existsSync, + openSync, +} from "fs"; +import { mkdir, unlink } from "fs/promises"; +import path from "path"; +import { Readable } from "stream"; +import invariant from "invariant"; +import JWT from "jsonwebtoken"; +import env from "@server/env"; +import Logger from "@server/logging/Logger"; +import BaseStorage from "./BaseStorage"; + +export default class LocalStorage extends BaseStorage { + public async getPresignedPost( + key: string, + acl: string, + maxUploadSize: number, + contentType = "image" + ) { + return Promise.resolve({ + url: this.getUrlForKey(key), + fields: { + key, + acl, + maxUploadSize, + contentType, + }, + } as any); + } + + public getUploadUrl() { + return "/api/files.create"; + } + + public getUrlForKey(key: string): string { + return `/api/files.get?key=${key}`; + } + + public store = async ({ + body, + key, + }: { + body: string | ReadStream | Buffer | Uint8Array | Blob; + contentLength?: number; + contentType?: string; + key: string; + acl?: string; + }) => { + const subdir = key.split("/").slice(0, -1).join("/"); + if (!existsSync(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir))) { + await mkdir(path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, subdir), { + recursive: true, + }); + } + + let src: NodeJS.ReadableStream; + if (body instanceof ReadStream) { + src = body; + } else if (body instanceof Blob) { + src = Readable.from(Buffer.from(await body.arrayBuffer())); + } else { + src = Readable.from(body); + } + + const destPath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key); + closeSync(openSync(destPath, "w")); + const dest = createWriteStream(destPath); + src.pipe(dest); + + return new Promise((resolve, reject) => { + src.once("end", () => resolve(this.getUrlForKey(key))); + src.once("err", (err) => { + dest.end(); + reject(err); + }); + }); + }; + + public async deleteFile(key: string) { + const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key); + try { + await unlink(filePath); + } catch (err) { + Logger.warn(`Couldn't delete ${filePath}`, err); + } + } + + public getSignedUrl = async ( + key: string, + expiresIn = LocalStorage.defaultSignedUrlExpires + ) => { + const sig = JWT.sign( + { + key, + type: "attachment", + }, + env.SECRET_KEY, + { + expiresIn, + } + ); + return Promise.resolve(`/api/files.get?sig=${sig}`); + }; + + public getFileStream(key: string) { + invariant( + env.FILE_STORAGE_LOCAL_ROOT_DIR, + "FILE_STORAGE_LOCAL_ROOT_DIR is required" + ); + + const filePath = path.join(env.FILE_STORAGE_LOCAL_ROOT_DIR, key); + return createReadStream(filePath); + } +} diff --git a/server/storage/files/S3Storage.ts b/server/storage/files/S3Storage.ts index ea2b6cf39..d1ba71386 100644 --- a/server/storage/files/S3Storage.ts +++ b/server/storage/files/S3Storage.ts @@ -47,7 +47,7 @@ export default class S3Storage extends BaseStorage { ); } - public getPublicEndpoint(isServerUpload?: boolean) { + private getPublicEndpoint(isServerUpload?: boolean) { if (env.AWS_S3_ACCELERATE_URL) { return env.AWS_S3_ACCELERATE_URL; } @@ -78,7 +78,15 @@ export default class S3Storage extends BaseStorage { }`; } - public upload = async ({ + public getUploadUrl(isServerUpload?: boolean) { + return this.getPublicEndpoint(isServerUpload); + } + + public getUrlForKey(key: string): string { + return `${this.getPublicEndpoint()}/${key}`; + } + + public store = async ({ body, contentLength, contentType, @@ -86,10 +94,10 @@ export default class S3Storage extends BaseStorage { acl, }: { body: S3.Body; - contentLength: number; - contentType: string; + contentLength?: number; + contentType?: string; key: string; - acl: string; + acl?: string; }) => { invariant( env.AWS_S3_UPLOAD_BUCKET_NAME, @@ -125,7 +133,10 @@ export default class S3Storage extends BaseStorage { .promise(); } - public getSignedUrl = async (key: string, expiresIn = 60) => { + public getSignedUrl = async ( + key: string, + expiresIn = S3Storage.defaultSignedUrlExpires + ) => { const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/); const params = { Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME, @@ -170,26 +181,6 @@ export default class S3Storage extends BaseStorage { return null; } - public async getFileBuffer(key: string) { - invariant( - env.AWS_S3_UPLOAD_BUCKET_NAME, - "AWS_S3_UPLOAD_BUCKET_NAME is required" - ); - - const response = await this.client - .getObject({ - Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME, - Key: key, - }) - .promise(); - - if (response.Body) { - return response.Body as Blob; - } - - throw new Error("Error getting file buffer from S3"); - } - private client: AWS.S3; private getEndpoint() { diff --git a/server/storage/files/__mocks__/index.ts b/server/storage/files/__mocks__/index.ts index 07ffc8db2..0c8716df8 100644 --- a/server/storage/files/__mocks__/index.ts +++ b/server/storage/files/__mocks__/index.ts @@ -1,7 +1,9 @@ export default { upload: jest.fn().mockReturnValue("/endpoint/key"), - getPublicEndpoint: jest.fn().mockReturnValue("http://mock"), + getUploadUrl: jest.fn().mockReturnValue("http://mock/create"), + + getUrlForKey: jest.fn().mockReturnValue("http://mock/get"), getSignedUrl: jest.fn().mockReturnValue("http://s3mock"), diff --git a/server/storage/files/index.ts b/server/storage/files/index.ts index c87b0b50c..08e11a8ac 100644 --- a/server/storage/files/index.ts +++ b/server/storage/files/index.ts @@ -1,3 +1,8 @@ +import env from "@server/env"; +import LocalStorage from "./LocalStorage"; import S3Storage from "./S3Storage"; -export default new S3Storage(); +const storage = + env.FILE_STORAGE === "local" ? new LocalStorage() : new S3Storage(); + +export default storage; diff --git a/server/test/TestServer.ts b/server/test/TestServer.ts new file mode 100644 index 000000000..1a2df84b2 --- /dev/null +++ b/server/test/TestServer.ts @@ -0,0 +1,84 @@ +import http from "http"; +import { AddressInfo } from "net"; +import Koa from "koa"; +// eslint-disable-next-line no-restricted-imports +import nodeFetch from "node-fetch"; + +class TestServer { + private server: http.Server; + private listener?: Promise | null; + + constructor(app: Koa) { + this.server = http.createServer(app.callback() as any); + } + + get address(): string { + const { port } = this.server.address() as AddressInfo; + return `http://localhost:${port}`; + } + + listen() { + if (!this.listener) { + this.listener = new Promise((resolve, reject) => { + this.server + .listen(0, () => resolve()) + .on("error", (err) => reject(err)); + }); + } + + return this.listener; + } + + fetch(path: string, opts: any) { + return this.listen().then(() => { + const url = `${this.address}${path}`; + const options = Object.assign({ headers: {} }, opts); + const contentType = + options.headers["Content-Type"] ?? options.headers["content-type"]; + // automatic JSON encoding + if (!contentType && typeof options.body === "object") { + options.headers["Content-Type"] = "application/json"; + options.body = JSON.stringify(options.body); + } + + return nodeFetch(url, options); + }); + } + + close() { + this.listener = null; + return new Promise((resolve, reject) => { + this.server.close((err) => (err ? reject(err) : resolve())); + }); + } + + delete(path: string, options?: any) { + return this.fetch(path, { ...options, method: "DELETE" }); + } + + get(path: string, options?: any) { + return this.fetch(path, { ...options, method: "GET" }); + } + + head(path: string, options?: any) { + return this.fetch(path, { ...options, method: "HEAD" }); + } + + options(path: string, options?: any) { + return this.fetch(path, { ...options, method: "OPTIONS" }); + } + + patch(path: string, options?: any) { + return this.fetch(path, { ...options, method: "PATCH" }); + } + + post(path: string, options?: any) { + return this.fetch(path, { ...options, method: "POST" }); + } + + put(path: string, options?: any) { + return this.fetch(path, { ...options, method: "PUT" }); + } +} + +export default TestServer; diff --git a/server/test/env.ts b/server/test/env.ts index d04695d9b..d5fab9ce7 100644 --- a/server/test/env.ts +++ b/server/test/env.ts @@ -18,6 +18,8 @@ env.OIDC_USERINFO_URI = "http://localhost/userinfo"; env.RATE_LIMITER_ENABLED = false; +env.FILE_STORAGE = "local"; +env.FILE_STORAGE_LOCAL_ROOT_DIR = "/tmp"; env.IFRAMELY_API_KEY = "123"; if (process.env.DATABASE_URL_TEST) { diff --git a/server/test/factories.ts b/server/test/factories.ts index 0f1e5597c..8a67f5d27 100644 --- a/server/test/factories.ts +++ b/server/test/factories.ts @@ -33,6 +33,7 @@ import { SearchQuery, Pin, } from "@server/models"; +import AttachmentHelper from "@server/models/helpers/AttachmentHelper"; export async function buildApiKey(overrides: Partial = {}) { if (!overrides.userId) { @@ -420,7 +421,10 @@ export async function buildFileOperation( }); } -export async function buildAttachment(overrides: Partial = {}) { +export async function buildAttachment( + overrides: Partial = {}, + fileName?: string +) { if (!overrides.teamId) { const team = await buildTeam(); overrides.teamId = team.id; @@ -441,11 +445,14 @@ export async function buildAttachment(overrides: Partial = {}) { overrides.documentId = document.id; } + const id = uuidv4(); + const acl = overrides.acl || "public-read"; + const name = fileName || faker.system.fileName(); return Attachment.create({ - key: `uploads/key/to/${faker.system.fileName}.png`, + key: AttachmentHelper.getKey({ acl, id, name, userId: overrides.userId }), contentType: "image/png", size: 100, - acl: "public-read", + acl, createdAt: new Date("2018-01-02T00:00:00.000Z"), updatedAt: new Date("2018-01-02T00:00:00.000Z"), ...overrides, diff --git a/server/test/support.ts b/server/test/support.ts index 14af26807..458a35931 100644 --- a/server/test/support.ts +++ b/server/test/support.ts @@ -1,22 +1,22 @@ import { faker } from "@faker-js/faker"; -import TestServer from "fetch-test-server"; import sharedEnv from "@shared/env"; import env from "@server/env"; import onerror from "@server/onerror"; import webService from "@server/services/web"; import { sequelize } from "@server/storage/database"; +import TestServer from "./TestServer"; export function getTestServer() { const app = webService(); onerror(app); - const server = new TestServer(app.callback()); + const server = new TestServer(app); - server.disconnect = async () => { + const disconnect = async () => { await sequelize.close(); - server.close(); + return server.close(); }; - afterAll(server.disconnect); + afterAll(disconnect); return server; } diff --git a/server/typings/index.d.ts b/server/typings/index.d.ts index a6fad124b..c1bdffcb8 100644 --- a/server/typings/index.d.ts +++ b/server/typings/index.d.ts @@ -6,8 +6,6 @@ declare module "formidable/lib/file"; declare module "oy-vey"; -declare module "fetch-test-server"; - declare module "dotenv"; declare module "email-providers" { diff --git a/server/utils/jwt.ts b/server/utils/jwt.ts index e3ccdb949..64f201a4e 100644 --- a/server/utils/jwt.ts +++ b/server/utils/jwt.ts @@ -4,7 +4,7 @@ import JWT from "jsonwebtoken"; import { Team, User } from "@server/models"; import { AuthenticationError } from "../errors"; -function getJWTPayload(token: string) { +export function getJWTPayload(token: string) { let payload; try { diff --git a/server/validation.test.ts b/server/validation.test.ts new file mode 100644 index 000000000..826b2e652 --- /dev/null +++ b/server/validation.test.ts @@ -0,0 +1,41 @@ +import { v4 as uuidv4 } from "uuid"; +import { ValidateKey } from "./validation"; + +describe("#ValidateKey.isValid", () => { + it("should return false if number of key components are not equal to 4", () => { + expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}`)).toBe(false); + expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}/foo/bar`)).toBe( + false + ); + }); + + it("should return false if the first key component is neither 'public' nor 'uploads' ", () => { + expect(ValidateKey.isValid(`foo/${uuidv4()}/${uuidv4()}/bar.png`)).toBe( + false + ); + }); + + it("should return false if second and third key components are not UUID", () => { + expect(ValidateKey.isValid(`uploads/foo/${uuidv4()}/bar.png`)).toBe(false); + expect(ValidateKey.isValid(`uploads/${uuidv4()}/foo/bar.png`)).toBe(false); + }); + + it("should return true successfully validating key", () => { + expect(ValidateKey.isValid(`public/${uuidv4()}/${uuidv4()}/foo.png`)).toBe( + true + ); + expect(ValidateKey.isValid(`uploads/${uuidv4()}/${uuidv4()}/foo.png`)).toBe( + true + ); + }); +}); + +describe("#ValidateKey.sanitize", () => { + it("should sanitize malicious looking keys", () => { + const uuid1 = uuidv4(); + const uuid2 = uuidv4(); + expect( + ValidateKey.sanitize(`public/${uuid1}/${uuid2}/~\.\u0000\malicious_key`) + ).toEqual(`public/${uuid1}/${uuid2}/~.malicious_key`); + }); +}); diff --git a/server/validation.ts b/server/validation.ts index a05f09e7e..c8eaa7b05 100644 --- a/server/validation.ts +++ b/server/validation.ts @@ -1,6 +1,8 @@ import isArrayLike from "lodash/isArrayLike"; +import sanitize from "sanitize-filename"; import { Primitive } from "utility-types"; import validator from "validator"; +import isIn from "validator/lib/isIn"; import isUUID from "validator/lib/isUUID"; import parseMentionUrl from "@shared/utils/parseMentionUrl"; import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers"; @@ -170,6 +172,30 @@ export const assertCollectionPermission = ( assertIn(value, [...Object.values(CollectionPermission), null], message); }; +export class ValidateKey { + public static isValid = (key: string) => { + const parts = key.split("/").slice(0, -1); + return ( + parts.length === 3 && + isIn(parts[0], ["uploads", "public"]) && + isUUID(parts[1]) && + isUUID(parts[2]) + ); + }; + + public static sanitize = (key: string) => { + const [filename] = key.split("/").slice(-1); + return key + .split("/") + .slice(0, -1) + .join("/") + .concat(`/${sanitize(filename)}`); + }; + + public static message = + "Must be of the form uploads/// or public///"; +} + export class ValidateDocumentId { /** * Checks if documentId is valid. A valid documentId is either diff --git a/yarn.lock b/yarn.lock index 14a642ca7..243498280 100644 --- a/yarn.lock +++ b/yarn.lock @@ -191,14 +191,7 @@ dependencies: "@babel/types" "^7.21.0" -"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.14.5", "@babel/helper-module-imports@^7.18.6", "@babel/helper-module-imports@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz#1a8f4c9f4027d23f520bd76b364d44434a72660c" - integrity sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg== - dependencies: - "@babel/types" "^7.22.5" - -"@babel/helper-module-imports@^7.22.15": +"@babel/helper-module-imports@^7.0.0", "@babel/helper-module-imports@^7.10.4", "@babel/helper-module-imports@^7.12.13", "@babel/helper-module-imports@^7.14.5", "@babel/helper-module-imports@^7.18.6", "@babel/helper-module-imports@^7.22.15", "@babel/helper-module-imports@^7.22.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz#16146307acdc40cc00c3b2c647713076464bdbf0" integrity sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w== @@ -279,12 +272,7 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== -"@babel/helper-validator-identifier@^7.19.1", "@babel/helper-validator-identifier@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" - integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ== - -"@babel/helper-validator-identifier@^7.22.19": +"@babel/helper-validator-identifier@^7.19.1", "@babel/helper-validator-identifier@^7.22.19", "@babel/helper-validator-identifier@^7.22.5": version "7.22.20" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== @@ -1095,16 +1083,7 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.22.5", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe" - integrity sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA== - dependencies: - "@babel/helper-string-parser" "^7.22.5" - "@babel/helper-validator-identifier" "^7.22.5" - to-fast-properties "^2.0.0" - -"@babel/types@^7.22.15": +"@babel/types@^7.0.0", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.20.0", "@babel/types@^7.20.2", "@babel/types@^7.20.5", "@babel/types@^7.20.7", "@babel/types@^7.21.0", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0": version "7.22.19" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.19.tgz#7425343253556916e440e662bb221a93ddb75684" integrity sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg== @@ -2921,6 +2900,13 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/form-data@^2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@types/form-data/-/form-data-2.5.0.tgz#5025f7433016f923348434c40006d9a797c1b0e8" + integrity sha512-23/wYiuckYYtFpL+4RPWiWmRQH2BjFuqCUi2+N3amB1a1Drv+i/byTrGvlLwRVLFNAZbwpbQ7JvTK+VCAPMbcg== + dependencies: + form-data "*" + "@types/formidable@^2.0.5", "@types/formidable@^2.0.6": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-2.0.6.tgz#811ed3cd8a8a7675e02420b3f861c317e055376a" @@ -3241,7 +3227,12 @@ "@types/node" "*" form-data "^4.0.0" -"@types/node@*", "@types/node@18.0.6", "@types/node@>=10.0.0", "@types/node@>=13.7.0": +"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=13.7.0": + version "20.5.9" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.5.9.tgz#a70ec9d8fa0180a314c3ede0e20ea56ff71aed9a" + integrity sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ== + +"@types/node@18.0.6": version "18.0.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.6.tgz#0ba49ac517ad69abe7a1508bc9b3a5483df9d5d7" integrity sha512-/xUq6H2aQm261exT6iZTMifUySEt4GR5KX8eYyY+C4MSNPqSh9oNIP7tz2GLKTlFaiBbgZNxffoR3CVRG+cljw== @@ -3437,6 +3428,13 @@ dependencies: "@types/node" "*" +"@types/sanitize-filename@^1.6.3": + version "1.6.3" + resolved "https://registry.yarnpkg.com/@types/sanitize-filename/-/sanitize-filename-1.6.3.tgz#182ebd5658fbd3fe36bcb771daad8b2623371705" + integrity sha512-1dAV8Va7KsiXNAstV2JmF4CRVG3Fsyl+VnBw87C9cCMccekpOqJBezS7MUnHYPChNAFee1WakwBXdfn7QJxzVg== + dependencies: + sanitize-filename "*" + "@types/scheduler@*": version "0.16.2" resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39" @@ -3992,7 +3990,7 @@ async@^3.2.3: asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== at-least-node@^1.0.0: version "1.0.0" @@ -5689,7 +5687,7 @@ delay@^5.0.0: delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== delegates@^1.0.0: version "1.0.0" @@ -6807,14 +6805,6 @@ fetch-retry@^5.0.5: resolved "https://registry.yarnpkg.com/fetch-retry/-/fetch-retry-5.0.5.tgz#61079b816b6651d88a022ebd45d51d83aa72b521" integrity sha512-q9SvpKH5Ka6h7X2C6r1sP31pQoeDb3o6/R9cg21ahfPAqbIOkW9tus1dXfwYb6G6dOI4F7nVS4Q+LSssBGIz0A== -fetch-test-server@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/fetch-test-server/-/fetch-test-server-1.2.0.tgz#65f23af1d030c293249a49bbd1b51e45fc68eb69" - integrity sha512-KjxYDGGfVC/paLya7UN+AFxb3wt0Mj79eOBjlpRdn9B1o0uo3vJCC9VGVTd17Q5kiBx+HvglP/BzBi8BZs18sA== - dependencies: - debug "^3.1.0" - node-fetch "^2.1.2" - fetch-with-proxy@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/fetch-with-proxy/-/fetch-with-proxy-3.0.1.tgz#29ed6d0e2550ef999d40b18de2ba476af4b7dee4" @@ -6944,19 +6934,19 @@ for-each@^0.3.3: dependencies: is-callable "^1.1.3" -form-data@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" - integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== +form-data@*, form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== +form-data@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" + integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" @@ -9877,7 +9867,7 @@ node-addon-api@^6.1.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== -node-fetch@2.6.7, node-fetch@2.7.0, node-fetch@^2.1.2, node-fetch@^2.6.1, node-fetch@^2.6.12: +node-fetch@2.6.7, node-fetch@2.7.0, node-fetch@^2.6.1, node-fetch@^2.6.12: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -11793,6 +11783,13 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sanitize-filename@*, sanitize-filename@^1.6.3: + version "1.6.3" + resolved "https://registry.yarnpkg.com/sanitize-filename/-/sanitize-filename-1.6.3.tgz#755ebd752045931977e30b2025d340d7c9090378" + integrity sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg== + dependencies: + truncate-utf8-bytes "^1.0.0" + sanitizer@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/sanitizer/-/sanitizer-0.1.3.tgz#d4f0af7475d9a7baf2a9e5a611718baa178a39e1" @@ -12839,7 +12836,7 @@ tr46@^4.1.1: tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== tree-kill@^1.2.2: version "1.2.2" @@ -12851,6 +12848,13 @@ triple-beam@^1.3.0: resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== +truncate-utf8-bytes@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b" + integrity sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ== + dependencies: + utf8-byte-length "^1.0.1" + ts-dedent@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.2.0.tgz#39e4bd297cd036292ae2394eb3412be63f563bb5" @@ -13194,6 +13198,11 @@ use-sidecar@^1.1.2: detect-node-es "^1.1.0" tslib "^2.0.0" +utf8-byte-length@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz#f45f150c4c66eee968186505ab93fcbb8ad6bf61" + integrity sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA== + utf8@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96" @@ -13421,7 +13430,7 @@ walker@^1.0.8: webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== webidl-conversions@^4.0.2: version "4.0.2" @@ -13477,7 +13486,7 @@ whatwg-url@^12.0.0, whatwg-url@^12.0.1: whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0"