diff --git a/app/components/Editor.tsx b/app/components/Editor.tsx index 67f86a73d..6a59aac9d 100644 --- a/app/components/Editor.tsx +++ b/app/components/Editor.tsx @@ -8,6 +8,7 @@ import { mergeRefs } from "react-merge-refs"; import { Optional } from "utility-types"; import insertFiles from "@shared/editor/commands/insertFiles"; import { Heading } from "@shared/editor/lib/getHeadings"; +import { AttachmentPreset } from "@shared/types"; import { getDataTransferFiles } from "@shared/utils/files"; import parseDocumentSlug from "@shared/utils/parseDocumentSlug"; import { isInternalUrl } from "@shared/utils/urls"; @@ -135,6 +136,7 @@ function Editor(props: Props, ref: React.RefObject | null) { async (file: File) => { const result = await uploadFile(file, { documentId: id, + preset: AttachmentPreset.DocumentAttachment, }); return result.url; }, diff --git a/app/scenes/Settings/components/DropToImport.tsx b/app/scenes/Settings/components/DropToImport.tsx index a30b1dada..7cd167853 100644 --- a/app/scenes/Settings/components/DropToImport.tsx +++ b/app/scenes/Settings/components/DropToImport.tsx @@ -4,6 +4,7 @@ import * as React from "react"; import Dropzone from "react-dropzone"; import { useTranslation } from "react-i18next"; import styled from "styled-components"; +import { AttachmentPreset } from "@shared/types"; import Flex from "~/components/Flex"; import LoadingIndicator from "~/components/LoadingIndicator"; import useStores from "~/hooks/useStores"; @@ -39,6 +40,7 @@ function DropToImport({ disabled, onSubmit, children, format }: Props) { try { const attachment = await uploadFile(file, { name: file.name, + preset: AttachmentPreset.Import, }); await collections.import(attachment.id, format); onSubmit(); diff --git a/app/scenes/Settings/components/ImageUpload.tsx b/app/scenes/Settings/components/ImageUpload.tsx index 90b4806c7..0640e9f71 100644 --- a/app/scenes/Settings/components/ImageUpload.tsx +++ b/app/scenes/Settings/components/ImageUpload.tsx @@ -5,6 +5,7 @@ import * as React from "react"; import AvatarEditor from "react-avatar-editor"; import Dropzone from "react-dropzone"; import styled from "styled-components"; +import { AttachmentPreset } from "@shared/types"; import { AttachmentValidation } from "@shared/validations"; import RootStore from "~/stores/RootStore"; import Button from "~/components/Button"; @@ -67,7 +68,7 @@ class ImageUpload extends React.Component { }); const attachment = await uploadFile(compressed, { name: this.file.name, - public: true, + preset: AttachmentPreset.Avatar, }); this.props.onSuccess(attachment.url); } catch (err) { diff --git a/app/utils/files.ts b/app/utils/files.ts index cc9864be8..564e3393a 100644 --- a/app/utils/files.ts +++ b/app/utils/files.ts @@ -1,4 +1,5 @@ import invariant from "invariant"; +import { AttachmentPreset } from "@shared/types"; import { client } from "./ApiClient"; import Logger from "./Logger"; @@ -7,8 +8,8 @@ type UploadOptions = { name?: string; /** The document that this file was uploaded in, if any */ documentId?: string; - /** Whether the file should be public in cloud storage */ - public?: boolean; + /** The preset to use for attachment configuration */ + preset: AttachmentPreset; /** Callback will be passed a number between 0-1 as upload progresses */ onProgress?: (fractionComplete: number) => void; }; @@ -17,11 +18,12 @@ export const uploadFile = async ( file: File | Blob, options: UploadOptions = { name: "", + preset: AttachmentPreset.DocumentAttachment, } ) => { const name = file instanceof File ? file.name : options.name; const response = await client.post("/attachments.create", { - public: options.public, + preset: options.preset, documentId: options.documentId, contentType: file.type, size: file.size, diff --git a/server/env.ts b/server/env.ts index 833aa38e1..d7e4c115d 100644 --- a/server/env.ts +++ b/server/env.ts @@ -534,6 +534,20 @@ export class Environment { public RATE_LIMITER_DURATION_WINDOW = this.toOptionalNumber(process.env.RATE_LIMITER_DURATION_WINDOW) ?? 60; + /** + * Set max allowed upload size for file attachments. + */ + @IsOptional() + @IsNumber() + public AWS_S3_UPLOAD_MAX_SIZE = + this.toOptionalNumber(process.env.AWS_S3_UPLOAD_MAX_SIZE) ?? 10000000000; + + /** + * Set default AWS S3 ACL for file attachments. + */ + @IsOptional() + public AWS_S3_ACL = process.env.AWS_S3_ACL ?? "private"; + private toOptionalString(value: string | undefined) { return value ? value : undefined; } diff --git a/server/migrations/20221120151710-attachment-expiry.js b/server/migrations/20221120151710-attachment-expiry.js new file mode 100644 index 000000000..8d2567307 --- /dev/null +++ b/server/migrations/20221120151710-attachment-expiry.js @@ -0,0 +1,22 @@ +"use strict"; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.addColumn("attachments", "expiresAt", { + type: Sequelize.DATE, + allowNull: true, + transaction, + }); + await queryInterface.addIndex("attachments", ["expiresAt"], { + transaction + }); + }); + }, + down: async (queryInterface) => { + await queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.removeColumn("attachments", "expiresAt", { transaction }); + await queryInterface.removeIndex("attachments", ["expiresAt"], { transaction }); + }); + }, +}; diff --git a/server/models/Attachment.ts b/server/models/Attachment.ts index 1da319a8a..3fe8dc103 100644 --- a/server/models/Attachment.ts +++ b/server/models/Attachment.ts @@ -10,7 +10,12 @@ import { DataType, IsNumeric, } from "sequelize-typescript"; -import { publicS3Endpoint, deleteFromS3, getFileByKey } from "@server/utils/s3"; +import { + publicS3Endpoint, + deleteFromS3, + getFileByKey, + getSignedUrl, +} from "@server/utils/s3"; import Document from "./Document"; import Team from "./Team"; import User from "./User"; @@ -47,26 +52,59 @@ class Attachment extends IdModel { @Column lastAccessedAt: Date | null; + @Column + expiresAt: Date | null; + // getters + /** + * Get the original uploaded file name. + */ get name() { return path.parse(this.key).base; } - get redirectUrl() { - return `/api/attachments.redirect?id=${this.id}`; - } - + /** + * Whether the attachment is private or not. + */ get isPrivate() { return this.acl === "private"; } + /** + * Get the contents of this attachment as a Buffer + */ get buffer() { return getFileByKey(this.key); } + /** + * Get a url that can be used to download the attachment if the user has a valid session. + */ + get url() { + return this.isPrivate ? this.redirectUrl : this.canonicalUrl; + } + + /** + * Get a url that can be used to download a private attachment if the user has a valid session. + */ + get redirectUrl() { + return `/api/attachments.redirect?id=${this.id}`; + } + + /** + * Get a direct URL to the attachment in storage. Note that this will not work for private attachments, + * a signed URL must be used. + */ get canonicalUrl() { - return `${publicS3Endpoint()}/${this.key}`; + return encodeURI(`${publicS3Endpoint()}/${this.key}`); + } + + /** + * Get a signed URL with the default expirt to download the attachment from storage. + */ + get signedUrl() { + return getSignedUrl(this.key); } // hooks diff --git a/server/models/FileOperation.ts b/server/models/FileOperation.ts index 589669ffa..f1340d9b1 100644 --- a/server/models/FileOperation.ts +++ b/server/models/FileOperation.ts @@ -73,7 +73,13 @@ class FileOperation extends IdModel { expire = async function () { this.state = "expired"; - await deleteFromS3(this.key); + try { + await deleteFromS3(this.key); + } catch (err) { + if (err.retryable) { + throw err; + } + } await this.save(); }; diff --git a/server/models/helpers/AttachmentHelper.ts b/server/models/helpers/AttachmentHelper.ts new file mode 100644 index 000000000..f9a0d4821 --- /dev/null +++ b/server/models/helpers/AttachmentHelper.ts @@ -0,0 +1,76 @@ +import { addHours } from "date-fns"; +import { AttachmentPreset } from "@shared/types"; +import env from "@server/env"; + +export default class AttachmentHelper { + /** + * Get the upload location for the given upload details + * + * @param acl The ACL to use + * @param id The ID of the attachment + * @param name The name of the attachment + * @param userId The ID of the user uploading the attachment + */ + static getKey({ + acl, + id, + name, + userId, + }: { + acl: string; + id: string; + name: string; + userId: string; + }) { + const bucket = acl === "public-read" ? "public" : "uploads"; + const keyPrefix = `${bucket}/${userId}/${id}`; + return `${keyPrefix}/${name}`; + } + + /** + * Get the ACL to use for a given attachment preset + * + * @param preset The preset to use + * @returns A valid S3 ACL + */ + static presetToAcl(preset: AttachmentPreset) { + switch (preset) { + case AttachmentPreset.Avatar: + return "public-read"; + default: + return env.AWS_S3_ACL; + } + } + + /** + * Get the expiration time for a given attachment preset + * + * @param preset The preset to use + * @returns An expiration time + */ + static presetToExpiry(preset: AttachmentPreset) { + switch (preset) { + case AttachmentPreset.Import: + return addHours(new Date(), 24); + default: + return undefined; + } + } + + /** + * Get the maximum upload size for a given attachment preset + * + * @param preset The preset to use + * @returns The maximum upload size in bytes + */ + static presetToMaxUploadSize(preset: AttachmentPreset) { + switch (preset) { + case AttachmentPreset.Avatar: + return Math.min(1024 * 1024 * 5, env.AWS_S3_UPLOAD_MAX_SIZE); + case AttachmentPreset.Import: + return env.MAXIMUM_IMPORT_SIZE; + default: + return env.AWS_S3_UPLOAD_MAX_SIZE; + } + } +} diff --git a/server/presenters/attachment.ts b/server/presenters/attachment.ts new file mode 100644 index 000000000..ebb568c91 --- /dev/null +++ b/server/presenters/attachment.ts @@ -0,0 +1,12 @@ +import { Attachment } from "@server/models"; + +export default function present(attachment: Attachment) { + return { + documentId: attachment.documentId, + contentType: attachment.contentType, + name: attachment.name, + id: attachment.id, + url: attachment.url, + size: attachment.size, + }; +} diff --git a/server/presenters/index.ts b/server/presenters/index.ts index 894739d2d..bb3c2ab5f 100644 --- a/server/presenters/index.ts +++ b/server/presenters/index.ts @@ -1,4 +1,5 @@ import presentApiKey from "./apiKey"; +import presentAttachment from "./attachment"; import presentAuthenticationProvider from "./authenticationProvider"; import presentAvailableTeam from "./availableTeam"; import presentCollection from "./collection"; @@ -27,6 +28,7 @@ import presentWebhookSubscription from "./webhookSubscription"; export { presentApiKey, + presentAttachment, presentAuthenticationProvider, presentAvailableTeam, presentCollection, diff --git a/server/queues/tasks/CleanupExpiredAttachmentsTask.ts b/server/queues/tasks/CleanupExpiredAttachmentsTask.ts new file mode 100644 index 000000000..951bcab4b --- /dev/null +++ b/server/queues/tasks/CleanupExpiredAttachmentsTask.ts @@ -0,0 +1,31 @@ +import { Op } from "sequelize"; +import Logger from "@server/logging/Logger"; +import { Attachment } from "@server/models"; +import BaseTask, { TaskPriority } from "./BaseTask"; + +type Props = { + limit: number; +}; + +export default class CleanupExpiredAttachmentsTask extends BaseTask { + public async perform({ limit }: Props) { + Logger.info("task", `Deleting expired attachments…`); + const attachments = await Attachment.unscoped().findAll({ + where: { + expiresAt: { + [Op.lt]: new Date(), + }, + }, + limit, + }); + await Promise.all(attachments.map((attachment) => attachment.destroy())); + Logger.info("task", `Removed ${attachments.length} attachments`); + } + + public get options() { + return { + attempts: 1, + priority: TaskPriority.Background, + }; + } +} diff --git a/server/routes/api/attachments.test.ts b/server/routes/api/attachments.test.ts index 7deab6300..7fb88a15f 100644 --- a/server/routes/api/attachments.test.ts +++ b/server/routes/api/attachments.test.ts @@ -1,3 +1,4 @@ +import { AttachmentPreset } from "@shared/types"; import Attachment from "@server/models/Attachment"; import { buildUser, @@ -34,6 +35,42 @@ describe("#attachments.create", () => { expect(res.status).toEqual(200); }); + it("should allow upload using avatar preset", async () => { + const user = await buildUser(); + const res = await server.post("/api/attachments.create", { + body: { + name: "test.png", + contentType: "image/png", + size: 1000, + preset: AttachmentPreset.Avatar, + token: user.getJwtToken(), + }, + }); + expect(res.status).toEqual(200); + + const body = await res.json(); + const attachment = await Attachment.findByPk(body.data.attachment.id); + expect(attachment!.expiresAt).toBeNull(); + }); + + it("should create expiring attachment using import preset", async () => { + const user = await buildUser(); + const res = await server.post("/api/attachments.create", { + body: { + name: "test.zip", + contentType: "application/zip", + size: 10000, + preset: AttachmentPreset.Import, + token: user.getJwtToken(), + }, + }); + expect(res.status).toEqual(200); + + const body = await res.json(); + const attachment = await Attachment.findByPk(body.data.attachment.id); + expect(attachment!.expiresAt).toBeTruthy(); + }); + it("should not allow file upload for public attachments", async () => { const user = await buildUser(); const res = await server.post("/api/attachments.create", { @@ -47,6 +84,20 @@ describe("#attachments.create", () => { }); expect(res.status).toEqual(400); }); + + it("should not allow file upload for avatar preset", async () => { + const user = await buildUser(); + const res = await server.post("/api/attachments.create", { + body: { + name: "test.pdf", + contentType: "application/pdf", + size: 1000, + preset: AttachmentPreset.Avatar, + token: user.getJwtToken(), + }, + }); + expect(res.status).toEqual(400); + }); }); describe("viewer", () => { @@ -63,6 +114,20 @@ describe("#attachments.create", () => { }); expect(res.status).toEqual(200); }); + + it("should allow upload using avatar preset", async () => { + const user = await buildViewer(); + const res = await server.post("/api/attachments.create", { + body: { + name: "test.png", + contentType: "image/png", + size: 1000, + preset: AttachmentPreset.Avatar, + token: user.getJwtToken(), + }, + }); + expect(res.status).toEqual(200); + }); }); }); diff --git a/server/routes/api/attachments.ts b/server/routes/api/attachments.ts index c2c646a8d..b7aea10b8 100644 --- a/server/routes/api/attachments.ts +++ b/server/routes/api/attachments.ts @@ -1,81 +1,87 @@ import Router from "koa-router"; import { v4 as uuidv4 } from "uuid"; +import { AttachmentPreset } from "@shared/types"; import { bytesToHumanReadable } from "@shared/utils/files"; import { AttachmentValidation } from "@shared/validations"; -import { sequelize } from "@server/database/sequelize"; import { AuthorizationError, ValidationError } from "@server/errors"; import auth from "@server/middlewares/authentication"; -import { Attachment, Document, Event } from "@server/models"; -import { authorize } from "@server/policies"; -import { ContextWithState } from "@server/types"; import { - getPresignedPost, - publicS3Endpoint, - getSignedUrl, -} from "@server/utils/s3"; + transaction, + TransactionContext, +} from "@server/middlewares/transaction"; +import { Attachment, Document, Event } from "@server/models"; +import AttachmentHelper from "@server/models/helpers/AttachmentHelper"; +import { authorize } from "@server/policies"; +import { presentAttachment } from "@server/presenters"; +import { ContextWithState } from "@server/types"; +import { getPresignedPost, publicS3Endpoint } from "@server/utils/s3"; import { assertIn, assertPresent, assertUuid } from "@server/validation"; const router = new Router(); -const AWS_S3_ACL = process.env.AWS_S3_ACL || "private"; -router.post("attachments.create", auth(), async (ctx) => { - const { - name, - documentId, - contentType = "application/octet-stream", - size, - public: isPublic, - } = ctx.request.body; - assertPresent(name, "name is required"); - assertPresent(size, "size is required"); +router.post( + "attachments.create", + auth(), + transaction(), + async (ctx: TransactionContext) => { + const { + name, + documentId, + contentType = "application/octet-stream", + size, + // 'public' is now deprecated and can be removed on December 1 2022. + public: isPublicDeprecated, + preset = isPublicDeprecated + ? AttachmentPreset.Avatar + : AttachmentPreset.DocumentAttachment, + } = ctx.request.body; + const { user, transaction } = ctx.state; - const { user } = ctx.state; + assertPresent(name, "name is required"); + assertPresent(size, "size is required"); - // Public attachments are only used for avatars, so this is loosely coupled – - // all user types can upload an avatar so no additional authorization is needed. - if (isPublic) { - assertIn(contentType, AttachmentValidation.avatarContentTypes); - } else { - authorize(user, "createAttachment", user.team); - } + // Public attachments are only used for avatars, so this is loosely coupled – + // all user types can upload an avatar so no additional authorization is needed. + if (preset === AttachmentPreset.Avatar) { + assertIn(contentType, AttachmentValidation.avatarContentTypes); + } else { + authorize(user, "createAttachment", user.team); + } - if ( - process.env.AWS_S3_UPLOAD_MAX_SIZE && - size > process.env.AWS_S3_UPLOAD_MAX_SIZE - ) { - throw ValidationError( - `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable( - parseInt(process.env.AWS_S3_UPLOAD_MAX_SIZE, 10) - )}` - ); - } + const maxUploadSize = AttachmentHelper.presetToMaxUploadSize(preset); - const modelId = uuidv4(); - const acl = - isPublic === undefined ? AWS_S3_ACL : isPublic ? "public-read" : "private"; - const bucket = acl === "public-read" ? "public" : "uploads"; - const keyPrefix = `${bucket}/${user.id}/${modelId}`; - const key = `${keyPrefix}/${name}`; - const presignedPost = await getPresignedPost(key, acl, contentType); - const endpoint = publicS3Endpoint(); - const url = `${endpoint}/${keyPrefix}/${encodeURIComponent(name)}`; + if (size > maxUploadSize) { + throw ValidationError( + `Sorry, this file is too large – the maximum size is ${bytesToHumanReadable( + maxUploadSize + )}` + ); + } - if (documentId !== undefined) { - assertUuid(documentId, "documentId must be a uuid"); - const document = await Document.findByPk(documentId, { + if (documentId !== undefined) { + assertUuid(documentId, "documentId must be a uuid"); + const document = await Document.findByPk(documentId, { + userId: user.id, + }); + authorize(user, "update", document); + } + + const modelId = uuidv4(); + const acl = AttachmentHelper.presetToAcl(preset); + const key = AttachmentHelper.getKey({ + acl, + id: modelId, + name, userId: user.id, }); - authorize(user, "update", document); - } - const attachment = await sequelize.transaction(async (transaction) => { const attachment = await Attachment.create( { id: modelId, key, acl, size, - url, + expiresAt: AttachmentHelper.presetToExpiry(preset), contentType, documentId, teamId: user.teamId, @@ -97,29 +103,26 @@ router.post("attachments.create", auth(), async (ctx) => { { transaction } ); - return attachment; - }); + const presignedPost = await getPresignedPost( + key, + acl, + maxUploadSize, + contentType + ); - ctx.body = { - data: { - maxUploadSize: process.env.AWS_S3_UPLOAD_MAX_SIZE, - uploadUrl: endpoint, - form: { - "Cache-Control": "max-age=31557600", - "Content-Type": contentType, - ...presignedPost.fields, + ctx.body = { + data: { + uploadUrl: publicS3Endpoint(), + form: { + "Cache-Control": "max-age=31557600", + "Content-Type": contentType, + ...presignedPost.fields, + }, + attachment: presentAttachment(attachment), }, - attachment: { - documentId, - contentType, - name, - id: attachment.id, - url: isPublic ? url : attachment.redirectUrl, - size, - }, - }, - }; -}); + }; + } +); router.post("attachments.delete", auth(), async (ctx) => { const { id } = ctx.request.body; @@ -168,8 +171,7 @@ const handleAttachmentsRedirect = async (ctx: ContextWithState) => { }); if (attachment.isPrivate) { - const accessUrl = await getSignedUrl(attachment.key); - ctx.redirect(accessUrl); + ctx.redirect(await attachment.signedUrl); } else { ctx.redirect(attachment.canonicalUrl); } diff --git a/server/routes/api/cron.ts b/server/routes/api/cron.ts index 065c61f0e..6a12294e9 100644 --- a/server/routes/api/cron.ts +++ b/server/routes/api/cron.ts @@ -5,6 +5,7 @@ import env from "@server/env"; import { AuthenticationError } from "@server/errors"; import CleanupDeletedDocumentsTask from "@server/queues/tasks/CleanupDeletedDocumentsTask"; import CleanupDeletedTeamsTask from "@server/queues/tasks/CleanupDeletedTeamsTask"; +import CleanupExpiredAttachmentsTask from "@server/queues/tasks/CleanupExpiredAttachmentsTask"; import CleanupExpiredFileOperationsTask from "@server/queues/tasks/CleanupExpiredFileOperationsTask"; import CleanupWebhookDeliveriesTask from "@server/queues/tasks/CleanupWebhookDeliveriesTask"; import InviteReminderTask from "@server/queues/tasks/InviteReminderTask"; @@ -37,6 +38,8 @@ const cronHandler = async (ctx: Context) => { await CleanupExpiredFileOperationsTask.schedule({ limit }); + await CleanupExpiredAttachmentsTask.schedule({ limit }); + await CleanupDeletedTeamsTask.schedule({ limit }); await CleanupWebhookDeliveriesTask.schedule({ limit }); diff --git a/server/routes/api/documents.ts b/server/routes/api/documents.ts index 59d401be9..fe20c05b9 100644 --- a/server/routes/api/documents.ts +++ b/server/routes/api/documents.ts @@ -1114,10 +1114,10 @@ router.post("documents.import", auth(), async (ctx) => { throw InvalidRequestError("Request must include a file parameter"); } - if (env.MAXIMUM_IMPORT_SIZE && file.size > env.MAXIMUM_IMPORT_SIZE) { + if (file.size > env.AWS_S3_UPLOAD_MAX_SIZE) { throw InvalidRequestError( `The selected file was larger than the ${bytesToHumanReadable( - env.MAXIMUM_IMPORT_SIZE + env.AWS_S3_UPLOAD_MAX_SIZE )} maximum size` ); } diff --git a/server/utils/s3.ts b/server/utils/s3.ts index fa898f861..c5c7053f6 100644 --- a/server/utils/s3.ts +++ b/server/utils/s3.ts @@ -107,14 +107,13 @@ export const getSignature = (policy: string) => { export const getPresignedPost = ( key: string, acl: string, + maxUploadSize: number, contentType = "image" ) => { const params = { Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME, Conditions: compact([ - process.env.AWS_S3_UPLOAD_MAX_SIZE - ? ["content-length-range", 0, +process.env.AWS_S3_UPLOAD_MAX_SIZE] - : undefined, + ["content-length-range", 0, maxUploadSize], ["starts-with", "$Content-Type", contentType], ["starts-with", "$Cache-Control", ""], ]), diff --git a/shared/types.ts b/shared/types.ts index f56e4744a..7954137a6 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -22,6 +22,12 @@ export type PublicEnv = { RELEASE: string | undefined; }; +export enum AttachmentPreset { + DocumentAttachment = "documentAttachment", + Import = "import", + Avatar = "avatar", +} + export enum IntegrationType { Post = "post", Command = "command",