diff --git a/package.json b/package.json index 4cc6e9546..74b7beb61 100644 --- a/package.json +++ b/package.json @@ -280,6 +280,7 @@ "@types/react-table": "^7.7.14", "@types/react-virtualized-auto-sizer": "^1.0.1", "@types/react-window": "^1.8.5", + "@types/readable-stream": "^4.0.2", "@types/redis-info": "^3.0.0", "@types/refractor": "^3.0.2", "@types/semver": "^7.5.0", diff --git a/server/queues/tasks/ExportDocumentTreeTask.ts b/server/queues/tasks/ExportDocumentTreeTask.ts index e597fb0c0..383869fc3 100644 --- a/server/queues/tasks/ExportDocumentTreeTask.ts +++ b/server/queues/tasks/ExportDocumentTreeTask.ts @@ -1,5 +1,6 @@ import path from "path"; import JSZip from "jszip"; +import escapeRegExp from "lodash/escapeRegExp"; import { FileOperationFormat, NavigationNode } from "@shared/types"; import Logger from "@server/logging/Logger"; import { Collection } from "@server/models"; @@ -73,14 +74,17 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { date: attachment.updatedAt, createFolders: true, }); + + text = text.replace( + new RegExp(escapeRegExp(attachment.redirectUrl), "g"), + encodeURI(attachment.key) + ); } catch (err) { Logger.error( `Failed to add attachment to archive: ${attachment.key}`, err ); } - - text = text.replace(attachment.redirectUrl, encodeURI(attachment.key)); }) ); diff --git a/server/routes/api/documents/documents.ts b/server/routes/api/documents/documents.ts index 793ec665b..5c0844b0a 100644 --- a/server/routes/api/documents/documents.ts +++ b/server/routes/api/documents/documents.ts @@ -1,6 +1,9 @@ +import path from "path"; import fs from "fs-extra"; import invariant from "invariant"; +import JSZip from "jszip"; import Router from "koa-router"; +import escapeRegExp from "lodash/escapeRegExp"; import mime from "mime-types"; import { Op, ScopeOptions, WhereOptions } from "sequelize"; import { TeamPreference } from "@shared/types"; @@ -20,11 +23,13 @@ import { ValidationError, IncorrectEditionError, } from "@server/errors"; +import Logger from "@server/logging/Logger"; import auth from "@server/middlewares/authentication"; import { rateLimiter } from "@server/middlewares/rateLimiter"; import { transaction } from "@server/middlewares/transaction"; import validate from "@server/middlewares/validate"; import { + Attachment, Backlink, Collection, Document, @@ -46,7 +51,9 @@ import { } from "@server/presenters"; import { APIContext } from "@server/types"; import { RateLimiterStrategy } from "@server/utils/RateLimiter"; +import ZipHelper from "@server/utils/ZipHelper"; import { getFileFromRequest } from "@server/utils/koa"; +import parseAttachmentIds from "@server/utils/parseAttachmentIds"; import { getTeamFromContext } from "@server/utils/passport"; import slugify from "@server/utils/slugify"; import { assertPresent } from "@server/validation"; @@ -516,8 +523,8 @@ router.post( includeState: !accept?.includes("text/markdown"), }); - let contentType; - let content; + let contentType: string; + let content: string; if (accept?.includes("text/html")) { contentType = "text/html"; @@ -537,26 +544,70 @@ router.post( content = DocumentHelper.toMarkdown(document); } - if (contentType !== "application/json") { - // Override the extension for Markdown as it's incorrect in the mime-types - // library until a new release > 2.1.35 - const extension = - contentType === "text/markdown" ? "md" : mime.extension(contentType); + if (contentType === "application/json") { + ctx.body = { + data: content, + }; + return; + } + // Override the extension for Markdown as it's incorrect in the mime-types + // library until a new release > 2.1.35 + const extension = + contentType === "text/markdown" ? "md" : mime.extension(contentType); + + const fileName = slugify(document.titleWithDefault); + const attachmentIds = parseAttachmentIds(document.text); + const attachments = attachmentIds.length + ? await Attachment.findAll({ + where: { + teamId: document.teamId, + id: attachmentIds, + }, + }) + : []; + + if (attachments.length === 0) { ctx.set("Content-Type", contentType); - ctx.set( - "Content-Disposition", - `attachment; filename="${slugify( - document.titleWithDefault - )}.${extension}"` - ); + ctx.attachment(`${fileName}.${extension}`); ctx.body = content; return; } - ctx.body = { - data: content, - }; + const zip = new JSZip(); + + await Promise.all( + attachments.map(async (attachment) => { + try { + const location = path.join( + "attachments", + `${attachment.id}.${mime.extension(attachment.contentType)}` + ); + zip.file(location, attachment.buffer, { + date: attachment.updatedAt, + createFolders: true, + }); + + content = content.replace( + new RegExp(escapeRegExp(attachment.redirectUrl), "g"), + location + ); + } catch (err) { + Logger.error( + `Failed to add attachment to archive: ${attachment.id}`, + err + ); + } + }) + ); + + zip.file(`${fileName}.${extension}`, content, { + date: document.updatedAt, + }); + + ctx.set("Content-Type", "application/zip"); + ctx.attachment(`${fileName}.zip`); + ctx.body = zip.generateNodeStream(ZipHelper.defaultStreamOptions); } ); diff --git a/server/routes/api/middlewares/apiWrapper.ts b/server/routes/api/middlewares/apiWrapper.ts index ea76bf6ce..757d51162 100644 --- a/server/routes/api/middlewares/apiWrapper.ts +++ b/server/routes/api/middlewares/apiWrapper.ts @@ -1,5 +1,6 @@ import stream from "stream"; import { Context, Next } from "koa"; +import { Readable } from "readable-stream"; export default function apiWrapper() { return async function apiWrapperMiddleware(ctx: Context, next: Next) { @@ -8,6 +9,7 @@ export default function apiWrapper() { if ( typeof ctx.body === "object" && + !(ctx.body instanceof Readable) && !(ctx.body instanceof stream.Readable) && !(ctx.body instanceof Buffer) ) { diff --git a/server/routes/api/revisions/revisions.ts b/server/routes/api/revisions/revisions.ts index ed02a0f32..e34201f6d 100644 --- a/server/routes/api/revisions/revisions.ts +++ b/server/routes/api/revisions/revisions.ts @@ -101,13 +101,9 @@ router.post( const content = await DocumentHelper.diff(before, revision); if (accept?.includes("text/html")) { + const name = `${slugify(document.titleWithDefault)}-${revision.id}.html`; ctx.set("Content-Type", "text/html"); - ctx.set( - "Content-Disposition", - `attachment; filename="${slugify(document.titleWithDefault)}-${ - revision.id - }.html"` - ); + ctx.attachment(name); ctx.body = content; return; } diff --git a/server/utils/ZipHelper.ts b/server/utils/ZipHelper.ts index 20666269a..e6fa8944c 100644 --- a/server/utils/ZipHelper.ts +++ b/server/utils/ZipHelper.ts @@ -22,6 +22,16 @@ export type FileTreeNode = { @trace() export default class ZipHelper { + public static defaultStreamOptions: JSZip.JSZipGeneratorOptions<"nodebuffer"> = + { + type: "nodebuffer", + streamFiles: true, + compression: "DEFLATE", + compressionOptions: { + level: 5, + }, + }; + /** * Converts the flat structure returned by JSZIP into a nested file structure * for easier processing. @@ -84,7 +94,10 @@ export default class ZipHelper { * @param zip JSZip object * @returns pathname of the temporary file where the zip was written to disk */ - public static async toTmpFile(zip: JSZip): Promise { + public static async toTmpFile( + zip: JSZip, + options?: JSZip.JSZipGeneratorOptions<"nodebuffer"> + ): Promise { Logger.debug("utils", "Creating tmp file…"); return new Promise((resolve, reject) => { tmp.file( @@ -105,17 +118,18 @@ export default class ZipHelper { zip .generateNodeStream( { - type: "nodebuffer", - streamFiles: true, + ...this.defaultStreamOptions, + ...options, }, (metadata) => { - const percent = Math.round(metadata.percent); - if (percent !== previousMetadata.percent) { + if (metadata.currentFile !== previousMetadata.currentFile) { + const percent = Math.round(metadata.percent); + const memory = process.memoryUsage(); + previousMetadata = { currentFile: metadata.currentFile, percent, }; - const memory = process.memoryUsage(); Logger.debug( "utils", `Writing zip file progress… ${percent}%`, diff --git a/yarn.lock b/yarn.lock index 0b2ab6be8..4931d15df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1822,7 +1822,7 @@ "@types/node" "*" jest-mock "^29.6.3" -"@jest/expect-utils@^29.6.1", "@jest/expect-utils@^29.6.4": +"@jest/expect-utils@^29.6.4": version "29.6.4" resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.6.4.tgz#17c7dfe6cec106441f218b0aff4b295f98346679" integrity sha512-FEhkJhqtvBwgSpiTrocquJCdXPsyvNKcl/n7A3u7X4pVoF4bswm11c9d4AV+kfq2Gpv/mM8x7E7DsRvH+djkrg== @@ -1889,7 +1889,7 @@ strip-ansi "^6.0.0" v8-to-istanbul "^9.0.1" -"@jest/schemas@^29.6.0", "@jest/schemas@^29.6.3": +"@jest/schemas@^29.6.3": version "29.6.3" resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03" integrity sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA== @@ -1946,7 +1946,7 @@ slash "^3.0.0" write-file-atomic "^4.0.2" -"@jest/types@^29.6.1", "@jest/types@^29.6.3": +"@jest/types@^29.6.3": version "29.6.3" resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59" integrity sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw== @@ -3397,6 +3397,14 @@ dependencies: "@types/react" "*" +"@types/readable-stream@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-4.0.2.tgz#5199cfeef35ea16d0e85076b1c6daa15766634c0" + integrity sha512-hhzOsMEISZ+mX1l+01F0duYt9wHEbCGmjARed0PcQoVS5zAdu7u5YbWYuNGhw09M1MgGr3kfsto+ut/MnAdKqA== + dependencies: + "@types/node" "*" + safe-buffer "~5.1.1" + "@types/redis-info@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@types/redis-info/-/redis-info-3.0.0.tgz#c925cd7249d71c3d9e12ef5b15168bdf26111b1d" @@ -5732,7 +5740,7 @@ diagnostics_channel@^1.1.0: resolved "https://registry.yarnpkg.com/diagnostics_channel/-/diagnostics_channel-1.1.0.tgz#bd66c49124ce3bac697dff57466464487f57cea5" integrity sha512-OE1ngLDjSBPG6Tx0YATELzYzy3RKHC+7veQ8gLa8yS7AAgw65mFbVdcsu3501abqOZCEZqZyAIemB0zXlqDSuw== -diff-sequences@^29.4.3, diff-sequences@^29.6.3: +diff-sequences@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921" integrity sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q== @@ -8373,7 +8381,7 @@ jest-config@^29.6.4: slash "^3.0.0" strip-json-comments "^3.1.1" -jest-diff@^29.6.1, jest-diff@^29.6.4: +jest-diff@^29.6.4: version "29.6.4" resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.6.4.tgz#85aaa6c92a79ae8cd9a54ebae8d5b6d9a513314a" integrity sha512-9F48UxR9e4XOEZvoUXEHSWY4qC4zERJaOfrbBg9JpbJOO43R1vN76REt/aMGZoY6GD5g84nnJiBIVlscegefpw== @@ -8435,7 +8443,7 @@ jest-fetch-mock@^3.0.3: cross-fetch "^3.0.4" promise-polyfill "^8.1.3" -jest-get-type@^29.4.3, jest-get-type@^29.6.3: +jest-get-type@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1" integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw== @@ -8467,7 +8475,7 @@ jest-leak-detector@^29.6.3: jest-get-type "^29.6.3" pretty-format "^29.6.3" -jest-matcher-utils@^29.6.1, jest-matcher-utils@^29.6.4: +jest-matcher-utils@^29.6.4: version "29.6.4" resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.6.4.tgz#327db7ababea49455df3b23e5d6109fe0c709d24" integrity sha512-KSzwyzGvK4HcfnserYqJHYi7sZVqdREJ9DMPAKVbS98JsIAvumihaNUbjrWw0St7p9IY7A9UskCW5MYlGmBQFQ== @@ -8477,7 +8485,7 @@ jest-matcher-utils@^29.6.1, jest-matcher-utils@^29.6.4: jest-get-type "^29.6.3" pretty-format "^29.6.3" -jest-message-util@^29.6.1, jest-message-util@^29.6.3: +jest-message-util@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.6.3.tgz#bce16050d86801b165f20cfde34dc01d3cf85fbf" integrity sha512-FtzaEEHzjDpQp51HX4UMkPZjy46ati4T5pEMyM6Ik48ztu4T9LQplZ6OsimHx7EuM9dfEh5HJa6D3trEftu3dA== @@ -8615,7 +8623,7 @@ jest-snapshot@^29.6.4: pretty-format "^29.6.3" semver "^7.5.3" -jest-util@^29.6.1, jest-util@^29.6.3: +jest-util@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.6.3.tgz#e15c3eac8716440d1ed076f09bc63ace1aebca63" integrity sha512-QUjna/xSy4B32fzcKTSz1w7YYzgiHrjjJjevdRf61HYk998R5vVMMNmrHESYZVDS5DSWs+1srPLPKxXPkeSDOA== @@ -10673,7 +10681,7 @@ pretty-bytes@^6.0.0: resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-6.1.0.tgz#1d1cc9aae1939012c74180b679da6684616bf804" integrity sha512-Rk753HI8f4uivXi4ZCIYdhmG1V+WKzvRMg/X+M42a6t7D07RcmopXJMDNk6N++7Bl75URRGsb40ruvg7Hcp2wQ== -pretty-format@^29.0.0, pretty-format@^29.6.1, pretty-format@^29.6.3: +pretty-format@^29.0.0, pretty-format@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.6.3.tgz#d432bb4f1ca6f9463410c3fb25a0ba88e594ace7" integrity sha512-ZsBgjVhFAj5KeK+nHfF1305/By3lechHQSMWCTl8iHSbfOm2TN5nHEtFc/+W7fAyUeCs2n5iow72gld4gW0xDw==