yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Return attachments when exporting an individual file (#5778)

Browse Source
This commit is contained in:
Tom Moor
2023-09-06 20:53:30 -04:00
committed by GitHub
parent d1de5871de
commit 127115272a
7 changed files with 116 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@@ -280,6 +280,7 @@
"@types/react-table": "^7.7.14",
"@types/react-virtualized-auto-sizer": "^1.0.1",
"@types/react-window": "^1.8.5",
"@types/readable-stream": "^4.0.2",
"@types/redis-info": "^3.0.0",
"@types/refractor": "^3.0.2",
"@types/semver": "^7.5.0",

8
server/queues/tasks/ExportDocumentTreeTask.ts
View File

@@ -1,5 +1,6 @@
import path from "path";
import JSZip from "jszip";
import escapeRegExp from "lodash/escapeRegExp";
import { FileOperationFormat, NavigationNode } from "@shared/types";
import Logger from "@server/logging/Logger";
import { Collection } from "@server/models";
@@ -73,14 +74,17 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
date: attachment.updatedAt,
createFolders: true,
});
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
encodeURI(attachment.key)
);
} catch (err) {
Logger.error(
`Failed to add attachment to archive: ${attachment.key}`,
err
);
}
text = text.replace(attachment.redirectUrl, encodeURI(attachment.key));
})
);

83
server/routes/api/documents/documents.ts
View File

@@ -1,6 +1,9 @@
import path from "path";
import fs from "fs-extra";
import invariant from "invariant";
import JSZip from "jszip";
import Router from "koa-router";
import escapeRegExp from "lodash/escapeRegExp";
import mime from "mime-types";
import { Op, ScopeOptions, WhereOptions } from "sequelize";
import { TeamPreference } from "@shared/types";
@@ -20,11 +23,13 @@ import {
ValidationError,
IncorrectEditionError,
} from "@server/errors";
import Logger from "@server/logging/Logger";
import auth from "@server/middlewares/authentication";
import { rateLimiter } from "@server/middlewares/rateLimiter";
import { transaction } from "@server/middlewares/transaction";
import validate from "@server/middlewares/validate";
import {
Attachment,
Backlink,
Collection,
Document,
@@ -46,7 +51,9 @@ import {
} from "@server/presenters";
import { APIContext } from "@server/types";
import { RateLimiterStrategy } from "@server/utils/RateLimiter";
import ZipHelper from "@server/utils/ZipHelper";
import { getFileFromRequest } from "@server/utils/koa";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import { getTeamFromContext } from "@server/utils/passport";
import slugify from "@server/utils/slugify";
import { assertPresent } from "@server/validation";
@@ -516,8 +523,8 @@ router.post(
includeState: !accept?.includes("text/markdown"),
});
let contentType;
let content;
let contentType: string;
let content: string;
if (accept?.includes("text/html")) {
contentType = "text/html";
@@ -537,26 +544,70 @@ router.post(
content = DocumentHelper.toMarkdown(document);
}
if (contentType !== "application/json") {
// Override the extension for Markdown as it's incorrect in the mime-types
// library until a new release > 2.1.35
const extension =
contentType === "text/markdown" ? "md" : mime.extension(contentType);
if (contentType === "application/json") {
ctx.body = {
data: content,
};
return;
}
// Override the extension for Markdown as it's incorrect in the mime-types
// library until a new release > 2.1.35
const extension =
contentType === "text/markdown" ? "md" : mime.extension(contentType);
const fileName = slugify(document.titleWithDefault);
const attachmentIds = parseAttachmentIds(document.text);
const attachments = attachmentIds.length
? await Attachment.findAll({
where: {
teamId: document.teamId,
id: attachmentIds,
},
})
: [];
if (attachments.length === 0) {
ctx.set("Content-Type", contentType);
ctx.set(
"Content-Disposition",
`attachment; filename="${slugify(
document.titleWithDefault
)}.${extension}"`
);
ctx.attachment(`${fileName}.${extension}`);
ctx.body = content;
return;
}
ctx.body = {
data: content,
};
const zip = new JSZip();
await Promise.all(
attachments.map(async (attachment) => {
try {
const location = path.join(
"attachments",
`${attachment.id}.${mime.extension(attachment.contentType)}`
);
zip.file(location, attachment.buffer, {
date: attachment.updatedAt,
createFolders: true,
});
content = content.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
location
);
} catch (err) {
Logger.error(
`Failed to add attachment to archive: ${attachment.id}`,
err
);
}
})
);
zip.file(`${fileName}.${extension}`, content, {
date: document.updatedAt,
});
ctx.set("Content-Type", "application/zip");
ctx.attachment(`${fileName}.zip`);
ctx.body = zip.generateNodeStream(ZipHelper.defaultStreamOptions);
}
);

2
server/routes/api/middlewares/apiWrapper.ts
View File

@@ -1,5 +1,6 @@
import stream from "stream";
import { Context, Next } from "koa";
import { Readable } from "readable-stream";
export default function apiWrapper() {
return async function apiWrapperMiddleware(ctx: Context, next: Next) {
@@ -8,6 +9,7 @@ export default function apiWrapper() {
if (
typeof ctx.body === "object" &&
!(ctx.body instanceof Readable) &&
!(ctx.body instanceof stream.Readable) &&
!(ctx.body instanceof Buffer)
) {

8
server/routes/api/revisions/revisions.ts
View File

@@ -101,13 +101,9 @@ router.post(
const content = await DocumentHelper.diff(before, revision);
if (accept?.includes("text/html")) {
const name = `${slugify(document.titleWithDefault)}-${revision.id}.html`;
ctx.set("Content-Type", "text/html");
ctx.set(
"Content-Disposition",
`attachment; filename="${slugify(document.titleWithDefault)}-${
revision.id
}.html"`
);
ctx.attachment(name);
ctx.body = content;
return;
}

26
server/utils/ZipHelper.ts
View File

@@ -22,6 +22,16 @@ export type FileTreeNode = {
@trace()
export default class ZipHelper {
public static defaultStreamOptions: JSZip.JSZipGeneratorOptions<"nodebuffer"> =
{
type: "nodebuffer",
streamFiles: true,
compression: "DEFLATE",
compressionOptions: {
level: 5,
},
};
/**
* Converts the flat structure returned by JSZIP into a nested file structure
* for easier processing.
@@ -84,7 +94,10 @@ export default class ZipHelper {
* @param zip JSZip object
* @returns pathname of the temporary file where the zip was written to disk
*/
public static async toTmpFile(zip: JSZip): Promise<string> {
public static async toTmpFile(
zip: JSZip,
options?: JSZip.JSZipGeneratorOptions<"nodebuffer">
): Promise<string> {
Logger.debug("utils", "Creating tmp file…");
return new Promise((resolve, reject) => {
tmp.file(
@@ -105,17 +118,18 @@ export default class ZipHelper {
zip
.generateNodeStream(
{
type: "nodebuffer",
streamFiles: true,
...this.defaultStreamOptions,
...options,
},
(metadata) => {
const percent = Math.round(metadata.percent);
if (percent !== previousMetadata.percent) {
if (metadata.currentFile !== previousMetadata.currentFile) {
const percent = Math.round(metadata.percent);
const memory = process.memoryUsage();
previousMetadata = {
currentFile: metadata.currentFile,
percent,
};
const memory = process.memoryUsage();
Logger.debug(
"utils",
`Writing zip file progress… ${percent}%`,

28
yarn.lock
View File

@@ -1822,7 +1822,7 @@
"@types/node" "*"
jest-mock "^29.6.3"
"@jest/expect-utils@^29.6.1", "@jest/expect-utils@^29.6.4":
"@jest/expect-utils@^29.6.4":
version "29.6.4"
resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.6.4.tgz#17c7dfe6cec106441f218b0aff4b295f98346679"
integrity sha512-FEhkJhqtvBwgSpiTrocquJCdXPsyvNKcl/n7A3u7X4pVoF4bswm11c9d4AV+kfq2Gpv/mM8x7E7DsRvH+djkrg==
@@ -1889,7 +1889,7 @@
strip-ansi "^6.0.0"
v8-to-istanbul "^9.0.1"
"@jest/schemas@^29.6.0", "@jest/schemas@^29.6.3":
"@jest/schemas@^29.6.3":
version "29.6.3"
resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03"
integrity sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==
@@ -1946,7 +1946,7 @@
slash "^3.0.0"
write-file-atomic "^4.0.2"
"@jest/types@^29.6.1", "@jest/types@^29.6.3":
"@jest/types@^29.6.3":
version "29.6.3"
resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59"
integrity sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==
@@ -3397,6 +3397,14 @@
dependencies:
"@types/react" "*"
"@types/readable-stream@^4.0.2":
version "4.0.2"
resolved "https://registry.yarnpkg.com/@types/readable-stream/-/readable-stream-4.0.2.tgz#5199cfeef35ea16d0e85076b1c6daa15766634c0"
integrity sha512-hhzOsMEISZ+mX1l+01F0duYt9wHEbCGmjARed0PcQoVS5zAdu7u5YbWYuNGhw09M1MgGr3kfsto+ut/MnAdKqA==
dependencies:
"@types/node" "*"
safe-buffer "~5.1.1"
"@types/redis-info@^3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/redis-info/-/redis-info-3.0.0.tgz#c925cd7249d71c3d9e12ef5b15168bdf26111b1d"
@@ -5732,7 +5740,7 @@ diagnostics_channel@^1.1.0:
resolved "https://registry.yarnpkg.com/diagnostics_channel/-/diagnostics_channel-1.1.0.tgz#bd66c49124ce3bac697dff57466464487f57cea5"
integrity sha512-OE1ngLDjSBPG6Tx0YATELzYzy3RKHC+7veQ8gLa8yS7AAgw65mFbVdcsu3501abqOZCEZqZyAIemB0zXlqDSuw==
diff-sequences@^29.4.3, diff-sequences@^29.6.3:
diff-sequences@^29.6.3:
version "29.6.3"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921"
integrity sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==
@@ -8373,7 +8381,7 @@ jest-config@^29.6.4:
slash "^3.0.0"
strip-json-comments "^3.1.1"
jest-diff@^29.6.1, jest-diff@^29.6.4:
jest-diff@^29.6.4:
version "29.6.4"
resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.6.4.tgz#85aaa6c92a79ae8cd9a54ebae8d5b6d9a513314a"
integrity sha512-9F48UxR9e4XOEZvoUXEHSWY4qC4zERJaOfrbBg9JpbJOO43R1vN76REt/aMGZoY6GD5g84nnJiBIVlscegefpw==
@@ -8435,7 +8443,7 @@ jest-fetch-mock@^3.0.3:
cross-fetch "^3.0.4"
promise-polyfill "^8.1.3"
jest-get-type@^29.4.3, jest-get-type@^29.6.3:
jest-get-type@^29.6.3:
version "29.6.3"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1"
integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==
@@ -8467,7 +8475,7 @@ jest-leak-detector@^29.6.3:
jest-get-type "^29.6.3"
pretty-format "^29.6.3"
jest-matcher-utils@^29.6.1, jest-matcher-utils@^29.6.4:
jest-matcher-utils@^29.6.4:
version "29.6.4"
resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.6.4.tgz#327db7ababea49455df3b23e5d6109fe0c709d24"
integrity sha512-KSzwyzGvK4HcfnserYqJHYi7sZVqdREJ9DMPAKVbS98JsIAvumihaNUbjrWw0St7p9IY7A9UskCW5MYlGmBQFQ==
@@ -8477,7 +8485,7 @@ jest-matcher-utils@^29.6.1, jest-matcher-utils@^29.6.4:
jest-get-type "^29.6.3"
pretty-format "^29.6.3"
jest-message-util@^29.6.1, jest-message-util@^29.6.3:
jest-message-util@^29.6.3:
version "29.6.3"
resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.6.3.tgz#bce16050d86801b165f20cfde34dc01d3cf85fbf"
integrity sha512-FtzaEEHzjDpQp51HX4UMkPZjy46ati4T5pEMyM6Ik48ztu4T9LQplZ6OsimHx7EuM9dfEh5HJa6D3trEftu3dA==
@@ -8615,7 +8623,7 @@ jest-snapshot@^29.6.4:
pretty-format "^29.6.3"
semver "^7.5.3"
jest-util@^29.6.1, jest-util@^29.6.3:
jest-util@^29.6.3:
version "29.6.3"
resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.6.3.tgz#e15c3eac8716440d1ed076f09bc63ace1aebca63"
integrity sha512-QUjna/xSy4B32fzcKTSz1w7YYzgiHrjjJjevdRf61HYk998R5vVMMNmrHESYZVDS5DSWs+1srPLPKxXPkeSDOA==
@@ -10673,7 +10681,7 @@ pretty-bytes@^6.0.0:
resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-6.1.0.tgz#1d1cc9aae1939012c74180b679da6684616bf804"
integrity sha512-Rk753HI8f4uivXi4ZCIYdhmG1V+WKzvRMg/X+M42a6t7D07RcmopXJMDNk6N++7Bl75URRGsb40ruvg7Hcp2wQ==
pretty-format@^29.0.0, pretty-format@^29.6.1, pretty-format@^29.6.3:
pretty-format@^29.0.0, pretty-format@^29.6.3:
version "29.6.3"
resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.6.3.tgz#d432bb4f1ca6f9463410c3fb25a0ba88e594ace7"
integrity sha512-ZsBgjVhFAj5KeK+nHfF1305/By3lechHQSMWCTl8iHSbfOm2TN5nHEtFc/+W7fAyUeCs2n5iow72gld4gW0xDw==