diff --git a/app/components/ExportDialog.tsx b/app/components/ExportDialog.tsx index 7b58210b9..fc51b9243 100644 --- a/app/components/ExportDialog.tsx +++ b/app/components/ExportDialog.tsx @@ -21,6 +21,8 @@ function ExportDialog({ collection, onSubmit }: Props) { const [format, setFormat] = React.useState( FileOperationFormat.MarkdownZip ); + const [includeAttachments, setIncludeAttachments] = + React.useState(true); const user = useCurrentUser(); const { showToast } = useToasts(); const { collections } = useStores(); @@ -34,11 +36,18 @@ function ExportDialog({ collection, onSubmit }: Props) { [] ); + const handleIncludeAttachmentsChange = React.useCallback( + (ev: React.ChangeEvent) => { + setIncludeAttachments(ev.target.checked); + }, + [] + ); + const handleSubmit = async () => { if (collection) { - await collection.export(format); + await collection.export(format, includeAttachments); } else { - await collections.export(format); + await collections.export(format, includeAttachments); } onSubmit(); showToast(t("Export started"), { type: "success" }); @@ -107,6 +116,23 @@ function ExportDialog({ collection, onSubmit }: Props) { ))} +
+ ); } diff --git a/app/models/Collection.ts b/app/models/Collection.ts index 79b05631e..dc4350d35 100644 --- a/app/models/Collection.ts +++ b/app/models/Collection.ts @@ -266,9 +266,10 @@ export default class Collection extends ParanoidModel { @action unstar = async () => this.store.unstar(this); - export = (format: FileOperationFormat) => + export = (format: FileOperationFormat, includeAttachments: boolean) => client.post("/collections.export", { id: this.id, format, + includeAttachments, }); } diff --git a/app/stores/CollectionsStore.ts b/app/stores/CollectionsStore.ts index 553c274ba..0cc0d7e15 100644 --- a/app/stores/CollectionsStore.ts +++ b/app/stores/CollectionsStore.ts @@ -236,8 +236,9 @@ export default class CollectionsStore extends BaseStore { this.rootStore.documents.fetchRecentlyViewed(); }; - export = (format: FileOperationFormat) => + export = (format: FileOperationFormat, includeAttachments: boolean) => client.post("/collections.export_all", { format, + includeAttachments, }); } diff --git a/server/commands/collectionExporter.ts b/server/commands/collectionExporter.ts index 80da27e7d..1e884082a 100644 --- a/server/commands/collectionExporter.ts +++ b/server/commands/collectionExporter.ts @@ -13,6 +13,7 @@ type Props = { team: Team; user: User; format?: FileOperationFormat; + includeAttachments?: boolean; ip: string; transaction: Transaction; }; @@ -22,6 +23,7 @@ async function collectionExporter({ team, user, format = FileOperationFormat.MarkdownZip, + includeAttachments = true, ip, transaction, }: Props) { @@ -36,6 +38,7 @@ async function collectionExporter({ url: null, size: 0, collectionId, + includeAttachments, userId: user.id, teamId: user.teamId, }, diff --git a/server/migrations/20230621004649-add-include-attachments-file-operation.js b/server/migrations/20230621004649-add-include-attachments-file-operation.js new file mode 100644 index 000000000..3249c5f0c --- /dev/null +++ b/server/migrations/20230621004649-add-include-attachments-file-operation.js @@ -0,0 +1,15 @@ +'use strict'; + +module.exports = { + async up (queryInterface, Sequelize) { + await queryInterface.addColumn("file_operations", "includeAttachments", { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: true, + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.removeColumn("file_operations", "includeAttachments"); + } +}; diff --git a/server/models/FileOperation.ts b/server/models/FileOperation.ts index 8ed634504..43ee87261 100644 --- a/server/models/FileOperation.ts +++ b/server/models/FileOperation.ts @@ -58,6 +58,9 @@ class FileOperation extends IdModel { @Column(DataType.BIGINT) size: number; + @Column(DataType.BOOLEAN) + includeAttachments: boolean; + /** * Mark the current file operation as expired and remove the file from storage. */ diff --git a/server/queues/tasks/ExportDocumentTreeTask.ts b/server/queues/tasks/ExportDocumentTreeTask.ts index b3e97cc16..c61bd63ff 100644 --- a/server/queues/tasks/ExportDocumentTreeTask.ts +++ b/server/queues/tasks/ExportDocumentTreeTask.ts @@ -25,12 +25,14 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { pathInZip, documentId, format = FileOperationFormat.MarkdownZip, + includeAttachments, pathMap, }: { zip: JSZip; pathInZip: string; documentId: string; format: FileOperationFormat; + includeAttachments: boolean; pathMap: Map; }) { Logger.debug("task", `Adding document to archive`, { documentId }); @@ -44,7 +46,9 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { ? await DocumentHelper.toHTML(document, { centered: true }) : await DocumentHelper.toMarkdown(document); - const attachmentIds = parseAttachmentIds(document.text); + const attachmentIds = includeAttachments + ? parseAttachmentIds(document.text) + : []; const attachments = attachmentIds.length ? await Attachment.findAll({ where: { @@ -117,13 +121,15 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { * @param zip The JSZip instance to add files to * @param collections The collections to export * @param format The format to export in + * @param includeAttachments Whether to include attachments in the export * * @returns The path to the zip file in tmp. */ protected async addCollectionsToArchive( zip: JSZip, collections: Collection[], - format: FileOperationFormat + format: FileOperationFormat, + includeAttachments = true ) { const pathMap = this.createPathMap(collections, format); Logger.debug( @@ -139,6 +145,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { zip, pathInZip, documentId, + includeAttachments, format, pathMap, }); diff --git a/server/queues/tasks/ExportHTMLZipTask.ts b/server/queues/tasks/ExportHTMLZipTask.ts index fc6a84ef9..2d16cf597 100644 --- a/server/queues/tasks/ExportHTMLZipTask.ts +++ b/server/queues/tasks/ExportHTMLZipTask.ts @@ -1,16 +1,17 @@ import JSZip from "jszip"; import { FileOperationFormat } from "@shared/types"; -import { Collection } from "@server/models"; +import { Collection, FileOperation } from "@server/models"; import ExportDocumentTreeTask from "./ExportDocumentTreeTask"; export default class ExportHTMLZipTask extends ExportDocumentTreeTask { - public async export(collections: Collection[]) { + public async export(collections: Collection[], fileOperation: FileOperation) { const zip = new JSZip(); return await this.addCollectionsToArchive( zip, collections, - FileOperationFormat.HTMLZip + FileOperationFormat.HTMLZip, + fileOperation.includeAttachments ); } } diff --git a/server/queues/tasks/ExportJSONTask.ts b/server/queues/tasks/ExportJSONTask.ts index dcb8d254d..8ee6e64d5 100644 --- a/server/queues/tasks/ExportJSONTask.ts +++ b/server/queues/tasks/ExportJSONTask.ts @@ -25,7 +25,11 @@ export default class ExportJSONTask extends ExportTask { // serial to avoid overloading, slow and steady wins the race for (const collection of collections) { - await this.addCollectionToArchive(zip, collection); + await this.addCollectionToArchive( + zip, + collection, + fileOperation.includeAttachments + ); } await this.addMetadataToArchive(zip, fileOperation); @@ -52,7 +56,11 @@ export default class ExportJSONTask extends ExportTask { ); } - private async addCollectionToArchive(zip: JSZip, collection: Collection) { + private async addCollectionToArchive( + zip: JSZip, + collection: Collection, + includeAttachments: boolean + ) { const output: CollectionJSONExport = { collection: { ...omit(presentCollection(collection), ["url"]), @@ -75,12 +83,14 @@ export default class ExportJSONTask extends ExportTask { continue; } - const attachments = await Attachment.findAll({ - where: { - teamId: document.teamId, - id: parseAttachmentIds(document.text), - }, - }); + const attachments = includeAttachments + ? await Attachment.findAll({ + where: { + teamId: document.teamId, + id: parseAttachmentIds(document.text), + }, + }) + : []; await Promise.all( attachments.map(async (attachment) => { diff --git a/server/queues/tasks/ExportMarkdownZipTask.ts b/server/queues/tasks/ExportMarkdownZipTask.ts index 6ea69eefe..8da8b5530 100644 --- a/server/queues/tasks/ExportMarkdownZipTask.ts +++ b/server/queues/tasks/ExportMarkdownZipTask.ts @@ -1,16 +1,17 @@ import JSZip from "jszip"; import { FileOperationFormat } from "@shared/types"; -import { Collection } from "@server/models"; +import { Collection, FileOperation } from "@server/models"; import ExportDocumentTreeTask from "./ExportDocumentTreeTask"; export default class ExportMarkdownZipTask extends ExportDocumentTreeTask { - public async export(collections: Collection[]) { + public async export(collections: Collection[], fileOperation: FileOperation) { const zip = new JSZip(); return await this.addCollectionsToArchive( zip, collections, - FileOperationFormat.MarkdownZip + FileOperationFormat.MarkdownZip, + fileOperation.includeAttachments ); } } diff --git a/server/queues/tasks/ExportTask.ts b/server/queues/tasks/ExportTask.ts index b59790699..159168bed 100644 --- a/server/queues/tasks/ExportTask.ts +++ b/server/queues/tasks/ExportTask.ts @@ -41,7 +41,9 @@ export default abstract class ExportTask extends BaseTask { }); try { - Logger.info("task", `ExportTask processing data for ${fileOperationId}`); + Logger.info("task", `ExportTask processing data for ${fileOperationId}`, { + includeAttachments: fileOperation.includeAttachments, + }); await this.updateFileOperation(fileOperation, { state: FileOperationState.Creating, diff --git a/server/routes/api/collections.ts b/server/routes/api/collections.ts index 16ef9b33f..5f72978ce 100644 --- a/server/routes/api/collections.ts +++ b/server/routes/api/collections.ts @@ -49,6 +49,7 @@ import { assertHexColor, assertIndexCharacters, assertCollectionPermission, + assertBoolean, } from "@server/validation"; import pagination from "./middlewares/pagination"; @@ -562,10 +563,14 @@ router.post( auth(), async (ctx: APIContext) => { const { id } = ctx.request.body; - const { format = FileOperationFormat.MarkdownZip } = ctx.request.body; + const { + format = FileOperationFormat.MarkdownZip, + includeAttachments = true, + } = ctx.request.body; assertUuid(id, "id is required"); assertIn(format, Object.values(FileOperationFormat), "Invalid format"); + assertBoolean(includeAttachments, "includeAttachments must be a boolean"); const { user } = ctx.state.auth; const team = await Team.findByPk(user.teamId); @@ -582,6 +587,7 @@ router.post( user, team, format, + includeAttachments, ip: ctx.request.ip, transaction, }) @@ -601,18 +607,23 @@ router.post( rateLimiter(RateLimiterStrategy.FivePerHour), auth(), async (ctx: APIContext) => { - const { format = FileOperationFormat.MarkdownZip } = ctx.request.body; + const { + format = FileOperationFormat.MarkdownZip, + includeAttachments = true, + } = ctx.request.body; const { user } = ctx.state.auth; const team = await Team.findByPk(user.teamId); authorize(user, "createExport", team); assertIn(format, Object.values(FileOperationFormat), "Invalid format"); + assertBoolean(includeAttachments, "includeAttachments must be a boolean"); const fileOperation = await sequelize.transaction(async (transaction) => collectionExporter({ user, team, format, + includeAttachments, ip: ctx.request.ip, transaction, }) diff --git a/shared/i18n/locales/en_US/translation.json b/shared/i18n/locales/en_US/translation.json index 4a0ed6b3e..007796dee 100644 --- a/shared/i18n/locales/en_US/translation.json +++ b/shared/i18n/locales/en_US/translation.json @@ -181,6 +181,8 @@ "Export": "Export", "Exporting the collection {{collectionName}} may take some time.": "Exporting the collection {{collectionName}} may take some time.", "You will receive an email when it's complete.": "You will receive an email when it's complete.", + "Include attachments": "Include attachments", + "Including uploaded images and files in the exported data": "Including uploaded images and files in the exported data", "{{ count }} member": "{{ count }} member", "{{ count }} member_plural": "{{ count }} members", "Group members": "Group members",