feat: Add option to not include attachments in exported data (#5463)

app/components/ExportDialog.tsx

@@ -21,6 +21,8 @@ function ExportDialog({ collection, onSubmit }: Props) {
   const [format, setFormat] = React.useState<FileOperationFormat>(
     FileOperationFormat.MarkdownZip
   );
+  const [includeAttachments, setIncludeAttachments] =
+    React.useState<boolean>(true);
   const user = useCurrentUser();
   const { showToast } = useToasts();
   const { collections } = useStores();
@@ -34,11 +36,18 @@ function ExportDialog({ collection, onSubmit }: Props) {
     []
   );
 
+  const handleIncludeAttachmentsChange = React.useCallback(
+    (ev: React.ChangeEvent<HTMLInputElement>) => {
+      setIncludeAttachments(ev.target.checked);
+    },
+    []
+  );
+
   const handleSubmit = async () => {
     if (collection) {
-      await collection.export(format);
+      await collection.export(format, includeAttachments);
     } else {
-      await collections.export(format);
+      await collections.export(format, includeAttachments);
     }
     onSubmit();
     showToast(t("Export started"), { type: "success" });
@@ -107,6 +116,23 @@ function ExportDialog({ collection, onSubmit }: Props) {
           </Option>
         ))}
       </Flex>
+      <hr />
+      <Option>
+        <input
+          type="checkbox"
+          name="includeAttachments"
+          checked={includeAttachments}
+          onChange={handleIncludeAttachmentsChange}
+        />
+        <div>
+          <Text size="small" weight="bold">
+            {t("Include attachments")}
+          </Text>
+          <Text size="small">
+            {t("Including uploaded images and files in the exported data")}.
+          </Text>{" "}
+        </div>
+      </Option>
     </ConfirmationDialog>
   );
 }

app/models/Collection.ts

@@ -266,9 +266,10 @@ export default class Collection extends ParanoidModel {
   @action
   unstar = async () => this.store.unstar(this);
 
-  export = (format: FileOperationFormat) =>
+  export = (format: FileOperationFormat, includeAttachments: boolean) =>
     client.post("/collections.export", {
       id: this.id,
       format,
+      includeAttachments,
     });
 }

app/stores/CollectionsStore.ts

@@ -236,8 +236,9 @@ export default class CollectionsStore extends BaseStore<Collection> {
     this.rootStore.documents.fetchRecentlyViewed();
   };
 
-  export = (format: FileOperationFormat) =>
+  export = (format: FileOperationFormat, includeAttachments: boolean) =>
     client.post("/collections.export_all", {
       format,
+      includeAttachments,
     });
 }

server/commands/collectionExporter.ts

@@ -13,6 +13,7 @@ type Props = {
   team: Team;
   user: User;
   format?: FileOperationFormat;
+  includeAttachments?: boolean;
   ip: string;
   transaction: Transaction;
 };
@@ -22,6 +23,7 @@ async function collectionExporter({
   team,
   user,
   format = FileOperationFormat.MarkdownZip,
+  includeAttachments = true,
   ip,
   transaction,
 }: Props) {
@@ -36,6 +38,7 @@ async function collectionExporter({
       url: null,
       size: 0,
       collectionId,
+      includeAttachments,
       userId: user.id,
       teamId: user.teamId,
     },

server/migrations/20230621004649-add-include-attachments-file-operation.js

@@ -0,0 +1,15 @@
+'use strict';
+
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    await queryInterface.addColumn("file_operations", "includeAttachments", {
+      type: Sequelize.BOOLEAN,
+      allowNull: false,
+      defaultValue: true,
+    });
+  },
+
+  async down (queryInterface, Sequelize) {
+    await queryInterface.removeColumn("file_operations", "includeAttachments");
+  }
+};

server/models/FileOperation.ts

@@ -58,6 +58,9 @@ class FileOperation extends IdModel {
   @Column(DataType.BIGINT)
   size: number;
 
+  @Column(DataType.BOOLEAN)
+  includeAttachments: boolean;
+
   /**
    * Mark the current file operation as expired and remove the file from storage.
    */

server/queues/tasks/ExportDocumentTreeTask.ts

@@ -25,12 +25,14 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
     pathInZip,
     documentId,
     format = FileOperationFormat.MarkdownZip,
+    includeAttachments,
     pathMap,
   }: {
     zip: JSZip;
     pathInZip: string;
     documentId: string;
     format: FileOperationFormat;
+    includeAttachments: boolean;
     pathMap: Map<string, string>;
   }) {
     Logger.debug("task", `Adding document to archive`, { documentId });
@@ -44,7 +46,9 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
         ? await DocumentHelper.toHTML(document, { centered: true })
         : await DocumentHelper.toMarkdown(document);
 
-    const attachmentIds = parseAttachmentIds(document.text);
+    const attachmentIds = includeAttachments
+      ? parseAttachmentIds(document.text)
+      : [];
     const attachments = attachmentIds.length
       ? await Attachment.findAll({
           where: {
@@ -117,13 +121,15 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
    * @param zip The JSZip instance to add files to
    * @param collections The collections to export
    * @param format The format to export in
+   * @param includeAttachments Whether to include attachments in the export
    *
    * @returns The path to the zip file in tmp.
    */
   protected async addCollectionsToArchive(
     zip: JSZip,
     collections: Collection[],
-    format: FileOperationFormat
+    format: FileOperationFormat,
+    includeAttachments = true
   ) {
     const pathMap = this.createPathMap(collections, format);
     Logger.debug(
@@ -139,6 +145,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
         zip,
         pathInZip,
         documentId,
+        includeAttachments,
         format,
         pathMap,
       });

server/queues/tasks/ExportHTMLZipTask.ts

@@ -1,16 +1,17 @@
 import JSZip from "jszip";
 import { FileOperationFormat } from "@shared/types";
-import { Collection } from "@server/models";
+import { Collection, FileOperation } from "@server/models";
 import ExportDocumentTreeTask from "./ExportDocumentTreeTask";
 
 export default class ExportHTMLZipTask extends ExportDocumentTreeTask {
-  public async export(collections: Collection[]) {
+  public async export(collections: Collection[], fileOperation: FileOperation) {
     const zip = new JSZip();
 
     return await this.addCollectionsToArchive(
       zip,
       collections,
-      FileOperationFormat.HTMLZip
+      FileOperationFormat.HTMLZip,
+      fileOperation.includeAttachments
     );
   }
 }

server/queues/tasks/ExportJSONTask.ts

@@ -25,7 +25,11 @@ export default class ExportJSONTask extends ExportTask {
 
     // serial to avoid overloading, slow and steady wins the race
     for (const collection of collections) {
-      await this.addCollectionToArchive(zip, collection);
+      await this.addCollectionToArchive(
+        zip,
+        collection,
+        fileOperation.includeAttachments
+      );
     }
 
     await this.addMetadataToArchive(zip, fileOperation);
@@ -52,7 +56,11 @@ export default class ExportJSONTask extends ExportTask {
     );
   }
 
-  private async addCollectionToArchive(zip: JSZip, collection: Collection) {
+  private async addCollectionToArchive(
+    zip: JSZip,
+    collection: Collection,
+    includeAttachments: boolean
+  ) {
     const output: CollectionJSONExport = {
       collection: {
         ...omit(presentCollection(collection), ["url"]),
@@ -75,12 +83,14 @@ export default class ExportJSONTask extends ExportTask {
           continue;
         }
 
-        const attachments = await Attachment.findAll({
-          where: {
-            teamId: document.teamId,
-            id: parseAttachmentIds(document.text),
-          },
-        });
+        const attachments = includeAttachments
+          ? await Attachment.findAll({
+              where: {
+                teamId: document.teamId,
+                id: parseAttachmentIds(document.text),
+              },
+            })
+          : [];
 
         await Promise.all(
           attachments.map(async (attachment) => {

server/queues/tasks/ExportMarkdownZipTask.ts

@@ -1,16 +1,17 @@
 import JSZip from "jszip";
 import { FileOperationFormat } from "@shared/types";
-import { Collection } from "@server/models";
+import { Collection, FileOperation } from "@server/models";
 import ExportDocumentTreeTask from "./ExportDocumentTreeTask";
 
 export default class ExportMarkdownZipTask extends ExportDocumentTreeTask {
-  public async export(collections: Collection[]) {
+  public async export(collections: Collection[], fileOperation: FileOperation) {
     const zip = new JSZip();
 
     return await this.addCollectionsToArchive(
       zip,
       collections,
-      FileOperationFormat.MarkdownZip
+      FileOperationFormat.MarkdownZip,
+      fileOperation.includeAttachments
     );
   }
 }

server/queues/tasks/ExportTask.ts

@@ -41,7 +41,9 @@ export default abstract class ExportTask extends BaseTask<Props> {
     });
 
     try {
-      Logger.info("task", `ExportTask processing data for ${fileOperationId}`);
+      Logger.info("task", `ExportTask processing data for ${fileOperationId}`, {
+        includeAttachments: fileOperation.includeAttachments,
+      });
 
       await this.updateFileOperation(fileOperation, {
         state: FileOperationState.Creating,

server/routes/api/collections.ts

@@ -49,6 +49,7 @@ import {
   assertHexColor,
   assertIndexCharacters,
   assertCollectionPermission,
+  assertBoolean,
 } from "@server/validation";
 import pagination from "./middlewares/pagination";
 
@@ -562,10 +563,14 @@ router.post(
   auth(),
   async (ctx: APIContext) => {
     const { id } = ctx.request.body;
-    const { format = FileOperationFormat.MarkdownZip } = ctx.request.body;
+    const {
+      format = FileOperationFormat.MarkdownZip,
+      includeAttachments = true,
+    } = ctx.request.body;
 
     assertUuid(id, "id is required");
     assertIn(format, Object.values(FileOperationFormat), "Invalid format");
+    assertBoolean(includeAttachments, "includeAttachments must be a boolean");
 
     const { user } = ctx.state.auth;
     const team = await Team.findByPk(user.teamId);
@@ -582,6 +587,7 @@ router.post(
         user,
         team,
         format,
+        includeAttachments,
         ip: ctx.request.ip,
         transaction,
       })
@@ -601,18 +607,23 @@ router.post(
   rateLimiter(RateLimiterStrategy.FivePerHour),
   auth(),
   async (ctx: APIContext) => {
-    const { format = FileOperationFormat.MarkdownZip } = ctx.request.body;
+    const {
+      format = FileOperationFormat.MarkdownZip,
+      includeAttachments = true,
+    } = ctx.request.body;
     const { user } = ctx.state.auth;
     const team = await Team.findByPk(user.teamId);
     authorize(user, "createExport", team);
 
     assertIn(format, Object.values(FileOperationFormat), "Invalid format");
+    assertBoolean(includeAttachments, "includeAttachments must be a boolean");
 
     const fileOperation = await sequelize.transaction(async (transaction) =>
       collectionExporter({
         user,
         team,
         format,
+        includeAttachments,
         ip: ctx.request.ip,
         transaction,
       })

shared/i18n/locales/en_US/translation.json

@@ -181,6 +181,8 @@
   "Export": "Export",
   "Exporting the collection <em>{{collectionName}}</em> may take some time.": "Exporting the collection <em>{{collectionName}}</em> may take some time.",
   "You will receive an email when it's complete.": "You will receive an email when it's complete.",
+  "Include attachments": "Include attachments",
+  "Including uploaded images and files in the exported data": "Including uploaded images and files in the exported data",
   "{{ count }} member": "{{ count }} member",
   "{{ count }} member_plural": "{{ count }} members",
   "Group members": "Group members",