JSON to client (#5553)

server/models/helpers/DocumentHelper.test.ts

@@ -1,6 +1,6 @@
 import Revision from "@server/models/Revision";
 import { buildDocument } from "@server/test/factories";
-import DocumentHelper from "./DocumentHelper";
+import { DocumentHelper } from "./DocumentHelper";
 
 describe("DocumentHelper", () => {
   beforeAll(() => {

server/models/helpers/DocumentHelper.tsx

@@ -7,14 +7,14 @@ import { JSDOM } from "jsdom";
 import { Node } from "prosemirror-model";
 import * as Y from "yjs";
 import textBetween from "@shared/editor/lib/textBetween";
-import MarkdownHelper from "@shared/utils/MarkdownHelper";
-import { parser, schema } from "@server/editor";
+import { ProsemirrorData } from "@shared/types";
+import { parser, serializer, schema } from "@server/editor";
 import { addTags } from "@server/logging/tracer";
 import { trace } from "@server/logging/tracing";
 import { Document, Revision } from "@server/models";
 import diff from "@server/utils/diff";
-import ProsemirrorHelper from "./ProsemirrorHelper";
-import TextHelper from "./TextHelper";
+import { ProsemirrorHelper } from "./ProsemirrorHelper";
+import { TextHelper } from "./TextHelper";
 
 type HTMLOptions = {
   /** Whether to include the document title in the generated HTML (defaults to true) */
@@ -35,32 +35,18 @@ type HTMLOptions = {
 };
 
 @trace()
-export default class DocumentHelper {
+export class DocumentHelper {
   /**
-   * Returns the document as JSON content. This method uses the collaborative state if available,
-   * otherwise it falls back to Markdown.
-   *
-   * @param document The document or revision to convert
-   * @returns The document content as JSON
-   */
-  static toJSON(document: Document | Revision) {
-    if ("state" in document && document.state) {
-      const ydoc = new Y.Doc();
-      Y.applyUpdate(ydoc, document.state);
-      return yDocToProsemirrorJSON(ydoc, "default");
-    }
-    const node = parser.parse(document.text) || Node.fromJSON(schema, {});
-    return node.toJSON();
-  }
-
-  /**
-   * Returns the document as a Prosemirror Node. This method uses the collaborative state if
-   * available, otherwise it falls back to Markdown.
+   * Returns the document as a Prosemirror Node. This method uses the derived content if available
+   * then the collaborative state, otherwise it falls back to Markdown.
    *
    * @param document The document or revision to convert
    * @returns The document content as a Prosemirror Node
    */
   static toProsemirror(document: Document | Revision) {
+    if ("content" in document && document.content) {
+      return Node.fromJSON(schema, document.content);
+    }
     if ("state" in document && document.state) {
       const ydoc = new Y.Doc();
       Y.applyUpdate(ydoc, document.state);
@@ -69,6 +55,55 @@ export default class DocumentHelper {
     return parser.parse(document.text) || Node.fromJSON(schema, {});
   }
 
+  /**
+   * Returns the document as a plain JSON object. This method uses the derived content if available
+   * then the collaborative state, otherwise it falls back to Markdown.
+   *
+   * @param document The document or revision to convert
+   * @param options Options for the conversion
+   * @returns The document content as a plain JSON object
+   */
+  static async toJSON(
+    document: Document | Revision,
+    options?: {
+      /** The team context */
+      teamId: string;
+      /** Whether to sign attachment urls, and if so for how many seconds is the signature valid */
+      signedUrls: number;
+      /** Marks to remove from the document */
+      removeMarks?: string[];
+    }
+  ): Promise<ProsemirrorData> {
+    let doc: Node | null;
+    let json;
+
+    if ("content" in document && document.content) {
+      doc = Node.fromJSON(schema, document.content);
+    } else if ("state" in document && document.state) {
+      const ydoc = new Y.Doc();
+      Y.applyUpdate(ydoc, document.state);
+      doc = Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default"));
+    } else {
+      doc = parser.parse(document.text);
+    }
+
+    if (doc && options?.signedUrls) {
+      json = await ProsemirrorHelper.signAttachmentUrls(
+        doc,
+        options.teamId,
+        options.signedUrls
+      );
+    } else {
+      json = doc?.toJSON() ?? {};
+    }
+
+    if (options?.removeMarks) {
+      json = ProsemirrorHelper.removeMarks(json, options.removeMarks);
+    }
+
+    return json;
+  }
+
   /**
    * Returns the document as plain text. This method uses the
    * collaborative state if available, otherwise it falls back to Markdown.
@@ -88,19 +123,30 @@ export default class DocumentHelper {
   }
 
   /**
-   * Returns the document as Markdown. This is a lossy conversion and should
-   * only be used for export.
+   * Returns the document as Markdown. This is a lossy conversion and should nly be used for export.
    *
    * @param document The document or revision to convert
    * @returns The document title and content as a Markdown string
    */
   static toMarkdown(document: Document | Revision) {
-    return MarkdownHelper.toMarkdown(document);
+    const text = serializer
+      .serialize(DocumentHelper.toProsemirror(document))
+      .replace(/\n\\(\n|$)/g, "\n\n")
+      .replace(/“/g, '"')
+      .replace(/”/g, '"')
+      .replace(/‘/g, "'")
+      .replace(/’/g, "'")
+      .trim();
+
+    const title = `${document.emoji ? document.emoji + " " : ""}${
+      document.title
+    }`;
+
+    return `# ${title}\n\n${text}`;
   }
 
   /**
-   * Returns the document as plain HTML. This is a lossy conversion and should
-   * only be used for export.
+   * Returns the document as plain HTML. This is a lossy conversion and should only be used for export.
    *
    * @param document The document or revision to convert
    * @param options Options for the HTML output

server/models/helpers/ProsemirrorHelper.tsx

@@ -1,5 +1,8 @@
 import { prosemirrorToYDoc } from "@getoutline/y-prosemirror";
 import { JSDOM } from "jsdom";
+import compact from "lodash/compact";
+import flatten from "lodash/flatten";
+import uniq from "lodash/uniq";
 import { Node, DOMSerializer, Fragment, Mark } from "prosemirror-model";
 import * as React from "react";
 import { renderToString } from "react-dom/server";
@@ -9,10 +12,14 @@ import EditorContainer from "@shared/editor/components/Styles";
 import embeds from "@shared/editor/embeds";
 import GlobalStyles from "@shared/styles/globals";
 import light from "@shared/styles/theme";
+import { ProsemirrorData } from "@shared/types";
+import { attachmentRedirectRegex } from "@shared/utils/ProsemirrorHelper";
 import { isRTL } from "@shared/utils/rtl";
 import { schema, parser } from "@server/editor";
 import Logger from "@server/logging/Logger";
 import { trace } from "@server/logging/tracing";
+import Attachment from "@server/models/Attachment";
+import FileStorage from "@server/storage/files";
 
 export type HTMLOptions = {
   /** A title, if it should be included */
@@ -36,15 +43,22 @@ type MentionAttrs = {
 };
 
 @trace()
-export default class ProsemirrorHelper {
+export class ProsemirrorHelper {
   /**
    * Returns the input text as a Y.Doc.
    *
    * @param markdown The text to parse
    * @returns The content as a Y.Doc.
    */
-  static toYDoc(markdown: string, fieldName = "default"): Y.Doc {
-    let node = parser.parse(markdown);
+  static toYDoc(input: string | ProsemirrorData, fieldName = "default"): Y.Doc {
+    if (typeof input === "object") {
+      return prosemirrorToYDoc(
+        ProsemirrorHelper.toProsemirror(input),
+        fieldName
+      );
+    }
+
+    let node = parser.parse(input);
 
     // in the editor embeds are created at runtime by converting links into
     // embeds where they match.Because we're converting to a CRDT structure on
@@ -106,7 +120,7 @@ export default class ProsemirrorHelper {
    * @param data The object to parse
    * @returns The content as a Prosemirror Node
    */
-  static toProsemirror(data: Record<string, any>) {
+  static toProsemirror(data: ProsemirrorData) {
     return Node.fromJSON(schema, data);
   }
 
@@ -116,10 +130,10 @@ export default class ProsemirrorHelper {
    * @param node The node to parse mentions from
    * @returns An array of mention attributes
    */
-  static parseMentions(node: Node) {
+  static parseMentions(doc: Node) {
     const mentions: MentionAttrs[] = [];
 
-    node.descendants((node: Node) => {
+    doc.descendants((node: Node) => {
       if (
         node.type.name === "mention" &&
         !mentions.some((m) => m.id === node.attrs.id)
@@ -138,6 +152,117 @@ export default class ProsemirrorHelper {
     return mentions;
   }
 
+  /**
+   * Removes all marks from the node that match the given types.
+   *
+   * @param data The ProsemirrorData object to remove marks from
+   * @param marks The mark types to remove
+   * @returns The content with marks removed
+   */
+  static removeMarks(data: ProsemirrorData, marks: string[]) {
+    function removeMarksInner(node: ProsemirrorData) {
+      if (node.marks) {
+        node.marks = node.marks.filter((mark) => !marks.includes(mark.type));
+      }
+      if (node.content) {
+        node.content.forEach(removeMarksInner);
+      }
+      return node;
+    }
+    return removeMarksInner(data);
+  }
+
+  /**
+   * Returns the document as a plain JSON object with attachment URLs signed.
+   *
+   * @param node The node to convert to JSON
+   * @param teamId The team ID to use for signing
+   * @param expiresIn The number of seconds until the signed URL expires
+   * @returns The content as a JSON object
+   */
+  static async signAttachmentUrls(doc: Node, teamId: string, expiresIn = 60) {
+    const attachmentIds = ProsemirrorHelper.parseAttachmentIds(doc);
+    const attachments = await Attachment.findAll({
+      where: {
+        id: attachmentIds,
+        teamId,
+      },
+    });
+
+    const mapping: Record<string, string> = {};
+
+    await Promise.all(
+      attachments.map(async (attachment) => {
+        const signedUrl = await FileStorage.getSignedUrl(
+          attachment.key,
+          expiresIn
+        );
+        mapping[attachment.redirectUrl] = signedUrl;
+      })
+    );
+
+    const json = doc.toJSON() as ProsemirrorData;
+
+    function replaceAttachmentUrls(node: ProsemirrorData) {
+      if (node.attrs?.src) {
+        node.attrs.src = mapping[node.attrs.src as string] || node.attrs.src;
+      } else if (node.attrs?.href) {
+        node.attrs.href = mapping[node.attrs.href as string] || node.attrs.href;
+      } else if (node.marks) {
+        node.marks.forEach((mark) => {
+          if (mark.attrs?.href) {
+            mark.attrs.href =
+              mapping[mark.attrs.href as string] || mark.attrs.href;
+          }
+        });
+      }
+
+      if (node.content) {
+        node.content.forEach(replaceAttachmentUrls);
+      }
+
+      return node;
+    }
+
+    return replaceAttachmentUrls(json);
+  }
+
+  /**
+   * Returns an array of attachment IDs in the node.
+   *
+   * @param node The node to parse attachments from
+   * @returns An array of attachment IDs
+   */
+  static parseAttachmentIds(doc: Node) {
+    const urls: string[] = [];
+
+    doc.descendants((node) => {
+      node.marks.forEach((mark) => {
+        if (mark.type.name === "link") {
+          urls.push(mark.attrs.href);
+        }
+      });
+      if (["image", "video"].includes(node.type.name)) {
+        urls.push(node.attrs.src);
+      }
+      if (node.type.name === "attachment") {
+        urls.push(node.attrs.href);
+      }
+    });
+
+    return uniq(
+      compact(
+        flatten(
+          urls.map((url) =>
+            [...url.matchAll(attachmentRedirectRegex)].map(
+              (match) => match.groups?.id
+            )
+          )
+        )
+      )
+    );
+  }
+
   /**
    * Returns the node as HTML. This is a lossy conversion and should only be used
    * for export.

server/models/helpers/SearchHelper.ts

@@ -12,7 +12,7 @@ import Share from "@server/models/Share";
 import Team from "@server/models/Team";
 import User from "@server/models/User";
 import { sequelize } from "@server/storage/database";
-import DocumentHelper from "./DocumentHelper";
+import { DocumentHelper } from "./DocumentHelper";
 
 type SearchResponse = {
   results: {

server/models/helpers/TextHelper.test.ts

@@ -1,5 +1,5 @@
 import { buildUser } from "@server/test/factories";
-import TextHelper from "./TextHelper";
+import { TextHelper } from "./TextHelper";
 
 describe("TextHelper", () => {
   beforeAll(() => {

server/models/helpers/TextHelper.ts

@@ -16,7 +16,7 @@ import parseAttachmentIds from "@server/utils/parseAttachmentIds";
 import parseImages from "@server/utils/parseImages";
 
 @trace()
-export default class TextHelper {
+export class TextHelper {
   /**
    * Replaces template variables in the given text with the current date and time.
    *