JSON to client (#5553)

app/actions/definitions/documents.tsx

@@ -31,7 +31,6 @@ import {
 import * as React from "react";
 import { toast } from "sonner";
 import { ExportContentType, TeamPreference } from "@shared/types";
-import MarkdownHelper from "@shared/utils/MarkdownHelper";
 import { getEventFiles } from "@shared/utils/files";
 import DocumentDelete from "~/scenes/DocumentDelete";
 import DocumentMove from "~/scenes/DocumentMove";
@@ -454,7 +453,7 @@ export const copyDocumentAsMarkdown = createAction({
       ? stores.documents.get(activeDocumentId)
       : undefined;
     if (document) {
-      copy(MarkdownHelper.toMarkdown(document));
+      copy(document.toMarkdown());
       toast.success(t("Markdown copied to clipboard"));
     }
   },

app/components/Facepile.tsx

@@ -5,6 +5,7 @@ import { s } from "@shared/styles";
 import User from "~/models/User";
 import Avatar from "~/components/Avatar";
 import Flex from "~/components/Flex";
+import { AvatarSize } from "./Avatar/Avatar";
 
 type Props = {
   users: User[];
@@ -17,7 +18,7 @@ type Props = {
 function Facepile({
   users,
   overflow = 0,
-  size = 32,
+  size = AvatarSize.Large,
   limit = 8,
   renderAvatar = DefaultAvatar,
   ...rest
@@ -43,7 +44,7 @@ function Facepile({
 }
 
 function DefaultAvatar(user: User) {
-  return <Avatar model={user} size={32} />;
+  return <Avatar model={user} size={AvatarSize.Large} />;
 }
 
 const AvatarWrapper = styled.div`
@@ -62,11 +63,11 @@ const More = styled.div<{ size: number }>`
   min-width: ${(props) => props.size}px;
   height: ${(props) => props.size}px;
   border-radius: 100%;
-  background: ${(props) => props.theme.slate};
-  color: ${s("text")};
+  background: ${(props) => props.theme.textTertiary};
+  color: ${s("white")};
   border: 2px solid ${s("background")};
   text-align: center;
-  font-size: 11px;
+  font-size: 12px;
   font-weight: 600;
 `;
 

app/editor/index.tsx

@@ -39,8 +39,8 @@ import { basicExtensions as extensions } from "@shared/editor/nodes";
 import Node from "@shared/editor/nodes/Node";
 import ReactNode from "@shared/editor/nodes/ReactNode";
 import { EventType } from "@shared/editor/types";
-import { UserPreferences } from "@shared/types";
-import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorData, UserPreferences } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
 import EventEmitter from "@shared/utils/events";
 import Flex from "~/components/Flex";
 import { PortalContext } from "~/components/Portal";
@@ -59,7 +59,7 @@ export type Props = {
   /** The user id of the current user */
   userId?: string;
   /** The editor content, should only be changed if you wish to reset the content */
-  value?: string;
+  value?: string | ProsemirrorData;
   /** The initial editor content as a markdown string or JSON object */
   defaultValue: string | object;
   /** Placeholder displayed when the editor is empty */

app/models/Document.ts

@@ -3,12 +3,19 @@ import i18n, { t } from "i18next";
 import capitalize from "lodash/capitalize";
 import floor from "lodash/floor";
 import { action, autorun, computed, observable, set } from "mobx";
+import { Node, Schema } from "prosemirror-model";
+import ExtensionManager from "@shared/editor/lib/ExtensionManager";
+import { richExtensions } from "@shared/editor/nodes";
+import type {
+  JSONObject,
+  NavigationNode,
+  ProsemirrorData,
+} from "@shared/types";
 import {
   ExportContentType,
   FileOperationFormat,
   NotificationEventType,
 } from "@shared/types";
-import type { JSONObject, NavigationNode } from "@shared/types";
 import Storage from "@shared/utils/Storage";
 import { isRTL } from "@shared/utils/rtl";
 import slugify from "@shared/utils/slugify";
@@ -61,6 +68,9 @@ export default class Document extends ParanoidModel {
   @observable
   id: string;
 
+  @observable.shallow
+  data: ProsemirrorData;
+
   /**
    * The original data source of the document, if imported.
    */
@@ -111,12 +121,6 @@ export default class Document extends ParanoidModel {
   @Relation(() => Collection, { onDelete: "cascade" })
   collection?: Collection;
 
-  /**
-   * The text content of the document as Markdown.
-   */
-  @observable
-  text: string;
-
   /**
    * The title of the document.
    */
@@ -515,6 +519,17 @@ export default class Document extends ParanoidModel {
     recursive?: boolean;
   }) => this.store.duplicate(this, options);
 
+  /**
+   * Returns the first blocks of the document, useful for displaying a preview.
+   *
+   * @param blocks The number of blocks to return, defaults to 4.
+   * @returns A new ProseMirror document.
+   */
+  getSummary = (blocks = 4) => ({
+    ...this.data,
+    content: this.data.content.slice(0, blocks),
+  });
+
   @computed
   get pinned(): boolean {
     return !!this.store.rootStore.pins.orderedData.find(
@@ -535,19 +550,40 @@ export default class Document extends ParanoidModel {
     return !this.isDeleted && !this.isTemplate && !this.isArchived;
   }
 
+  @computed
+  get childDocuments() {
+    return this.store.orderedData.filter(
+      (doc) => doc.parentDocumentId === this.id
+    );
+  }
+
   @computed
   get asNavigationNode(): NavigationNode {
     return {
       id: this.id,
       title: this.title,
-      children: this.store.orderedData
-        .filter((doc) => doc.parentDocumentId === this.id)
-        .map((doc) => doc.asNavigationNode),
+      children: this.childDocuments.map((doc) => doc.asNavigationNode),
       url: this.url,
       isDraft: this.isDraft,
     };
   }
 
+  /**
+   * Returns the markdown representation of the document derived from the ProseMirror data.
+   *
+   * @returns The markdown representation of the document as a string.
+   */
+  toMarkdown = () => {
+    const extensionManager = new ExtensionManager(richExtensions);
+    const serializer = extensionManager.serializer();
+    const schema = new Schema({
+      nodes: extensionManager.nodes,
+      marks: extensionManager.marks,
+    });
+    const markdown = serializer.serialize(Node.fromJSON(schema, this.data));
+    return markdown;
+  };
+
   download = (contentType: ExportContentType) =>
     client.post(
       `/documents.export`,

app/models/Revision.ts

@@ -1,4 +1,5 @@
 import { computed } from "mobx";
+import { ProsemirrorData } from "@shared/types";
 import { isRTL } from "@shared/utils/rtl";
 import Document from "./Document";
 import User from "./User";
@@ -18,8 +19,8 @@ class Revision extends Model {
   /** The document title when the revision was created */
   title: string;
 
-  /** Markdown string of the content when revision was created */
-  text: string;
+  /** Prosemirror data of the content when revision was created */
+  data: ProsemirrorData;
 
   /** The emoji of the document when the revision was created */
   emoji: string | null;

app/scenes/Collection/components/MembershipPreview.tsx

@@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next";
 import { PAGINATION_SYMBOL } from "~/stores/base/Store";
 import Collection from "~/models/Collection";
 import Avatar from "~/components/Avatar";
+import { AvatarSize } from "~/components/Avatar/Avatar";
 import Facepile from "~/components/Facepile";
 import Fade from "~/components/Fade";
 import NudeButton from "~/components/NudeButton";
@@ -66,7 +67,7 @@ const MembershipPreview = ({ collection, limit = 8 }: Props) => {
     return null;
   }
 
-  const overflow = usersCount - groupsCount - collectionUsers.length;
+  const overflow = usersCount + groupsCount - collectionUsers.length;
 
   return (
     <NudeButton
@@ -107,7 +108,9 @@ const MembershipPreview = ({ collection, limit = 8 }: Props) => {
           users={sortBy(collectionUsers, "lastActiveAt")}
           overflow={overflow}
           limit={limit}
-          renderAvatar={(user) => <Avatar model={user} size={32} />}
+          renderAvatar={(item) => (
+            <Avatar model={item} size={AvatarSize.Large} />
+          )}
         />
       </Fade>
     </NudeButton>

app/scenes/Document/Shared.tsx

@@ -123,10 +123,10 @@ function SharedDocumentScene(props: Props) {
   React.useEffect(() => {
     async function fetchData() {
       try {
-        const response = await documents.fetchWithSharedTree(documentSlug, {
+        const res = await documents.fetchWithSharedTree(documentSlug, {
           shareId,
         });
-        setResponse(response);
+        setResponse(res);
       } catch (err) {
         setError(err);
       }

app/scenes/Document/components/CommentThreadItem.tsx

@@ -8,7 +8,7 @@ import { toast } from "sonner";
 import styled, { css } from "styled-components";
 import breakpoint from "styled-components-breakpoint";
 import { s } from "@shared/styles";
-import { JSONObject } from "@shared/types";
+import { ProsemirrorData } from "@shared/types";
 import { dateToRelative } from "@shared/utils/date";
 import { Minute } from "@shared/utils/time";
 import Comment from "~/models/Comment";
@@ -100,7 +100,7 @@ function CommentThreadItem({
   const [isEditing, setEditing, setReadOnly] = useBoolean();
   const formRef = React.useRef<HTMLFormElement>(null);
 
-  const handleChange = (value: (asString: boolean) => JSONObject) => {
+  const handleChange = (value: (asString: boolean) => ProsemirrorData) => {
     setData(value(false));
   };
 

app/scenes/Document/components/DataLoader.tsx

@@ -2,6 +2,7 @@ import { observer } from "mobx-react";
 import * as React from "react";
 import { useLocation, RouteComponentProps, StaticContext } from "react-router";
 import { NavigationNode, TeamPreference } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
 import { RevisionHelper } from "@shared/utils/RevisionHelper";
 import Document from "~/models/Document";
 import Revision from "~/models/Revision";
@@ -92,7 +93,7 @@ function DataLoader({ match, children }: Props) {
       }
     }
     void fetchDocument();
-  }, [ui, documents, document, shareId, documentSlug]);
+  }, [ui, documents, shareId, documentSlug]);
 
   React.useEffect(() => {
     async function fetchRevision() {
@@ -161,7 +162,7 @@ function DataLoader({ match, children }: Props) {
         collectionId: document.collectionId,
         parentDocumentId: nested ? document.id : document.parentDocumentId,
         title,
-        text: "",
+        data: ProsemirrorHelper.getEmptyDocument(),
       });
 
       return newDocument.url;

app/scenes/Document/components/Document.tsx

@@ -1,6 +1,9 @@
+import cloneDeep from "lodash/cloneDeep";
 import debounce from "lodash/debounce";
+import isEqual from "lodash/isEqual";
 import { action, observable } from "mobx";
 import { observer } from "mobx-react";
+import { Node } from "prosemirror-model";
 import { AllSelection } from "prosemirror-state";
 import * as React from "react";
 import { WithTranslation, withTranslation } from "react-i18next";
@@ -16,9 +19,8 @@ import styled from "styled-components";
 import breakpoint from "styled-components-breakpoint";
 import { s } from "@shared/styles";
 import { NavigationNode } from "@shared/types";
-import { Heading } from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorHelper, Heading } from "@shared/utils/ProsemirrorHelper";
 import { parseDomain } from "@shared/utils/domains";
-import getTasks from "@shared/utils/getTasks";
 import RootStore from "~/stores/RootStore";
 import Document from "~/models/Document";
 import Revision from "~/models/Revision";
@@ -34,6 +36,7 @@ import PlaceholderDocument from "~/components/PlaceholderDocument";
 import RegisterKeyDown from "~/components/RegisterKeyDown";
 import withStores from "~/components/withStores";
 import type { Editor as TEditor } from "~/editor";
+import { SearchResult } from "~/editor/components/LinkEditor";
 import { client } from "~/utils/ApiClient";
 import { replaceTitleVariables } from "~/utils/date";
 import { emojiToUrl } from "~/utils/emoji";
@@ -73,13 +76,13 @@ type Props = WithTranslation &
   RootStore &
   RouteComponentProps<Params, StaticContext, LocationState> & {
     sharedTree?: NavigationNode;
-    abilities: Record<string, any>;
+    abilities: Record<string, boolean>;
     document: Document;
     revision?: Revision;
     readOnly: boolean;
     shareId?: string;
     onCreateLink?: (title: string, nested?: boolean) => Promise<string>;
-    onSearchLink?: (term: string) => any;
+    onSearchLink?: (term: string) => Promise<SearchResult[]>;
   };
 
 @observer
@@ -108,8 +111,6 @@ class DocumentScene extends React.Component<Props> {
   @observable
   headings: Heading[] = [];
 
-  getEditorText: () => string = () => this.props.document.text;
-
   componentDidMount() {
     this.updateIsDirty();
   }
@@ -140,8 +141,8 @@ class DocumentScene extends React.Component<Props> {
       return;
     }
 
-    const { view, parser } = editorRef;
-    const doc = parser.parse(template.text);
+    const { view, schema } = editorRef;
+    const doc = Node.fromJSON(schema, template.data);
 
     if (doc) {
       view.dispatch(
@@ -168,10 +169,8 @@ class DocumentScene extends React.Component<Props> {
     if (template.emoji) {
       this.props.document.emoji = template.emoji;
     }
-    if (template.text) {
-      this.props.document.text = template.text;
-    }
 
+    this.props.document.data = cloneDeep(template.data);
     this.updateIsDirty();
 
     return this.onSave({
@@ -292,15 +291,18 @@ class DocumentScene extends React.Component<Props> {
     }
 
     // get the latest version of the editor text value
-    const text = this.getEditorText ? this.getEditorText() : document.text;
-
-    // prevent save before anything has been written (single hash is empty doc)
-    if (text.trim() === "" && document.title.trim() === "") {
+    const doc = this.editor.current?.view.state.doc;
+    if (!doc) {
       return;
     }
 
-    document.text = text;
-    document.tasks = getTasks(document.text);
+    // prevent save before anything has been written (single hash is empty doc)
+    if (ProsemirrorHelper.isEmpty(doc) && document.title.trim() === "") {
+      return;
+    }
+
+    document.data = doc.toJSON();
+    document.tasks = ProsemirrorHelper.getTasksSummary(doc);
 
     // prevent autosave if nothing has changed
     if (options.autosave && !this.isEditorDirty && !document.isDirty()) {
@@ -340,12 +342,11 @@ class DocumentScene extends React.Component<Props> {
 
   updateIsDirty = () => {
     const { document } = this.props;
-    const editorText = this.getEditorText().trim();
-    this.isEditorDirty = editorText !== document.text.trim();
+    const doc = this.editor.current?.view.state.doc;
+    this.isEditorDirty = !isEqual(doc?.toJSON(), document.data);
 
     // a single hash is a doc with just an empty title
-    this.isEmpty =
-      (!editorText || editorText === "#" || editorText === "\\") && !this.title;
+    this.isEmpty = (!doc || ProsemirrorHelper.isEmpty(doc)) && !this.title;
   };
 
   updateIsDirtyDebounced = debounce(this.updateIsDirty, 500);
@@ -358,9 +359,8 @@ class DocumentScene extends React.Component<Props> {
     this.isUploading = false;
   };
 
-  handleChange = (getEditorText: () => string) => {
+  handleChange = () => {
     const { document } = this.props;
-    this.getEditorText = getEditorText;
 
     // Keep derived task list in sync
     const tasks = this.editor.current?.getTasks();
@@ -503,8 +503,8 @@ class DocumentScene extends React.Component<Props> {
                         isDraft={document.isDraft}
                         template={document.isTemplate}
                         document={document}
-                        value={readOnly ? document.text : undefined}
-                        defaultValue={document.text}
+                        value={readOnly ? document.data : undefined}
+                        defaultValue={document.data}
                         embedsDisabled={embedsDisabled}
                         onSynced={this.onSynced}
                         onFileUploadStart={this.onFileUploadStart}

app/scenes/DocumentNew.tsx

@@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next";
 import { useHistory, useLocation, useRouteMatch } from "react-router-dom";
 import { toast } from "sonner";
 import { UserPreference } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
 import CenteredContent from "~/components/CenteredContent";
 import Flex from "~/components/Flex";
 import PlaceholderDocument from "~/components/PlaceholderDocument";
@@ -49,7 +50,7 @@ function DocumentNew({ template }: Props) {
           templateId: query.get("templateId") ?? undefined,
           template,
           title: "",
-          text: "",
+          data: ProsemirrorHelper.getEmptyDocument(),
         });
         history.replace(
           template || !user.separateEditMode

app/stores/DocumentsStore.ts

@@ -22,6 +22,7 @@ import env from "~/env";
 import type {
   FetchOptions,
   PaginationParams,
+  PartialWithId,
   Properties,
   SearchResult,
 } from "~/types";
@@ -473,6 +474,14 @@ export default class DocumentsStore extends Store<Document> {
     return this.data.get(res.data.id);
   };
 
+  override fetch = (id: string, options: FetchOptions = {}) =>
+    super.fetch(
+      id,
+      options,
+      (res: { data: { document: PartialWithId<Document> } }) =>
+        res.data.document
+    );
+
   @action
   fetchWithSharedTree = async (
     id: string,
@@ -507,7 +516,6 @@ export default class DocumentsStore extends Store<Document> {
       const res = await client.post("/documents.info", {
         id,
         shareId: options.shareId,
-        apiVersion: 2,
       });
 
       invariant(res?.data, "Document not available");

app/stores/RevisionsStore.ts

@@ -35,7 +35,6 @@ export default class RevisionsStore extends Store<Revision> {
             id: "latest",
             documentId: document.id,
             title: document.title,
-            text: document.text,
             createdAt: document.updatedAt,
             createdBy: document.createdBy,
           },

app/stores/base/Store.ts

@@ -229,7 +229,11 @@ export default abstract class Store<T extends Model> {
   }
 
   @action
-  async fetch(id: string, options: JSONObject = {}): Promise<T> {
+  async fetch(
+    id: string,
+    options: JSONObject = {},
+    accessor = (res: unknown) => (res as { data: PartialWithId<T> }).data
+  ): Promise<T> {
     if (!this.actions.includes(RPCAction.Info)) {
       throw new Error(`Cannot fetch ${this.modelName}`);
     }
@@ -248,7 +252,7 @@ export default abstract class Store<T extends Model> {
       return runInAction(`info#${this.modelName}`, () => {
         invariant(res?.data, "Data should be available");
         this.addPolicies(res.policies);
-        return this.add(res.data);
+        return this.add(accessor(res));
       });
     } catch (err) {
       if (err instanceof AuthorizationError || err instanceof NotFoundError) {

app/utils/ApiClient.ts

@@ -84,6 +84,7 @@ class ApiClient {
       Accept: "application/json",
       "cache-control": "no-cache",
       "x-editor-version": EDITOR_VERSION,
+      "x-api-version": "3",
       pragma: "no-cache",
       ...options?.headers,
     };