diff --git a/server/commands/documentCollaborativeUpdater.ts b/server/commands/documentCollaborativeUpdater.ts index da7c84878..31e9b59be 100644 --- a/server/commands/documentCollaborativeUpdater.ts +++ b/server/commands/documentCollaborativeUpdater.ts @@ -41,7 +41,8 @@ export default async function documentCollaborativeUpdater({ }); const state = Y.encodeStateAsUpdate(ydoc); - const node = Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default")); + const content = yDocToProsemirrorJSON(ydoc, "default"); + const node = Node.fromJSON(schema, content); const text = serializer.serialize(node, undefined); const isUnchanged = document.text === text; const lastModifiedById = userId ?? document.lastModifiedById; @@ -63,6 +64,7 @@ export default async function documentCollaborativeUpdater({ await document.update( { text, + content, state: Buffer.from(state), lastModifiedById, collaboratorIds, diff --git a/server/commands/documentCreator.ts b/server/commands/documentCreator.ts index 0b73ceded..cd8f88fd1 100644 --- a/server/commands/documentCreator.ts +++ b/server/commands/documentCreator.ts @@ -1,7 +1,7 @@ import { Transaction } from "sequelize"; import { Optional } from "utility-types"; import { Document, Event, User } from "@server/models"; -import DocumentHelper from "@server/models/helpers/DocumentHelper"; +import TextHelper from "@server/models/helpers/TextHelper"; type Props = Optional< Pick< @@ -90,12 +90,12 @@ export default async function documentCreator({ sourceMetadata, fullWidth: templateDocument ? templateDocument.fullWidth : fullWidth, emoji: templateDocument ? templateDocument.emoji : emoji, - title: DocumentHelper.replaceTemplateVariables( + title: TextHelper.replaceTemplateVariables( templateDocument ? templateDocument.title : title, user ), - text: await DocumentHelper.replaceImagesWithAttachments( - DocumentHelper.replaceTemplateVariables( + text: await TextHelper.replaceImagesWithAttachments( + TextHelper.replaceTemplateVariables( templateDocument ? templateDocument.text : text, user ), diff --git a/server/commands/documentImporter.ts b/server/commands/documentImporter.ts index 53554e6f9..94f9a8dda 100644 --- a/server/commands/documentImporter.ts +++ b/server/commands/documentImporter.ts @@ -10,8 +10,8 @@ import parseTitle from "@shared/utils/parseTitle"; import { DocumentValidation } from "@shared/validations"; import { traceFunction } from "@server/logging/tracing"; import { User } from "@server/models"; -import DocumentHelper from "@server/models/helpers/DocumentHelper"; import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper"; +import TextHelper from "@server/models/helpers/TextHelper"; import turndownService from "@server/utils/turndown"; import { FileImportError, InvalidRequestError } from "../errors"; @@ -203,7 +203,7 @@ async function documentImporter({ // to match our hardbreak parser. text = text.trim().replace(/
/gi, "\\n"); - text = await DocumentHelper.replaceImagesWithAttachments( + text = await TextHelper.replaceImagesWithAttachments( text, user, ip, diff --git a/server/emails/templates/CommentCreatedEmail.tsx b/server/emails/templates/CommentCreatedEmail.tsx index 42db92728..2212369ab 100644 --- a/server/emails/templates/CommentCreatedEmail.tsx +++ b/server/emails/templates/CommentCreatedEmail.tsx @@ -2,10 +2,10 @@ import * as React from "react"; import { NotificationEventType } from "@shared/types"; import { Day } from "@shared/utils/time"; import { Collection, Comment, Document } from "@server/models"; -import DocumentHelper from "@server/models/helpers/DocumentHelper"; import HTMLHelper from "@server/models/helpers/HTMLHelper"; import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper"; import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper"; +import TextHelper from "@server/models/helpers/TextHelper"; import BaseEmail, { EmailProps } from "./BaseEmail"; import Body from "./components/Body"; import Button from "./components/Button"; @@ -74,7 +74,7 @@ export default class CommentCreatedEmail extends BaseEmail< } ); - content = await DocumentHelper.attachmentsToSignedUrls( + content = await TextHelper.attachmentsToSignedUrls( content, document.teamId, (4 * Day) / 1000 diff --git a/server/emails/templates/CommentMentionedEmail.tsx b/server/emails/templates/CommentMentionedEmail.tsx index f97239fef..e356b81db 100644 --- a/server/emails/templates/CommentMentionedEmail.tsx +++ b/server/emails/templates/CommentMentionedEmail.tsx @@ -2,10 +2,10 @@ import * as React from "react"; import { NotificationEventType } from "@shared/types"; import { Day } from "@shared/utils/time"; import { Collection, Comment, Document } from "@server/models"; -import DocumentHelper from "@server/models/helpers/DocumentHelper"; import HTMLHelper from "@server/models/helpers/HTMLHelper"; import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper"; import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper"; +import TextHelper from "@server/models/helpers/TextHelper"; import BaseEmail, { EmailProps } from "./BaseEmail"; import Body from "./components/Body"; import Button from "./components/Button"; @@ -66,7 +66,7 @@ export default class CommentMentionedEmail extends BaseEmail< } ); - content = await DocumentHelper.attachmentsToSignedUrls( + content = await TextHelper.attachmentsToSignedUrls( content, document.teamId, (4 * Day) / 1000 diff --git a/server/migrations/20231118195149-add-content-to-documents.js b/server/migrations/20231118195149-add-content-to-documents.js new file mode 100644 index 000000000..5b74caf24 --- /dev/null +++ b/server/migrations/20231118195149-add-content-to-documents.js @@ -0,0 +1,19 @@ +'use strict'; + +module.exports = { + async up (queryInterface, Sequelize) { + await queryInterface.addColumn("documents", "content", { + type: Sequelize.JSONB, + allowNull: true, + }); + await queryInterface.addColumn("revisions", "content", { + type: Sequelize.JSONB, + allowNull: true, + }); + }, + + async down (queryInterface) { + await queryInterface.removeColumn("revisions", "content"); + await queryInterface.removeColumn("documents", "content"); + } +}; diff --git a/server/models/Document.ts b/server/models/Document.ts index 8cbaaeff1..67ee3092a 100644 --- a/server/models/Document.ts +++ b/server/models/Document.ts @@ -35,7 +35,11 @@ import { AllowNull, } from "sequelize-typescript"; import isUUID from "validator/lib/isUUID"; -import type { NavigationNode, SourceMetadata } from "@shared/types"; +import type { + NavigationNode, + ProsemirrorData, + SourceMetadata, +} from "@shared/types"; import getTasks from "@shared/utils/getTasks"; import slugify from "@shared/utils/slugify"; import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers"; @@ -206,15 +210,18 @@ class Document extends ParanoidModel { @Column(DataType.SMALLINT) version: number; + @Default(false) @Column template: boolean; + @Default(false) @Column fullWidth: boolean; @Column insightsEnabled: boolean; + /** The version of the editor last used to edit this document. */ @SimpleLength({ max: 255, msg: `editorVersion must be 255 characters or less`, @@ -222,6 +229,7 @@ class Document extends ParanoidModel { @Column editorVersion: string; + /** An emoji to use as the document icon. */ @Length({ max: 1, msg: `Emoji must be a single character`, @@ -229,9 +237,25 @@ class Document extends ParanoidModel { @Column emoji: string | null; + /** + * The content of the document as Markdown. + * + * @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown. + * This column will be removed in a future migration. + */ @Column(DataType.TEXT) text: string; + /** + * The content of the document as JSON, this is a snapshot at the last time the state was saved. + */ + @Column(DataType.JSONB) + content: ProsemirrorData; + + /** + * The content of the document as YJS collaborative state, this column can be quite large and + * should only be selected from the DB when the `content` snapshot cannot be used. + */ @SimpleLength({ max: DocumentValidation.maxStateLength, msg: `Document collaborative state is too large, you must create a new document`, @@ -239,28 +263,38 @@ class Document extends ParanoidModel { @Column(DataType.BLOB) state: Uint8Array; + /** Whether this document is part of onboarding. */ @Default(false) @Column isWelcome: boolean; + /** How many versions there are in the history of this document. */ @IsNumeric @Default(0) @Column(DataType.INTEGER) revisionCount: number; + /** Whether the document is archvied, and if so when. */ @IsDate @Column archivedAt: Date | null; + /** Whether the document is published, and if so when. */ @IsDate @Column publishedAt: Date | null; + /** An array of user IDs that have edited this document. */ @Column(DataType.ARRAY(DataType.UUID)) collaboratorIds: string[] = []; // getters + /** + * The frontend path to this document. + * + * @deprecated Use `path` instead. + */ get url() { if (!this.title) { return `/doc/untitled-${this.urlId}`; @@ -269,6 +303,11 @@ class Document extends ParanoidModel { return `/doc/${slugifiedTitle}-${this.urlId}`; } + /** The frontend path to this document. */ + get path() { + return this.url; + } + get tasks() { return getTasks(this.text || ""); } @@ -363,6 +402,11 @@ class Document extends ParanoidModel { model.collaboratorIds = []; } + // backfill content if it's missing + if (!model.content) { + model.content = DocumentHelper.toJSON(model); + } + // ensure the last modifying user is a collaborator model.collaboratorIds = uniq( model.collaboratorIds.concat(model.lastModifiedById) @@ -589,6 +633,22 @@ class Document extends ParanoidModel { return !!(this.importId && this.sourceMetadata?.trial); } + /** + * Revert the state of the document to match the passed revision. + * + * @param revision The revision to revert to. + */ + restoreFromRevision = (revision: Revision) => { + if (revision.documentId !== this.id) { + throw new Error("Revision does not belong to this document"); + } + + this.content = revision.content; + this.text = revision.text; + this.title = revision.title; + this.emoji = revision.emoji; + }; + /** * Get a list of users that have collaborated on this document * diff --git a/server/models/Revision.ts b/server/models/Revision.ts index 4336a02e3..ff01926d1 100644 --- a/server/models/Revision.ts +++ b/server/models/Revision.ts @@ -9,6 +9,7 @@ import { IsNumeric, Length as SimpleLength, } from "sequelize-typescript"; +import type { ProsemirrorData } from "@shared/types"; import { DocumentValidation } from "@shared/validations"; import Document from "./Document"; import User from "./User"; @@ -46,9 +47,21 @@ class Revision extends IdModel { @Column title: string; + /** + * The content of the revision as Markdown. + * + * @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown. + * This column will be removed in a future migration. + */ @Column(DataType.TEXT) text: string; + /** + * The content of the revision as JSON. + */ + @Column(DataType.JSONB) + content: ProsemirrorData; + @Length({ max: 1, msg: `Emoji must be a single character`, @@ -100,6 +113,7 @@ class Revision extends IdModel { title: document.title, text: document.text, emoji: document.emoji, + content: document.content, userId: document.lastModifiedById, editorVersion: document.editorVersion, version: document.version, diff --git a/server/models/helpers/DocumentHelper.test.ts b/server/models/helpers/DocumentHelper.test.ts index 0c36260b8..3a4dc64cd 100644 --- a/server/models/helpers/DocumentHelper.test.ts +++ b/server/models/helpers/DocumentHelper.test.ts @@ -1,5 +1,5 @@ import Revision from "@server/models/Revision"; -import { buildDocument, buildUser } from "@server/test/factories"; +import { buildDocument } from "@server/test/factories"; import DocumentHelper from "./DocumentHelper"; describe("DocumentHelper", () => { @@ -12,28 +12,6 @@ describe("DocumentHelper", () => { jest.useRealTimers(); }); - describe("replaceTemplateVariables", () => { - it("should replace {time} with current time", async () => { - const user = await buildUser(); - const result = DocumentHelper.replaceTemplateVariables( - "Hello {time}", - user - ); - - expect(result).toBe("Hello 12 00 AM"); - }); - - it("should replace {date} with current date", async () => { - const user = await buildUser(); - const result = DocumentHelper.replaceTemplateVariables( - "Hello {date}", - user - ); - - expect(result).toBe("Hello January 1 2021"); - }); - }); - describe("parseMentions", () => { it("should not parse normal links as mentions", async () => { const document = await buildDocument({ diff --git a/server/models/helpers/DocumentHelper.tsx b/server/models/helpers/DocumentHelper.tsx index ad2543dd1..38e159086 100644 --- a/server/models/helpers/DocumentHelper.tsx +++ b/server/models/helpers/DocumentHelper.tsx @@ -1,33 +1,20 @@ import { updateYFragment, + yDocToProsemirror, yDocToProsemirrorJSON, } from "@getoutline/y-prosemirror"; import { JSDOM } from "jsdom"; -import escapeRegExp from "lodash/escapeRegExp"; -import startCase from "lodash/startCase"; import { Node } from "prosemirror-model"; -import { Transaction } from "sequelize"; import * as Y from "yjs"; import textBetween from "@shared/editor/lib/textBetween"; -import { AttachmentPreset } from "@shared/types"; import MarkdownHelper from "@shared/utils/MarkdownHelper"; -import { - getCurrentDateAsString, - getCurrentDateTimeAsString, - getCurrentTimeAsString, - unicodeCLDRtoBCP47, -} from "@shared/utils/date"; -import attachmentCreator from "@server/commands/attachmentCreator"; import { parser, schema } from "@server/editor"; import { addTags } from "@server/logging/tracer"; import { trace } from "@server/logging/tracing"; -import { Document, Revision, User } from "@server/models"; -import FileStorage from "@server/storage/files"; +import { Document, Revision } from "@server/models"; import diff from "@server/utils/diff"; -import parseAttachmentIds from "@server/utils/parseAttachmentIds"; -import parseImages from "@server/utils/parseImages"; -import Attachment from "../Attachment"; import ProsemirrorHelper from "./ProsemirrorHelper"; +import TextHelper from "./TextHelper"; type HTMLOptions = { /** Whether to include the document title in the generated HTML (defaults to true) */ @@ -48,8 +35,25 @@ type HTMLOptions = { @trace() export default class DocumentHelper { /** - * Returns the document as a Prosemirror Node. This method uses the - * collaborative state if available, otherwise it falls back to Markdown. + * Returns the document as JSON content. This method uses the collaborative state if available, + * otherwise it falls back to Markdown. + * + * @param document The document or revision to convert + * @returns The document content as JSON + */ + static toJSON(document: Document | Revision) { + if ("state" in document && document.state) { + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, document.state); + return yDocToProsemirrorJSON(ydoc, "default"); + } + const node = parser.parse(document.text) || Node.fromJSON(schema, {}); + return node.toJSON(); + } + + /** + * Returns the document as a Prosemirror Node. This method uses the collaborative state if + * available, otherwise it falls back to Markdown. * * @param document The document or revision to convert * @returns The document content as a Prosemirror Node @@ -124,7 +128,7 @@ export default class DocumentHelper { return output; } - output = await DocumentHelper.attachmentsToSignedUrls( + output = await TextHelper.attachmentsToSignedUrls( output, teamId, typeof options.signedUrls === "number" ? options.signedUrls : undefined @@ -188,7 +192,7 @@ export default class DocumentHelper { : (await before.$get("document"))?.teamId; if (teamId) { - diffedContentAsHTML = await DocumentHelper.attachmentsToSignedUrls( + diffedContentAsHTML = await TextHelper.attachmentsToSignedUrls( diffedContentAsHTML, teamId, typeof signedUrls === "number" ? signedUrls : undefined @@ -336,114 +340,6 @@ export default class DocumentHelper { return `${head?.innerHTML} ${body?.innerHTML}`; } - /** - * Converts attachment urls in documents to signed equivalents that allow - * direct access without a session cookie - * - * @param text The text either html or markdown which contains urls to be converted - * @param teamId The team context - * @param expiresIn The time that signed urls should expire (in seconds) - * @returns The replaced text - */ - static async attachmentsToSignedUrls( - text: string, - teamId: string, - expiresIn = 3000 - ) { - const attachmentIds = parseAttachmentIds(text); - - await Promise.all( - attachmentIds.map(async (id) => { - const attachment = await Attachment.findOne({ - where: { - id, - teamId, - }, - }); - - if (attachment) { - const signedUrl = await FileStorage.getSignedUrl( - attachment.key, - expiresIn - ); - - text = text.replace( - new RegExp(escapeRegExp(attachment.redirectUrl), "g"), - signedUrl - ); - } - }) - ); - return text; - } - - /** - * Replaces template variables in the given text with the current date and time. - * - * @param text The text to replace the variables in - * @param user The user to get the language/locale from - * @returns The text with the variables replaced - */ - static replaceTemplateVariables(text: string, user: User) { - const locales = user.language - ? unicodeCLDRtoBCP47(user.language) - : undefined; - - return text - .replace(/{date}/g, startCase(getCurrentDateAsString(locales))) - .replace(/{time}/g, startCase(getCurrentTimeAsString(locales))) - .replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales))); - } - - /** - * Replaces remote and base64 encoded images in the given text with attachment - * urls and uploads the images to the storage provider. - * - * @param text The text to replace the images in - * @param user The user context - * @param ip The IP address of the user - * @param transaction The transaction to use for the database operations - * @returns The text with the images replaced - */ - static async replaceImagesWithAttachments( - text: string, - user: User, - ip?: string, - transaction?: Transaction - ) { - let output = text; - const images = parseImages(text); - - await Promise.all( - images.map(async (image) => { - // Skip attempting to fetch images that are not valid urls - try { - new URL(image.src); - } catch { - return; - } - - const attachment = await attachmentCreator({ - name: image.alt ?? "image", - url: image.src, - preset: AttachmentPreset.DocumentAttachment, - user, - ip, - transaction, - }); - - if (attachment) { - output = output.replace( - new RegExp(escapeRegExp(image.src), "g"), - attachment.redirectUrl - ); - } - }) - ); - - return output; - } - /** * Applies the given Markdown to the document, this essentially creates a * single change in the collaborative state that makes all the edits to get @@ -461,12 +357,12 @@ export default class DocumentHelper { append = false ) { document.text = append ? document.text + text : text; + const doc = parser.parse(document.text); if (document.state) { const ydoc = new Y.Doc(); Y.applyUpdate(ydoc, document.state); const type = ydoc.get("default", Y.XmlFragment) as Y.XmlFragment; - const doc = parser.parse(document.text); if (!type.doc) { throw new Error("type.doc not found"); @@ -476,8 +372,13 @@ export default class DocumentHelper { updateYFragment(type.doc, type, doc, new Map()); const state = Y.encodeStateAsUpdate(ydoc); + const node = yDocToProsemirror(schema, ydoc); + + document.content = node.toJSON(); document.state = Buffer.from(state); document.changed("state", true); + } else if (doc) { + document.content = doc.toJSON(); } return document; diff --git a/server/models/helpers/TextHelper.test.ts b/server/models/helpers/TextHelper.test.ts new file mode 100644 index 000000000..f36acd887 --- /dev/null +++ b/server/models/helpers/TextHelper.test.ts @@ -0,0 +1,29 @@ +import { buildUser } from "@server/test/factories"; +import TextHelper from "./TextHelper"; + +describe("TextHelper", () => { + beforeAll(() => { + jest.useFakeTimers(); + jest.setSystemTime(Date.parse("2021-01-01T00:00:00.000Z")); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + describe("replaceTemplateVariables", () => { + it("should replace {time} with current time", async () => { + const user = await buildUser(); + const result = TextHelper.replaceTemplateVariables("Hello {time}", user); + + expect(result).toBe("Hello 12 00 AM"); + }); + + it("should replace {date} with current date", async () => { + const user = await buildUser(); + const result = TextHelper.replaceTemplateVariables("Hello {date}", user); + + expect(result).toBe("Hello January 1 2021"); + }); + }); +}); diff --git a/server/models/helpers/TextHelper.ts b/server/models/helpers/TextHelper.ts new file mode 100644 index 000000000..2b7b3172e --- /dev/null +++ b/server/models/helpers/TextHelper.ts @@ -0,0 +1,125 @@ +import escapeRegExp from "lodash/escapeRegExp"; +import startCase from "lodash/startCase"; +import { Transaction } from "sequelize"; +import { AttachmentPreset } from "@shared/types"; +import { + getCurrentDateAsString, + getCurrentDateTimeAsString, + getCurrentTimeAsString, + unicodeCLDRtoBCP47, +} from "@shared/utils/date"; +import attachmentCreator from "@server/commands/attachmentCreator"; +import { Attachment, User } from "@server/models"; +import FileStorage from "@server/storage/files"; +import parseAttachmentIds from "@server/utils/parseAttachmentIds"; +import parseImages from "@server/utils/parseImages"; + +export default class TextHelper { + /** + * Replaces template variables in the given text with the current date and time. + * + * @param text The text to replace the variables in + * @param user The user to get the language/locale from + * @returns The text with the variables replaced + */ + static replaceTemplateVariables(text: string, user: User) { + const locales = user.language + ? unicodeCLDRtoBCP47(user.language) + : undefined; + + return text + .replace(/{date}/g, startCase(getCurrentDateAsString(locales))) + .replace(/{time}/g, startCase(getCurrentTimeAsString(locales))) + .replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales))); + } + + /** + * Converts attachment urls in documents to signed equivalents that allow + * direct access without a session cookie + * + * @param text The text either html or markdown which contains urls to be converted + * @param teamId The team context + * @param expiresIn The time that signed urls should expire (in seconds) + * @returns The replaced text + */ + static async attachmentsToSignedUrls( + text: string, + teamId: string, + expiresIn = 3000 + ) { + const attachmentIds = parseAttachmentIds(text); + + await Promise.all( + attachmentIds.map(async (id) => { + const attachment = await Attachment.findOne({ + where: { + id, + teamId, + }, + }); + + if (attachment) { + const signedUrl = await FileStorage.getSignedUrl( + attachment.key, + expiresIn + ); + + text = text.replace( + new RegExp(escapeRegExp(attachment.redirectUrl), "g"), + signedUrl + ); + } + }) + ); + return text; + } + + /** + * Replaces remote and base64 encoded images in the given text with attachment + * urls and uploads the images to the storage provider. + * + * @param markdown The text to replace the images in + * @param user The user context + * @param ip The IP address of the user + * @param transaction The transaction to use for the database operations + * @returns The text with the images replaced + */ + static async replaceImagesWithAttachments( + markdown: string, + user: User, + ip?: string, + transaction?: Transaction + ) { + let output = markdown; + const images = parseImages(markdown); + + await Promise.all( + images.map(async (image) => { + // Skip attempting to fetch images that are not valid urls + try { + new URL(image.src); + } catch { + return; + } + + const attachment = await attachmentCreator({ + name: image.alt ?? "image", + url: image.src, + preset: AttachmentPreset.DocumentAttachment, + user, + ip, + transaction, + }); + + if (attachment) { + output = output.replace( + new RegExp(escapeRegExp(image.src), "g"), + attachment.redirectUrl + ); + } + }) + ); + + return output; + } +} diff --git a/server/presenters/document.ts b/server/presenters/document.ts index 71211c996..50932466c 100644 --- a/server/presenters/document.ts +++ b/server/presenters/document.ts @@ -1,6 +1,6 @@ import { traceFunction } from "@server/logging/tracing"; import { Document } from "@server/models"; -import DocumentHelper from "@server/models/helpers/DocumentHelper"; +import TextHelper from "@server/models/helpers/TextHelper"; import presentUser from "./user"; type Options = { @@ -16,10 +16,7 @@ async function presentDocument( ...options, }; const text = options.isPublic - ? await DocumentHelper.attachmentsToSignedUrls( - document.text, - document.teamId - ) + ? await TextHelper.attachmentsToSignedUrls(document.text, document.teamId) : document.text; const data: Record = { diff --git a/server/presenters/revision.ts b/server/presenters/revision.ts index 6c6505f70..24cdc4ad8 100644 --- a/server/presenters/revision.ts +++ b/server/presenters/revision.ts @@ -1,6 +1,7 @@ import parseTitle from "@shared/utils/parseTitle"; import { traceFunction } from "@server/logging/tracing"; import { Revision } from "@server/models"; +import DocumentHelper from "@server/models/helpers/DocumentHelper"; import presentUser from "./user"; async function presentRevision(revision: Revision, diff?: string) { @@ -11,7 +12,7 @@ async function presentRevision(revision: Revision, diff?: string) { id: revision.id, documentId: revision.documentId, title: strippedTitle, - text: revision.text, + text: DocumentHelper.toMarkdown(revision), emoji: revision.emoji ?? emoji, html: diff, createdAt: revision.createdAt, diff --git a/server/routes/api/documents/documents.ts b/server/routes/api/documents/documents.ts index cfc3fc4fb..acfbbef7b 100644 --- a/server/routes/api/documents/documents.ts +++ b/server/routes/api/documents/documents.ts @@ -684,12 +684,11 @@ router.post( // restore a document to a specific revision authorize(user, "update", document); const revision = await Revision.findByPk(revisionId); - authorize(document, "restore", revision); - document.text = revision.text; - document.title = revision.title; + document.restoreFromRevision(revision); await document.save(); + await Event.create({ name: "documents.restore", documentId: document.id, diff --git a/server/scripts/20221008000000-backfill-crdt.ts b/server/scripts/20221008000000-backfill-crdt.ts index 48cd99c97..8f0b48e44 100644 --- a/server/scripts/20221008000000-backfill-crdt.ts +++ b/server/scripts/20221008000000-backfill-crdt.ts @@ -27,7 +27,6 @@ export default async function main(exit = false) { offset: page * limit, where: { ...(teamId ? { teamId } : {}), - state: null, }, order: [["createdAt", "ASC"]], paranoid: false, diff --git a/server/scripts/20231119000000-backfill-document-content.ts b/server/scripts/20231119000000-backfill-document-content.ts new file mode 100644 index 000000000..9dd6d160a --- /dev/null +++ b/server/scripts/20231119000000-backfill-document-content.ts @@ -0,0 +1,61 @@ +import "./bootstrap"; +import { yDocToProsemirrorJSON } from "@getoutline/y-prosemirror"; +import { Node } from "prosemirror-model"; +import * as Y from "yjs"; +import { parser, schema } from "@server/editor"; +import { Document } from "@server/models"; + +const limit = 100; +const page = 0; + +export default async function main(exit = false) { + const work = async (page: number): Promise => { + console.log(`Backfill content… page ${page}`); + + // Retrieve all documents within set limit. + const documents = await Document.unscoped().findAll({ + attributes: ["id", "urlId", "content", "text", "state"], + limit, + offset: page * limit, + order: [["createdAt", "ASC"]], + paranoid: false, + }); + + for (const document of documents) { + if (document.content || !document.text) { + continue; + } + + console.log(`Writing content for ${document.id}`); + + if ("state" in document && document.state) { + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, document.state); + document.content = yDocToProsemirrorJSON(ydoc, "default"); + } else { + const node = parser.parse(document.text) || Node.fromJSON(schema, {}); + document.content = node.toJSON(); + } + + document.changed("content", true); + + await document.save({ + hooks: false, + silent: true, + }); + } + + return documents.length === limit ? work(page + 1) : undefined; + }; + + await work(page); + + if (exit) { + console.log("Backfill complete"); + process.exit(0); + } +} + +if (process.env.NODE_ENV !== "test") { + void main(true); +} diff --git a/server/scripts/20231119000000-backfill-revision-content.ts b/server/scripts/20231119000000-backfill-revision-content.ts new file mode 100644 index 000000000..9c689edd5 --- /dev/null +++ b/server/scripts/20231119000000-backfill-revision-content.ts @@ -0,0 +1,52 @@ +import "./bootstrap"; +import { Node } from "prosemirror-model"; +import { parser, schema } from "@server/editor"; +import { Revision } from "@server/models"; + +const limit = 100; +const page = 0; + +export default async function main(exit = false) { + const work = async (page: number): Promise => { + console.log(`Backfill content… page ${page}`); + + // Retrieve all revisions within set limit. + const revisions = await Revision.unscoped().findAll({ + attributes: ["id", "content", "text"], + limit, + offset: page * limit, + order: [["createdAt", "ASC"]], + paranoid: false, + }); + + for (const revision of revisions) { + if (revision.content || !revision.text) { + continue; + } + + console.log(`Writing content for ${revision.id}`); + + const node = parser.parse(revision.text) || Node.fromJSON(schema, {}); + revision.content = node.toJSON(); + revision.changed("content", true); + + await revision.save({ + hooks: false, + silent: true, + }); + } + + return revisions.length === limit ? work(page + 1) : undefined; + }; + + await work(page); + + if (exit) { + console.log("Backfill complete"); + process.exit(0); + } +} + +if (process.env.NODE_ENV !== "test") { + void main(true); +}