yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adds content column to documents and revisions as JSON snapshot (#6179)

Browse Source
This commit is contained in:
Tom Moor
2023-12-17 10:51:11 -05:00
committed by GitHub
parent 78b9322a28
commit 1840370e6f
18 changed files with 411 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/commands/documentCollaborativeUpdater.ts
View File

@@ -41,7 +41,8 @@ export default async function documentCollaborativeUpdater({
});
const state = Y.encodeStateAsUpdate(ydoc);
const node = Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default"));
const content = yDocToProsemirrorJSON(ydoc, "default");
const node = Node.fromJSON(schema, content);
const text = serializer.serialize(node, undefined);
const isUnchanged = document.text === text;
const lastModifiedById = userId ?? document.lastModifiedById;
@@ -63,6 +64,7 @@ export default async function documentCollaborativeUpdater({
await document.update(
{
text,
content,
state: Buffer.from(state),
lastModifiedById,
collaboratorIds,

8
server/commands/documentCreator.ts
View File

@@ -1,7 +1,7 @@
import { Transaction } from "sequelize";
import { Optional } from "utility-types";
import { Document, Event, User } from "@server/models";
import DocumentHelper from "@server/models/helpers/DocumentHelper";
import TextHelper from "@server/models/helpers/TextHelper";
type Props = Optional<
Pick<
@@ -90,12 +90,12 @@ export default async function documentCreator({
sourceMetadata,
fullWidth: templateDocument ? templateDocument.fullWidth : fullWidth,
emoji: templateDocument ? templateDocument.emoji : emoji,
title: DocumentHelper.replaceTemplateVariables(
title: TextHelper.replaceTemplateVariables(
templateDocument ? templateDocument.title : title,
user
),
text: await DocumentHelper.replaceImagesWithAttachments(
DocumentHelper.replaceTemplateVariables(
text: await TextHelper.replaceImagesWithAttachments(
TextHelper.replaceTemplateVariables(
templateDocument ? templateDocument.text : text,
user
),

4
server/commands/documentImporter.ts
View File

@@ -10,8 +10,8 @@ import parseTitle from "@shared/utils/parseTitle";
import { DocumentValidation } from "@shared/validations";
import { traceFunction } from "@server/logging/tracing";
import { User } from "@server/models";
import DocumentHelper from "@server/models/helpers/DocumentHelper";
import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
import TextHelper from "@server/models/helpers/TextHelper";
import turndownService from "@server/utils/turndown";
import { FileImportError, InvalidRequestError } from "../errors";
@@ -203,7 +203,7 @@ async function documentImporter({
// to match our hardbreak parser.
text = text.trim().replace(/<br>/gi, "\\n");
text = await DocumentHelper.replaceImagesWithAttachments(
text = await TextHelper.replaceImagesWithAttachments(
text,
user,
ip,

4
server/emails/templates/CommentCreatedEmail.tsx
View File

@@ -2,10 +2,10 @@ import * as React from "react";
import { NotificationEventType } from "@shared/types";
import { Day } from "@shared/utils/time";
import { Collection, Comment, Document } from "@server/models";
import DocumentHelper from "@server/models/helpers/DocumentHelper";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
import TextHelper from "@server/models/helpers/TextHelper";
import BaseEmail, { EmailProps } from "./BaseEmail";
import Body from "./components/Body";
import Button from "./components/Button";
@@ -74,7 +74,7 @@ export default class CommentCreatedEmail extends BaseEmail<
}
);
content = await DocumentHelper.attachmentsToSignedUrls(
content = await TextHelper.attachmentsToSignedUrls(
content,
document.teamId,
(4 * Day) / 1000

4
server/emails/templates/CommentMentionedEmail.tsx
View File

@@ -2,10 +2,10 @@ import * as React from "react";
import { NotificationEventType } from "@shared/types";
import { Day } from "@shared/utils/time";
import { Collection, Comment, Document } from "@server/models";
import DocumentHelper from "@server/models/helpers/DocumentHelper";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import NotificationSettingsHelper from "@server/models/helpers/NotificationSettingsHelper";
import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
import TextHelper from "@server/models/helpers/TextHelper";
import BaseEmail, { EmailProps } from "./BaseEmail";
import Body from "./components/Body";
import Button from "./components/Button";
@@ -66,7 +66,7 @@ export default class CommentMentionedEmail extends BaseEmail<
}
);
content = await DocumentHelper.attachmentsToSignedUrls(
content = await TextHelper.attachmentsToSignedUrls(
content,
document.teamId,
(4 * Day) / 1000

19
server/migrations/20231118195149-add-content-to-documents.js Normal file
View File

@@ -0,0 +1,19 @@
'use strict';
module.exports = {
async up (queryInterface, Sequelize) {
await queryInterface.addColumn("documents", "content", {
type: Sequelize.JSONB,
allowNull: true,
});
await queryInterface.addColumn("revisions", "content", {
type: Sequelize.JSONB,
allowNull: true,
});
},
async down (queryInterface) {
await queryInterface.removeColumn("revisions", "content");
await queryInterface.removeColumn("documents", "content");
}
};

62
server/models/Document.ts
View File

@@ -35,7 +35,11 @@ import {
AllowNull,
} from "sequelize-typescript";
import isUUID from "validator/lib/isUUID";
import type { NavigationNode, SourceMetadata } from "@shared/types";
import type {
NavigationNode,
ProsemirrorData,
SourceMetadata,
} from "@shared/types";
import getTasks from "@shared/utils/getTasks";
import slugify from "@shared/utils/slugify";
import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
@@ -206,15 +210,18 @@ class Document extends ParanoidModel {
@Column(DataType.SMALLINT)
version: number;
@Default(false)
@Column
template: boolean;
@Default(false)
@Column
fullWidth: boolean;
@Column
insightsEnabled: boolean;
/** The version of the editor last used to edit this document. */
@SimpleLength({
max: 255,
msg: `editorVersion must be 255 characters or less`,
@@ -222,6 +229,7 @@ class Document extends ParanoidModel {
@Column
editorVersion: string;
/** An emoji to use as the document icon. */
@Length({
max: 1,
msg: `Emoji must be a single character`,
@@ -229,9 +237,25 @@ class Document extends ParanoidModel {
@Column
emoji: string | null;
/**
* The content of the document as Markdown.
*
* @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown.
* This column will be removed in a future migration.
*/
@Column(DataType.TEXT)
text: string;
/**
* The content of the document as JSON, this is a snapshot at the last time the state was saved.
*/
@Column(DataType.JSONB)
content: ProsemirrorData;
/**
* The content of the document as YJS collaborative state, this column can be quite large and
* should only be selected from the DB when the `content` snapshot cannot be used.
*/
@SimpleLength({
max: DocumentValidation.maxStateLength,
msg: `Document collaborative state is too large, you must create a new document`,
@@ -239,28 +263,38 @@ class Document extends ParanoidModel {
@Column(DataType.BLOB)
state: Uint8Array;
/** Whether this document is part of onboarding. */
@Default(false)
@Column
isWelcome: boolean;
/** How many versions there are in the history of this document. */
@IsNumeric
@Default(0)
@Column(DataType.INTEGER)
revisionCount: number;
/** Whether the document is archvied, and if so when. */
@IsDate
@Column
archivedAt: Date | null;
/** Whether the document is published, and if so when. */
@IsDate
@Column
publishedAt: Date | null;
/** An array of user IDs that have edited this document. */
@Column(DataType.ARRAY(DataType.UUID))
collaboratorIds: string[] = [];
// getters
/**
* The frontend path to this document.
*
* @deprecated Use `path` instead.
*/
get url() {
if (!this.title) {
return `/doc/untitled-${this.urlId}`;
@@ -269,6 +303,11 @@ class Document extends ParanoidModel {
return `/doc/${slugifiedTitle}-${this.urlId}`;
}
/** The frontend path to this document. */
get path() {
return this.url;
}
get tasks() {
return getTasks(this.text || "");
}
@@ -363,6 +402,11 @@ class Document extends ParanoidModel {
model.collaboratorIds = [];
}
// backfill content if it's missing
if (!model.content) {
model.content = DocumentHelper.toJSON(model);
}
// ensure the last modifying user is a collaborator
model.collaboratorIds = uniq(
model.collaboratorIds.concat(model.lastModifiedById)
@@ -589,6 +633,22 @@ class Document extends ParanoidModel {
return !!(this.importId && this.sourceMetadata?.trial);
}
/**
* Revert the state of the document to match the passed revision.
*
* @param revision The revision to revert to.
*/
restoreFromRevision = (revision: Revision) => {
if (revision.documentId !== this.id) {
throw new Error("Revision does not belong to this document");
}
this.content = revision.content;
this.text = revision.text;
this.title = revision.title;
this.emoji = revision.emoji;
};
/**
* Get a list of users that have collaborated on this document
*

14
server/models/Revision.ts
View File

@@ -9,6 +9,7 @@ import {
IsNumeric,
Length as SimpleLength,
} from "sequelize-typescript";
import type { ProsemirrorData } from "@shared/types";
import { DocumentValidation } from "@shared/validations";
import Document from "./Document";
import User from "./User";
@@ -46,9 +47,21 @@ class Revision extends IdModel {
@Column
title: string;
/**
* The content of the revision as Markdown.
*
* @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown.
* This column will be removed in a future migration.
*/
@Column(DataType.TEXT)
text: string;
/**
* The content of the revision as JSON.
*/
@Column(DataType.JSONB)
content: ProsemirrorData;
@Length({
max: 1,
msg: `Emoji must be a single character`,
@@ -100,6 +113,7 @@ class Revision extends IdModel {
title: document.title,
text: document.text,
emoji: document.emoji,
content: document.content,
userId: document.lastModifiedById,
editorVersion: document.editorVersion,
version: document.version,

24
server/models/helpers/DocumentHelper.test.ts
View File

@@ -1,5 +1,5 @@
import Revision from "@server/models/Revision";
import { buildDocument, buildUser } from "@server/test/factories";
import { buildDocument } from "@server/test/factories";
import DocumentHelper from "./DocumentHelper";
describe("DocumentHelper", () => {
@@ -12,28 +12,6 @@ describe("DocumentHelper", () => {
jest.useRealTimers();
});
describe("replaceTemplateVariables", () => {
it("should replace {time} with current time", async () => {
const user = await buildUser();
const result = DocumentHelper.replaceTemplateVariables(
"Hello {time}",
user
);
expect(result).toBe("Hello 12 00 AM");
});
it("should replace {date} with current date", async () => {
const user = await buildUser();
const result = DocumentHelper.replaceTemplateVariables(
"Hello {date}",
user
);
expect(result).toBe("Hello January 1 2021");
});
});
describe("parseMentions", () => {
it("should not parse normal links as mentions", async () => {
const document = await buildDocument({

159
server/models/helpers/DocumentHelper.tsx
View File

@@ -1,33 +1,20 @@
import {
updateYFragment,
yDocToProsemirror,
yDocToProsemirrorJSON,
} from "@getoutline/y-prosemirror";
import { JSDOM } from "jsdom";
import escapeRegExp from "lodash/escapeRegExp";
import startCase from "lodash/startCase";
import { Node } from "prosemirror-model";
import { Transaction } from "sequelize";
import * as Y from "yjs";
import textBetween from "@shared/editor/lib/textBetween";
import { AttachmentPreset } from "@shared/types";
import MarkdownHelper from "@shared/utils/MarkdownHelper";
import {
getCurrentDateAsString,
getCurrentDateTimeAsString,
getCurrentTimeAsString,
unicodeCLDRtoBCP47,
} from "@shared/utils/date";
import attachmentCreator from "@server/commands/attachmentCreator";
import { parser, schema } from "@server/editor";
import { addTags } from "@server/logging/tracer";
import { trace } from "@server/logging/tracing";
import { Document, Revision, User } from "@server/models";
import FileStorage from "@server/storage/files";
import { Document, Revision } from "@server/models";
import diff from "@server/utils/diff";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
import Attachment from "../Attachment";
import ProsemirrorHelper from "./ProsemirrorHelper";
import TextHelper from "./TextHelper";
type HTMLOptions = {
/** Whether to include the document title in the generated HTML (defaults to true) */
@@ -48,8 +35,25 @@ type HTMLOptions = {
@trace()
export default class DocumentHelper {
/**
* Returns the document as a Prosemirror Node. This method uses the
* collaborative state if available, otherwise it falls back to Markdown.
* Returns the document as JSON content. This method uses the collaborative state if available,
* otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as JSON
*/
static toJSON(document: Document | Revision) {
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
return yDocToProsemirrorJSON(ydoc, "default");
}
const node = parser.parse(document.text) || Node.fromJSON(schema, {});
return node.toJSON();
}
/**
* Returns the document as a Prosemirror Node. This method uses the collaborative state if
* available, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as a Prosemirror Node
@@ -124,7 +128,7 @@ export default class DocumentHelper {
return output;
}
output = await DocumentHelper.attachmentsToSignedUrls(
output = await TextHelper.attachmentsToSignedUrls(
output,
teamId,
typeof options.signedUrls === "number" ? options.signedUrls : undefined
@@ -188,7 +192,7 @@ export default class DocumentHelper {
: (await before.$get("document"))?.teamId;
if (teamId) {
diffedContentAsHTML = await DocumentHelper.attachmentsToSignedUrls(
diffedContentAsHTML = await TextHelper.attachmentsToSignedUrls(
diffedContentAsHTML,
teamId,
typeof signedUrls === "number" ? signedUrls : undefined
@@ -336,114 +340,6 @@ export default class DocumentHelper {
return `${head?.innerHTML} ${body?.innerHTML}`;
}
/**
* Converts attachment urls in documents to signed equivalents that allow
* direct access without a session cookie
*
* @param text The text either html or markdown which contains urls to be converted
* @param teamId The team context
* @param expiresIn The time that signed urls should expire (in seconds)
* @returns The replaced text
*/
static async attachmentsToSignedUrls(
text: string,
teamId: string,
expiresIn = 3000
) {
const attachmentIds = parseAttachmentIds(text);
await Promise.all(
attachmentIds.map(async (id) => {
const attachment = await Attachment.findOne({
where: {
id,
teamId,
},
});
if (attachment) {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
signedUrl
);
}
})
);
return text;
}
/**
* Replaces template variables in the given text with the current date and time.
*
* @param text The text to replace the variables in
* @param user The user to get the language/locale from
* @returns The text with the variables replaced
*/
static replaceTemplateVariables(text: string, user: User) {
const locales = user.language
? unicodeCLDRtoBCP47(user.language)
: undefined;
return text
.replace(/{date}/g, startCase(getCurrentDateAsString(locales)))
.replace(/{time}/g, startCase(getCurrentTimeAsString(locales)))
.replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales)));
}
/**
* Replaces remote and base64 encoded images in the given text with attachment
* urls and uploads the images to the storage provider.
*
* @param text The text to replace the images in
* @param user The user context
* @param ip The IP address of the user
* @param transaction The transaction to use for the database operations
* @returns The text with the images replaced
*/
static async replaceImagesWithAttachments(
text: string,
user: User,
ip?: string,
transaction?: Transaction
) {
let output = text;
const images = parseImages(text);
await Promise.all(
images.map(async (image) => {
// Skip attempting to fetch images that are not valid urls
try {
new URL(image.src);
} catch {
return;
}
const attachment = await attachmentCreator({
name: image.alt ?? "image",
url: image.src,
preset: AttachmentPreset.DocumentAttachment,
user,
ip,
transaction,
});
if (attachment) {
output = output.replace(
new RegExp(escapeRegExp(image.src), "g"),
attachment.redirectUrl
);
}
})
);
return output;
}
/**
* Applies the given Markdown to the document, this essentially creates a
* single change in the collaborative state that makes all the edits to get
@@ -461,12 +357,12 @@ export default class DocumentHelper {
append = false
) {
document.text = append ? document.text + text : text;
const doc = parser.parse(document.text);
if (document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
const type = ydoc.get("default", Y.XmlFragment) as Y.XmlFragment;
const doc = parser.parse(document.text);
if (!type.doc) {
throw new Error("type.doc not found");
@@ -476,8 +372,13 @@ export default class DocumentHelper {
updateYFragment(type.doc, type, doc, new Map());
const state = Y.encodeStateAsUpdate(ydoc);
const node = yDocToProsemirror(schema, ydoc);
document.content = node.toJSON();
document.state = Buffer.from(state);
document.changed("state", true);
} else if (doc) {
document.content = doc.toJSON();
}
return document;

29
server/models/helpers/TextHelper.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { buildUser } from "@server/test/factories";
import TextHelper from "./TextHelper";
describe("TextHelper", () => {
beforeAll(() => {
jest.useFakeTimers();
jest.setSystemTime(Date.parse("2021-01-01T00:00:00.000Z"));
});
afterAll(() => {
jest.useRealTimers();
});
describe("replaceTemplateVariables", () => {
it("should replace {time} with current time", async () => {
const user = await buildUser();
const result = TextHelper.replaceTemplateVariables("Hello {time}", user);
expect(result).toBe("Hello 12 00 AM");
});
it("should replace {date} with current date", async () => {
const user = await buildUser();
const result = TextHelper.replaceTemplateVariables("Hello {date}", user);
expect(result).toBe("Hello January 1 2021");
});
});
});

125
server/models/helpers/TextHelper.ts Normal file
View File

@@ -0,0 +1,125 @@
import escapeRegExp from "lodash/escapeRegExp";
import startCase from "lodash/startCase";
import { Transaction } from "sequelize";
import { AttachmentPreset } from "@shared/types";
import {
getCurrentDateAsString,
getCurrentDateTimeAsString,
getCurrentTimeAsString,
unicodeCLDRtoBCP47,
} from "@shared/utils/date";
import attachmentCreator from "@server/commands/attachmentCreator";
import { Attachment, User } from "@server/models";
import FileStorage from "@server/storage/files";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
export default class TextHelper {
/**
* Replaces template variables in the given text with the current date and time.
*
* @param text The text to replace the variables in
* @param user The user to get the language/locale from
* @returns The text with the variables replaced
*/
static replaceTemplateVariables(text: string, user: User) {
const locales = user.language
? unicodeCLDRtoBCP47(user.language)
: undefined;
return text
.replace(/{date}/g, startCase(getCurrentDateAsString(locales)))
.replace(/{time}/g, startCase(getCurrentTimeAsString(locales)))
.replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales)));
}
/**
* Converts attachment urls in documents to signed equivalents that allow
* direct access without a session cookie
*
* @param text The text either html or markdown which contains urls to be converted
* @param teamId The team context
* @param expiresIn The time that signed urls should expire (in seconds)
* @returns The replaced text
*/
static async attachmentsToSignedUrls(
text: string,
teamId: string,
expiresIn = 3000
) {
const attachmentIds = parseAttachmentIds(text);
await Promise.all(
attachmentIds.map(async (id) => {
const attachment = await Attachment.findOne({
where: {
id,
teamId,
},
});
if (attachment) {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
signedUrl
);
}
})
);
return text;
}
/**
* Replaces remote and base64 encoded images in the given text with attachment
* urls and uploads the images to the storage provider.
*
* @param markdown The text to replace the images in
* @param user The user context
* @param ip The IP address of the user
* @param transaction The transaction to use for the database operations
* @returns The text with the images replaced
*/
static async replaceImagesWithAttachments(
markdown: string,
user: User,
ip?: string,
transaction?: Transaction
) {
let output = markdown;
const images = parseImages(markdown);
await Promise.all(
images.map(async (image) => {
// Skip attempting to fetch images that are not valid urls
try {
new URL(image.src);
} catch {
return;
}
const attachment = await attachmentCreator({
name: image.alt ?? "image",
url: image.src,
preset: AttachmentPreset.DocumentAttachment,
user,
ip,
transaction,
});
if (attachment) {
output = output.replace(
new RegExp(escapeRegExp(image.src), "g"),
attachment.redirectUrl
);
}
})
);
return output;
}
}

7
server/presenters/document.ts
View File

@@ -1,6 +1,6 @@
import { traceFunction } from "@server/logging/tracing";
import { Document } from "@server/models";
import DocumentHelper from "@server/models/helpers/DocumentHelper";
import TextHelper from "@server/models/helpers/TextHelper";
import presentUser from "./user";
type Options = {
@@ -16,10 +16,7 @@ async function presentDocument(
...options,
};
const text = options.isPublic
? await DocumentHelper.attachmentsToSignedUrls(
document.text,
document.teamId
)
? await TextHelper.attachmentsToSignedUrls(document.text, document.teamId)
: document.text;
const data: Record<string, any> = {

3
server/presenters/revision.ts
View File

@@ -1,6 +1,7 @@
import parseTitle from "@shared/utils/parseTitle";
import { traceFunction } from "@server/logging/tracing";
import { Revision } from "@server/models";
import DocumentHelper from "@server/models/helpers/DocumentHelper";
import presentUser from "./user";
async function presentRevision(revision: Revision, diff?: string) {
@@ -11,7 +12,7 @@ async function presentRevision(revision: Revision, diff?: string) {
id: revision.id,
documentId: revision.documentId,
title: strippedTitle,
text: revision.text,
text: DocumentHelper.toMarkdown(revision),
emoji: revision.emoji ?? emoji,
html: diff,
createdAt: revision.createdAt,

5
server/routes/api/documents/documents.ts
View File

@@ -684,12 +684,11 @@ router.post(
// restore a document to a specific revision
authorize(user, "update", document);
const revision = await Revision.findByPk(revisionId);
authorize(document, "restore", revision);
document.text = revision.text;
document.title = revision.title;
document.restoreFromRevision(revision);
await document.save();
await Event.create({
name: "documents.restore",
documentId: document.id,

1
server/scripts/20221008000000-backfill-crdt.ts
View File

@@ -27,7 +27,6 @@ export default async function main(exit = false) {
offset: page * limit,
where: {
...(teamId ? { teamId } : {}),
state: null,
},
order: [["createdAt", "ASC"]],
paranoid: false,

61
server/scripts/20231119000000-backfill-document-content.ts Normal file
View File

@@ -0,0 +1,61 @@
import "./bootstrap";
import { yDocToProsemirrorJSON } from "@getoutline/y-prosemirror";
import { Node } from "prosemirror-model";
import * as Y from "yjs";
import { parser, schema } from "@server/editor";
import { Document } from "@server/models";
const limit = 100;
const page = 0;
export default async function main(exit = false) {
const work = async (page: number): Promise<void> => {
console.log(`Backfill content… page ${page}`);
// Retrieve all documents within set limit.
const documents = await Document.unscoped().findAll({
attributes: ["id", "urlId", "content", "text", "state"],
limit,
offset: page * limit,
order: [["createdAt", "ASC"]],
paranoid: false,
});
for (const document of documents) {
if (document.content || !document.text) {
continue;
}
console.log(`Writing content for ${document.id}`);
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
document.content = yDocToProsemirrorJSON(ydoc, "default");
} else {
const node = parser.parse(document.text) || Node.fromJSON(schema, {});
document.content = node.toJSON();
}
document.changed("content", true);
await document.save({
hooks: false,
silent: true,
});
}
return documents.length === limit ? work(page + 1) : undefined;
};
await work(page);
if (exit) {
console.log("Backfill complete");
process.exit(0);
}
}
if (process.env.NODE_ENV !== "test") {
void main(true);
}

52
server/scripts/20231119000000-backfill-revision-content.ts Normal file
View File

@@ -0,0 +1,52 @@
import "./bootstrap";
import { Node } from "prosemirror-model";
import { parser, schema } from "@server/editor";
import { Revision } from "@server/models";
const limit = 100;
const page = 0;
export default async function main(exit = false) {
const work = async (page: number): Promise<void> => {
console.log(`Backfill content… page ${page}`);
// Retrieve all revisions within set limit.
const revisions = await Revision.unscoped().findAll({
attributes: ["id", "content", "text"],
limit,
offset: page * limit,
order: [["createdAt", "ASC"]],
paranoid: false,
});
for (const revision of revisions) {
if (revision.content || !revision.text) {
continue;
}
console.log(`Writing content for ${revision.id}`);
const node = parser.parse(revision.text) || Node.fromJSON(schema, {});
revision.content = node.toJSON();
revision.changed("content", true);
await revision.save({
hooks: false,
silent: true,
});
}
return revisions.length === limit ? work(page + 1) : undefined;
};
await work(page);
if (exit) {
console.log("Backfill complete");
process.exit(0);
}
}
if (process.env.NODE_ENV !== "test") {
void main(true);
}