yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adds content column to documents and revisions as JSON snapshot (#6179)

Browse Source
This commit is contained in:
Tom Moor
2023-12-17 10:51:11 -05:00
committed by GitHub
parent 78b9322a28
commit 1840370e6f
18 changed files with 411 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
server/models/Document.ts
View File

@@ -35,7 +35,11 @@ import {
AllowNull,
} from "sequelize-typescript";
import isUUID from "validator/lib/isUUID";
import type { NavigationNode, SourceMetadata } from "@shared/types";
import type {
NavigationNode,
ProsemirrorData,
SourceMetadata,
} from "@shared/types";
import getTasks from "@shared/utils/getTasks";
import slugify from "@shared/utils/slugify";
import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
@@ -206,15 +210,18 @@ class Document extends ParanoidModel {
@Column(DataType.SMALLINT)
version: number;
@Default(false)
@Column
template: boolean;
@Default(false)
@Column
fullWidth: boolean;
@Column
insightsEnabled: boolean;
/** The version of the editor last used to edit this document. */
@SimpleLength({
max: 255,
msg: `editorVersion must be 255 characters or less`,
@@ -222,6 +229,7 @@ class Document extends ParanoidModel {
@Column
editorVersion: string;
/** An emoji to use as the document icon. */
@Length({
max: 1,
msg: `Emoji must be a single character`,
@@ -229,9 +237,25 @@ class Document extends ParanoidModel {
@Column
emoji: string | null;
/**
* The content of the document as Markdown.
*
* @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown.
* This column will be removed in a future migration.
*/
@Column(DataType.TEXT)
text: string;
/**
* The content of the document as JSON, this is a snapshot at the last time the state was saved.
*/
@Column(DataType.JSONB)
content: ProsemirrorData;
/**
* The content of the document as YJS collaborative state, this column can be quite large and
* should only be selected from the DB when the `content` snapshot cannot be used.
*/
@SimpleLength({
max: DocumentValidation.maxStateLength,
msg: `Document collaborative state is too large, you must create a new document`,
@@ -239,28 +263,38 @@ class Document extends ParanoidModel {
@Column(DataType.BLOB)
state: Uint8Array;
/** Whether this document is part of onboarding. */
@Default(false)
@Column
isWelcome: boolean;
/** How many versions there are in the history of this document. */
@IsNumeric
@Default(0)
@Column(DataType.INTEGER)
revisionCount: number;
/** Whether the document is archvied, and if so when. */
@IsDate
@Column
archivedAt: Date | null;
/** Whether the document is published, and if so when. */
@IsDate
@Column
publishedAt: Date | null;
/** An array of user IDs that have edited this document. */
@Column(DataType.ARRAY(DataType.UUID))
collaboratorIds: string[] = [];
// getters
/**
* The frontend path to this document.
*
* @deprecated Use `path` instead.
*/
get url() {
if (!this.title) {
return `/doc/untitled-${this.urlId}`;
@@ -269,6 +303,11 @@ class Document extends ParanoidModel {
return `/doc/${slugifiedTitle}-${this.urlId}`;
}
/** The frontend path to this document. */
get path() {
return this.url;
}
get tasks() {
return getTasks(this.text || "");
}
@@ -363,6 +402,11 @@ class Document extends ParanoidModel {
model.collaboratorIds = [];
}
// backfill content if it's missing
if (!model.content) {
model.content = DocumentHelper.toJSON(model);
}
// ensure the last modifying user is a collaborator
model.collaboratorIds = uniq(
model.collaboratorIds.concat(model.lastModifiedById)
@@ -589,6 +633,22 @@ class Document extends ParanoidModel {
return !!(this.importId && this.sourceMetadata?.trial);
}
/**
* Revert the state of the document to match the passed revision.
*
* @param revision The revision to revert to.
*/
restoreFromRevision = (revision: Revision) => {
if (revision.documentId !== this.id) {
throw new Error("Revision does not belong to this document");
}
this.content = revision.content;
this.text = revision.text;
this.title = revision.title;
this.emoji = revision.emoji;
};
/**
* Get a list of users that have collaborated on this document
*

14
server/models/Revision.ts
View File

@@ -9,6 +9,7 @@ import {
IsNumeric,
Length as SimpleLength,
} from "sequelize-typescript";
import type { ProsemirrorData } from "@shared/types";
import { DocumentValidation } from "@shared/validations";
import Document from "./Document";
import User from "./User";
@@ -46,9 +47,21 @@ class Revision extends IdModel {
@Column
title: string;
/**
* The content of the revision as Markdown.
*
* @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown.
* This column will be removed in a future migration.
*/
@Column(DataType.TEXT)
text: string;
/**
* The content of the revision as JSON.
*/
@Column(DataType.JSONB)
content: ProsemirrorData;
@Length({
max: 1,
msg: `Emoji must be a single character`,
@@ -100,6 +113,7 @@ class Revision extends IdModel {
title: document.title,
text: document.text,
emoji: document.emoji,
content: document.content,
userId: document.lastModifiedById,
editorVersion: document.editorVersion,
version: document.version,

24
server/models/helpers/DocumentHelper.test.ts
View File

@@ -1,5 +1,5 @@
import Revision from "@server/models/Revision";
import { buildDocument, buildUser } from "@server/test/factories";
import { buildDocument } from "@server/test/factories";
import DocumentHelper from "./DocumentHelper";
describe("DocumentHelper", () => {
@@ -12,28 +12,6 @@ describe("DocumentHelper", () => {
jest.useRealTimers();
});
describe("replaceTemplateVariables", () => {
it("should replace {time} with current time", async () => {
const user = await buildUser();
const result = DocumentHelper.replaceTemplateVariables(
"Hello {time}",
user
);
expect(result).toBe("Hello 12 00 AM");
});
it("should replace {date} with current date", async () => {
const user = await buildUser();
const result = DocumentHelper.replaceTemplateVariables(
"Hello {date}",
user
);
expect(result).toBe("Hello January 1 2021");
});
});
describe("parseMentions", () => {
it("should not parse normal links as mentions", async () => {
const document = await buildDocument({

159
server/models/helpers/DocumentHelper.tsx
View File

@@ -1,33 +1,20 @@
import {
updateYFragment,
yDocToProsemirror,
yDocToProsemirrorJSON,
} from "@getoutline/y-prosemirror";
import { JSDOM } from "jsdom";
import escapeRegExp from "lodash/escapeRegExp";
import startCase from "lodash/startCase";
import { Node } from "prosemirror-model";
import { Transaction } from "sequelize";
import * as Y from "yjs";
import textBetween from "@shared/editor/lib/textBetween";
import { AttachmentPreset } from "@shared/types";
import MarkdownHelper from "@shared/utils/MarkdownHelper";
import {
getCurrentDateAsString,
getCurrentDateTimeAsString,
getCurrentTimeAsString,
unicodeCLDRtoBCP47,
} from "@shared/utils/date";
import attachmentCreator from "@server/commands/attachmentCreator";
import { parser, schema } from "@server/editor";
import { addTags } from "@server/logging/tracer";
import { trace } from "@server/logging/tracing";
import { Document, Revision, User } from "@server/models";
import FileStorage from "@server/storage/files";
import { Document, Revision } from "@server/models";
import diff from "@server/utils/diff";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
import Attachment from "../Attachment";
import ProsemirrorHelper from "./ProsemirrorHelper";
import TextHelper from "./TextHelper";
type HTMLOptions = {
/** Whether to include the document title in the generated HTML (defaults to true) */
@@ -48,8 +35,25 @@ type HTMLOptions = {
@trace()
export default class DocumentHelper {
/**
* Returns the document as a Prosemirror Node. This method uses the
* collaborative state if available, otherwise it falls back to Markdown.
* Returns the document as JSON content. This method uses the collaborative state if available,
* otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as JSON
*/
static toJSON(document: Document | Revision) {
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
return yDocToProsemirrorJSON(ydoc, "default");
}
const node = parser.parse(document.text) || Node.fromJSON(schema, {});
return node.toJSON();
}
/**
* Returns the document as a Prosemirror Node. This method uses the collaborative state if
* available, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as a Prosemirror Node
@@ -124,7 +128,7 @@ export default class DocumentHelper {
return output;
}
output = await DocumentHelper.attachmentsToSignedUrls(
output = await TextHelper.attachmentsToSignedUrls(
output,
teamId,
typeof options.signedUrls === "number" ? options.signedUrls : undefined
@@ -188,7 +192,7 @@ export default class DocumentHelper {
: (await before.$get("document"))?.teamId;
if (teamId) {
diffedContentAsHTML = await DocumentHelper.attachmentsToSignedUrls(
diffedContentAsHTML = await TextHelper.attachmentsToSignedUrls(
diffedContentAsHTML,
teamId,
typeof signedUrls === "number" ? signedUrls : undefined
@@ -336,114 +340,6 @@ export default class DocumentHelper {
return `${head?.innerHTML} ${body?.innerHTML}`;
}
/**
* Converts attachment urls in documents to signed equivalents that allow
* direct access without a session cookie
*
* @param text The text either html or markdown which contains urls to be converted
* @param teamId The team context
* @param expiresIn The time that signed urls should expire (in seconds)
* @returns The replaced text
*/
static async attachmentsToSignedUrls(
text: string,
teamId: string,
expiresIn = 3000
) {
const attachmentIds = parseAttachmentIds(text);
await Promise.all(
attachmentIds.map(async (id) => {
const attachment = await Attachment.findOne({
where: {
id,
teamId,
},
});
if (attachment) {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
signedUrl
);
}
})
);
return text;
}
/**
* Replaces template variables in the given text with the current date and time.
*
* @param text The text to replace the variables in
* @param user The user to get the language/locale from
* @returns The text with the variables replaced
*/
static replaceTemplateVariables(text: string, user: User) {
const locales = user.language
? unicodeCLDRtoBCP47(user.language)
: undefined;
return text
.replace(/{date}/g, startCase(getCurrentDateAsString(locales)))
.replace(/{time}/g, startCase(getCurrentTimeAsString(locales)))
.replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales)));
}
/**
* Replaces remote and base64 encoded images in the given text with attachment
* urls and uploads the images to the storage provider.
*
* @param text The text to replace the images in
* @param user The user context
* @param ip The IP address of the user
* @param transaction The transaction to use for the database operations
* @returns The text with the images replaced
*/
static async replaceImagesWithAttachments(
text: string,
user: User,
ip?: string,
transaction?: Transaction
) {
let output = text;
const images = parseImages(text);
await Promise.all(
images.map(async (image) => {
// Skip attempting to fetch images that are not valid urls
try {
new URL(image.src);
} catch {
return;
}
const attachment = await attachmentCreator({
name: image.alt ?? "image",
url: image.src,
preset: AttachmentPreset.DocumentAttachment,
user,
ip,
transaction,
});
if (attachment) {
output = output.replace(
new RegExp(escapeRegExp(image.src), "g"),
attachment.redirectUrl
);
}
})
);
return output;
}
/**
* Applies the given Markdown to the document, this essentially creates a
* single change in the collaborative state that makes all the edits to get
@@ -461,12 +357,12 @@ export default class DocumentHelper {
append = false
) {
document.text = append ? document.text + text : text;
const doc = parser.parse(document.text);
if (document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
const type = ydoc.get("default", Y.XmlFragment) as Y.XmlFragment;
const doc = parser.parse(document.text);
if (!type.doc) {
throw new Error("type.doc not found");
@@ -476,8 +372,13 @@ export default class DocumentHelper {
updateYFragment(type.doc, type, doc, new Map());
const state = Y.encodeStateAsUpdate(ydoc);
const node = yDocToProsemirror(schema, ydoc);
document.content = node.toJSON();
document.state = Buffer.from(state);
document.changed("state", true);
} else if (doc) {
document.content = doc.toJSON();
}
return document;

29
server/models/helpers/TextHelper.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { buildUser } from "@server/test/factories";
import TextHelper from "./TextHelper";
describe("TextHelper", () => {
beforeAll(() => {
jest.useFakeTimers();
jest.setSystemTime(Date.parse("2021-01-01T00:00:00.000Z"));
});
afterAll(() => {
jest.useRealTimers();
});
describe("replaceTemplateVariables", () => {
it("should replace {time} with current time", async () => {
const user = await buildUser();
const result = TextHelper.replaceTemplateVariables("Hello {time}", user);
expect(result).toBe("Hello 12 00 AM");
});
it("should replace {date} with current date", async () => {
const user = await buildUser();
const result = TextHelper.replaceTemplateVariables("Hello {date}", user);
expect(result).toBe("Hello January 1 2021");
});
});
});

125
server/models/helpers/TextHelper.ts Normal file
View File

@@ -0,0 +1,125 @@
import escapeRegExp from "lodash/escapeRegExp";
import startCase from "lodash/startCase";
import { Transaction } from "sequelize";
import { AttachmentPreset } from "@shared/types";
import {
getCurrentDateAsString,
getCurrentDateTimeAsString,
getCurrentTimeAsString,
unicodeCLDRtoBCP47,
} from "@shared/utils/date";
import attachmentCreator from "@server/commands/attachmentCreator";
import { Attachment, User } from "@server/models";
import FileStorage from "@server/storage/files";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
export default class TextHelper {
/**
* Replaces template variables in the given text with the current date and time.
*
* @param text The text to replace the variables in
* @param user The user to get the language/locale from
* @returns The text with the variables replaced
*/
static replaceTemplateVariables(text: string, user: User) {
const locales = user.language
? unicodeCLDRtoBCP47(user.language)
: undefined;
return text
.replace(/{date}/g, startCase(getCurrentDateAsString(locales)))
.replace(/{time}/g, startCase(getCurrentTimeAsString(locales)))
.replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales)));
}
/**
* Converts attachment urls in documents to signed equivalents that allow
* direct access without a session cookie
*
* @param text The text either html or markdown which contains urls to be converted
* @param teamId The team context
* @param expiresIn The time that signed urls should expire (in seconds)
* @returns The replaced text
*/
static async attachmentsToSignedUrls(
text: string,
teamId: string,
expiresIn = 3000
) {
const attachmentIds = parseAttachmentIds(text);
await Promise.all(
attachmentIds.map(async (id) => {
const attachment = await Attachment.findOne({
where: {
id,
teamId,
},
});
if (attachment) {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
signedUrl
);
}
})
);
return text;
}
/**
* Replaces remote and base64 encoded images in the given text with attachment
* urls and uploads the images to the storage provider.
*
* @param markdown The text to replace the images in
* @param user The user context
* @param ip The IP address of the user
* @param transaction The transaction to use for the database operations
* @returns The text with the images replaced
*/
static async replaceImagesWithAttachments(
markdown: string,
user: User,
ip?: string,
transaction?: Transaction
) {
let output = markdown;
const images = parseImages(markdown);
await Promise.all(
images.map(async (image) => {
// Skip attempting to fetch images that are not valid urls
try {
new URL(image.src);
} catch {
return;
}
const attachment = await attachmentCreator({
name: image.alt ?? "image",
url: image.src,
preset: AttachmentPreset.DocumentAttachment,
user,
ip,
transaction,
});
if (attachment) {
output = output.replace(
new RegExp(escapeRegExp(image.src), "g"),
attachment.redirectUrl
);
}
})
);
return output;
}
}