yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adds content column to documents and revisions as JSON snapshot (#6179)

Browse Source
This commit is contained in:
Tom Moor
2023-12-17 10:51:11 -05:00
committed by GitHub
parent 78b9322a28
commit 1840370e6f
18 changed files with 411 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
server/models/helpers/DocumentHelper.test.ts
View File

@@ -1,5 +1,5 @@
import Revision from "@server/models/Revision";
import { buildDocument, buildUser } from "@server/test/factories";
import { buildDocument } from "@server/test/factories";
import DocumentHelper from "./DocumentHelper";
describe("DocumentHelper", () => {
@@ -12,28 +12,6 @@ describe("DocumentHelper", () => {
jest.useRealTimers();
});
describe("replaceTemplateVariables", () => {
it("should replace {time} with current time", async () => {
const user = await buildUser();
const result = DocumentHelper.replaceTemplateVariables(
"Hello {time}",
user
);
expect(result).toBe("Hello 12 00 AM");
});
it("should replace {date} with current date", async () => {
const user = await buildUser();
const result = DocumentHelper.replaceTemplateVariables(
"Hello {date}",
user
);
expect(result).toBe("Hello January 1 2021");
});
});
describe("parseMentions", () => {
it("should not parse normal links as mentions", async () => {
const document = await buildDocument({

159
server/models/helpers/DocumentHelper.tsx
View File

@@ -1,33 +1,20 @@
import {
updateYFragment,
yDocToProsemirror,
yDocToProsemirrorJSON,
} from "@getoutline/y-prosemirror";
import { JSDOM } from "jsdom";
import escapeRegExp from "lodash/escapeRegExp";
import startCase from "lodash/startCase";
import { Node } from "prosemirror-model";
import { Transaction } from "sequelize";
import * as Y from "yjs";
import textBetween from "@shared/editor/lib/textBetween";
import { AttachmentPreset } from "@shared/types";
import MarkdownHelper from "@shared/utils/MarkdownHelper";
import {
getCurrentDateAsString,
getCurrentDateTimeAsString,
getCurrentTimeAsString,
unicodeCLDRtoBCP47,
} from "@shared/utils/date";
import attachmentCreator from "@server/commands/attachmentCreator";
import { parser, schema } from "@server/editor";
import { addTags } from "@server/logging/tracer";
import { trace } from "@server/logging/tracing";
import { Document, Revision, User } from "@server/models";
import FileStorage from "@server/storage/files";
import { Document, Revision } from "@server/models";
import diff from "@server/utils/diff";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
import Attachment from "../Attachment";
import ProsemirrorHelper from "./ProsemirrorHelper";
import TextHelper from "./TextHelper";
type HTMLOptions = {
/** Whether to include the document title in the generated HTML (defaults to true) */
@@ -48,8 +35,25 @@ type HTMLOptions = {
@trace()
export default class DocumentHelper {
/**
* Returns the document as a Prosemirror Node. This method uses the
* collaborative state if available, otherwise it falls back to Markdown.
* Returns the document as JSON content. This method uses the collaborative state if available,
* otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as JSON
*/
static toJSON(document: Document | Revision) {
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
return yDocToProsemirrorJSON(ydoc, "default");
}
const node = parser.parse(document.text) || Node.fromJSON(schema, {});
return node.toJSON();
}
/**
* Returns the document as a Prosemirror Node. This method uses the collaborative state if
* available, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as a Prosemirror Node
@@ -124,7 +128,7 @@ export default class DocumentHelper {
return output;
}
output = await DocumentHelper.attachmentsToSignedUrls(
output = await TextHelper.attachmentsToSignedUrls(
output,
teamId,
typeof options.signedUrls === "number" ? options.signedUrls : undefined
@@ -188,7 +192,7 @@ export default class DocumentHelper {
: (await before.$get("document"))?.teamId;
if (teamId) {
diffedContentAsHTML = await DocumentHelper.attachmentsToSignedUrls(
diffedContentAsHTML = await TextHelper.attachmentsToSignedUrls(
diffedContentAsHTML,
teamId,
typeof signedUrls === "number" ? signedUrls : undefined
@@ -336,114 +340,6 @@ export default class DocumentHelper {
return `${head?.innerHTML} ${body?.innerHTML}`;
}
/**
* Converts attachment urls in documents to signed equivalents that allow
* direct access without a session cookie
*
* @param text The text either html or markdown which contains urls to be converted
* @param teamId The team context
* @param expiresIn The time that signed urls should expire (in seconds)
* @returns The replaced text
*/
static async attachmentsToSignedUrls(
text: string,
teamId: string,
expiresIn = 3000
) {
const attachmentIds = parseAttachmentIds(text);
await Promise.all(
attachmentIds.map(async (id) => {
const attachment = await Attachment.findOne({
where: {
id,
teamId,
},
});
if (attachment) {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
signedUrl
);
}
})
);
return text;
}
/**
* Replaces template variables in the given text with the current date and time.
*
* @param text The text to replace the variables in
* @param user The user to get the language/locale from
* @returns The text with the variables replaced
*/
static replaceTemplateVariables(text: string, user: User) {
const locales = user.language
? unicodeCLDRtoBCP47(user.language)
: undefined;
return text
.replace(/{date}/g, startCase(getCurrentDateAsString(locales)))
.replace(/{time}/g, startCase(getCurrentTimeAsString(locales)))
.replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales)));
}
/**
* Replaces remote and base64 encoded images in the given text with attachment
* urls and uploads the images to the storage provider.
*
* @param text The text to replace the images in
* @param user The user context
* @param ip The IP address of the user
* @param transaction The transaction to use for the database operations
* @returns The text with the images replaced
*/
static async replaceImagesWithAttachments(
text: string,
user: User,
ip?: string,
transaction?: Transaction
) {
let output = text;
const images = parseImages(text);
await Promise.all(
images.map(async (image) => {
// Skip attempting to fetch images that are not valid urls
try {
new URL(image.src);
} catch {
return;
}
const attachment = await attachmentCreator({
name: image.alt ?? "image",
url: image.src,
preset: AttachmentPreset.DocumentAttachment,
user,
ip,
transaction,
});
if (attachment) {
output = output.replace(
new RegExp(escapeRegExp(image.src), "g"),
attachment.redirectUrl
);
}
})
);
return output;
}
/**
* Applies the given Markdown to the document, this essentially creates a
* single change in the collaborative state that makes all the edits to get
@@ -461,12 +357,12 @@ export default class DocumentHelper {
append = false
) {
document.text = append ? document.text + text : text;
const doc = parser.parse(document.text);
if (document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
const type = ydoc.get("default", Y.XmlFragment) as Y.XmlFragment;
const doc = parser.parse(document.text);
if (!type.doc) {
throw new Error("type.doc not found");
@@ -476,8 +372,13 @@ export default class DocumentHelper {
updateYFragment(type.doc, type, doc, new Map());
const state = Y.encodeStateAsUpdate(ydoc);
const node = yDocToProsemirror(schema, ydoc);
document.content = node.toJSON();
document.state = Buffer.from(state);
document.changed("state", true);
} else if (doc) {
document.content = doc.toJSON();
}
return document;

29
server/models/helpers/TextHelper.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { buildUser } from "@server/test/factories";
import TextHelper from "./TextHelper";
describe("TextHelper", () => {
beforeAll(() => {
jest.useFakeTimers();
jest.setSystemTime(Date.parse("2021-01-01T00:00:00.000Z"));
});
afterAll(() => {
jest.useRealTimers();
});
describe("replaceTemplateVariables", () => {
it("should replace {time} with current time", async () => {
const user = await buildUser();
const result = TextHelper.replaceTemplateVariables("Hello {time}", user);
expect(result).toBe("Hello 12 00 AM");
});
it("should replace {date} with current date", async () => {
const user = await buildUser();
const result = TextHelper.replaceTemplateVariables("Hello {date}", user);
expect(result).toBe("Hello January 1 2021");
});
});
});

125
server/models/helpers/TextHelper.ts Normal file
View File

@@ -0,0 +1,125 @@
import escapeRegExp from "lodash/escapeRegExp";
import startCase from "lodash/startCase";
import { Transaction } from "sequelize";
import { AttachmentPreset } from "@shared/types";
import {
getCurrentDateAsString,
getCurrentDateTimeAsString,
getCurrentTimeAsString,
unicodeCLDRtoBCP47,
} from "@shared/utils/date";
import attachmentCreator from "@server/commands/attachmentCreator";
import { Attachment, User } from "@server/models";
import FileStorage from "@server/storage/files";
import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
export default class TextHelper {
/**
* Replaces template variables in the given text with the current date and time.
*
* @param text The text to replace the variables in
* @param user The user to get the language/locale from
* @returns The text with the variables replaced
*/
static replaceTemplateVariables(text: string, user: User) {
const locales = user.language
? unicodeCLDRtoBCP47(user.language)
: undefined;
return text
.replace(/{date}/g, startCase(getCurrentDateAsString(locales)))
.replace(/{time}/g, startCase(getCurrentTimeAsString(locales)))
.replace(/{datetime}/g, startCase(getCurrentDateTimeAsString(locales)));
}
/**
* Converts attachment urls in documents to signed equivalents that allow
* direct access without a session cookie
*
* @param text The text either html or markdown which contains urls to be converted
* @param teamId The team context
* @param expiresIn The time that signed urls should expire (in seconds)
* @returns The replaced text
*/
static async attachmentsToSignedUrls(
text: string,
teamId: string,
expiresIn = 3000
) {
const attachmentIds = parseAttachmentIds(text);
await Promise.all(
attachmentIds.map(async (id) => {
const attachment = await Attachment.findOne({
where: {
id,
teamId,
},
});
if (attachment) {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
signedUrl
);
}
})
);
return text;
}
/**
* Replaces remote and base64 encoded images in the given text with attachment
* urls and uploads the images to the storage provider.
*
* @param markdown The text to replace the images in
* @param user The user context
* @param ip The IP address of the user
* @param transaction The transaction to use for the database operations
* @returns The text with the images replaced
*/
static async replaceImagesWithAttachments(
markdown: string,
user: User,
ip?: string,
transaction?: Transaction
) {
let output = markdown;
const images = parseImages(markdown);
await Promise.all(
images.map(async (image) => {
// Skip attempting to fetch images that are not valid urls
try {
new URL(image.src);
} catch {
return;
}
const attachment = await attachmentCreator({
name: image.alt ?? "image",
url: image.src,
preset: AttachmentPreset.DocumentAttachment,
user,
ip,
transaction,
});
if (attachment) {
output = output.replace(
new RegExp(escapeRegExp(image.src), "g"),
attachment.redirectUrl
);
}
})
);
return output;
}
}