yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

JSON to client (#5553)

Browse Source
This commit is contained in:
Tom Moor
2024-05-24 08:29:00 -04:00
committed by GitHub
parent e1e8257df7
commit d51267b8bc
71 changed files with 651 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/models/Comment.ts
View File

@@ -10,7 +10,7 @@ import {
DefaultScope,
} from "sequelize-typescript";
import type { ProsemirrorData } from "@shared/types";
import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { CommentValidation } from "@shared/validations";
import { schema } from "@server/editor";
import Document from "./Document";

46
server/models/Document.test.ts
View File

@@ -1,5 +1,6 @@
import { EmptyResultError } from "sequelize";
import slugify from "@shared/utils/slugify";
import { parser } from "@server/editor";
import Document from "@server/models/Document";
import {
buildDocument,
@@ -208,20 +209,7 @@ describe("#findByPk", () => {
});
describe("tasks", () => {
test("should consider all the possible checkTtems", async () => {
const document = await buildDocument({
text: `- [x] test
- [X] test
- [ ] test
- [-] test
- [_] test`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(4);
expect(tasks.total).toBe(5);
});
test("should return tasks keys set to 0 if checkItems isn't present", async () => {
test("should return tasks keys set to 0 if check items isn't present", async () => {
const document = await buildDocument({
text: `text`,
});
@@ -230,11 +218,12 @@ describe("tasks", () => {
expect(tasks.total).toBe(0);
});
test("should return tasks keys set to 0 if the text contains broken checkItems", async () => {
test("should return tasks keys set to 0 if the text contains broken check items", async () => {
const document = await buildDocument({
text: `- [x ] test
- [ x ] test
- [ ] test`,
text: `
- [x ] test
- [ x ] test
- [ ] test`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(0);
@@ -243,8 +232,9 @@ describe("tasks", () => {
test("should return tasks", async () => {
const document = await buildDocument({
text: `- [x] list item
- [ ] list item`,
text: `
- [x] list item
- [ ] list item`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(1);
@@ -253,15 +243,21 @@ describe("tasks", () => {
test("should update tasks on save", async () => {
const document = await buildDocument({
text: `- [x] list item
- [ ] list item`,
text: `
- [x] list item
- [ ] list item`,
});
const tasks = document.tasks;
expect(tasks.completed).toBe(1);
expect(tasks.total).toBe(2);
document.text = `- [x] list item
- [ ] list item
- [ ] list item`;
document.content = parser
.parse(
`
- [x] list item
- [ ] list item
- [ ] list item`
)
?.toJSON();
await document.save();
const newTasks = document.tasks;
expect(newTasks.completed).toBe(1);

16
server/models/Document.ts
View File

@@ -47,8 +47,8 @@ import type {
ProsemirrorData,
SourceMetadata,
} from "@shared/types";
import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { UrlHelper } from "@shared/utils/UrlHelper";
import getTasks from "@shared/utils/getTasks";
import slugify from "@shared/utils/slugify";
import { DocumentValidation } from "@shared/validations";
import { ValidationError } from "@server/errors";
@@ -63,7 +63,7 @@ import UserMembership from "./UserMembership";
import View from "./View";
import ParanoidModel from "./base/ParanoidModel";
import Fix from "./decorators/Fix";
import DocumentHelper from "./helpers/DocumentHelper";
import { DocumentHelper } from "./helpers/DocumentHelper";
import Length from "./validators/Length";
export const DOCUMENT_VERSION = 2;
@@ -75,9 +75,6 @@ type AdditionalFindOptions = {
};
@DefaultScope(() => ({
attributes: {
exclude: ["state"],
},
include: [
{
model: User,
@@ -337,7 +334,9 @@ class Document extends ParanoidModel<
}
get tasks() {
return getTasks(this.text || "");
return ProsemirrorHelper.getTasksSummary(
DocumentHelper.toProsemirror(this)
);
}
// hooks
@@ -411,7 +410,7 @@ class Document extends ParanoidModel<
}
@BeforeUpdate
static processUpdate(model: Document) {
static async processUpdate(model: Document) {
// ensure documents have a title
model.title = model.title || "";
@@ -431,7 +430,7 @@ class Document extends ParanoidModel<
// backfill content if it's missing
if (!model.content) {
model.content = DocumentHelper.toJSON(model);
model.content = await DocumentHelper.toJSON(model);
}
// ensure the last modifying user is a collaborator
@@ -608,7 +607,6 @@ class Document extends ParanoidModel<
// allow default preloading of collection membership if `userId` is passed in find options
// almost every endpoint needs the collection membership to determine policy permissions.
const scope = this.scope([
...(includeState ? [] : ["withoutState"]),
"withDrafts",
{
method: ["withCollectionPermissions", userId, rest.paranoid],

63
server/models/Team.ts
View File

@@ -1,8 +1,5 @@
import crypto from "crypto";
import fs from "fs";
import path from "path";
import { URL } from "url";
import util from "util";
import { subMinutes } from "date-fns";
import {
InferAttributes,
@@ -30,12 +27,7 @@ import {
BeforeCreate,
} from "sequelize-typescript";
import { TeamPreferenceDefaults } from "@shared/constants";
import {
CollectionPermission,
TeamPreference,
TeamPreferences,
UserRole,
} from "@shared/types";
import { TeamPreference, TeamPreferences, UserRole } from "@shared/types";
import { getBaseDomain, RESERVED_SUBDOMAINS } from "@shared/utils/domains";
import env from "@server/env";
import { ValidationError } from "@server/errors";
@@ -55,8 +47,6 @@ import IsUrlOrRelativePath from "./validators/IsUrlOrRelativePath";
import Length from "./validators/Length";
import NotContainsUrl from "./validators/NotContainsUrl";
const readFile = util.promisify(fs.readFile);
@Scopes(() => ({
withDomains: {
include: [{ model: TeamDomain }],
@@ -279,57 +269,6 @@ class Team extends ParanoidModel<
});
};
provisionFirstCollection = async (userId: string) => {
await this.sequelize!.transaction(async (transaction) => {
const collection = await Collection.create(
{
name: "Welcome",
description: `This collection is a quick guide to what ${env.APP_NAME} is all about. Feel free to delete this collection once your team is up to speed with the basics!`,
teamId: this.id,
createdById: userId,
sort: Collection.DEFAULT_SORT,
permission: CollectionPermission.ReadWrite,
},
{
transaction,
}
);
// For the first collection we go ahead and create some intitial documents to get
// the team started. You can edit these in /server/onboarding/x.md
const onboardingDocs = [
"Integrations & API",
"Our Editor",
"Getting Started",
"What is Outline",
];
for (const title of onboardingDocs) {
const text = await readFile(
path.join(process.cwd(), "server", "onboarding", `${title}.md`),
"utf8"
);
const document = await Document.create(
{
version: 2,
isWelcome: true,
parentDocumentId: null,
collectionId: collection.id,
teamId: collection.teamId,
lastModifiedById: collection.createdById,
createdById: collection.createdById,
title,
text,
},
{ transaction }
);
await document.publish(collection.createdById, collection.id, {
transaction,
});
}
});
};
public collectionIds = async function (paranoid = true) {
const models = await Collection.findAll({
attributes: ["id"],

2
server/models/helpers/DocumentHelper.test.ts
View File

@@ -1,6 +1,6 @@
import Revision from "@server/models/Revision";
import { buildDocument } from "@server/test/factories";
import DocumentHelper from "./DocumentHelper";
import { DocumentHelper } from "./DocumentHelper";
describe("DocumentHelper", () => {
beforeAll(() => {

104
server/models/helpers/DocumentHelper.tsx
View File

@@ -7,14 +7,14 @@ import { JSDOM } from "jsdom";
import { Node } from "prosemirror-model";
import * as Y from "yjs";
import textBetween from "@shared/editor/lib/textBetween";
import MarkdownHelper from "@shared/utils/MarkdownHelper";
import { parser, schema } from "@server/editor";
import { ProsemirrorData } from "@shared/types";
import { parser, serializer, schema } from "@server/editor";
import { addTags } from "@server/logging/tracer";
import { trace } from "@server/logging/tracing";
import { Document, Revision } from "@server/models";
import diff from "@server/utils/diff";
import ProsemirrorHelper from "./ProsemirrorHelper";
import TextHelper from "./TextHelper";
import { ProsemirrorHelper } from "./ProsemirrorHelper";
import { TextHelper } from "./TextHelper";
type HTMLOptions = {
/** Whether to include the document title in the generated HTML (defaults to true) */
@@ -35,32 +35,18 @@ type HTMLOptions = {
};
@trace()
export default class DocumentHelper {
export class DocumentHelper {
/**
* Returns the document as JSON content. This method uses the collaborative state if available,
* otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as JSON
*/
static toJSON(document: Document | Revision) {
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
return yDocToProsemirrorJSON(ydoc, "default");
}
const node = parser.parse(document.text) || Node.fromJSON(schema, {});
return node.toJSON();
}
/**
* Returns the document as a Prosemirror Node. This method uses the collaborative state if
* available, otherwise it falls back to Markdown.
* Returns the document as a Prosemirror Node. This method uses the derived content if available
* then the collaborative state, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @returns The document content as a Prosemirror Node
*/
static toProsemirror(document: Document | Revision) {
if ("content" in document && document.content) {
return Node.fromJSON(schema, document.content);
}
if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
@@ -69,6 +55,55 @@ export default class DocumentHelper {
return parser.parse(document.text) || Node.fromJSON(schema, {});
}
/**
* Returns the document as a plain JSON object. This method uses the derived content if available
* then the collaborative state, otherwise it falls back to Markdown.
*
* @param document The document or revision to convert
* @param options Options for the conversion
* @returns The document content as a plain JSON object
*/
static async toJSON(
document: Document | Revision,
options?: {
/** The team context */
teamId: string;
/** Whether to sign attachment urls, and if so for how many seconds is the signature valid */
signedUrls: number;
/** Marks to remove from the document */
removeMarks?: string[];
}
): Promise<ProsemirrorData> {
let doc: Node | null;
let json;
if ("content" in document && document.content) {
doc = Node.fromJSON(schema, document.content);
} else if ("state" in document && document.state) {
const ydoc = new Y.Doc();
Y.applyUpdate(ydoc, document.state);
doc = Node.fromJSON(schema, yDocToProsemirrorJSON(ydoc, "default"));
} else {
doc = parser.parse(document.text);
}
if (doc && options?.signedUrls) {
json = await ProsemirrorHelper.signAttachmentUrls(
doc,
options.teamId,
options.signedUrls
);
} else {
json = doc?.toJSON() ?? {};
}
if (options?.removeMarks) {
json = ProsemirrorHelper.removeMarks(json, options.removeMarks);
}
return json;
}
/**
* Returns the document as plain text. This method uses the
* collaborative state if available, otherwise it falls back to Markdown.
@@ -88,19 +123,30 @@ export default class DocumentHelper {
}
/**
* Returns the document as Markdown. This is a lossy conversion and should
* only be used for export.
* Returns the document as Markdown. This is a lossy conversion and should nly be used for export.
*
* @param document The document or revision to convert
* @returns The document title and content as a Markdown string
*/
static toMarkdown(document: Document | Revision) {
return MarkdownHelper.toMarkdown(document);
const text = serializer
.serialize(DocumentHelper.toProsemirror(document))
.replace(/\n\\(\n|$)/g, "\n\n")
.replace(/“/g, '"')
.replace(/”/g, '"')
.replace(/‘/g, "'")
.replace(/’/g, "'")
.trim();
const title = `${document.emoji ? document.emoji + " " : ""}${
document.title
}`;
return `# ${title}\n\n${text}`;
}
/**
* Returns the document as plain HTML. This is a lossy conversion and should
* only be used for export.
* Returns the document as plain HTML. This is a lossy conversion and should only be used for export.
*
* @param document The document or revision to convert
* @param options Options for the HTML output

137
server/models/helpers/ProsemirrorHelper.tsx
View File

@@ -1,5 +1,8 @@
import { prosemirrorToYDoc } from "@getoutline/y-prosemirror";
import { JSDOM } from "jsdom";
import compact from "lodash/compact";
import flatten from "lodash/flatten";
import uniq from "lodash/uniq";
import { Node, DOMSerializer, Fragment, Mark } from "prosemirror-model";
import * as React from "react";
import { renderToString } from "react-dom/server";
@@ -9,10 +12,14 @@ import EditorContainer from "@shared/editor/components/Styles";
import embeds from "@shared/editor/embeds";
import GlobalStyles from "@shared/styles/globals";
import light from "@shared/styles/theme";
import { ProsemirrorData } from "@shared/types";
import { attachmentRedirectRegex } from "@shared/utils/ProsemirrorHelper";
import { isRTL } from "@shared/utils/rtl";
import { schema, parser } from "@server/editor";
import Logger from "@server/logging/Logger";
import { trace } from "@server/logging/tracing";
import Attachment from "@server/models/Attachment";
import FileStorage from "@server/storage/files";
export type HTMLOptions = {
/** A title, if it should be included */
@@ -36,15 +43,22 @@ type MentionAttrs = {
};
@trace()
export default class ProsemirrorHelper {
export class ProsemirrorHelper {
/**
* Returns the input text as a Y.Doc.
*
* @param markdown The text to parse
* @returns The content as a Y.Doc.
*/
static toYDoc(markdown: string, fieldName = "default"): Y.Doc {
let node = parser.parse(markdown);
static toYDoc(input: string | ProsemirrorData, fieldName = "default"): Y.Doc {
if (typeof input === "object") {
return prosemirrorToYDoc(
ProsemirrorHelper.toProsemirror(input),
fieldName
);
}
let node = parser.parse(input);
// in the editor embeds are created at runtime by converting links into
// embeds where they match.Because we're converting to a CRDT structure on
@@ -106,7 +120,7 @@ export default class ProsemirrorHelper {
* @param data The object to parse
* @returns The content as a Prosemirror Node
*/
static toProsemirror(data: Record<string, any>) {
static toProsemirror(data: ProsemirrorData) {
return Node.fromJSON(schema, data);
}
@@ -116,10 +130,10 @@ export default class ProsemirrorHelper {
* @param node The node to parse mentions from
* @returns An array of mention attributes
*/
static parseMentions(node: Node) {
static parseMentions(doc: Node) {
const mentions: MentionAttrs[] = [];
node.descendants((node: Node) => {
doc.descendants((node: Node) => {
if (
node.type.name === "mention" &&
!mentions.some((m) => m.id === node.attrs.id)
@@ -138,6 +152,117 @@ export default class ProsemirrorHelper {
return mentions;
}
/**
* Removes all marks from the node that match the given types.
*
* @param data The ProsemirrorData object to remove marks from
* @param marks The mark types to remove
* @returns The content with marks removed
*/
static removeMarks(data: ProsemirrorData, marks: string[]) {
function removeMarksInner(node: ProsemirrorData) {
if (node.marks) {
node.marks = node.marks.filter((mark) => !marks.includes(mark.type));
}
if (node.content) {
node.content.forEach(removeMarksInner);
}
return node;
}
return removeMarksInner(data);
}
/**
* Returns the document as a plain JSON object with attachment URLs signed.
*
* @param node The node to convert to JSON
* @param teamId The team ID to use for signing
* @param expiresIn The number of seconds until the signed URL expires
* @returns The content as a JSON object
*/
static async signAttachmentUrls(doc: Node, teamId: string, expiresIn = 60) {
const attachmentIds = ProsemirrorHelper.parseAttachmentIds(doc);
const attachments = await Attachment.findAll({
where: {
id: attachmentIds,
teamId,
},
});
const mapping: Record<string, string> = {};
await Promise.all(
attachments.map(async (attachment) => {
const signedUrl = await FileStorage.getSignedUrl(
attachment.key,
expiresIn
);
mapping[attachment.redirectUrl] = signedUrl;
})
);
const json = doc.toJSON() as ProsemirrorData;
function replaceAttachmentUrls(node: ProsemirrorData) {
if (node.attrs?.src) {
node.attrs.src = mapping[node.attrs.src as string] || node.attrs.src;
} else if (node.attrs?.href) {
node.attrs.href = mapping[node.attrs.href as string] || node.attrs.href;
} else if (node.marks) {
node.marks.forEach((mark) => {
if (mark.attrs?.href) {
mark.attrs.href =
mapping[mark.attrs.href as string] || mark.attrs.href;
}
});
}
if (node.content) {
node.content.forEach(replaceAttachmentUrls);
}
return node;
}
return replaceAttachmentUrls(json);
}
/**
* Returns an array of attachment IDs in the node.
*
* @param node The node to parse attachments from
* @returns An array of attachment IDs
*/
static parseAttachmentIds(doc: Node) {
const urls: string[] = [];
doc.descendants((node) => {
node.marks.forEach((mark) => {
if (mark.type.name === "link") {
urls.push(mark.attrs.href);
}
});
if (["image", "video"].includes(node.type.name)) {
urls.push(node.attrs.src);
}
if (node.type.name === "attachment") {
urls.push(node.attrs.href);
}
});
return uniq(
compact(
flatten(
urls.map((url) =>
[...url.matchAll(attachmentRedirectRegex)].map(
(match) => match.groups?.id
)
)
)
)
);
}
/**
* Returns the node as HTML. This is a lossy conversion and should only be used
* for export.

2
server/models/helpers/SearchHelper.ts
View File

@@ -12,7 +12,7 @@ import Share from "@server/models/Share";
import Team from "@server/models/Team";
import User from "@server/models/User";
import { sequelize } from "@server/storage/database";
import DocumentHelper from "./DocumentHelper";
import { DocumentHelper } from "./DocumentHelper";
type SearchResponse = {
results: {

2
server/models/helpers/TextHelper.test.ts
View File

@@ -1,5 +1,5 @@
import { buildUser } from "@server/test/factories";
import TextHelper from "./TextHelper";
import { TextHelper } from "./TextHelper";
describe("TextHelper", () => {
beforeAll(() => {

2
server/models/helpers/TextHelper.ts
View File

@@ -16,7 +16,7 @@ import parseAttachmentIds from "@server/utils/parseAttachmentIds";
import parseImages from "@server/utils/parseImages";
@trace()
export default class TextHelper {
export class TextHelper {
/**
* Replaces template variables in the given text with the current date and time.
*

2
server/models/validators/TextLength.ts
View File

@@ -2,7 +2,7 @@ import size from "lodash/size";
import { Node } from "prosemirror-model";
import { addAttributeOptions } from "sequelize-typescript";
import { ProsemirrorData } from "@shared/types";
import ProsemirrorHelper from "@shared/utils/ProsemirrorHelper";
import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { schema } from "@server/editor";
/**