Adds content column to documents and revisions as JSON snapshot (#6179)

server/scripts/20221008000000-backfill-crdt.ts

@@ -27,7 +27,6 @@ export default async function main(exit = false) {
       offset: page * limit,
       where: {
         ...(teamId ? { teamId } : {}),
-        state: null,
       },
       order: [["createdAt", "ASC"]],
       paranoid: false,

server/scripts/20231119000000-backfill-document-content.ts

@@ -0,0 +1,61 @@
+import "./bootstrap";
+import { yDocToProsemirrorJSON } from "@getoutline/y-prosemirror";
+import { Node } from "prosemirror-model";
+import * as Y from "yjs";
+import { parser, schema } from "@server/editor";
+import { Document } from "@server/models";
+
+const limit = 100;
+const page = 0;
+
+export default async function main(exit = false) {
+  const work = async (page: number): Promise<void> => {
+    console.log(`Backfill content… page ${page}`);
+
+    // Retrieve all documents within set limit.
+    const documents = await Document.unscoped().findAll({
+      attributes: ["id", "urlId", "content", "text", "state"],
+      limit,
+      offset: page * limit,
+      order: [["createdAt", "ASC"]],
+      paranoid: false,
+    });
+
+    for (const document of documents) {
+      if (document.content || !document.text) {
+        continue;
+      }
+
+      console.log(`Writing content for ${document.id}`);
+
+      if ("state" in document && document.state) {
+        const ydoc = new Y.Doc();
+        Y.applyUpdate(ydoc, document.state);
+        document.content = yDocToProsemirrorJSON(ydoc, "default");
+      } else {
+        const node = parser.parse(document.text) || Node.fromJSON(schema, {});
+        document.content = node.toJSON();
+      }
+
+      document.changed("content", true);
+
+      await document.save({
+        hooks: false,
+        silent: true,
+      });
+    }
+
+    return documents.length === limit ? work(page + 1) : undefined;
+  };
+
+  await work(page);
+
+  if (exit) {
+    console.log("Backfill complete");
+    process.exit(0);
+  }
+}
+
+if (process.env.NODE_ENV !== "test") {
+  void main(true);
+}

server/scripts/20231119000000-backfill-revision-content.ts

@@ -0,0 +1,52 @@
+import "./bootstrap";
+import { Node } from "prosemirror-model";
+import { parser, schema } from "@server/editor";
+import { Revision } from "@server/models";
+
+const limit = 100;
+const page = 0;
+
+export default async function main(exit = false) {
+  const work = async (page: number): Promise<void> => {
+    console.log(`Backfill content… page ${page}`);
+
+    // Retrieve all revisions within set limit.
+    const revisions = await Revision.unscoped().findAll({
+      attributes: ["id", "content", "text"],
+      limit,
+      offset: page * limit,
+      order: [["createdAt", "ASC"]],
+      paranoid: false,
+    });
+
+    for (const revision of revisions) {
+      if (revision.content || !revision.text) {
+        continue;
+      }
+
+      console.log(`Writing content for ${revision.id}`);
+
+      const node = parser.parse(revision.text) || Node.fromJSON(schema, {});
+      revision.content = node.toJSON();
+      revision.changed("content", true);
+
+      await revision.save({
+        hooks: false,
+        silent: true,
+      });
+    }
+
+    return revisions.length === limit ? work(page + 1) : undefined;
+  };
+
+  await work(page);
+
+  if (exit) {
+    console.log("Backfill complete");
+    process.exit(0);
+  }
+}
+
+if (process.env.NODE_ENV !== "test") {
+  void main(true);
+}