Merge branch 'main' of github.com:outline/outline

server/commands/documentUpdater.ts

@@ -9,6 +9,8 @@ type Props = {
   document: Document;
   /** The new title */
   title?: string;
+  /** The document emoji */
+  emoji?: string | null;
   /** The new text content */
   text?: string;
   /** Whether the editing session is complete */
@@ -44,6 +46,7 @@ export default async function documentUpdater({
   user,
   document,
   title,
+  emoji,
   text,
   editorVersion,
   templateId,
@@ -62,6 +65,9 @@ export default async function documentUpdater({
   if (title !== undefined) {
     document.title = title.trim();
   }
+  if (emoji !== undefined) {
+    document.emoji = emoji;
+  }
   if (editorVersion) {
     document.editorVersion = editorVersion;
   }

server/migrations/20230815063830-add-emoji-to-revisions.js

@@ -0,0 +1,14 @@
+'use strict';
+
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    await queryInterface.addColumn("revisions", "emoji", {
+      type: Sequelize.STRING,
+      allowNull: true,
+    });
+  },
+
+  async down (queryInterface) {
+    await queryInterface.removeColumn("revisions", "emoji");
+  }
+};

server/migrations/20230815063834-migrate-emoji-in-document-title.js

@@ -0,0 +1,28 @@
+"use strict";
+
+const { execSync } = require("child_process");
+const path = require("path");
+
+module.exports = {
+  async up() {
+    if (
+      process.env.NODE_ENV === "test" ||
+      process.env.DEPLOYMENT === "hosted"
+    ) {
+      return;
+    }
+
+    const scriptName = path.basename(__filename);
+    const scriptPath = path.join(
+      process.cwd(),
+      "build",
+      `server/scripts/${scriptName}`
+    );
+
+    execSync(`node ${scriptPath}`, { stdio: "inherit" });
+  },
+
+  async down() {
+    // noop
+  },
+};

server/migrations/20230827234031-migrate-emoji-in-revision-title.js

@@ -0,0 +1,28 @@
+"use strict";
+
+const { execSync } = require("child_process");
+const path = require("path");
+
+module.exports = {
+  async up() {
+    if (
+      process.env.NODE_ENV === "test" ||
+      process.env.DEPLOYMENT === "hosted"
+    ) {
+      return;
+    }
+
+    const scriptName = path.basename(__filename);
+    const scriptPath = path.join(
+      process.cwd(),
+      "build",
+      `server/scripts/${scriptName}`
+    );
+
+    execSync(`node ${scriptPath}`, { stdio: "inherit" });
+  },
+
+  async down() {
+    // noop
+  },
+};

server/models/Document.ts

@@ -1,4 +1,5 @@
 import compact from "lodash/compact";
+import isNil from "lodash/isNil";
 import uniq from "lodash/uniq";
 import randomstring from "randomstring";
 import type { SaveOptions } from "sequelize";
@@ -33,7 +34,6 @@ import {
 import isUUID from "validator/lib/isUUID";
 import type { NavigationNode } from "@shared/types";
 import getTasks from "@shared/utils/getTasks";
-import parseTitle from "@shared/utils/parseTitle";
 import { SLUG_URL_REGEX } from "@shared/utils/urlHelpers";
 import { DocumentValidation } from "@shared/validations";
 import slugify from "@server/utils/slugify";
@@ -261,7 +261,7 @@ class Document extends ParanoidModel {
   // hooks
 
   @BeforeSave
-  static async updateTitleInCollectionStructure(
+  static async updateCollectionStructure(
     model: Document,
     { transaction }: SaveOptions<Document>
   ) {
@@ -271,7 +271,7 @@ class Document extends ParanoidModel {
       model.archivedAt ||
       model.template ||
       !model.publishedAt ||
-      !model.changed("title") ||
+      !(model.changed("title") || model.changed("emoji")) ||
       !model.collectionId
     ) {
       return;
@@ -330,10 +330,6 @@ class Document extends ParanoidModel {
 
   @BeforeUpdate
   static processUpdate(model: Document) {
-    const { emoji } = parseTitle(model.title);
-    // emoji in the title is split out for easier display
-    model.emoji = emoji || null;
-
     // ensure documents have a title
     model.title = model.title || "";
 
@@ -795,6 +791,7 @@ class Document extends ParanoidModel {
       id: this.id,
       title: this.title,
       url: this.url,
+      emoji: isNil(this.emoji) ? undefined : this.emoji,
       children,
     };
   };

server/models/Revision.ts

@@ -49,6 +49,13 @@ class Revision extends IdModel {
   @Column(DataType.TEXT)
   text: string;
 
+  @Length({
+    max: 1,
+    msg: `Emoji must be a single character`,
+  })
+  @Column
+  emoji: string | null;
+
   // associations
 
   @BelongsTo(() => Document, "documentId")
@@ -65,6 +72,14 @@ class Revision extends IdModel {
   @Column(DataType.UUID)
   userId: string;
 
+  // static methods
+
+  /**
+   * Find the latest revision for a given document
+   *
+   * @param documentId The document id to find the latest revision for
+   * @returns A Promise that resolves to a Revision model
+   */
   static findLatest(documentId: string) {
     return this.findOne({
       where: {
@@ -74,10 +89,17 @@ class Revision extends IdModel {
     });
   }
 
+  /**
+   * Build a Revision model from a Document model
+   *
+   * @param document The document to build from
+   * @returns A Revision model
+   */
   static buildFromDocument(document: Document) {
     return this.build({
       title: document.title,
       text: document.text,
+      emoji: document.emoji,
       userId: document.lastModifiedById,
       editorVersion: document.editorVersion,
       version: document.version,
@@ -88,6 +110,13 @@ class Revision extends IdModel {
     });
   }
 
+  /**
+   * Create a Revision model from a Document model and save it to the database
+   *
+   * @param document The document to create from
+   * @param options Options passed to the save method
+   * @returns A Promise that resolves when saved
+   */
   static createFromDocument(
     document: Document,
     options?: SaveOptions<Revision>

server/presenters/document.ts

@@ -27,6 +27,7 @@ async function presentDocument(
     url: document.url,
     urlId: document.urlId,
     title: document.title,
+    emoji: document.emoji,
     text,
     tasks: document.tasks,
     createdAt: document.createdAt,

server/presenters/revision.ts

@@ -1,13 +1,18 @@
+import parseTitle from "@shared/utils/parseTitle";
 import { traceFunction } from "@server/logging/tracing";
 import { Revision } from "@server/models";
 import presentUser from "./user";
 
 async function presentRevision(revision: Revision, diff?: string) {
+  // TODO: Remove this fallback once all revisions have been migrated
+  const { emoji, strippedTitle } = parseTitle(revision.title);
+
   return {
     id: revision.id,
     documentId: revision.documentId,
-    title: revision.title,
+    title: strippedTitle,
     text: revision.text,
+    emoji: revision.emoji ?? emoji,
     html: diff,
     createdAt: revision.createdAt,
     createdBy: presentUser(revision.user),

server/routes/api/documents/documents.test.ts

@@ -2492,6 +2492,7 @@ describe("#documents.update", () => {
     const document = await buildDraftDocument({
       teamId: team.id,
     });
+
     const res = await server.post("/api/documents.update", {
       body: {
         token: user.getJwtToken(),
@@ -2503,6 +2504,7 @@ describe("#documents.update", () => {
     });
     const body = await res.json();
     expect(res.status).toEqual(400);
+
     expect(body.message).toBe(
       "collectionId is required to publish a draft without collection"
     );
@@ -2515,7 +2517,6 @@ describe("#documents.update", () => {
       text: "text",
       teamId: team.id,
     });
-
     const res = await server.post("/api/documents.update", {
       body: {
         token: user.getJwtToken(),
@@ -2551,6 +2552,36 @@ describe("#documents.update", () => {
     expect(res.status).toEqual(403);
   });
 
+  it("should fail to update an invalid emoji value", async () => {
+    const { user, document } = await seed();
+
+    const res = await server.post("/api/documents.update", {
+      body: {
+        token: user.getJwtToken(),
+        id: document.id,
+        emoji: ":)",
+      },
+    });
+    const body = await res.json();
+    expect(res.status).toEqual(400);
+
+    expect(body.message).toBe("emoji: Invalid");
+  });
+
+  it("should successfully update the emoji", async () => {
+    const { user, document } = await seed();
+    const res = await server.post("/api/documents.update", {
+      body: {
+        token: user.getJwtToken(),
+        id: document.id,
+        emoji: "😂",
+      },
+    });
+    const body = await res.json();
+    expect(res.status).toEqual(200);
+    expect(body.data.emoji).toBe("😂");
+  });
+
   it("should not add template to collection structure when publishing", async () => {
     const user = await buildUser();
     const collection = await buildCollection({

server/routes/api/documents/documents.ts

@@ -861,6 +861,7 @@ router.post(
       lastModifiedById: user.id,
       createdById: user.id,
       template: true,
+      emoji: original.emoji,
       title: original.title,
       text: original.text,
     });

server/routes/api/documents/schema.ts

@@ -1,3 +1,4 @@
+import emojiRegex from "emoji-regex";
 import isEmpty from "lodash/isEmpty";
 import isUUID from "validator/lib/isUUID";
 import { z } from "zod";
@@ -186,6 +187,9 @@ export const DocumentsUpdateSchema = BaseSchema.extend({
     /** Doc text to be updated */
     text: z.string().optional(),
 
+    /** Emoji displayed alongside doc title */
+    emoji: z.string().regex(emojiRegex()).nullish(),
+
     /** Boolean to denote if the doc should occupy full width */
     fullWidth: z.boolean().optional(),
 

server/scripts/20230815063834-migrate-emoji-in-document-title.test.ts

@@ -0,0 +1,120 @@
+import { Document } from "@server/models";
+import { buildDocument, buildDraftDocument } from "@server/test/factories";
+import { setupTestDatabase } from "@server/test/support";
+import script from "./20230815063834-migrate-emoji-in-document-title";
+
+setupTestDatabase();
+
+describe("#work", () => {
+  it("should correctly update title and emoji for a draft document", async () => {
+    const document = await buildDraftDocument({
+      title: "😵 Title draft",
+    });
+    expect(document.publishedAt).toBeNull();
+    expect(document.emoji).toBeNull();
+
+    await script();
+    const draft = await Document.unscoped().findByPk(document.id);
+    expect(draft).not.toBeNull();
+    expect(draft?.title).toEqual("Title draft");
+    expect(draft?.emoji).toEqual("😵");
+  });
+
+  it("should correctly update title and emoji for a published document", async () => {
+    const document = await buildDocument({
+      title: "👱🏽‍♀️ Title published",
+    });
+    expect(document.publishedAt).toBeTruthy();
+    expect(document.emoji).toBeNull();
+
+    await script();
+    const published = await Document.unscoped().findByPk(document.id);
+    expect(published).not.toBeNull();
+    expect(published?.title).toEqual("Title published");
+    expect(published?.emoji).toEqual("👱🏽‍♀️");
+  });
+
+  it("should correctly update title and emoji for an archived document", async () => {
+    const document = await buildDocument({
+      title: "🍇 Title archived",
+    });
+    await document.archive(document.createdById);
+    expect(document.archivedAt).toBeTruthy();
+    expect(document.emoji).toBeNull();
+
+    await script();
+    const archived = await Document.unscoped().findByPk(document.id);
+    expect(archived).not.toBeNull();
+    expect(archived?.title).toEqual("Title archived");
+    expect(archived?.emoji).toEqual("🍇");
+  });
+
+  it("should correctly update title and emoji for a template", async () => {
+    const document = await buildDocument({
+      title: "🐹 Title template",
+      template: true,
+    });
+    expect(document.template).toBe(true);
+    expect(document.emoji).toBeNull();
+
+    await script();
+    const template = await Document.unscoped().findByPk(document.id);
+    expect(template).not.toBeNull();
+    expect(template?.title).toEqual("Title template");
+    expect(template?.emoji).toEqual("🐹");
+  });
+
+  it("should correctly update title and emoji for a deleted document", async () => {
+    const document = await buildDocument({
+      title: "🚵🏼‍♂️ Title deleted",
+    });
+    await document.destroy();
+    expect(document.deletedAt).toBeTruthy();
+    expect(document.emoji).toBeNull();
+
+    await script();
+    const deleted = await Document.unscoped().findByPk(document.id, {
+      paranoid: false,
+    });
+    expect(deleted).not.toBeNull();
+    expect(deleted?.title).toEqual("Title deleted");
+    expect(deleted?.emoji).toEqual("🚵🏼‍♂️");
+  });
+
+  it("should correctly update title emoji when there are leading spaces", async () => {
+    const document = await buildDocument({
+      title: "    🤨    Title with spaces",
+    });
+    expect(document.emoji).toBeNull();
+
+    await script();
+
+    const doc = await Document.unscoped().findByPk(document.id);
+    expect(doc).not.toBeNull();
+    expect(doc?.title).toEqual("Title with spaces");
+    expect(doc?.emoji).toEqual("🤨");
+  });
+
+  it("should correctly paginate and update title emojis", async () => {
+    const buildManyDocuments = [];
+    for (let i = 1; i <= 10; i++) {
+      buildManyDocuments.push(buildDocument({ title: "🚵🏼‍♂️ Title" }));
+    }
+
+    const manyDocuments = await Promise.all(buildManyDocuments);
+
+    for (const document of manyDocuments) {
+      expect(document.title).toEqual("🚵🏼‍♂️ Title");
+      expect(document.emoji).toBeNull();
+    }
+
+    await script(false, 2);
+
+    const documents = await Document.unscoped().findAll();
+
+    for (const document of documents) {
+      expect(document.title).toEqual("Title");
+      expect(document.emoji).toEqual("🚵🏼‍♂️");
+    }
+  });
+});

server/scripts/20230815063834-migrate-emoji-in-document-title.ts

@@ -0,0 +1,69 @@
+import "./bootstrap";
+import { Transaction, Op } from "sequelize";
+import parseTitle from "@shared/utils/parseTitle";
+import { Document } from "@server/models";
+import { sequelize } from "@server/storage/database";
+
+let page = parseInt(process.argv[2], 10);
+page = Number.isNaN(page) ? 0 : page;
+
+export default async function main(exit = false, limit = 1000) {
+  const work = async (page: number): Promise<void> => {
+    console.log(`Backfill document emoji from title… page ${page}`);
+    let documents: Document[] = [];
+    await sequelize.transaction(async (transaction) => {
+      documents = await Document.unscoped().findAll({
+        attributes: {
+          exclude: ["state"],
+        },
+        where: {
+          version: {
+            [Op.ne]: null,
+          },
+        },
+        limit,
+        offset: page * limit,
+        order: [["createdAt", "ASC"]],
+        paranoid: false,
+        lock: Transaction.LOCK.UPDATE,
+        transaction,
+      });
+
+      for (const document of documents) {
+        try {
+          const { emoji, strippedTitle } = parseTitle(document.title);
+          if (emoji) {
+            document.emoji = emoji;
+            document.title = strippedTitle;
+
+            if (document.changed()) {
+              console.log(`Migrating ${document.id}…`);
+
+              await document.save({
+                silent: true,
+                transaction,
+              });
+            }
+          }
+        } catch (err) {
+          console.error(`Failed at ${document.id}:`, err);
+          continue;
+        }
+      }
+    });
+    return documents.length === limit ? work(page + 1) : undefined;
+  };
+
+  await work(page);
+
+  console.log("Backfill complete");
+
+  if (exit) {
+    process.exit(0);
+  }
+}
+
+// In the test suite we import the script rather than run via node CLI
+if (process.env.NODE_ENV !== "test") {
+  void main(true);
+}

server/scripts/20230827234031-migrate-emoji-in-revision-title.ts

@@ -0,0 +1,64 @@
+import "./bootstrap";
+import { Transaction } from "sequelize";
+import parseTitle from "@shared/utils/parseTitle";
+import { Revision } from "@server/models";
+import { sequelize } from "@server/storage/database";
+
+let page = parseInt(process.argv[2], 10);
+page = Number.isNaN(page) ? 0 : page;
+
+export default async function main(exit = false, limit = 1000) {
+  const work = async (page: number): Promise<void> => {
+    console.log(`Backfill revision emoji from title… page ${page}`);
+    let revisions: Revision[] = [];
+    await sequelize.transaction(async (transaction) => {
+      revisions = await Revision.unscoped().findAll({
+        attributes: {
+          exclude: ["text"],
+        },
+        limit,
+        offset: page * limit,
+        order: [["createdAt", "ASC"]],
+        paranoid: false,
+        lock: Transaction.LOCK.UPDATE,
+        transaction,
+      });
+
+      for (const revision of revisions) {
+        try {
+          const { emoji, strippedTitle } = parseTitle(revision.title);
+          if (emoji) {
+            revision.emoji = emoji;
+            revision.title = strippedTitle;
+
+            if (revision.changed()) {
+              console.log(`Migrating ${revision.id}…`);
+
+              await revision.save({
+                silent: true,
+                transaction,
+              });
+            }
+          }
+        } catch (err) {
+          console.error(`Failed at ${revision.id}:`, err);
+          continue;
+        }
+      }
+    });
+    return revisions.length === limit ? work(page + 1) : undefined;
+  };
+
+  await work(page);
+
+  console.log("Backfill complete");
+
+  if (exit) {
+    process.exit(0);
+  }
+}
+
+// In the test suite we import the script rather than run via node CLI
+if (process.env.NODE_ENV !== "test") {
+  void main(true);
+}

server/test/factories.ts

@@ -377,6 +377,7 @@ export async function buildDocument(
       publishedAt: isNull(overrides.collectionId) ? null : new Date(),
       lastModifiedById: overrides.userId,
       createdById: overrides.userId,
+      editorVersion: 2,
       ...overrides,
     },
     {