chore: Update documentImporter with changes from enterprise, improved Confluence compat

server/utils/turndown/breaks.ts

@@ -0,0 +1,15 @@
+import TurndownService from "turndown";
+
+/**
+ * A turndown plugin for converting break tags to newlines.
+ *
+ * @param turndownService The TurndownService instance.
+ */
+export default function breaks(turndownService: TurndownService) {
+  turndownService.addRule("breaks", {
+    filter: ["br"],
+    replacement: function () {
+      return "\n";
+    },
+  });
+}

server/utils/turndown/confluence-code-block.ts

@@ -0,0 +1,57 @@
+import { repeat } from "lodash";
+import TurndownService from "turndown";
+
+const highlightRegExp = /brush: ([a-z0-9]+);/;
+
+/**
+ * A turndown plugin for converting a confluence code block to markdown.
+ *
+ * @param turndownService The TurndownService instance.
+ */
+export default function confluenceCodeBlock(turndownService: TurndownService) {
+  turndownService.addRule("fencedConfluenceHighlightedCodeBlock", {
+    filter: function (node) {
+      const firstChild = node.firstChild;
+      return (
+        node.nodeName === "DIV" &&
+        firstChild?.nodeName === "PRE" &&
+        // @ts-expect-error className exists
+        firstChild.className === "syntaxhighlighter-pre"
+      );
+    },
+    replacement: function (content, node) {
+      const dataSyntaxhighlighterParams =
+        // @ts-expect-error getAttribute exists
+        node.firstChild?.getAttribute("data-syntaxhighlighter-params") ?? "";
+      const language = (dataSyntaxhighlighterParams.match(highlightRegExp) || [
+        null,
+        "",
+      ])[1];
+      const code = node.firstChild?.textContent ?? "";
+
+      const fenceChar = "`";
+      let fenceSize = 3;
+      const fenceInCodeRegex = new RegExp("^" + fenceChar + "{3,}", "gm");
+
+      let match;
+      while ((match = fenceInCodeRegex.exec(code))) {
+        if (match[0].length >= fenceSize) {
+          fenceSize = match[0].length + 1;
+        }
+      }
+
+      const fence = repeat(fenceChar, fenceSize);
+
+      return (
+        "\n\n" +
+        fence +
+        language +
+        "\n" +
+        code.replace(/\n$/, "") +
+        "\n" +
+        fence +
+        "\n\n"
+      );
+    },
+  });
+}

server/utils/turndown/confluence-task-list.ts

@@ -0,0 +1,25 @@
+import TurndownService from "turndown";
+
+/**
+ * A turndown plugin for converting a confluence task list to markdown.
+ *
+ * @param turndownService The TurndownService instance.
+ */
+export default function confluenceTaskList(turndownService: TurndownService) {
+  turndownService.addRule("confluenceTaskList", {
+    filter: function (node) {
+      return (
+        node.nodeName === "LI" &&
+        node.parentNode?.nodeName === "UL" &&
+        // @ts-expect-error className exists
+        node.parentNode?.className.includes("inline-task-list")
+      );
+    },
+    replacement: function (content, node) {
+      return (
+        // @ts-expect-error className exists
+        (node.className === "checked" ? "- [x]" : "- [ ]") + ` ${content} \n`
+      );
+    },
+  });
+}

server/utils/turndown/index.ts

@@ -0,0 +1,30 @@
+import { gfm } from "@joplin/turndown-plugin-gfm";
+import TurndownService from "turndown";
+import breaks from "./breaks";
+import confluenceCodeBlock from "./confluence-code-block";
+import confluenceTaskList from "./confluence-task-list";
+
+/**
+ * Turndown converts HTML to Markdown and is used in the importer code.
+ *
+ * For options, see: https://github.com/domchristie/turndown#options
+ */
+const service = new TurndownService({
+  hr: "---",
+  bulletListMarker: "-",
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  blankReplacement: (content, node) => {
+    if (node.nodeName === "P") {
+      return "\n\n\\\n";
+    }
+    return "";
+  },
+})
+  .remove(["script", "style", "title", "head"])
+  .use(gfm)
+  .use(confluenceTaskList)
+  .use(confluenceCodeBlock)
+  .use(breaks);
+
+export default service;