fix: Improve logic for word import (#6361)

* Refactor DocumentConverter * Support parsing images from Confluence exported .doc files * fix: Bring across 2 fixes from enterprise codebase * Bust dependency cache
2024-01-09 20:29:47 -08:00
parent a032f2e7e5
commit 7d7d0fd9ca
6 changed files with 297 additions and 227 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -36,12 +36,12 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: install-deps
          command: yarn install --frozen-lockfile
      - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
          paths:
            - ./node_modules
  lint:
@@ -49,7 +49,7 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: lint
          command: yarn lint
@@ -58,7 +58,7 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: typescript
          command: yarn tsc
@@ -67,7 +67,7 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: test
          command: yarn test:app
@@ -76,7 +76,7 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: test
          command: yarn test:shared
@@ -86,7 +86,7 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: migrate
          command: ./node_modules/.bin/sequelize db:migrate --url $DATABASE_URL_TEST
@@ -102,7 +102,7 @@ jobs:
    steps:
      - checkout
      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-v1-{{ checksum "package.json" }}
      - run:
          name: build-vite
          command: yarn vite:build
--- a/app/stores/CollectionsStore.ts
+++ b/app/stores/CollectionsStore.ts
@@ -230,11 +230,11 @@ export default class CollectionsStore extends Store<Collection> {
    return find(this.orderedData, (col: Collection) => url.endsWith(col.urlId));
  }

-  delete = async (collection: Collection) => {
+  async delete(collection: Collection) {
    await super.delete(collection);
    await this.rootStore.documents.fetchRecentlyUpdated();
    await this.rootStore.documents.fetchRecentlyViewed();
-  };
+  }

  export = (format: FileOperationFormat, includeAttachments: boolean) =>
    client.post("/collections.export_all", {
--- a/package.json
+++ b/package.json
@@ -76,6 +76,7 @@
    "@tippyjs/react": "^4.2.6",
    "@tommoor/remove-markdown": "^0.3.2",
    "@types/form-data": "^2.5.0",
+    "@types/mailparser": "^3.4.4",
    "@types/sanitize-filename": "^1.6.3",
    "@vitejs/plugin-react": "^3.1.0",
    "addressparser": "^1.0.1",
@@ -136,6 +137,7 @@
    "koa-sslify": "5.0.1",
    "koa-useragent": "^4.1.0",
    "lodash": "^4.17.21",
+    "mailparser": "^3.6.6",
    "mammoth": "^1.6.0",
    "markdown-it": "^13.0.1",
    "markdown-it-container": "^3.0.0",
@@ -175,7 +177,6 @@
    "prosemirror-transform": "^1.8.0",
    "prosemirror-view": "^1.32.0",
    "query-string": "^7.1.3",
-    "quoted-printable": "^1.0.1",
    "randomstring": "1.2.3",
    "rate-limiter-flexible": "^2.4.2",
    "react": "^17.0.2",
@@ -224,7 +225,6 @@
    "tmp": "^0.2.1",
    "turndown": "^7.1.2",
    "umzug": "^3.2.1",
-    "utf8": "^3.0.0",
    "utility-types": "^3.10.0",
    "uuid": "^8.3.2",
    "validator": "13.11.0",
--- a/server/commands/documentImporter.ts
+++ b/server/commands/documentImporter.ts
@@ -1,136 +1,15 @@
-import path from "path";
 import emojiRegex from "emoji-regex";
 import escapeRegExp from "lodash/escapeRegExp";
 import truncate from "lodash/truncate";
-import mammoth from "mammoth";
-import quotedPrintable from "quoted-printable";
 import { Transaction } from "sequelize";
-import utf8 from "utf8";
 import parseTitle from "@shared/utils/parseTitle";
 import { DocumentValidation } from "@shared/validations";
 import { traceFunction } from "@server/logging/tracing";
 import { User } from "@server/models";
 import ProsemirrorHelper from "@server/models/helpers/ProsemirrorHelper";
 import TextHelper from "@server/models/helpers/TextHelper";
-import turndownService from "@server/utils/turndown";
-import { FileImportError, InvalidRequestError } from "../errors";
-
-interface ImportableFile {
-  type: string;
-  getMarkdown: (content: Buffer | string) => Promise<string>;
-}
-
-const importMapping: ImportableFile[] = [
-  {
-    type: "application/msword",
-    getMarkdown: confluenceToMarkdown,
-  },
-  {
-    type: "application/octet-stream",
-    getMarkdown: docxToMarkdown,
-  },
-  {
-    type: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-    getMarkdown: docxToMarkdown,
-  },
-  {
-    type: "text/html",
-    getMarkdown: htmlToMarkdown,
-  },
-  {
-    type: "text/plain",
-    getMarkdown: fileToMarkdown,
-  },
-  {
-    type: "text/markdown",
-    getMarkdown: fileToMarkdown,
-  },
-];
-
-async function fileToMarkdown(content: Buffer | string): Promise<string> {
-  if (content instanceof Buffer) {
-    content = content.toString("utf8");
-  }
-  return content;
-}
-
-async function docxToMarkdown(content: Buffer | string): Promise<string> {
-  if (content instanceof Buffer) {
-    const { value: html } = await mammoth.convertToHtml({
-      buffer: content,
-    });
-
-    return turndownService.turndown(html);
-  }
-
-  throw new Error("docxToMarkdown: content must be a Buffer");
-}
-
-async function htmlToMarkdown(content: Buffer | string): Promise<string> {
-  if (content instanceof Buffer) {
-    content = content.toString("utf8");
-  }
-
-  return turndownService.turndown(content);
-}
-
-async function confluenceToMarkdown(value: Buffer | string): Promise<string> {
-  if (value instanceof Buffer) {
-    value = value.toString("utf8");
-  }
-
-  // We're only supporting the ridiculous output from Confluence here, regular
-  // Word documents should call into the docxToMarkdown importer.
-  // See: https://jira.atlassian.com/browse/CONFSERVER-38237
-  if (!value.includes("Content-Type: multipart/related")) {
-    throw FileImportError("Unsupported Word file");
-  }
-
-  // get boundary marker
-  const boundaryMarker = value.match(/boundary="(.+)"/);
-
-  if (!boundaryMarker) {
-    throw FileImportError("Unsupported Word file (No boundary marker)");
-  }
-
-  // get content between multipart boundaries
-  let boundaryReached = 0;
-  const lines = value.split("\n").filter((line) => {
-    if (line.includes(boundaryMarker[1])) {
-      boundaryReached++;
-      return false;
-    }
-
-    if (line.startsWith("Content-")) {
-      return false;
-    }
-
-    // 1 == definition
-    // 2 == content
-    // 3 == ending
-    if (boundaryReached === 2) {
-      return true;
-    }
-
-    return false;
-  });
-
-  if (!lines.length) {
-    throw FileImportError("Unsupported Word file (No content found)");
-  }
-
-  // Mime attachment is "quoted printable" encoded, must be decoded first
-  // https://en.wikipedia.org/wiki/Quoted-printable
-  value = utf8.decode(quotedPrintable.decode(lines.join("\n")));
-
-  // If we don't remove the title here it becomes printed in the document
-  // body by turndown
-  turndownService.remove(["style", "title"]);
-
-  // Now we should have something that looks like HTML
-  const html = turndownService.turndown(value);
-  return html.replace(/<br>/g, " \\n ");
-}
+import { DocumentConverter } from "@server/utils/DocumentConverter";
+import { InvalidRequestError } from "../errors";

 type Props = {
  user: User;
@@ -154,31 +33,12 @@ async function documentImporter({
  title: string;
  state: Buffer;
 }> {
-  const fileInfo = importMapping.filter((item) => {
-    if (item.type === mimeType) {
-      if (
-        mimeType === "application/octet-stream" &&
-        path.extname(fileName) !== ".docx"
-      ) {
-        return false;
-      }
-
-      return true;
-    }
-
-    if (item.type === "text/markdown" && path.extname(fileName) === ".md") {
-      return true;
-    }
-
-    return false;
-  })[0];
-
-  if (!fileInfo) {
-    throw InvalidRequestError(`File type ${mimeType} not supported`);
-  }
-
+  let text = await DocumentConverter.convertToMarkdown(
+    content,
+    fileName,
+    mimeType
+  );
  let title = fileName.replace(/\.[^/.]+$/, "");
-  let text = await fileInfo.getMarkdown(content);

  // find and extract emoji near the beginning of the document.
  const regex = emojiRegex();
@@ -203,6 +63,13 @@ async function documentImporter({
  // to match our hardbreak parser.
  text = text.trim().replace(/<br>/gi, "\\n");

+  // Escape any dollar signs in the text to prevent them being interpreted as
+  // math blocks
+  text = text.replace(/\$/g, "\\$");
+
+  // Remove any closed and immediately reopened formatting marks
+  text = text.replace(/\*\*\*\*/gi, "").replace(/____/gi, "");
+
  text = await TextHelper.replaceImagesWithAttachments(
    text,
    user,
--- a/server/utils/DocumentConverter.ts
+++ b/server/utils/DocumentConverter.ts
@@ -0,0 +1,125 @@
+import escapeRegExp from "lodash/escapeRegExp";
+import { simpleParser } from "mailparser";
+import mammoth from "mammoth";
+import { FileImportError } from "@server/errors";
+import turndownService from "@server/utils/turndown";
+
+export class DocumentConverter {
+  /**
+   * Convert an incoming file to markdown.
+   * @param content The content of the file.
+   * @param fileName The name of the file, including extension.
+   * @param mimeType The mime type of the file.
+   * @returns The markdown representation of the file.
+   */
+  public static async convertToMarkdown(
+    content: Buffer | string,
+    fileName: string,
+    mimeType: string
+  ) {
+    // First try to convert the file based on the mime type.
+    switch (mimeType) {
+      case "application/msword":
+        return this.confluenceToMarkdown(content);
+      case "application/octet-stream":
+        if (fileName.endsWith(".docx")) {
+          return this.docXToMarkdown(content);
+        }
+        throw FileImportError(`File type ${mimeType} not supported`);
+      case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        return this.docXToMarkdown(content);
+      case "text/html":
+        return this.htmlToMarkdown(content);
+      case "text/plain":
+      case "text/markdown":
+        return this.fileToMarkdown(content);
+      default:
+        break;
+    }
+
+    // If the mime type doesn't work, try to convert based on the file extension.
+    const extension = fileName.split(".").pop();
+    switch (extension) {
+      case "docx":
+        return this.docXToMarkdown(content);
+      case "html":
+        return this.htmlToMarkdown(content);
+      case "md":
+      case "markdown":
+        return this.fileToMarkdown(content);
+      default:
+        throw FileImportError(`File type ${mimeType} not supported`);
+    }
+  }
+
+  public static async docXToMarkdown(content: Buffer | string) {
+    if (content instanceof Buffer) {
+      const { value } = await mammoth.convertToHtml({
+        buffer: content,
+      });
+
+      return turndownService.turndown(value);
+    }
+
+    throw FileImportError("Unsupported Word file");
+  }
+
+  public static async htmlToMarkdown(content: Buffer | string) {
+    if (content instanceof Buffer) {
+      content = content.toString("utf8");
+    }
+
+    return turndownService.turndown(content);
+  }
+
+  public static async fileToMarkdown(content: Buffer | string) {
+    if (content instanceof Buffer) {
+      content = content.toString("utf8");
+    }
+    return content;
+  }
+
+  public static async confluenceToMarkdown(value: Buffer | string) {
+    if (value instanceof Buffer) {
+      value = value.toString("utf8");
+    }
+
+    // We're only supporting the output from Confluence here, regular Word documents should call
+    // into the docxToMarkdown importer. See: https://jira.atlassian.com/browse/CONFSERVER-38237
+    if (!value.includes("Content-Type: multipart/related")) {
+      throw FileImportError("Unsupported Word file");
+    }
+
+    // Confluence "Word" documents are actually just multi-part email messages, so we can use
+    // mailparser to parse the content.
+    const parsed = await simpleParser(value);
+    if (!parsed.html) {
+      throw FileImportError("Unsupported Word file (No content found)");
+    }
+
+    // Replace the content-location with a data URI for each attachment.
+    for (const attachment of parsed.attachments) {
+      const contentLocation = String(
+        attachment.headers.get("content-location") ?? ""
+      );
+
+      const id = contentLocation.split("/").pop();
+      if (!id) {
+        continue;
+      }
+
+      parsed.html = parsed.html.replace(
+        new RegExp(escapeRegExp(id), "g"),
+        `data:image/png;base64,${attachment.content.toString("base64")}`
+      );
+    }
+
+    // If we don't remove the title here it becomes printed in the document
+    // body by turndown
+    turndownService.remove(["style", "title"]);
+
+    // Now we should have something that looks like HTML
+    const html = turndownService.turndown(parsed.html);
+    return html.replace(/<br>/g, " \\n ");
+  }
+}
--- a/yarn.lock
+++ b/yarn.lock
@@ -2563,6 +2563,14 @@
    colors "~1.2.1"
    string-argv "~0.3.1"

+"@selderee/plugin-htmlparser2@^0.11.0":
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz#d5b5e29a7ba6d3958a1972c7be16f4b2c188c517"
+  integrity sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==
+  dependencies:
+    domhandler "^5.0.3"
+    selderee "^0.11.0"
+
 "@sentry-internal/feedback@7.85.0":
  version "7.85.0"
  resolved "https://registry.yarnpkg.com/@sentry-internal/feedback/-/feedback-7.85.0.tgz#94ef44d59a01f145895525a9bd737dc68f4c7d64"
@@ -3157,6 +3165,14 @@
  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.172.tgz#aad774c28e7bfd7a67de25408e03ee5a8c3d028a"
  integrity sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==

+"@types/mailparser@^3.4.4":
+  version "3.4.4"
+  resolved "https://registry.yarnpkg.com/@types/mailparser/-/mailparser-3.4.4.tgz#0bd71e205573b9dd9a445e10a8b8cb0e45420998"
+  integrity sha512-C6Znp2QVS25JqtuPyxj38Qh+QoFcLycdxsvcc6IZCGekhaMBzbdTXzwGzhGoYb3TfKu8IRCNV0sV1o3Od97cEQ==
+  dependencies:
+    "@types/node" "*"
+    iconv-lite "^0.6.3"
+
 "@types/markdown-it-container@^2.0.9":
  version "2.0.9"
  resolved "https://registry.yarnpkg.com/@types/markdown-it-container/-/markdown-it-container-2.0.9.tgz#2bc616eefdc5969b10c35645cd93e71152b8cfd2"
@@ -4459,17 +4475,7 @@ browserslist-to-esbuild@^1.2.0:
  dependencies:
    browserslist "^4.17.3"

-browserslist@^4.17.3, browserslist@^4.21.4:
-  version "4.21.5"
-  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.5.tgz#75c5dae60063ee641f977e00edd3cfb2fb7af6a7"
-  integrity sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==
-  dependencies:
-    caniuse-lite "^1.0.30001449"
-    electron-to-chromium "^1.4.284"
-    node-releases "^2.0.8"
-    update-browserslist-db "^1.0.10"
-
-browserslist@^4.22.2:
+browserslist@^4.17.3, browserslist@^4.21.4, browserslist@^4.22.2:
  version "4.22.2"
  resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.2.tgz#704c4943072bd81ea18997f3bd2180e89c77874b"
  integrity sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A==
@@ -4594,11 +4600,6 @@ cancan@3.1.0:
    auto-bind "^1.1.0"
    is-plain-obj "^1.1.0"

-caniuse-lite@^1.0.30001449:
-  version "1.0.30001451"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001451.tgz#2e197c698fc1373d63e1406d6607ea4617c613f1"
-  integrity sha512-XY7UbUpGRatZzoRft//5xOa69/1iGJRBlrieH6QYrkKLIFn3m7OVEJ81dSrKoy2BnKsdbX5cLrOispZNYo9v2w==
-
 caniuse-lite@^1.0.30001565:
  version "1.0.30001576"
  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001576.tgz#893be772cf8ee6056d6c1e2d07df365b9ec0a5c4"
@@ -5664,10 +5665,10 @@ deep-is@^0.1.3, deep-is@~0.1.3:
  resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831"
  integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==

-deepmerge@^4.2.2:
-  version "4.2.2"
-  resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
-  integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==
+deepmerge@^4.2.2, deepmerge@^4.3.1:
+  version "4.3.1"
+  resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a"
+  integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==

 define-data-property@^1.0.1, define-data-property@^1.1.1:
  version "1.1.1"
@@ -5952,11 +5953,6 @@ ejs@^3.1.6, ejs@^3.1.7:
  dependencies:
    jake "^10.8.5"

-electron-to-chromium@^1.4.284:
-  version "1.4.295"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.295.tgz#911d5df67542bf7554336142eb302c5ec90bba66"
-  integrity sha512-lEO94zqf1bDA3aepxwnWoHUjA8sZ+2owgcSZjYQy0+uOSEclJX0VieZC+r+wLpSxUHRd6gG32znTWmr+5iGzFw==
-
 electron-to-chromium@^1.4.601:
  version "1.4.623"
  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.623.tgz#0f7400114ac3425500e9244d2b0e9c3107c331cb"
@@ -6012,6 +6008,11 @@ encodeurl@^1.0.2:
  resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
  integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=

+encoding-japanese@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/encoding-japanese/-/encoding-japanese-2.0.0.tgz#fa0226e5469e7b5b69a04fea7d5481bd1fa56936"
+  integrity sha512-++P0RhebUC8MJAwJOsT93dT+5oc5oPImp1HubZpAuCZ5kTLnhuuBhKHj2jJeO/Gj93idPBWmIuQ9QWMe5rX3pQ==
+
 end-of-stream@^1.0.0, end-of-stream@^1.1.0:
  version "1.4.4"
  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
@@ -6069,7 +6070,7 @@ ensure-posix-path@^1.1.0:
  resolved "https://registry.yarnpkg.com/ensure-posix-path/-/ensure-posix-path-1.1.1.tgz#3c62bdb19fa4681544289edb2b382adc029179ce"
  integrity sha512-VWU0/zXzVbeJNXvME/5EmLuEj2TauvoaTz6aFYK1Z92JCBlDlZ3Gu0tuGR42kpW1754ywTs+QB0g5TP0oj9Zaw==

-entities@^4.2.0, entities@^4.3.0, entities@^4.4.0:
+entities@^4.2.0, entities@^4.4.0:
  version "4.4.0"
  resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174"
  integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==
@@ -7436,7 +7437,7 @@ hastscript@^6.0.0:
    property-information "^5.0.0"
    space-separated-tokens "^1.0.0"

-he@^1.1.0:
+he@1.2.0, he@^1.1.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
  integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
@@ -7522,15 +7523,26 @@ html-parse-stringify@^3.0.1:
  dependencies:
    void-elements "3.1.0"

-htmlparser2@^8.0.1:
-  version "8.0.1"
-  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.1.tgz#abaa985474fcefe269bc761a779b544d7196d010"
-  integrity sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==
+html-to-text@9.0.5:
+  version "9.0.5"
+  resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-9.0.5.tgz#6149a0f618ae7a0db8085dca9bbf96d32bb8368d"
+  integrity sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==
+  dependencies:
+    "@selderee/plugin-htmlparser2" "^0.11.0"
+    deepmerge "^4.3.1"
+    dom-serializer "^2.0.0"
+    htmlparser2 "^8.0.2"
+    selderee "^0.11.0"
+
+htmlparser2@^8.0.1, htmlparser2@^8.0.2:
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21"
+  integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==
  dependencies:
    domelementtype "^2.3.0"
-    domhandler "^5.0.2"
+    domhandler "^5.0.3"
    domutils "^3.0.1"
-    entities "^4.3.0"
+    entities "^4.4.0"

 http-assert@^1.3.0:
  version "1.4.1"
@@ -7672,7 +7684,7 @@ iconv-lite@0.4.24:
  dependencies:
    safer-buffer ">= 2.1.2 < 3"

-iconv-lite@0.6, iconv-lite@0.6.3:
+iconv-lite@0.6, iconv-lite@0.6.3, iconv-lite@^0.6.3:
  version "0.6.3"
  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
  integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==
@@ -9173,6 +9185,11 @@ lazystream@^1.0.0:
  dependencies:
    readable-stream "^2.0.5"

+leac@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/leac/-/leac-0.6.0.tgz#dcf136e382e666bd2475f44a1096061b70dc0912"
+  integrity sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==
+
 lead@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/lead/-/lead-1.0.0.tgz#6f14f99a37be3a9dd784f5495690e5903466ee42"
@@ -9208,11 +9225,41 @@ lib0@^0.2.42, lib0@^0.2.46, lib0@^0.2.47, lib0@^0.2.74:
  dependencies:
    isomorphic.js "^0.2.4"

+libbase64@1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/libbase64/-/libbase64-1.2.1.tgz#fb93bf4cb6d730f29b92155b6408d1bd2176a8c8"
+  integrity sha512-l+nePcPbIG1fNlqMzrh68MLkX/gTxk/+vdvAb388Ssi7UuUN31MI44w4Yf33mM3Cm4xDfw48mdf3rkdHszLNew==
+
+libmime@5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/libmime/-/libmime-5.2.0.tgz#c4ed5cbd2d9fdd27534543a68bb8d17c658d51d8"
+  integrity sha512-X2U5Wx0YmK0rXFbk67ASMeqYIkZ6E5vY7pNWRKtnNzqjvdYYG8xtPDpCnuUEnPU9vlgNev+JoSrcaKSUaNvfsw==
+  dependencies:
+    encoding-japanese "2.0.0"
+    iconv-lite "0.6.3"
+    libbase64 "1.2.1"
+    libqp "2.0.1"
+
+libmime@5.2.1:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/libmime/-/libmime-5.2.1.tgz#a1075eaf702fa597161948dcae3afd03be383ac4"
+  integrity sha512-A0z9O4+5q+ZTj7QwNe/Juy1KARNb4WaviO4mYeFC4b8dBT2EEqK2pkM+GC8MVnkOjqhl5nYQxRgnPYRRTNmuSQ==
+  dependencies:
+    encoding-japanese "2.0.0"
+    iconv-lite "0.6.3"
+    libbase64 "1.2.1"
+    libqp "2.0.1"
+
 libphonenumber-js@^1.10.14:
  version "1.10.15"
  resolved "https://registry.yarnpkg.com/libphonenumber-js/-/libphonenumber-js-1.10.15.tgz#cad454adb5bf271bc820bbf7dd66776afcda7be6"
  integrity sha512-sLeVLmWX17VCKKulc+aDIRHS95TxoTsKMRJi5s5gJdwlqNzMWcBCtSHHruVyXjqfi67daXM2SnLf2juSrdx5Sg==

+libqp@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/libqp/-/libqp-2.0.1.tgz#b8fed76cc1ea6c9ceff8888169e4e0de70cd5cf2"
+  integrity sha512-Ka0eC5LkF3IPNQHJmYBWljJsw0UvM6j+QdKRbWyCdTmYwvIDE6a7bCm0UkTAL/K+3KXK5qXT/ClcInU01OpdLg==
+
 lie@~3.3.0:
  version "3.3.0"
  resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
@@ -9235,6 +9282,13 @@ lines-and-columns@^1.1.6:
  resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
  integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=

+linkify-it@5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-5.0.0.tgz#9ef238bfa6dc70bd8e7f9572b52d369af569b421"
+  integrity sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==
+  dependencies:
+    uc.micro "^2.0.0"
+
 linkify-it@^4.0.1:
  version "4.0.1"
  resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-4.0.1.tgz#01f1d5e508190d06669982ba31a7d9f56a5751ec"
@@ -9464,6 +9518,30 @@ magic-string@^0.27.0:
  dependencies:
    "@jridgewell/sourcemap-codec" "^1.4.13"

+mailparser@^3.6.6:
+  version "3.6.6"
+  resolved "https://registry.yarnpkg.com/mailparser/-/mailparser-3.6.6.tgz#055dbac39ce549e94836b9ab8c184a60e4ac9b61"
+  integrity sha512-noCjBl3FToxmqTP2fp7z17hQsiCroWNntfTd8O+UejOAF59xeN5WGZK27ilexXV2e2X/cbUhG3L8sfEKaz0/sw==
+  dependencies:
+    encoding-japanese "2.0.0"
+    he "1.2.0"
+    html-to-text "9.0.5"
+    iconv-lite "0.6.3"
+    libmime "5.2.1"
+    linkify-it "5.0.0"
+    mailsplit "5.4.0"
+    nodemailer "6.9.8"
+    tlds "1.248.0"
+
+mailsplit@5.4.0:
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/mailsplit/-/mailsplit-5.4.0.tgz#9f4692fadd9013e9ce632147d996931d2abac6ba"
+  integrity sha512-wnYxX5D5qymGIPYLwnp6h8n1+6P6vz/MJn5AzGjZ8pwICWssL+CCQjWBIToOVHASmATot4ktvlLo6CyLfOXWYA==
+  dependencies:
+    libbase64 "1.2.1"
+    libmime "5.2.0"
+    libqp "2.0.1"
+
 make-dir@^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
@@ -9860,15 +9938,10 @@ node-releases@^2.0.14:
  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b"
  integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==

-node-releases@^2.0.8:
-  version "2.0.10"
-  resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f"
-  integrity sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==
-
-nodemailer@^6.9.4:
-  version "6.9.4"
-  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.4.tgz#93bd4a60eb0be6fa088a0483340551ebabfd2abf"
-  integrity sha512-CXjQvrQZV4+6X5wP6ZIgdehJamI63MFoYFGGPtHudWym9qaEHDNdPzaj5bfMCvxG1vhAileSWW90q7nL0N36mA==
+nodemailer@6.9.8, nodemailer@^6.9.4:
+  version "6.9.8"
+  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.8.tgz#29601e80440f2af7aa62b32758fdac7c6b784143"
+  integrity sha512-cfrYUk16e67Ks051i4CntM9kshRYei1/o/Gi8K1d+R34OIs21xdFnW7Pt7EucmVKA0LKtqUGNcjMZ7ehjl49mQ==

 nodemon@^2.0.22:
  version "2.0.22"
@@ -10259,6 +10332,14 @@ parse5@^7.0.0, parse5@^7.1.1, parse5@^7.1.2:
  dependencies:
    entities "^4.4.0"

+parseley@^0.12.0:
+  version "0.12.1"
+  resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.12.1.tgz#4afd561d50215ebe259e3e7a853e62f600683aef"
+  integrity sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==
+  dependencies:
+    leac "^0.6.0"
+    peberminta "^0.9.0"
+
 parseurl@^1.3.2:
  version "1.3.3"
  resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
@@ -10412,6 +10493,11 @@ pause@0.0.1:
  resolved "https://registry.yarnpkg.com/pause/-/pause-0.0.1.tgz#1d408b3fdb76923b9543d96fb4c9dfd535d9cb5d"
  integrity sha1-HUCLP9t2kjuVQ9lvtMnf1TXZy10=

+peberminta@^0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/peberminta/-/peberminta-0.9.0.tgz#8ec9bc0eb84b7d368126e71ce9033501dca2a352"
+  integrity sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==
+
 pend@~1.2.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
@@ -10945,13 +11031,6 @@ quick-temp@^0.1.8:
    rimraf "^2.5.4"
    underscore.string "~3.3.4"

-quoted-printable@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/quoted-printable/-/quoted-printable-1.0.1.tgz#9eebf5eb3d11eef022b264fd2d2b6b2bb3b84cc3"
-  integrity sha1-nuv16z0R7vAismT9LStrK7O4TMM=
-  dependencies:
-    utf8 "^2.1.0"
-
 raf@^3.4.1:
  version "3.4.1"
  resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"
@@ -11791,6 +11870,13 @@ scroll-into-view-if-needed@^2.2.28:
  dependencies:
    compute-scroll-into-view "^1.0.17"

+selderee@^0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.11.0.tgz#6af0c7983e073ad3e35787ffe20cefd9daf0ec8a"
+  integrity sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==
+  dependencies:
+    parseley "^0.12.0"
+
 semver@^5.6.0, semver@^5.7.0, semver@^5.7.1:
  version "5.7.1"
  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
@@ -12607,6 +12693,11 @@ tippy.js@^6.3.1:
  dependencies:
    "@popperjs/core" "^2.9.0"

+tlds@1.248.0:
+  version "1.248.0"
+  resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.248.0.tgz#65bf56eee6d0ace1e918fbc653227ef18a9ddf8d"
+  integrity sha512-noj0KdpWTBhwsKxMOXk0rN9otg4kTgLm4WohERRHbJ9IY+kSDKr3RmjitaQ3JFzny+DyvBOQKlFZhp0G0qNSfg==
+
 tmp@^0.0.33:
  version "0.0.33"
  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"
@@ -12915,6 +13006,11 @@ uc.micro@^1.0.1, uc.micro@^1.0.5:
  resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
  integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==

+uc.micro@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.0.0.tgz#84b3c335c12b1497fd9e80fcd3bfa7634c363ff1"
+  integrity sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==
+
 uid2@0.0.3, uid2@0.0.x:
  version "0.0.3"
  resolved "https://registry.yarnpkg.com/uid2/-/uid2-0.0.3.tgz#483126e11774df2f71b8b639dcd799c376162b82"
@@ -13040,14 +13136,6 @@ upath@^1.2.0:
  resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"
  integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==

-update-browserslist-db@^1.0.10:
-  version "1.0.10"
-  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3"
-  integrity sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==
-  dependencies:
-    escalade "^3.1.1"
-    picocolors "^1.0.0"
-
 update-browserslist-db@^1.0.13:
  version "1.0.13"
  resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4"
@@ -13104,16 +13192,6 @@ utf8-byte-length@^1.0.1:
  resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz#f45f150c4c66eee968186505ab93fcbb8ad6bf61"
  integrity sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA==

-utf8@^2.1.0:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96"
-  integrity sha1-H6DZJw6b6FDZsFAn9jUZv0ZFfZY=
-
-utf8@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/utf8/-/utf8-3.0.0.tgz#f052eed1364d696e769ef058b183df88c87f69d1"
-  integrity sha512-E8VjFIQ/TyQgp+TZfS6l8yp/xWppSAHzidGiRrqe4bK4XP9pTRyKFgGJpO3SN7zdX4DeomTrwaseCHovfpFcqQ==
-
 util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"