feat: Support importing .docx or .html files as new documents (#1551)

* Support importing .docx as new documents

* Add html file support, build types and interface for easily adding file types to importer

* fix: Upload embedded images in docx to storage

* refactor: Bulk of logic to command

* refactor: Do all importing on server, so we're not splitting logic for import into two places

* test: Add documentImporter tests


Co-authored-by: Lance Whatley <whatl3y@gmail.com>

app/components/DropToImport.js

@@ -8,7 +8,6 @@ import { withRouter, type RouterHistory, type Match } from "react-router-dom";
 import { createGlobalStyle } from "styled-components";
 import DocumentsStore from "stores/DocumentsStore";
 import LoadingIndicator from "components/LoadingIndicator";
-import importFile from "utils/importFile";
 
 const EMPTY_OBJECT = {};
 let importingLock = false;
@@ -61,12 +60,12 @@ class DropToImport extends React.Component<Props> {
       }
 
       for (const file of files) {
-        const doc = await importFile({
-          documents: this.props.documents,
+        const doc = await this.props.documents.import(
           file,
           documentId,
           collectionId,
-        });
+          { publish: true }
+        );
 
         if (redirect) {
           this.props.history.push(doc.url);
@@ -95,7 +94,7 @@ class DropToImport extends React.Component<Props> {
 
     return (
       <Dropzone
-        accept="text/markdown, text/plain"
+        accept={documents.importFileTypes.join(", ")}
         onDropAccepted={this.onDropAccepted}
         style={EMPTY_OBJECT}
         disableClick

app/menus/CollectionMenu.js

@@ -15,7 +15,6 @@ import { DropdownMenu, DropdownMenuItem } from "components/DropdownMenu";
 import Modal from "components/Modal";
 import VisuallyHidden from "components/VisuallyHidden";
 import getDataTransferFiles from "utils/getDataTransferFiles";
-import importFile from "utils/importFile";
 import { newDocumentUrl } from "utils/routeHelpers";
 
 type Props = {
@@ -55,11 +54,13 @@ class CollectionMenu extends React.Component<Props> {
     const files = getDataTransferFiles(ev);
 
     try {
-      const document = await importFile({
-        file: files[0],
-        documents: this.props.documents,
-        collectionId: this.props.collection.id,
-      });
+      const file = files[0];
+      const document = await this.props.documents.import(
+        file,
+        null,
+        this.props.collection.id,
+        { publish: true }
+      );
       this.props.history.push(document.url);
     } catch (err) {
       this.props.ui.showToast(err.message);
@@ -103,7 +104,14 @@ class CollectionMenu extends React.Component<Props> {
   };
 
   render() {
-    const { policies, collection, position, onOpen, onClose } = this.props;
+    const {
+      policies,
+      documents,
+      collection,
+      position,
+      onOpen,
+      onClose,
+    } = this.props;
     const can = policies.abilities(collection.id);
 
     return (
@@ -114,7 +122,7 @@ class CollectionMenu extends React.Component<Props> {
             ref={(ref) => (this.file = ref)}
             onChange={this.onFilePicked}
             onClick={(ev) => ev.stopPropagation()}
-            accept="text/markdown, text/plain"
+            accept={documents.importFileTypes.join(", ")}
           />
         </VisuallyHidden>
 

app/stores/DocumentsStore.js

@@ -18,12 +18,23 @@ import Document from "models/Document";
 import type { FetchOptions, PaginationParams, SearchResult } from "types";
 import { client } from "utils/ApiClient";
 
+type ImportOptions = {
+  publish?: boolean,
+};
+
 export default class DocumentsStore extends BaseStore<Document> {
   @observable recentlyViewedIds: string[] = [];
   @observable searchCache: Map<string, SearchResult[]> = new Map();
   @observable starredIds: Map<string, boolean> = new Map();
   @observable backlinks: Map<string, string[]> = new Map();
 
+  importFileTypes: string[] = [
+    "text/markdown",
+    "text/plain",
+    "text/html",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ];
+
   constructor(rootStore: RootStore) {
     super(rootStore, Document);
   }
@@ -455,6 +466,41 @@ export default class DocumentsStore extends BaseStore<Document> {
     return this.add(res.data);
   };
 
+  @action
+  import = async (
+    file: File,
+    parentDocumentId: string,
+    collectionId: string,
+    options: ImportOptions
+  ) => {
+    const title = file.name.replace(/\.[^/.]+$/, "");
+    const formData = new FormData();
+
+    [
+      { key: "parentDocumentId", value: parentDocumentId },
+      { key: "collectionId", value: collectionId },
+      { key: "title", value: title },
+      { key: "publish", value: options.publish },
+      { key: "file", value: file },
+    ].map((info) => {
+      if (typeof info.value === "string" && info.value) {
+        formData.append(info.key, info.value);
+      }
+      if (typeof info.value === "boolean") {
+        formData.append(info.key, info.value.toString());
+      }
+      if (info.value instanceof File) {
+        formData.append(info.key, info.value);
+      }
+    });
+
+    const res = await client.post("/documents.import", formData);
+    invariant(res && res.data, "Data should be available");
+
+    this.addPolicies(res.policies);
+    return this.add(res.data);
+  };
+
   _add = this.add;
 
   @action

app/utils/ApiClient.js

@@ -28,12 +28,13 @@ class ApiClient {
   fetch = async (
     path: string,
     method: string,
-    data: ?Object,
+    data: ?Object | FormData | void,
     options: Object = {}
   ) => {
     let body;
     let modifiedPath;
     let urlToFetch;
+    let isJson;
 
     if (method === "GET") {
       if (data) {
@@ -42,7 +43,18 @@ class ApiClient {
         modifiedPath = path;
       }
     } else if (method === "POST" || method === "PUT") {
-      body = data ? JSON.stringify(data) : undefined;
+      body = data || undefined;
+
+      // Only stringify data if its a normal object and
+      // not if it's [object FormData], in addition to
+      // toggling Content-Type to application/json
+      if (
+        typeof data === "object" &&
+        (data || "").toString() === "[object Object]"
+      ) {
+        isJson = true;
+        body = JSON.stringify(data);
+      }
     }
 
     if (path.match(/^http/)) {
@@ -51,14 +63,20 @@ class ApiClient {
       urlToFetch = this.baseUrl + (modifiedPath || path);
     }
 
-    // Construct headers
-    const headers = new Headers({
+    let headerOptions: any = {
       Accept: "application/json",
-      "Content-Type": "application/json",
       "cache-control": "no-cache",
       "x-editor-version": EDITOR_VERSION,
       pragma: "no-cache",
-    });
+    };
+    // for multipart forms or other non JSON requests fetch
+    // populates the Content-Type without needing to explicitly
+    // set it.
+    if (isJson) {
+      headerOptions["Content-Type"] = "application/json";
+    }
+    const headers = new Headers(headerOptions);
+
     if (stores.auth.authenticated) {
       invariant(stores.auth.token, "JWT token not set properly");
       headers.set("Authorization", `Bearer ${stores.auth.token}`);

app/utils/importFile.js

@@ -1,58 +0,0 @@
-// @flow
-import parseTitle from "shared/utils/parseTitle";
-import DocumentsStore from "stores/DocumentsStore";
-import Document from "models/Document";
-
-type Options = {
-  file: File,
-  documents: DocumentsStore,
-  collectionId: string,
-  documentId?: string,
-};
-
-const importFile = async ({
-  documents,
-  file,
-  documentId,
-  collectionId,
-}: Options): Promise<Document> => {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader();
-
-    reader.onload = async (ev) => {
-      let text = ev.target.result;
-      let title;
-
-      // If the first line of the imported file looks like a markdown heading
-      // then we can use this as the document title
-      if (text.trim().startsWith("# ")) {
-        const result = parseTitle(text);
-        title = result.title;
-        text = text.replace(`# ${title}\n`, "");
-
-        // otherwise, just use the filename without the extension as our best guess
-      } else {
-        title = file.name.replace(/\.[^/.]+$/, "");
-      }
-
-      let document = new Document(
-        {
-          parentDocumentId: documentId,
-          collectionId,
-          text,
-          title,
-        },
-        documents
-      );
-      try {
-        document = await document.save({ publish: true });
-        resolve(document);
-      } catch (err) {
-        reject(err);
-      }
-    };
-    reader.readAsText(file);
-  });
-};
-
-export default importFile;