From ea5d2ea9e014eb386dda0f44beb7c4f679c60345 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Sun, 27 Dec 2020 23:00:26 -0800 Subject: [PATCH] refactor, add preview --- app/scenes/Settings/ImportExport.js | 117 ++++++++++++++++++--- server/commands/documentBatchImporter.js | 98 +++++++---------- shared/i18n/locales/en_US/translation.json | 3 +- shared/utils/zip.js | 76 +++++++++++++ 4 files changed, 215 insertions(+), 79 deletions(-) create mode 100644 shared/utils/zip.js diff --git a/app/scenes/Settings/ImportExport.js b/app/scenes/Settings/ImportExport.js index 6d4686d50..0ba4966ab 100644 --- a/app/scenes/Settings/ImportExport.js +++ b/app/scenes/Settings/ImportExport.js @@ -1,10 +1,14 @@ // @flow import { observer } from "mobx-react"; +import { CollectionIcon } from "outline-icons"; import * as React from "react"; import { useTranslation, Trans } from "react-i18next"; +import styled from "styled-components"; +import { parseOutlineExport } from "shared/utils/zip"; import Button from "components/Button"; import CenteredContent from "components/CenteredContent"; import HelpText from "components/HelpText"; +import Notice from "components/Notice"; import PageTitle from "components/PageTitle"; import VisuallyHidden from "components/VisuallyHidden"; import useCurrentUser from "hooks/useCurrentUser"; @@ -20,14 +24,14 @@ function ImportExport() { const [isLoading, setLoading] = React.useState(false); const [isImporting, setImporting] = React.useState(false); const [isExporting, setExporting] = React.useState(false); + const [file, setFile] = React.useState(); + const [importDetails, setImportDetails] = React.useState(); - const handleFilePicked = React.useCallback( + const handleImport = React.useCallback( async (ev) => { - const files = getDataTransferFiles(ev); setImporting(true); try { - const file = files[0]; await documents.batchImport(file); showToast(t("Import completed")); } catch (err) { @@ -37,16 +41,37 @@ function ImportExport() { fileRef.current.value = ""; } setImporting(false); + setFile(undefined); + setImportDetails(undefined); } }, - [t, documents, showToast] + [t, file, documents, showToast] ); - const handleImport = React.useCallback(() => { - if (fileRef.current) { - fileRef.current.click(); + const handleFilePicked = React.useCallback(async (ev) => { + ev.preventDefault(); + + const files = getDataTransferFiles(ev); + const file = files[0]; + setFile(file); + + try { + setImportDetails(await parseOutlineExport(file)); + } catch (err) { + setImportDetails([]); } - }, [fileRef]); + }, []); + + const handlePickFile = React.useCallback( + (ev) => { + ev.preventDefault(); + + if (fileRef.current) { + fileRef.current.click(); + } + }, + [fileRef] + ); const handleExport = React.useCallback( async (ev: SyntheticEvent<>) => { @@ -64,6 +89,14 @@ function ImportExport() { [t, collections, showToast] ); + const hasCollections = importDetails + ? !!importDetails.filter((detail) => detail.type === "collection").length + : false; + const hasDocuments = importDetails + ? !!importDetails.filter((detail) => detail.type === "document").length + : false; + const isImportable = hasCollections && hasDocuments; + return ( @@ -83,14 +116,46 @@ function ImportExport() { accept="application/zip" /> - + {file && !isImportable && ( + + + Sorry, the file {{ fileName: file.name }} is + missing valid collections or documents. + + + )} + {file && importDetails && isImportable ? ( + <> + + + {{ fileName: file.name }} looks good, the + following collections and their documents will be imported: + + + {importDetails + .filter((detail) => detail.type === "collection") + .map((detail) => ( + + + {detail.name} + + ))} + + + + + ) : ( + + )}

{t("Export")}

@@ -117,4 +182,24 @@ function ImportExport() { ); } +const List = styled.ul` + padding: 0; + margin: 8px 0 0; +`; + +const ImportPreview = styled(Notice)` + margin-bottom: 16px; +`; + +const ImportPreviewItem = styled.li` + display: flex; + align-items: center; + list-style: none; +`; + +const CollectionName = styled.span` + font-weight: 500; + margin-left: 4px; +`; + export default observer(ImportExport); diff --git a/server/commands/documentBatchImporter.js b/server/commands/documentBatchImporter.js index 2b24b7efb..9de70ab0c 100644 --- a/server/commands/documentBatchImporter.js +++ b/server/commands/documentBatchImporter.js @@ -1,11 +1,13 @@ // @flow import fs from "fs"; +import os from "os"; import path from "path"; import debug from "debug"; import File from "formidable/lib/file"; import invariant from "invariant"; -import JSZip from "jszip"; import { values, keys } from "lodash"; +import uuid from "uuid"; +import { parseOutlineExport } from "../../shared/utils/zip"; import { InvalidRequestError } from "../errors"; import { Attachment, Document, Collection, User } from "../models"; import attachmentCreator from "./attachmentCreator"; @@ -27,57 +29,26 @@ export default async function documentBatchImporter({ }) { // load the zip structure into memory const zipData = await fs.promises.readFile(file.path); - const zip = await JSZip.loadAsync(zipData); + + let items; + try { + items = await await parseOutlineExport(zipData); + } catch (err) { + throw new InvalidRequestError(err.message); + } // store progress and pointers let collections: { string: Collection } = {}; let documents: { string: Document } = {}; let attachments: { string: Attachment } = {}; - // this is so we can use async / await a little easier - let folders = []; - zip.forEach(async function (path, item) { - // known skippable items - if (path.startsWith("__MACOSX") || path.endsWith(".DS_Store")) { - return; - } - - folders.push([path, item]); - }); - - for (const [rawPath, item] of folders) { - const itemPath = rawPath.replace(/\/$/, ""); - const depth = itemPath.split("/").length - 1; - - if (depth === 0 && !item.dir) { - throw new InvalidRequestError( - "Root of zip file must only contain folders representing collections" - ); - } - } - - for (const [rawPath, item] of folders) { - const itemPath = rawPath.replace(/\/$/, ""); - const itemDir = path.dirname(itemPath); - const name = path.basename(item.name); - const depth = itemPath.split("/").length - 1; - - // metadata - let metadata = {}; - try { - metadata = item.comment ? JSON.parse(item.comment) : {}; - } catch (err) { - log( - `ZIP comment found for ${item.name}, but could not be parsed as metadata: ${item.comment}` - ); - } - - if (depth === 0 && item.dir && name) { + for (const item of items) { + if (item.type === "collection") { // check if collection with name exists let [collection, isCreated] = await Collection.findOrCreate({ where: { teamId: user.teamId, - name, + name: item.name, }, defaults: { creatorId: user.id, @@ -92,28 +63,31 @@ export default async function documentBatchImporter({ collection = await Collection.create({ teamId: user.teamId, creatorId: user.id, - name: `${name} (Imported)`, + name: `${item.name} (Imported)`, private: false, }); } - collections[itemPath] = collection; + collections[item.path] = collection; continue; } - if (depth > 0 && !item.dir && item.name.endsWith(".md")) { - const collectionDir = itemDir.split("/")[0]; + if (item.type === "document") { + const collectionDir = item.dir.split("/")[0]; const collection = collections[collectionDir]; - invariant(collection, `Collection must exist for document ${itemDir}`); + invariant(collection, `Collection must exist for document ${item.dir}`); // we have a document - const content = await item.async("string"); + const content = await item.item.async("string"); const name = path.basename(item.name); - await fs.promises.writeFile(`/tmp/${name}`, content); + const tmpDir = os.tmpdir(); + const tmpFilePath = `${tmpDir}/upload-${uuid.v4()}`; + + await fs.promises.writeFile(tmpFilePath, content); const file = new File({ name, type: "text/markdown", - path: `/tmp/${name}`, + path: tmpFilePath, }); const { text, title } = await documentImporter({ @@ -124,9 +98,10 @@ export default async function documentBatchImporter({ // must be a nested document, find and reference the parent document let parentDocumentId; - if (depth > 1) { - const parentDocument = documents[`${itemDir}.md`] || documents[itemDir]; - invariant(parentDocument, `Document must exist for parent ${itemDir}`); + if (item.depth > 1) { + const parentDocument = + documents[`${item.dir}.md`] || documents[item.dir]; + invariant(parentDocument, `Document must exist for parent ${item.dir}`); parentDocumentId = parentDocument.id; } @@ -135,8 +110,8 @@ export default async function documentBatchImporter({ text, publish: true, collectionId: collection.id, - createdAt: metadata.createdAt - ? new Date(metadata.createdAt) + createdAt: item.metadata.createdAt + ? new Date(item.metadata.createdAt) : item.date, updatedAt: item.date, parentDocumentId, @@ -144,25 +119,24 @@ export default async function documentBatchImporter({ ip, }); - documents[itemPath] = document; + documents[item.path] = document; continue; } - if (depth > 0 && !item.dir && itemPath.includes("uploads")) { - // we have an attachment - const buffer = await item.async("nodebuffer"); + if (item.type === "attachment") { + const buffer = await item.item.async("nodebuffer"); const attachment = await attachmentCreator({ - name, + name: item.name, type, buffer, user, ip, }); - attachments[itemPath] = attachment; + attachments[item.path] = attachment; continue; } - log(`Skipped importing ${itemPath}`); + log(`Skipped importing ${item.path}`); } // All collections, documents, and attachments have been created – time to diff --git a/shared/i18n/locales/en_US/translation.json b/shared/i18n/locales/en_US/translation.json index 87afdbf7e..c5aa5ea3e 100644 --- a/shared/i18n/locales/en_US/translation.json +++ b/shared/i18n/locales/en_US/translation.json @@ -280,7 +280,8 @@ "Import": "Import", "It is possible to import a zip file of folders and Markdown files previously exported from an Outline instance. Support will soon be added for importing from other services.": "It is possible to import a zip file of folders and Markdown files previously exported from an Outline instance. Support will soon be added for importing from other services.", "Importing": "Importing", - "Import Data": "Import Data", + "Confirm & Import": "Confirm & Import", + "Choose File…": "Choose File…", "A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <2>{{userEmail}}.": "A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <2>{{userEmail}}.", "Export Requested": "Export Requested", "Requesting Export": "Requesting Export", diff --git a/shared/utils/zip.js b/shared/utils/zip.js new file mode 100644 index 000000000..d6464f3f2 --- /dev/null +++ b/shared/utils/zip.js @@ -0,0 +1,76 @@ +// @flow +import path from "path"; +import JSZip, { ZipObject } from "jszip"; + +export type Item = {| + path: string, + dir: string, + name: string, + depth: number, + metadata: Object, + type: "collection" | "document" | "attachment", + item: ZipObject, +|}; + +export async function parseOutlineExport( + input: File | Buffer +): Promise { + const zip = await JSZip.loadAsync(input); + + // this is so we can use async / await a little easier + let items: Item[] = []; + zip.forEach(async function (rawPath, item) { + const itemPath = rawPath.replace(/\/$/, ""); + const dir = path.dirname(itemPath); + const name = path.basename(item.name); + const depth = itemPath.split("/").length - 1; + + // known skippable items + if (itemPath.startsWith("__MACOSX") || itemPath.endsWith(".DS_Store")) { + return; + } + + // attempt to parse extra metadata from zip comment + let metadata = {}; + try { + metadata = item.comment ? JSON.parse(item.comment) : {}; + } catch (err) { + console.log( + `ZIP comment found for ${item.name}, but could not be parsed as metadata: ${item.comment}` + ); + } + + if (depth === 0 && !item.dir) { + throw new Error( + "Root of zip file must only contain folders representing collections" + ); + } + + let type; + if (depth === 0 && item.dir && name) { + type = "collection"; + } + if (depth > 0 && !item.dir && item.name.endsWith(".md")) { + type = "document"; + } + if (depth > 0 && !item.dir && itemPath.includes("uploads")) { + type = "attachment"; + } + + if (!type) { + return; + } + + items.push({ + path: itemPath, + dir, + name, + depth, + type, + metadata, + item, + }); + }); + + return items; +}