diff --git a/app/actions/definitions/documents.tsx b/app/actions/definitions/documents.tsx index baf7a30c2..871e7406e 100644 --- a/app/actions/definitions/documents.tsx +++ b/app/actions/definitions/documents.tsx @@ -22,6 +22,7 @@ import { LightBulbIcon, } from "outline-icons"; import * as React from "react"; +import { ExportContentType } from "@shared/types"; import { getEventFiles } from "@shared/utils/files"; import DocumentDelete from "~/scenes/DocumentDelete"; import DocumentMove from "~/scenes/DocumentMove"; @@ -203,7 +204,7 @@ export const downloadDocumentAsHTML = createAction({ } const document = stores.documents.get(activeDocumentId); - document?.download("text/html"); + document?.download(ExportContentType.Html); }, }); @@ -229,7 +230,7 @@ export const downloadDocumentAsPDF = createAction({ const document = stores.documents.get(activeDocumentId); document - ?.download("application/pdf") + ?.download(ExportContentType.Pdf) .finally(() => id && stores.toasts.hideToast(id)); }, }); @@ -248,7 +249,7 @@ export const downloadDocumentAsMarkdown = createAction({ } const document = stores.documents.get(activeDocumentId); - document?.download("text/markdown"); + document?.download(ExportContentType.Markdown); }, }); diff --git a/app/components/ConfirmationDialog.tsx b/app/components/ConfirmationDialog.tsx index 590be3d4f..317648985 100644 --- a/app/components/ConfirmationDialog.tsx +++ b/app/components/ConfirmationDialog.tsx @@ -51,7 +51,7 @@ const ConfirmationDialog: React.FC = ({
{children}
diff --git a/app/components/Icons/Markdown.tsx b/app/components/Icons/Markdown.tsx new file mode 100644 index 000000000..61b1d098d --- /dev/null +++ b/app/components/Icons/Markdown.tsx @@ -0,0 +1,31 @@ +import * as React from "react"; + +type Props = { + size?: number; + color?: string; +}; + +export default function MarkdownIcon({ + size = 24, + color = "currentColor", +}: Props) { + return ( + + + + + ); +} diff --git a/app/components/InputSelect.tsx b/app/components/InputSelect.tsx index 200c07700..613ac0936 100644 --- a/app/components/InputSelect.tsx +++ b/app/components/InputSelect.tsx @@ -42,7 +42,7 @@ export type Props = { icon?: React.ReactNode; options: Option[]; note?: React.ReactNode; - onChange: (value: string | null) => void; + onChange?: (value: string | null) => void; }; const getOptionFromValue = (options: Option[], value: string | null) => { @@ -109,7 +109,7 @@ const InputSelect = (props: Props) => { previousValue.current = select.selectedValue; async function load() { - await onChange(select.selectedValue); + await onChange?.(select.selectedValue); } load(); diff --git a/app/components/InputSelectPermission.tsx b/app/components/InputSelectPermission.tsx index 6d406e161..cc307cdef 100644 --- a/app/components/InputSelectPermission.tsx +++ b/app/components/InputSelectPermission.tsx @@ -21,7 +21,7 @@ export default function InputSelectPermission( value = ""; } - onChange(value); + onChange?.(value); }, [onChange] ); diff --git a/app/models/Document.ts b/app/models/Document.ts index 0dd53d754..bea547a95 100644 --- a/app/models/Document.ts +++ b/app/models/Document.ts @@ -1,6 +1,7 @@ import { addDays, differenceInDays } from "date-fns"; import { floor } from "lodash"; import { action, autorun, computed, observable, set } from "mobx"; +import { ExportContentType } from "@shared/types"; import parseTitle from "@shared/utils/parseTitle"; import { isRTL } from "@shared/utils/rtl"; import DocumentsStore from "~/stores/DocumentsStore"; @@ -419,10 +420,8 @@ export default class Document extends ParanoidModel { }; } - download = async ( - contentType: "text/html" | "text/markdown" | "application/pdf" - ) => { - await client.post( + download = (contentType: ExportContentType) => { + return client.post( `/documents.export`, { id: this.id, diff --git a/app/models/FileOperation.ts b/app/models/FileOperation.ts index 6328eadf1..cf0f9d484 100644 --- a/app/models/FileOperation.ts +++ b/app/models/FileOperation.ts @@ -1,4 +1,5 @@ import { computed } from "mobx"; +import { FileOperationFormat, FileOperationType } from "@shared/types"; import { bytesToHumanReadable } from "@shared/utils/files"; import BaseModel from "./BaseModel"; import User from "./User"; @@ -16,7 +17,9 @@ class FileOperation extends BaseModel { size: number; - type: "import" | "export"; + type: FileOperationType; + + format: FileOperationFormat; user: User; diff --git a/app/scenes/Settings/Export.tsx b/app/scenes/Settings/Export.tsx index 898acdb2d..8f3e45fe7 100644 --- a/app/scenes/Settings/Export.tsx +++ b/app/scenes/Settings/Export.tsx @@ -11,30 +11,26 @@ import Text from "~/components/Text"; import useCurrentUser from "~/hooks/useCurrentUser"; import useStores from "~/hooks/useStores"; import useToasts from "~/hooks/useToasts"; +import ExportDialog from "./components/ExportDialog"; import FileOperationListItem from "./components/FileOperationListItem"; function Export() { const { t } = useTranslation(); const user = useCurrentUser(); - const { fileOperations, collections } = useStores(); + const { fileOperations, dialogs } = useStores(); const { showToast } = useToasts(); - const [isLoading, setLoading] = React.useState(false); - const [isExporting, setExporting] = React.useState(false); - const handleExport = React.useCallback( + const handleOpenDialog = React.useCallback( async (ev: React.SyntheticEvent) => { ev.preventDefault(); - setLoading(true); - try { - await collections.export(); - setExporting(true); - showToast(t("Export in progress…")); - } finally { - setLoading(false); - } + dialogs.openModal({ + title: t("Export data"), + isCentered: true, + content: , + }); }, - [t, collections, showToast] + [dialogs, t] ); const handleDelete = React.useCallback( @@ -65,16 +61,8 @@ function Export() { }} /> -
diff --git a/app/scenes/Settings/components/ExportDialog.tsx b/app/scenes/Settings/components/ExportDialog.tsx new file mode 100644 index 000000000..1ce9a7ac0 --- /dev/null +++ b/app/scenes/Settings/components/ExportDialog.tsx @@ -0,0 +1,114 @@ +import { observer } from "mobx-react"; +import { CodeIcon } from "outline-icons"; +import * as React from "react"; +import { Trans, useTranslation } from "react-i18next"; +import styled from "styled-components"; +import { FileOperationFormat } from "@shared/types"; +import ConfirmationDialog from "~/components/ConfirmationDialog"; +import Flex from "~/components/Flex"; +import MarkdownIcon from "~/components/Icons/Markdown"; +import Text from "~/components/Text"; +import useStores from "~/hooks/useStores"; + +type Props = { + onSubmit: () => void; +}; + +function ExportDialog({ onSubmit }: Props) { + const [format, setFormat] = React.useState( + FileOperationFormat.MarkdownZip + ); + const { collections } = useStores(); + const { t } = useTranslation(); + + const handleFormatChange = React.useCallback( + (ev: React.ChangeEvent) => { + setFormat(ev.target.value as FileOperationFormat); + }, + [] + ); + + const handleSubmit = React.useCallback(async () => { + await collections.export(format); + onSubmit(); + }, [collections, format, onSubmit]); + + return ( + + + + + + + ); +} + +const Format = styled.div` + display: flex; + flex-direction: column; + align-items: center; + flex-shrink: 0; + background: ${(props) => props.theme.secondaryBackground}; + border-radius: 6px; + width: 25%; + font-weight: 500; + font-size: 14px; + text-align: center; + padding: 10px 8px; + cursor: var(--pointer); +`; + +const Option = styled.label` + display: flex; + align-items: center; + gap: 16px; + + p { + margin: 0; + } +`; + +const Input = styled.input` + display: none; + + &:checked + ${Format} { + box-shadow: inset 0 0 0 2px ${(props) => props.theme.inputBorderFocused}; + } +`; + +export default observer(ExportDialog); diff --git a/app/scenes/Settings/components/FileOperationListItem.tsx b/app/scenes/Settings/components/FileOperationListItem.tsx index 0f2e475fb..b0eca8958 100644 --- a/app/scenes/Settings/components/FileOperationListItem.tsx +++ b/app/scenes/Settings/components/FileOperationListItem.tsx @@ -3,6 +3,7 @@ import { ArchiveIcon, DoneIcon, WarningIcon } from "outline-icons"; import * as React from "react"; import { useTranslation } from "react-i18next"; import { useTheme } from "styled-components"; +import { FileOperationFormat, FileOperationType } from "@shared/types"; import FileOperation from "~/models/FileOperation"; import { Action } from "~/components/Actions"; import ListItem from "~/components/List/Item"; @@ -36,10 +37,19 @@ const FileOperationListItem = ({ fileOperation, handleDelete }: Props) => { error: , }; + const formatToReadable = { + [FileOperationFormat.MarkdownZip]: "Markdown", + [FileOperationFormat.HTMLZip]: "HTML", + [FileOperationFormat.PDFZip]: "PDF", + }; + + const format = formatToReadable[fileOperation.format]; + const title = - fileOperation.type === "import" || fileOperation.collectionId + fileOperation.type === FileOperationType.Import || + fileOperation.collectionId ? fileOperation.name - : t("All collections"); + : t("All collections") + (format ? ` • ${format}` : ""); return ( { this.rootStore.documents.fetchRecentlyViewed(); }; - export = () => { - return client.post("/collections.export_all"); + export = (format: string) => { + return client.post("/collections.export_all", { + format, + }); }; } diff --git a/app/stores/FileOperationsStore.ts b/app/stores/FileOperationsStore.ts index 6191f6722..18133a922 100644 --- a/app/stores/FileOperationsStore.ts +++ b/app/stores/FileOperationsStore.ts @@ -1,5 +1,6 @@ import { orderBy } from "lodash"; import { computed } from "mobx"; +import { FileOperationType } from "@shared/types"; import FileOperation from "~/models/FileOperation"; import BaseStore, { RPCAction } from "./BaseStore"; import RootStore from "./RootStore"; @@ -15,7 +16,8 @@ export default class FileOperationsStore extends BaseStore { get imports(): FileOperation[] { return orderBy( Array.from(this.data.values()).reduce( - (acc, fileOp) => (fileOp.type === "import" ? [...acc, fileOp] : acc), + (acc, fileOp) => + fileOp.type === FileOperationType.Import ? [...acc, fileOp] : acc, [] ), "createdAt", @@ -27,7 +29,8 @@ export default class FileOperationsStore extends BaseStore { get exports(): FileOperation[] { return orderBy( Array.from(this.data.values()).reduce( - (acc, fileOp) => (fileOp.type === "export" ? [...acc, fileOp] : acc), + (acc, fileOp) => + fileOp.type === FileOperationType.Export ? [...acc, fileOp] : acc, [] ), "createdAt", diff --git a/server/commands/attachmentCreator.ts b/server/commands/attachmentCreator.ts index 221d1dc35..e10ec8a0e 100644 --- a/server/commands/attachmentCreator.ts +++ b/server/commands/attachmentCreator.ts @@ -1,7 +1,7 @@ import { Transaction } from "sequelize"; import { v4 as uuidv4 } from "uuid"; import { Attachment, Event, User } from "@server/models"; -import { uploadToS3FromBuffer } from "@server/utils/s3"; +import { uploadToS3 } from "@server/utils/s3"; export default async function attachmentCreator({ id, @@ -24,7 +24,13 @@ export default async function attachmentCreator({ }) { const key = `uploads/${user.id}/${uuidv4()}/${name}`; const acl = process.env.AWS_S3_ACL || "private"; - const url = await uploadToS3FromBuffer(buffer, type, key, acl); + const url = await uploadToS3({ + body: buffer, + contentType: type, + contentLength: buffer.length, + key, + acl, + }); const attachment = await Attachment.create( { id, diff --git a/server/commands/collectionExporter.ts b/server/commands/collectionExporter.ts index de00418fc..c5dafd63b 100644 --- a/server/commands/collectionExporter.ts +++ b/server/commands/collectionExporter.ts @@ -1,33 +1,37 @@ import { Transaction } from "sequelize"; -import { APM } from "@server/logging/tracing"; -import { Collection, Event, Team, User, FileOperation } from "@server/models"; import { + FileOperationFormat, FileOperationType, FileOperationState, - FileOperationFormat, -} from "@server/models/FileOperation"; +} from "@shared/types"; +import { APM } from "@server/logging/tracing"; +import { Collection, Event, Team, User, FileOperation } from "@server/models"; import { getAWSKeyForFileOp } from "@server/utils/s3"; +type Props = { + collection?: Collection; + team: Team; + user: User; + format?: FileOperationFormat; + ip: string; + transaction: Transaction; +}; + async function collectionExporter({ collection, team, user, + format = FileOperationFormat.MarkdownZip, ip, transaction, -}: { - collection?: Collection; - team: Team; - user: User; - ip: string; - transaction: Transaction; -}) { +}: Props) { const collectionId = collection?.id; const key = getAWSKeyForFileOp(user.teamId, collection?.name || team.name); const fileOperation = await FileOperation.create( { type: FileOperationType.Export, state: FileOperationState.Creating, - format: FileOperationFormat.MarkdownZip, + format, key, url: null, size: 0, @@ -49,7 +53,8 @@ async function collectionExporter({ collectionId, ip, data: { - type: FileOperationType.Import, + type: FileOperationType.Export, + format, }, }, { diff --git a/server/models/FileOperation.ts b/server/models/FileOperation.ts index f1340d9b1..f5889e40b 100644 --- a/server/models/FileOperation.ts +++ b/server/models/FileOperation.ts @@ -8,6 +8,11 @@ import { Table, DataType, } from "sequelize-typescript"; +import { + FileOperationFormat, + FileOperationState, + FileOperationType, +} from "@shared/types"; import { deleteFromS3, getFileByKey } from "@server/utils/s3"; import Collection from "./Collection"; import Team from "./Team"; @@ -15,24 +20,6 @@ import User from "./User"; import IdModel from "./base/IdModel"; import Fix from "./decorators/Fix"; -export enum FileOperationType { - Import = "import", - Export = "export", -} - -export enum FileOperationFormat { - MarkdownZip = "outline-markdown", - Notion = "notion", -} - -export enum FileOperationState { - Creating = "creating", - Uploading = "uploading", - Complete = "complete", - Error = "error", - Expired = "expired", -} - @DefaultScope(() => ({ include: [ { diff --git a/server/queues/processors/FileOperationsProcessor.ts b/server/queues/processors/FileOperationsProcessor.ts index 933f85750..75db65f8f 100644 --- a/server/queues/processors/FileOperationsProcessor.ts +++ b/server/queues/processors/FileOperationsProcessor.ts @@ -1,10 +1,8 @@ import invariant from "invariant"; +import { FileOperationFormat, FileOperationType } from "@shared/types"; import { FileOperation } from "@server/models"; -import { - FileOperationFormat, - FileOperationType, -} from "@server/models/FileOperation"; import { Event as TEvent, FileOperationEvent } from "@server/types"; +import ExportHTMLZipTask from "../tasks/ExportHTMLZipTask"; import ExportMarkdownZipTask from "../tasks/ExportMarkdownZipTask"; import ImportMarkdownZipTask from "../tasks/ImportMarkdownZipTask"; import ImportNotionTask from "../tasks/ImportNotionTask"; @@ -40,6 +38,11 @@ export default class FileOperationsProcessor extends BaseProcessor { if (fileOperation.type === FileOperationType.Export) { switch (fileOperation.format) { + case FileOperationFormat.HTMLZip: + await ExportHTMLZipTask.schedule({ + fileOperationId: event.modelId, + }); + break; case FileOperationFormat.MarkdownZip: await ExportMarkdownZipTask.schedule({ fileOperationId: event.modelId, diff --git a/server/queues/tasks/CleanupExpiredFileOperationsTask.test.ts b/server/queues/tasks/CleanupExpiredFileOperationsTask.test.ts index 034eda0c1..bef1625d9 100644 --- a/server/queues/tasks/CleanupExpiredFileOperationsTask.test.ts +++ b/server/queues/tasks/CleanupExpiredFileOperationsTask.test.ts @@ -1,9 +1,6 @@ import { subDays } from "date-fns"; +import { FileOperationState, FileOperationType } from "@shared/types"; import { FileOperation } from "@server/models"; -import { - FileOperationState, - FileOperationType, -} from "@server/models/FileOperation"; import { buildFileOperation } from "@server/test/factories"; import { setupTestDatabase } from "@server/test/support"; import CleanupExpiredFileOperationsTask from "./CleanupExpiredFileOperationsTask"; diff --git a/server/queues/tasks/CleanupExpiredFileOperationsTask.ts b/server/queues/tasks/CleanupExpiredFileOperationsTask.ts index 299e8d7de..af309f1c8 100644 --- a/server/queues/tasks/CleanupExpiredFileOperationsTask.ts +++ b/server/queues/tasks/CleanupExpiredFileOperationsTask.ts @@ -1,8 +1,8 @@ import { subDays } from "date-fns"; import { Op } from "sequelize"; +import { FileOperationState } from "@shared/types"; import Logger from "@server/logging/Logger"; import { FileOperation } from "@server/models"; -import { FileOperationState } from "@server/models/FileOperation"; import BaseTask, { TaskPriority } from "./BaseTask"; type Props = { diff --git a/server/queues/tasks/ExportHTMLZipTask.ts b/server/queues/tasks/ExportHTMLZipTask.ts new file mode 100644 index 000000000..aabdfafa6 --- /dev/null +++ b/server/queues/tasks/ExportHTMLZipTask.ts @@ -0,0 +1,10 @@ +import { FileOperationFormat } from "@shared/types"; +import { Collection } from "@server/models"; +import { archiveCollections } from "@server/utils/zip"; +import ExportTask from "./ExportTask"; + +export default class ExportHTMLZipTask extends ExportTask { + public async export(collections: Collection[]) { + return await archiveCollections(collections, FileOperationFormat.HTMLZip); + } +} diff --git a/server/queues/tasks/ExportMarkdownZipTask.ts b/server/queues/tasks/ExportMarkdownZipTask.ts index d5d562a65..a84a93cb6 100644 --- a/server/queues/tasks/ExportMarkdownZipTask.ts +++ b/server/queues/tasks/ExportMarkdownZipTask.ts @@ -1,130 +1,13 @@ -import fs from "fs"; -import { truncate } from "lodash"; -import ExportFailureEmail from "@server/emails/templates/ExportFailureEmail"; -import ExportSuccessEmail from "@server/emails/templates/ExportSuccessEmail"; -import Logger from "@server/logging/Logger"; -import { Collection, Event, FileOperation, Team, User } from "@server/models"; -import { FileOperationState } from "@server/models/FileOperation"; -import fileOperationPresenter from "@server/presenters/fileOperation"; -import { uploadToS3FromBuffer } from "@server/utils/s3"; +import { FileOperationFormat } from "@shared/types"; +import { Collection } from "@server/models"; import { archiveCollections } from "@server/utils/zip"; -import BaseTask, { TaskPriority } from "./BaseTask"; +import ExportTask from "./ExportTask"; -type Props = { - fileOperationId: string; -}; - -export default class ExportMarkdownZipTask extends BaseTask { - /** - * Runs the export task. - * - * @param props The props - */ - public async perform({ fileOperationId }: Props) { - const fileOperation = await FileOperation.findByPk(fileOperationId, { - rejectOnEmpty: true, - }); - - const [team, user] = await Promise.all([ - Team.findByPk(fileOperation.teamId, { rejectOnEmpty: true }), - User.findByPk(fileOperation.userId, { rejectOnEmpty: true }), - ]); - - const collectionIds = fileOperation.collectionId - ? [fileOperation.collectionId] - : await user.collectionIds(); - - const collections = await Collection.findAll({ - where: { - id: collectionIds, - }, - }); - - try { - Logger.info("task", `ExportTask processing data for ${fileOperationId}`); - - await this.updateFileOperation(fileOperation, { - state: FileOperationState.Creating, - }); - - const filePath = await archiveCollections(collections); - - Logger.info("task", `ExportTask uploading data for ${fileOperationId}`); - - await this.updateFileOperation(fileOperation, { - state: FileOperationState.Uploading, - }); - - const fileBuffer = await fs.promises.readFile(filePath); - const stat = await fs.promises.stat(filePath); - const url = await uploadToS3FromBuffer( - fileBuffer, - "application/zip", - fileOperation.key, - "private" - ); - - await this.updateFileOperation(fileOperation, { - size: stat.size, - state: FileOperationState.Complete, - url, - }); - - await ExportSuccessEmail.schedule({ - to: user.email, - userId: user.id, - id: fileOperation.id, - teamUrl: team.url, - teamId: team.id, - }); - } catch (error) { - await this.updateFileOperation(fileOperation, { - state: FileOperationState.Error, - error, - }); - await ExportFailureEmail.schedule({ - to: user.email, - userId: user.id, - teamUrl: team.url, - teamId: team.id, - }); - throw error; - } - } - - /** - * Update the state of the underlying FileOperation in the database and send - * an event to the client. - * - * @param fileOperation The FileOperation to update - */ - private async updateFileOperation( - fileOperation: FileOperation, - options: Partial & { error?: Error } - ) { - await fileOperation.update({ - ...options, - error: options.error - ? truncate(options.error.message, { length: 255 }) - : undefined, - }); - - await Event.schedule({ - name: "fileOperations.update", - modelId: fileOperation.id, - teamId: fileOperation.teamId, - actorId: fileOperation.userId, - data: fileOperationPresenter(fileOperation), - }); - } - - /** - * Job options such as priority and retry strategy, as defined by Bull. - */ - public get options() { - return { - priority: TaskPriority.Background, - attempts: 1, - }; +export default class ExportMarkdownZipTask extends ExportTask { + public async export(collections: Collection[]) { + return await archiveCollections( + collections, + FileOperationFormat.MarkdownZip + ); } } diff --git a/server/queues/tasks/ExportTask.ts b/server/queues/tasks/ExportTask.ts new file mode 100644 index 000000000..d4b6e692d --- /dev/null +++ b/server/queues/tasks/ExportTask.ts @@ -0,0 +1,138 @@ +import fs from "fs"; +import { truncate } from "lodash"; +import { FileOperationState } from "@shared/types"; +import ExportFailureEmail from "@server/emails/templates/ExportFailureEmail"; +import ExportSuccessEmail from "@server/emails/templates/ExportSuccessEmail"; +import Logger from "@server/logging/Logger"; +import { Collection, Event, FileOperation, Team, User } from "@server/models"; +import fileOperationPresenter from "@server/presenters/fileOperation"; +import { uploadToS3 } from "@server/utils/s3"; +import BaseTask, { TaskPriority } from "./BaseTask"; + +type Props = { + fileOperationId: string; +}; + +export default abstract class ExportTask extends BaseTask { + /** + * Transforms the data to be exported, uploads, and notifies user. + * + * @param props The props + */ + public async perform({ fileOperationId }: Props) { + Logger.info("task", `ExportTask fetching data for ${fileOperationId}`); + const fileOperation = await FileOperation.findByPk(fileOperationId, { + rejectOnEmpty: true, + }); + + const [team, user] = await Promise.all([ + Team.findByPk(fileOperation.teamId, { rejectOnEmpty: true }), + User.findByPk(fileOperation.userId, { rejectOnEmpty: true }), + ]); + + const collectionIds = fileOperation.collectionId + ? [fileOperation.collectionId] + : await user.collectionIds(); + + const collections = await Collection.findAll({ + where: { + id: collectionIds, + }, + }); + + try { + Logger.info("task", `ExportTask processing data for ${fileOperationId}`); + + await this.updateFileOperation(fileOperation, { + state: FileOperationState.Creating, + }); + + const filePath = await this.export(collections); + + Logger.info("task", `ExportTask uploading data for ${fileOperationId}`); + + await this.updateFileOperation(fileOperation, { + state: FileOperationState.Uploading, + }); + + const stat = await fs.promises.stat(filePath); + const url = await uploadToS3({ + body: fs.createReadStream(filePath), + contentLength: stat.size, + contentType: "application/zip", + key: fileOperation.key, + acl: "private", + }); + + await this.updateFileOperation(fileOperation, { + size: stat.size, + state: FileOperationState.Complete, + url, + }); + + await ExportSuccessEmail.schedule({ + to: user.email, + userId: user.id, + id: fileOperation.id, + teamUrl: team.url, + teamId: team.id, + }); + } catch (error) { + await this.updateFileOperation(fileOperation, { + state: FileOperationState.Error, + error, + }); + await ExportFailureEmail.schedule({ + to: user.email, + userId: user.id, + teamUrl: team.url, + teamId: team.id, + }); + throw error; + } + } + + /** + * Transform the data in all of the passed collections into a single Buffer. + * + * @param collections The collections to export + * @returns A promise that resolves to a temporary file path + */ + protected abstract export(collections: Collection[]): Promise; + + /** + * Update the state of the underlying FileOperation in the database and send + * an event to the client. + * + * @param fileOperation The FileOperation to update + */ + private async updateFileOperation( + fileOperation: FileOperation, + options: Partial & { error?: Error } + ) { + await fileOperation.update({ + ...options, + error: options.error + ? truncate(options.error.message, { length: 255 }) + : undefined, + }); + + await Event.schedule({ + name: "fileOperations.update", + modelId: fileOperation.id, + teamId: fileOperation.teamId, + actorId: fileOperation.userId, + data: fileOperationPresenter(fileOperation), + }); + } + + /** + * Job options such as priority and retry strategy, as defined by Bull. + */ + public get options() { + return { + priority: TaskPriority.Background, + attempts: 1, + }; + } +} diff --git a/server/queues/tasks/ImportTask.ts b/server/queues/tasks/ImportTask.ts index 2155200c5..1e72367e7 100644 --- a/server/queues/tasks/ImportTask.ts +++ b/server/queues/tasks/ImportTask.ts @@ -1,6 +1,6 @@ import { S3 } from "aws-sdk"; import { truncate } from "lodash"; -import { CollectionPermission } from "@shared/types"; +import { CollectionPermission, FileOperationState } from "@shared/types"; import { CollectionValidation } from "@shared/validations"; import attachmentCreator from "@server/commands/attachmentCreator"; import documentCreator from "@server/commands/documentCreator"; @@ -15,7 +15,6 @@ import { FileOperation, Attachment, } from "@server/models"; -import { FileOperationState } from "@server/models/FileOperation"; import BaseTask, { TaskPriority } from "./BaseTask"; type Props = { diff --git a/server/routes/api/collections.ts b/server/routes/api/collections.ts index 00b580069..b424a6206 100644 --- a/server/routes/api/collections.ts +++ b/server/routes/api/collections.ts @@ -3,7 +3,12 @@ import invariant from "invariant"; import Router from "koa-router"; import { Sequelize, Op, WhereOptions } from "sequelize"; import { randomElement } from "@shared/random"; -import { CollectionPermission } from "@shared/types"; +import { + CollectionPermission, + FileOperationFormat, + FileOperationState, + FileOperationType, +} from "@shared/types"; import { colorPalette } from "@shared/utils/collections"; import { RateLimiterStrategy } from "@server/RateLimiter"; import collectionExporter from "@server/commands/collectionExporter"; @@ -27,11 +32,6 @@ import { Attachment, FileOperation, } from "@server/models"; -import { - FileOperationFormat, - FileOperationState, - FileOperationType, -} from "@server/models/FileOperation"; import { authorize } from "@server/policies"; import { presentCollection, @@ -576,16 +576,20 @@ router.post( router.post( "collections.export_all", auth(), - rateLimiter(RateLimiterStrategy.TenPerHour), + rateLimiter(RateLimiterStrategy.FivePerHour), async (ctx) => { + const { format = FileOperationFormat.MarkdownZip } = ctx.request.body; const { user } = ctx.state; const team = await Team.findByPk(user.teamId); authorize(user, "createExport", team); + assertIn(format, Object.values(FileOperationFormat), "Invalid format"); + const fileOperation = await sequelize.transaction(async (transaction) => { return collectionExporter({ user, team, + format, ip: ctx.request.ip, transaction, }); diff --git a/server/routes/api/fileOperations.test.ts b/server/routes/api/fileOperations.test.ts index 3fd1f4c5c..2c0dbb216 100644 --- a/server/routes/api/fileOperations.test.ts +++ b/server/routes/api/fileOperations.test.ts @@ -1,8 +1,5 @@ +import { FileOperationState, FileOperationType } from "@shared/types"; import { Collection, User, Event, FileOperation } from "@server/models"; -import { - FileOperationState, - FileOperationType, -} from "@server/models/FileOperation"; import { buildAdmin, buildCollection, diff --git a/server/routes/api/fileOperations.ts b/server/routes/api/fileOperations.ts index f0483de75..e522e683d 100644 --- a/server/routes/api/fileOperations.ts +++ b/server/routes/api/fileOperations.ts @@ -1,10 +1,10 @@ import Router from "koa-router"; import { WhereOptions } from "sequelize/types"; +import { FileOperationType } from "@shared/types"; import fileOperationDeleter from "@server/commands/fileOperationDeleter"; import { ValidationError } from "@server/errors"; import auth from "@server/middlewares/authentication"; import { FileOperation, Team } from "@server/models"; -import { FileOperationType } from "@server/models/FileOperation"; import { authorize } from "@server/policies"; import { presentFileOperation } from "@server/presenters"; import { ContextWithState } from "@server/types"; diff --git a/server/test/factories.ts b/server/test/factories.ts index f5cfd1cb6..4776b4913 100644 --- a/server/test/factories.ts +++ b/server/test/factories.ts @@ -1,6 +1,10 @@ import { isNull } from "lodash"; import { v4 as uuidv4 } from "uuid"; -import { CollectionPermission } from "@shared/types"; +import { + CollectionPermission, + FileOperationState, + FileOperationType, +} from "@shared/types"; import { Share, Team, @@ -21,10 +25,6 @@ import { ApiKey, Subscription, } from "@server/models"; -import { - FileOperationState, - FileOperationType, -} from "@server/models/FileOperation"; let count = 1; diff --git a/server/utils/__mocks__/s3.ts b/server/utils/__mocks__/s3.ts index 407f94a80..59bfbedaa 100644 --- a/server/utils/__mocks__/s3.ts +++ b/server/utils/__mocks__/s3.ts @@ -1,4 +1,4 @@ -export const uploadToS3FromBuffer = jest.fn().mockReturnValue("/endpoint/key"); +export const uploadToS3 = jest.fn().mockReturnValue("/endpoint/key"); export const publicS3Endpoint = jest.fn().mockReturnValue("http://mock"); diff --git a/server/utils/s3.ts b/server/utils/s3.ts index 30345606d..88d79fb7e 100644 --- a/server/utils/s3.ts +++ b/server/utils/s3.ts @@ -85,21 +85,28 @@ export const publicS3Endpoint = (isServerUpload?: boolean) => { }${AWS_S3_UPLOAD_BUCKET_NAME}`; }; -export const uploadToS3FromBuffer = async ( - buffer: Buffer, - contentType: string, - key: string, - acl: string -) => { +export const uploadToS3 = async ({ + body, + contentLength, + contentType, + key, + acl, +}: { + body: S3.Body; + contentLength: number; + contentType: string; + key: string; + acl: string; +}) => { await s3 .putObject({ ACL: acl, Bucket: AWS_S3_UPLOAD_BUCKET_NAME, Key: key, ContentType: contentType, - ContentLength: buffer.length, + ContentLength: contentLength, ContentDisposition: "attachment", - Body: buffer, + Body: body, }) .promise(); const endpoint = publicS3Endpoint(true); diff --git a/server/utils/zip.ts b/server/utils/zip.ts index ca5193a4e..6758ddac6 100644 --- a/server/utils/zip.ts +++ b/server/utils/zip.ts @@ -3,6 +3,7 @@ import path from "path"; import JSZip, { JSZipObject } from "jszip"; import { find } from "lodash"; import tmp from "tmp"; +import { FileOperationFormat } from "@shared/types"; import { ValidationError } from "@server/errors"; import Logger from "@server/logging/Logger"; import Attachment from "@server/models/Attachment"; @@ -26,9 +27,21 @@ export type Item = { item: JSZipObject; }; +export type FileTreeNode = { + /** The title, extracted from the file name */ + title: string; + /** The file name including extension */ + name: string; + /** The full path to within the zip file */ + path: string; + /** The nested children */ + children: FileTreeNode[]; +}; + async function addDocumentTreeToArchive( zip: JSZip, - documents: NavigationNode[] + documents: NavigationNode[], + format = FileOperationFormat.MarkdownZip ) { for (const doc of documents) { const document = await Document.findByPk(doc.id); @@ -37,7 +50,10 @@ async function addDocumentTreeToArchive( continue; } - let text = DocumentHelper.toMarkdown(document); + let text = + format === FileOperationFormat.HTMLZip + ? await DocumentHelper.toHTML(document) + : await DocumentHelper.toMarkdown(document); const attachments = await Attachment.findAll({ where: { teamId: document.teamId, @@ -52,7 +68,9 @@ async function addDocumentTreeToArchive( let title = serializeFilename(document.title) || "Untitled"; - title = safeAddFileToArchive(zip, `${title}.md`, text, { + const extension = format === FileOperationFormat.HTMLZip ? "html" : "md"; + + title = safeAddFileToArchive(zip, `${title}.${extension}`, text, { date: document.updatedAt, comment: JSON.stringify({ createdAt: document.createdAt, @@ -161,7 +179,10 @@ async function archiveToPath(zip: JSZip): Promise { }); } -export async function archiveCollections(collections: Collection[]) { +export async function archiveCollections( + collections: Collection[], + format: FileOperationFormat +) { const zip = new JSZip(); for (const collection of collections) { @@ -169,7 +190,11 @@ export async function archiveCollections(collections: Collection[]) { const folder = zip.folder(serializeFilename(collection.name)); if (folder) { - await addDocumentTreeToArchive(folder, collection.documentStructure); + await addDocumentTreeToArchive( + folder, + collection.documentStructure, + format + ); } } } @@ -177,17 +202,6 @@ export async function archiveCollections(collections: Collection[]) { return archiveToPath(zip); } -export type FileTreeNode = { - /** The title, extracted from the file name */ - title: string; - /** The file name including extension */ - name: string; - /** The full path to within the zip file */ - path: string; - /** The nested children */ - children: FileTreeNode[]; -}; - /** * Converts the flat structure returned by JSZIP into a nested file structure * for easier processing. diff --git a/shared/i18n/locales/en_US/translation.json b/shared/i18n/locales/en_US/translation.json index 0faf7ad4a..2d96608f6 100644 --- a/shared/i18n/locales/en_US/translation.json +++ b/shared/i18n/locales/en_US/translation.json @@ -627,6 +627,8 @@ "Please choose a single file to import": "Please choose a single file to import", "Your import is being processed, you can safely leave this page": "Your import is being processed, you can safely leave this page", "File not supported – please upload a valid ZIP file": "File not supported – please upload a valid ZIP file", + "A ZIP file containing the images, and documents in the Markdown format.": "A ZIP file containing the images, and documents in the Markdown format.", + "A ZIP file containing the images, and documents as HTML files.": "A ZIP file containing the images, and documents as HTML files.", "Completed": "Completed", "Processing": "Processing", "Expired": "Expired", @@ -691,12 +693,9 @@ "Choose a subdomain to enable a login page just for your team.": "Choose a subdomain to enable a login page just for your team.", "Start view": "Start view", "This is the screen that workspace members will first see when they sign in.": "This is the screen that workspace members will first see when they sign in.", - "Export in progress…": "Export in progress…", + "Export data": "Export data", "Export deleted": "Export deleted", "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – if you have notifications enabled, we will email a link to {{ userEmail }} when it’s complete.": "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – if you have notifications enabled, we will email a link to {{ userEmail }} when it’s complete.", - "Export Requested": "Export Requested", - "Requesting Export": "Requesting Export", - "Export Data": "Export Data", "Recent exports": "Recent exports", "Manage optional and beta features. Changing these settings will affect the experience for all members of the workspace.": "Manage optional and beta features. Changing these settings will affect the experience for all members of the workspace.", "Seamless editing": "Seamless editing", diff --git a/shared/types.ts b/shared/types.ts index 8f42cadf6..43b04e8ca 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -7,6 +7,32 @@ export enum Client { Desktop = "desktop", } +export enum ExportContentType { + Markdown = "text/markdown", + Html = "text/html", + Pdf = "application/pdf", +} + +export enum FileOperationFormat { + MarkdownZip = "outline-markdown", + HTMLZip = "html", + PDFZip = "pdf", + Notion = "notion", +} + +export enum FileOperationType { + Import = "import", + Export = "export", +} + +export enum FileOperationState { + Creating = "creating", + Uploading = "uploading", + Complete = "complete", + Error = "error", + Expired = "expired", +} + export type PublicEnv = { URL: string; CDN_URL: string;