yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Bulk HTML export (#4620)

Browse Source 
* wip

* Working bulk html export

* Refactor

* test

* test
This commit is contained in:
Tom Moor
2022-12-30 17:42:20 +00:00
committed by GitHub
parent 1b8dd9399c
commit f3469d25fe
32 changed files with 485 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
server/queues/processors/FileOperationsProcessor.ts
View File

@@ -1,10 +1,8 @@
import invariant from "invariant";
import { FileOperationFormat, FileOperationType } from "@shared/types";
import { FileOperation } from "@server/models";
import {
FileOperationFormat,
FileOperationType,
} from "@server/models/FileOperation";
import { Event as TEvent, FileOperationEvent } from "@server/types";
import ExportHTMLZipTask from "../tasks/ExportHTMLZipTask";
import ExportMarkdownZipTask from "../tasks/ExportMarkdownZipTask";
import ImportMarkdownZipTask from "../tasks/ImportMarkdownZipTask";
import ImportNotionTask from "../tasks/ImportNotionTask";
@@ -40,6 +38,11 @@ export default class FileOperationsProcessor extends BaseProcessor {
if (fileOperation.type === FileOperationType.Export) {
switch (fileOperation.format) {
case FileOperationFormat.HTMLZip:
await ExportHTMLZipTask.schedule({
fileOperationId: event.modelId,
});
break;
case FileOperationFormat.MarkdownZip:
await ExportMarkdownZipTask.schedule({
fileOperationId: event.modelId,

5
server/queues/tasks/CleanupExpiredFileOperationsTask.test.ts
View File

@@ -1,9 +1,6 @@
import { subDays } from "date-fns";
import { FileOperationState, FileOperationType } from "@shared/types";
import { FileOperation } from "@server/models";
import {
FileOperationState,
FileOperationType,
} from "@server/models/FileOperation";
import { buildFileOperation } from "@server/test/factories";
import { setupTestDatabase } from "@server/test/support";
import CleanupExpiredFileOperationsTask from "./CleanupExpiredFileOperationsTask";

2
server/queues/tasks/CleanupExpiredFileOperationsTask.ts
View File

@@ -1,8 +1,8 @@
import { subDays } from "date-fns";
import { Op } from "sequelize";
import { FileOperationState } from "@shared/types";
import Logger from "@server/logging/Logger";
import { FileOperation } from "@server/models";
import { FileOperationState } from "@server/models/FileOperation";
import BaseTask, { TaskPriority } from "./BaseTask";
type Props = {

10
server/queues/tasks/ExportHTMLZipTask.ts Normal file
View File

@@ -0,0 +1,10 @@
import { FileOperationFormat } from "@shared/types";
import { Collection } from "@server/models";
import { archiveCollections } from "@server/utils/zip";
import ExportTask from "./ExportTask";
export default class ExportHTMLZipTask extends ExportTask {
public async export(collections: Collection[]) {
return await archiveCollections(collections, FileOperationFormat.HTMLZip);
}
}

135
server/queues/tasks/ExportMarkdownZipTask.ts
View File

@@ -1,130 +1,13 @@
import fs from "fs";
import { truncate } from "lodash";
import ExportFailureEmail from "@server/emails/templates/ExportFailureEmail";
import ExportSuccessEmail from "@server/emails/templates/ExportSuccessEmail";
import Logger from "@server/logging/Logger";
import { Collection, Event, FileOperation, Team, User } from "@server/models";
import { FileOperationState } from "@server/models/FileOperation";
import fileOperationPresenter from "@server/presenters/fileOperation";
import { uploadToS3FromBuffer } from "@server/utils/s3";
import { FileOperationFormat } from "@shared/types";
import { Collection } from "@server/models";
import { archiveCollections } from "@server/utils/zip";
import BaseTask, { TaskPriority } from "./BaseTask";
import ExportTask from "./ExportTask";
type Props = {
fileOperationId: string;
};
export default class ExportMarkdownZipTask extends BaseTask<Props> {
/**
* Runs the export task.
*
* @param props The props
*/
public async perform({ fileOperationId }: Props) {
const fileOperation = await FileOperation.findByPk(fileOperationId, {
rejectOnEmpty: true,
});
const [team, user] = await Promise.all([
Team.findByPk(fileOperation.teamId, { rejectOnEmpty: true }),
User.findByPk(fileOperation.userId, { rejectOnEmpty: true }),
]);
const collectionIds = fileOperation.collectionId
? [fileOperation.collectionId]
: await user.collectionIds();
const collections = await Collection.findAll({
where: {
id: collectionIds,
},
});
try {
Logger.info("task", `ExportTask processing data for ${fileOperationId}`);
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Creating,
});
const filePath = await archiveCollections(collections);
Logger.info("task", `ExportTask uploading data for ${fileOperationId}`);
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Uploading,
});
const fileBuffer = await fs.promises.readFile(filePath);
const stat = await fs.promises.stat(filePath);
const url = await uploadToS3FromBuffer(
fileBuffer,
"application/zip",
fileOperation.key,
"private"
);
await this.updateFileOperation(fileOperation, {
size: stat.size,
state: FileOperationState.Complete,
url,
});
await ExportSuccessEmail.schedule({
to: user.email,
userId: user.id,
id: fileOperation.id,
teamUrl: team.url,
teamId: team.id,
});
} catch (error) {
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Error,
error,
});
await ExportFailureEmail.schedule({
to: user.email,
userId: user.id,
teamUrl: team.url,
teamId: team.id,
});
throw error;
}
}
/**
* Update the state of the underlying FileOperation in the database and send
* an event to the client.
*
* @param fileOperation The FileOperation to update
*/
private async updateFileOperation(
fileOperation: FileOperation,
options: Partial<FileOperation> & { error?: Error }
) {
await fileOperation.update({
...options,
error: options.error
? truncate(options.error.message, { length: 255 })
: undefined,
});
await Event.schedule({
name: "fileOperations.update",
modelId: fileOperation.id,
teamId: fileOperation.teamId,
actorId: fileOperation.userId,
data: fileOperationPresenter(fileOperation),
});
}
/**
* Job options such as priority and retry strategy, as defined by Bull.
*/
public get options() {
return {
priority: TaskPriority.Background,
attempts: 1,
};
export default class ExportMarkdownZipTask extends ExportTask {
public async export(collections: Collection[]) {
return await archiveCollections(
collections,
FileOperationFormat.MarkdownZip
);
}
}

138
server/queues/tasks/ExportTask.ts Normal file
View File

@@ -0,0 +1,138 @@
import fs from "fs";
import { truncate } from "lodash";
import { FileOperationState } from "@shared/types";
import ExportFailureEmail from "@server/emails/templates/ExportFailureEmail";
import ExportSuccessEmail from "@server/emails/templates/ExportSuccessEmail";
import Logger from "@server/logging/Logger";
import { Collection, Event, FileOperation, Team, User } from "@server/models";
import fileOperationPresenter from "@server/presenters/fileOperation";
import { uploadToS3 } from "@server/utils/s3";
import BaseTask, { TaskPriority } from "./BaseTask";
type Props = {
fileOperationId: string;
};
export default abstract class ExportTask extends BaseTask<Props> {
/**
* Transforms the data to be exported, uploads, and notifies user.
*
* @param props The props
*/
public async perform({ fileOperationId }: Props) {
Logger.info("task", `ExportTask fetching data for ${fileOperationId}`);
const fileOperation = await FileOperation.findByPk(fileOperationId, {
rejectOnEmpty: true,
});
const [team, user] = await Promise.all([
Team.findByPk(fileOperation.teamId, { rejectOnEmpty: true }),
User.findByPk(fileOperation.userId, { rejectOnEmpty: true }),
]);
const collectionIds = fileOperation.collectionId
? [fileOperation.collectionId]
: await user.collectionIds();
const collections = await Collection.findAll({
where: {
id: collectionIds,
},
});
try {
Logger.info("task", `ExportTask processing data for ${fileOperationId}`);
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Creating,
});
const filePath = await this.export(collections);
Logger.info("task", `ExportTask uploading data for ${fileOperationId}`);
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Uploading,
});
const stat = await fs.promises.stat(filePath);
const url = await uploadToS3({
body: fs.createReadStream(filePath),
contentLength: stat.size,
contentType: "application/zip",
key: fileOperation.key,
acl: "private",
});
await this.updateFileOperation(fileOperation, {
size: stat.size,
state: FileOperationState.Complete,
url,
});
await ExportSuccessEmail.schedule({
to: user.email,
userId: user.id,
id: fileOperation.id,
teamUrl: team.url,
teamId: team.id,
});
} catch (error) {
await this.updateFileOperation(fileOperation, {
state: FileOperationState.Error,
error,
});
await ExportFailureEmail.schedule({
to: user.email,
userId: user.id,
teamUrl: team.url,
teamId: team.id,
});
throw error;
}
}
/**
* Transform the data in all of the passed collections into a single Buffer.
*
* @param collections The collections to export
* @returns A promise that resolves to a temporary file path
*/
protected abstract export(collections: Collection[]): Promise<string>;
/**
* Update the state of the underlying FileOperation in the database and send
* an event to the client.
*
* @param fileOperation The FileOperation to update
*/
private async updateFileOperation(
fileOperation: FileOperation,
options: Partial<FileOperation> & { error?: Error }
) {
await fileOperation.update({
...options,
error: options.error
? truncate(options.error.message, { length: 255 })
: undefined,
});
await Event.schedule({
name: "fileOperations.update",
modelId: fileOperation.id,
teamId: fileOperation.teamId,
actorId: fileOperation.userId,
data: fileOperationPresenter(fileOperation),
});
}
/**
* Job options such as priority and retry strategy, as defined by Bull.
*/
public get options() {
return {
priority: TaskPriority.Background,
attempts: 1,
};
}
}

3
server/queues/tasks/ImportTask.ts
View File

@@ -1,6 +1,6 @@
import { S3 } from "aws-sdk";
import { truncate } from "lodash";
import { CollectionPermission } from "@shared/types";
import { CollectionPermission, FileOperationState } from "@shared/types";
import { CollectionValidation } from "@shared/validations";
import attachmentCreator from "@server/commands/attachmentCreator";
import documentCreator from "@server/commands/documentCreator";
@@ -15,7 +15,6 @@ import {
FileOperation,
Attachment,
} from "@server/models";
import { FileOperationState } from "@server/models/FileOperation";
import BaseTask, { TaskPriority } from "./BaseTask";
type Props = {