yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: Refactor data import (#3434)

Browse Source 
* Complete refactor of import

* feat: Notion data import (#3442)
This commit is contained in:
Tom Moor
2022-04-23 10:07:35 -07:00
committed by GitHub
parent bdcfaae025
commit 33ce49cc33
45 changed files with 2217 additions and 1066 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
server/routes/api/collections.ts
View File

@@ -4,8 +4,10 @@ import Router from "koa-router";
import { Sequelize, Op, WhereOptions } from "sequelize";
import collectionExporter from "@server/commands/collectionExporter";
import teamUpdater from "@server/commands/teamUpdater";
import { sequelize } from "@server/database/sequelize";
import { ValidationError } from "@server/errors";
import auth from "@server/middlewares/authentication";
import {
Collection,
CollectionUser,
@@ -15,7 +17,13 @@ import {
User,
Group,
Attachment,
FileOperation,
} from "@server/models";
import {
FileOperationFormat,
FileOperationState,
FileOperationType,
} from "@server/models/FileOperation";
import { authorize } from "@server/policies";
import {
presentCollection,
@@ -134,22 +142,47 @@ router.post("collections.info", auth(), async (ctx) => {
});
router.post("collections.import", auth(), async (ctx) => {
const { type, attachmentId } = ctx.body;
assertIn(type, ["outline"], "type must be one of 'outline'");
const { attachmentId, format = FileOperationFormat.MarkdownZip } = ctx.body;
assertUuid(attachmentId, "attachmentId is required");
const { user } = ctx.state;
authorize(user, "importCollection", user.team);
const attachment = await Attachment.findByPk(attachmentId);
authorize(user, "read", attachment);
await Event.create({
name: "collections.import",
modelId: attachmentId,
teamId: user.teamId,
actorId: user.id,
data: {
type,
},
ip: ctx.request.ip,
assertIn(format, Object.values(FileOperationFormat), "Invalid format");
await sequelize.transaction(async (transaction) => {
const fileOperation = await FileOperation.create(
{
type: FileOperationType.Import,
state: FileOperationState.Creating,
format,
size: attachment.size,
key: attachment.key,
userId: user.id,
teamId: user.teamId,
},
{
transaction,
}
);
await Event.create(
{
name: "fileOperations.create",
teamId: user.teamId,
actorId: user.id,
modelId: fileOperation.id,
data: {
type: FileOperationType.Import,
},
},
{
transaction,
}
);
});
ctx.body = {

173
server/routes/api/documents.ts
View File

@@ -1,3 +1,4 @@
import fs from "fs-extra";
import invariant from "invariant";
import Router from "koa-router";
import { Op, ScopeOptions, WhereOptions } from "sequelize";
@@ -6,6 +7,7 @@ import documentCreator from "@server/commands/documentCreator";
import documentImporter from "@server/commands/documentImporter";
import documentMover from "@server/commands/documentMover";
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
import documentUpdater from "@server/commands/documentUpdater";
import { sequelize } from "@server/database/sequelize";
import {
NotFoundError,
@@ -999,8 +1001,6 @@ router.post("documents.update", auth(), async (ctx) => {
text,
fullWidth,
publish,
autosave,
done,
lastRevision,
templateId,
append,
@@ -1012,91 +1012,37 @@ router.post("documents.update", auth(), async (ctx) => {
}
const { user } = ctx.state;
const document = await Document.findByPk(id, {
userId: user.id,
});
authorize(user, "update", document);
let collection: Collection | null | undefined;
if (lastRevision && lastRevision !== document.revisionCount) {
throw InvalidRequestError("Document has changed since last revision");
}
const document = await sequelize.transaction(async (transaction) => {
const document = await Document.findByPk(id, {
userId: user.id,
transaction,
});
authorize(user, "update", document);
const previousTitle = document.title;
collection = document.collection;
// Update document
if (title !== undefined) {
document.title = title;
}
if (editorVersion) {
document.editorVersion = editorVersion;
}
if (templateId) {
document.templateId = templateId;
}
if (fullWidth !== undefined) {
document.fullWidth = fullWidth;
}
if (!user.team?.collaborativeEditing) {
if (append) {
document.text += text;
} else if (text !== undefined) {
document.text = text;
if (lastRevision && lastRevision !== document.revisionCount) {
throw InvalidRequestError("Document has changed since last revision");
}
}
document.lastModifiedById = user.id;
const { collection } = document;
const changed = document.changed();
if (publish) {
await document.publish(user.id);
} else {
await document.save();
}
if (publish) {
await Event.create({
name: "documents.publish",
documentId: document.id,
collectionId: document.collectionId,
teamId: document.teamId,
actorId: user.id,
data: {
title: document.title,
},
return documentUpdater({
document,
user,
title,
text,
fullWidth,
publish,
append,
templateId,
editorVersion,
transaction,
ip: ctx.request.ip,
});
} else if (changed) {
await Event.create({
name: "documents.update",
documentId: document.id,
collectionId: document.collectionId,
teamId: document.teamId,
actorId: user.id,
data: {
autosave,
done,
title: document.title,
},
ip: ctx.request.ip,
});
}
});
if (document.title !== previousTitle) {
Event.schedule({
name: "documents.title_change",
documentId: document.id,
collectionId: document.collectionId,
teamId: document.teamId,
actorId: user.id,
data: {
previousTitle,
title: document.title,
},
ip: ctx.request.ip,
});
}
invariant(collection, "collection not found");
document.updatedBy = user;
document.collection = collection;
@@ -1342,22 +1288,31 @@ router.post("documents.import", auth(), async (ctx) => {
});
}
const { text, title } = await documentImporter({
user,
file,
ip: ctx.request.ip,
});
const document = await documentCreator({
source: "import",
title,
text,
publish,
collectionId,
parentDocumentId,
index,
user,
ip: ctx.request.ip,
const content = await fs.readFile(file.path, "utf8");
const document = await sequelize.transaction(async (transaction) => {
const { text, title } = await documentImporter({
user,
fileName: file.name,
mimeType: file.type,
content,
ip: ctx.request.ip,
transaction,
});
return documentCreator({
source: "import",
title,
text,
publish,
collectionId,
parentDocumentId,
index,
user,
ip: ctx.request.ip,
transaction,
});
});
document.collection = collection;
return (ctx.body = {
@@ -1414,7 +1369,7 @@ router.post("documents.create", auth(), async (ctx) => {
});
}
let templateDocument;
let templateDocument: Document | null | undefined;
if (templateId) {
templateDocument = await Document.findByPk(templateId, {
@@ -1423,19 +1378,23 @@ router.post("documents.create", auth(), async (ctx) => {
authorize(user, "read", templateDocument);
}
const document = await documentCreator({
title,
text,
publish,
collectionId,
parentDocumentId,
templateDocument,
template,
index,
user,
editorVersion,
ip: ctx.request.ip,
const document = await sequelize.transaction(async (transaction) => {
return documentCreator({
title,
text,
publish,
collectionId,
parentDocumentId,
templateDocument,
template,
index,
user,
editorVersion,
ip: ctx.request.ip,
transaction,
});
});
document.collection = collection;
return (ctx.body = {

40
server/routes/api/fileOperations.test.ts
View File

@@ -1,5 +1,9 @@
import TestServer from "fetch-test-server";
import { Collection, User, Event, FileOperation } from "@server/models";
import {
FileOperationState,
FileOperationType,
} from "@server/models/FileOperation";
import webService from "@server/services/web";
import {
buildAdmin,
@@ -23,7 +27,7 @@ describe("#fileOperations.info", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
});
@@ -31,7 +35,7 @@ describe("#fileOperations.info", () => {
body: {
id: exportData.id,
token: admin.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
const body = await res.json();
@@ -49,7 +53,7 @@ describe("#fileOperations.info", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
});
@@ -57,7 +61,7 @@ describe("#fileOperations.info", () => {
body: {
id: exportData.id,
token: user.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
expect(res.status).toEqual(403);
@@ -71,14 +75,14 @@ describe("#fileOperations.list", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
const body = await res.json();
@@ -100,7 +104,7 @@ describe("#fileOperations.list", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
collectionId: collection.id,
@@ -108,7 +112,7 @@ describe("#fileOperations.list", () => {
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
const body = await res.json();
@@ -131,7 +135,7 @@ describe("#fileOperations.list", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
collectionId: collection.id,
@@ -142,7 +146,7 @@ describe("#fileOperations.list", () => {
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
const body = await res.json();
@@ -168,7 +172,7 @@ describe("#fileOperations.list", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
collectionId: collection.id,
@@ -179,7 +183,7 @@ describe("#fileOperations.list", () => {
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin2.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
const body = await res.json();
@@ -197,7 +201,7 @@ describe("#fileOperations.list", () => {
const res = await server.post("/api/fileOperations.list", {
body: {
token: user.getJwtToken(),
type: "export",
type: FileOperationType.Export,
},
});
expect(res.status).toEqual(403);
@@ -211,7 +215,7 @@ describe("#fileOperations.redirect", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
});
@@ -234,7 +238,7 @@ describe("#fileOperations.info", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
});
@@ -259,7 +263,7 @@ describe("#fileOperations.info", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
});
@@ -280,10 +284,10 @@ describe("#fileOperations.delete", () => {
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
type: FileOperationType.Export,
teamId: team.id,
userId: admin.id,
state: "complete",
state: FileOperationState.Complete,
});
const deleteResponse = await server.post("/api/fileOperations.delete", {
body: {