yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: Refactor worker, emails and data cleanup to task system (#3337)

Browse Source 
* Refactor worker, all emails on task system

* fix

* lint

* fix: Remove a bunch of expect-error comments in related tests

* refactor: Move work from utils.gc into tasks

* test

* Add tracing to tasks and processors
fix: DebounceProcessor triggering on all events
Event.add -> Event.schedule
This commit is contained in:
Tom Moor
2022-04-06 16:48:28 -07:00
committed by GitHub
parent 9c766362ed
commit dbfdcd6d23
41 changed files with 729 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/routes/api/documents.ts
View File

@@ -1024,7 +1024,7 @@ router.post("documents.update", auth(), async (ctx) => {
}
if (document.title !== previousTitle) {
Event.add({
Event.schedule({
name: "documents.title_change",
documentId: document.id,
collectionId: document.collectionId,

124
server/routes/api/utils.test.ts
View File

@@ -1,8 +1,5 @@
import { subDays } from "date-fns";
import TestServer from "fetch-test-server";
import { Document, FileOperation } from "@server/models";
import webService from "@server/services/web";
import { buildDocument, buildFileOperation } from "@server/test/factories";
import { flushdb } from "@server/test/support";
const app = webService();
@@ -12,127 +9,6 @@ beforeEach(() => flushdb());
afterAll(() => server.close());
describe("#utils.gc", () => {
it("should not destroy documents not deleted", async () => {
await buildDocument({
publishedAt: new Date(),
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
expect(res.status).toEqual(200);
expect(
await Document.unscoped().count({
paranoid: false,
})
).toEqual(1);
});
it("should not destroy documents deleted less than 30 days ago", async () => {
await buildDocument({
publishedAt: new Date(),
deletedAt: subDays(new Date(), 25),
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
expect(res.status).toEqual(200);
expect(
await Document.unscoped().count({
paranoid: false,
})
).toEqual(1);
});
it("should destroy documents deleted more than 30 days ago", async () => {
await buildDocument({
publishedAt: new Date(),
deletedAt: subDays(new Date(), 60),
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
expect(res.status).toEqual(200);
expect(
await Document.unscoped().count({
paranoid: false,
})
).toEqual(0);
});
it("should destroy draft documents deleted more than 30 days ago", async () => {
await buildDocument({
publishedAt: undefined,
deletedAt: subDays(new Date(), 60),
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
expect(res.status).toEqual(200);
expect(
await Document.unscoped().count({
paranoid: false,
})
).toEqual(0);
});
it("should expire exports older than 30 days ago", async () => {
await buildFileOperation({
type: "export",
state: "complete",
createdAt: subDays(new Date(), 30),
});
await buildFileOperation({
type: "export",
state: "complete",
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
const data = await FileOperation.count({
where: {
type: "export",
state: "expired",
},
});
expect(res.status).toEqual(200);
expect(data).toEqual(1);
});
it("should not expire exports made less than 30 days ago", async () => {
await buildFileOperation({
type: "export",
state: "complete",
createdAt: subDays(new Date(), 29),
});
await buildFileOperation({
type: "export",
state: "complete",
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
const data = await FileOperation.count({
where: {
type: "export",
state: "expired",
},
});
expect(res.status).toEqual(200);
expect(data).toEqual(0);
});
it("should require authentication", async () => {
const res = await server.post("/api/utils.gc");
expect(res.status).toEqual(401);

65
server/routes/api/utils.ts
View File

@@ -1,11 +1,8 @@
import { subDays } from "date-fns";
import Router from "koa-router";
import { Op } from "sequelize";
import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
import teamPermanentDeleter from "@server/commands/teamPermanentDeleter";
import { AuthenticationError } from "@server/errors";
import { Document, Team, FileOperation } from "@server/models";
import Logger from "../../logging/logger";
import CleanupDeletedDocumentsTask from "@server/queues/tasks/CleanupDeletedDocumentsTask";
import CleanupDeletedTeamsTask from "@server/queues/tasks/CleanupDeletedTeamsTask";
import CleanupExpiredFileOperationsTask from "@server/queues/tasks/CleanupExpiredFileOperationsTask";
const router = new Router();
@@ -16,59 +13,11 @@ router.post("utils.gc", async (ctx) => {
throw AuthenticationError("Invalid secret token");
}
Logger.info(
"utils",
`Permanently destroying upto ${limit} documents older than 30 days…`
);
const documents = await Document.scope("withDrafts").findAll({
attributes: ["id", "teamId", "text", "deletedAt"],
where: {
deletedAt: {
[Op.lt]: subDays(new Date(), 30),
},
},
paranoid: false,
limit,
});
const countDeletedDocument = await documentPermanentDeleter(documents);
Logger.info("utils", `Destroyed ${countDeletedDocument} documents`);
Logger.info(
"utils",
`Expiring all the collection export older than 30 days…`
);
const exports = await FileOperation.unscoped().findAll({
where: {
type: "export",
createdAt: {
[Op.lt]: subDays(new Date(), 30),
},
state: {
[Op.ne]: "expired",
},
},
});
await Promise.all(
exports.map(async (e) => {
await e.expire();
})
);
Logger.info(
"utils",
`Permanently destroying upto ${limit} teams older than 30 days…`
);
const teams = await Team.findAll({
where: {
deletedAt: {
[Op.lt]: subDays(new Date(), 30),
},
},
paranoid: false,
limit,
});
await CleanupDeletedDocumentsTask.schedule({ limit });
for (const team of teams) {
await teamPermanentDeleter(team);
}
await CleanupExpiredFileOperationsTask.schedule({ limit });
await CleanupDeletedTeamsTask.schedule({ limit });
ctx.body = {
success: true,