chore: Refactor worker, emails and data cleanup to task system (#3337)

* Refactor worker, all emails on task system

* fix

* lint

* fix: Remove a bunch of expect-error comments in related tests

* refactor: Move work from utils.gc into tasks

* test

* Add tracing to tasks and processors
fix: DebounceProcessor triggering on all events
Event.add -> Event.schedule

server/routes/api/documents.ts

@@ -1024,7 +1024,7 @@ router.post("documents.update", auth(), async (ctx) => {
   }
 
   if (document.title !== previousTitle) {
-    Event.add({
+    Event.schedule({
       name: "documents.title_change",
       documentId: document.id,
       collectionId: document.collectionId,

server/routes/api/utils.test.ts

@@ -1,8 +1,5 @@
-import { subDays } from "date-fns";
 import TestServer from "fetch-test-server";
-import { Document, FileOperation } from "@server/models";
 import webService from "@server/services/web";
-import { buildDocument, buildFileOperation } from "@server/test/factories";
 import { flushdb } from "@server/test/support";
 
 const app = webService();
@@ -12,127 +9,6 @@ beforeEach(() => flushdb());
 afterAll(() => server.close());
 
 describe("#utils.gc", () => {
-  it("should not destroy documents not deleted", async () => {
-    await buildDocument({
-      publishedAt: new Date(),
-    });
-    const res = await server.post("/api/utils.gc", {
-      body: {
-        token: process.env.UTILS_SECRET,
-      },
-    });
-    expect(res.status).toEqual(200);
-    expect(
-      await Document.unscoped().count({
-        paranoid: false,
-      })
-    ).toEqual(1);
-  });
-
-  it("should not destroy documents deleted less than 30 days ago", async () => {
-    await buildDocument({
-      publishedAt: new Date(),
-      deletedAt: subDays(new Date(), 25),
-    });
-    const res = await server.post("/api/utils.gc", {
-      body: {
-        token: process.env.UTILS_SECRET,
-      },
-    });
-    expect(res.status).toEqual(200);
-    expect(
-      await Document.unscoped().count({
-        paranoid: false,
-      })
-    ).toEqual(1);
-  });
-
-  it("should destroy documents deleted more than 30 days ago", async () => {
-    await buildDocument({
-      publishedAt: new Date(),
-      deletedAt: subDays(new Date(), 60),
-    });
-    const res = await server.post("/api/utils.gc", {
-      body: {
-        token: process.env.UTILS_SECRET,
-      },
-    });
-    expect(res.status).toEqual(200);
-    expect(
-      await Document.unscoped().count({
-        paranoid: false,
-      })
-    ).toEqual(0);
-  });
-
-  it("should destroy draft documents deleted more than 30 days ago", async () => {
-    await buildDocument({
-      publishedAt: undefined,
-      deletedAt: subDays(new Date(), 60),
-    });
-    const res = await server.post("/api/utils.gc", {
-      body: {
-        token: process.env.UTILS_SECRET,
-      },
-    });
-    expect(res.status).toEqual(200);
-    expect(
-      await Document.unscoped().count({
-        paranoid: false,
-      })
-    ).toEqual(0);
-  });
-
-  it("should expire exports older than 30 days ago", async () => {
-    await buildFileOperation({
-      type: "export",
-      state: "complete",
-      createdAt: subDays(new Date(), 30),
-    });
-    await buildFileOperation({
-      type: "export",
-      state: "complete",
-    });
-    const res = await server.post("/api/utils.gc", {
-      body: {
-        token: process.env.UTILS_SECRET,
-      },
-    });
-    const data = await FileOperation.count({
-      where: {
-        type: "export",
-        state: "expired",
-      },
-    });
-    expect(res.status).toEqual(200);
-    expect(data).toEqual(1);
-  });
-
-  it("should not expire exports made less than 30 days ago", async () => {
-    await buildFileOperation({
-      type: "export",
-      state: "complete",
-      createdAt: subDays(new Date(), 29),
-    });
-    await buildFileOperation({
-      type: "export",
-      state: "complete",
-    });
-    const res = await server.post("/api/utils.gc", {
-      body: {
-        token: process.env.UTILS_SECRET,
-      },
-    });
-    const data = await FileOperation.count({
-      where: {
-        type: "export",
-        state: "expired",
-      },
-    });
-    expect(res.status).toEqual(200);
-    expect(data).toEqual(0);
-  });
-
   it("should require authentication", async () => {
     const res = await server.post("/api/utils.gc");
     expect(res.status).toEqual(401);

server/routes/api/utils.ts

@@ -1,11 +1,8 @@
-import { subDays } from "date-fns";
 import Router from "koa-router";
-import { Op } from "sequelize";
-import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
-import teamPermanentDeleter from "@server/commands/teamPermanentDeleter";
 import { AuthenticationError } from "@server/errors";
-import { Document, Team, FileOperation } from "@server/models";
-import Logger from "../../logging/logger";
+import CleanupDeletedDocumentsTask from "@server/queues/tasks/CleanupDeletedDocumentsTask";
+import CleanupDeletedTeamsTask from "@server/queues/tasks/CleanupDeletedTeamsTask";
+import CleanupExpiredFileOperationsTask from "@server/queues/tasks/CleanupExpiredFileOperationsTask";
 
 const router = new Router();
 
@@ -16,59 +13,11 @@ router.post("utils.gc", async (ctx) => {
     throw AuthenticationError("Invalid secret token");
   }
 
-  Logger.info(
-    "utils",
-    `Permanently destroying upto ${limit} documents older than 30 days…`
-  );
-  const documents = await Document.scope("withDrafts").findAll({
-    attributes: ["id", "teamId", "text", "deletedAt"],
-    where: {
-      deletedAt: {
-        [Op.lt]: subDays(new Date(), 30),
-      },
-    },
-    paranoid: false,
-    limit,
-  });
-  const countDeletedDocument = await documentPermanentDeleter(documents);
-  Logger.info("utils", `Destroyed ${countDeletedDocument} documents`);
-  Logger.info(
-    "utils",
-    `Expiring all the collection export older than 30 days…`
-  );
-  const exports = await FileOperation.unscoped().findAll({
-    where: {
-      type: "export",
-      createdAt: {
-        [Op.lt]: subDays(new Date(), 30),
-      },
-      state: {
-        [Op.ne]: "expired",
-      },
-    },
-  });
-  await Promise.all(
-    exports.map(async (e) => {
-      await e.expire();
-    })
-  );
-  Logger.info(
-    "utils",
-    `Permanently destroying upto ${limit} teams older than 30 days…`
-  );
-  const teams = await Team.findAll({
-    where: {
-      deletedAt: {
-        [Op.lt]: subDays(new Date(), 30),
-      },
-    },
-    paranoid: false,
-    limit,
-  });
+  await CleanupDeletedDocumentsTask.schedule({ limit });
 
-  for (const team of teams) {
-    await teamPermanentDeleter(team);
-  }
+  await CleanupExpiredFileOperationsTask.schedule({ limit });
+
+  await CleanupDeletedTeamsTask.schedule({ limit });
 
   ctx.body = {
     success: true,

server/routes/auth/providers/email.test.ts

@@ -1,5 +1,5 @@
 import TestServer from "fetch-test-server";
-import mailer from "@server/mailer";
+import EmailTask from "@server/queues/tasks/EmailTask";
 import webService from "@server/services/web";
 import { buildUser, buildGuestUser, buildTeam } from "@server/test/factories";
 import { flushdb } from "@server/test/support";
@@ -24,7 +24,7 @@ describe("email", () => {
   });
 
   it("should respond with redirect location when user is SSO enabled", async () => {
-    const spy = jest.spyOn(mailer, "sendTemplate");
+    const spy = jest.spyOn(EmailTask, "schedule");
     const user = await buildUser();
     const res = await server.post("/auth/email", {
       body: {
@@ -42,7 +42,7 @@ describe("email", () => {
     process.env.URL = "http://localoutline.com";
     process.env.SUBDOMAINS_ENABLED = "true";
     const user = await buildUser();
-    const spy = jest.spyOn(mailer, "sendTemplate");
+    const spy = jest.spyOn(EmailTask, "schedule");
     await buildTeam({
       subdomain: "example",
     });
@@ -62,7 +62,7 @@ describe("email", () => {
   });
 
   it("should respond with success when user is not SSO enabled", async () => {
-    const spy = jest.spyOn(mailer, "sendTemplate");
+    const spy = jest.spyOn(EmailTask, "schedule");
     const user = await buildGuestUser();
     const res = await server.post("/auth/email", {
       body: {
@@ -77,7 +77,7 @@ describe("email", () => {
   });
 
   it("should respond with success regardless of whether successful to prevent crawling email logins", async () => {
-    const spy = jest.spyOn(mailer, "sendTemplate");
+    const spy = jest.spyOn(EmailTask, "schedule");
     const res = await server.post("/auth/email", {
       body: {
         email: "user@example.com",
@@ -91,7 +91,7 @@ describe("email", () => {
   });
   describe("with multiple users matching email", () => {
     it("should default to current subdomain with SSO", async () => {
-      const spy = jest.spyOn(mailer, "sendTemplate");
+      const spy = jest.spyOn(EmailTask, "schedule");
       process.env.URL = "http://localoutline.com";
       process.env.SUBDOMAINS_ENABLED = "true";
       const email = "sso-user@example.org";
@@ -121,7 +121,7 @@ describe("email", () => {
     });
 
     it("should default to current subdomain with guest email", async () => {
-      const spy = jest.spyOn(mailer, "sendTemplate");
+      const spy = jest.spyOn(EmailTask, "schedule");
       process.env.URL = "http://localoutline.com";
       process.env.SUBDOMAINS_ENABLED = "true";
       const email = "guest-user@example.org";
@@ -151,7 +151,7 @@ describe("email", () => {
     });
 
     it("should default to custom domain with SSO", async () => {
-      const spy = jest.spyOn(mailer, "sendTemplate");
+      const spy = jest.spyOn(EmailTask, "schedule");
       const email = "sso-user-2@example.org";
       const team = await buildTeam({
         domain: "docs.mycompany.com",
@@ -179,7 +179,7 @@ describe("email", () => {
     });
 
     it("should default to custom domain with guest email", async () => {
-      const spy = jest.spyOn(mailer, "sendTemplate");
+      const spy = jest.spyOn(EmailTask, "schedule");
       const email = "guest-user-2@example.org";
       const team = await buildTeam({
         domain: "docs.mycompany.com",

server/routes/auth/providers/email.ts

@@ -3,10 +3,10 @@ import Router from "koa-router";
 import { find } from "lodash";
 import { parseDomain, isCustomSubdomain } from "@shared/utils/domains";
 import { AuthorizationError } from "@server/errors";
-import mailer from "@server/mailer";
 import errorHandling from "@server/middlewares/errorHandling";
 import methodOverride from "@server/middlewares/methodOverride";
 import { User, Team } from "@server/models";
+import EmailTask from "@server/queues/tasks/EmailTask";
 import { signIn } from "@server/utils/authentication";
 import { isCustomDomain } from "@server/utils/domains";
 import { getUserForEmailSigninToken } from "@server/utils/jwt";
@@ -110,10 +110,13 @@ router.post("email", errorHandling(), async (ctx) => {
     }
 
     // send email to users registered address with a short-lived token
-    await mailer.sendTemplate("signin", {
-      to: user.email,
-      token: user.getEmailSigninToken(),
-      teamUrl: team.url,
+    await EmailTask.schedule({
+      type: "signin",
+      options: {
+        to: user.email,
+        token: user.getEmailSigninToken(),
+        teamUrl: team.url,
+      },
     });
     user.lastSigninEmailSentAt = new Date();
     await user.save();
@@ -147,9 +150,12 @@ router.get("email.callback", async (ctx) => {
   }
 
   if (user.isInvited) {
-    await mailer.sendTemplate("welcome", {
-      to: user.email,
-      teamUrl: user.team.url,
+    await EmailTask.schedule({
+      type: "welcome",
+      options: {
+        to: user.email,
+        teamUrl: user.team.url,
+      },
     });
   }