Move bulk of webhook logic to plugin (#4866)

* Move bulk of webhook logic to plugin

* Re-enable cleanup task

* cron tasks

app/hooks/useSettingsConfig.ts

@@ -10,7 +10,6 @@ import {
   TeamIcon,
   BeakerIcon,
   BuildingBlocksIcon,
-  WebhooksIcon,
   SettingsIcon,
   ExportIcon,
   ImportIcon,
@@ -32,7 +31,6 @@ import Security from "~/scenes/Settings/Security";
 import SelfHosted from "~/scenes/Settings/SelfHosted";
 import Shares from "~/scenes/Settings/Shares";
 import Tokens from "~/scenes/Settings/Tokens";
-import Webhooks from "~/scenes/Settings/Webhooks";
 import Zapier from "~/scenes/Settings/Zapier";
 import GoogleIcon from "~/components/Icons/GoogleIcon";
 import ZapierIcon from "~/components/Icons/ZapierIcon";
@@ -172,14 +170,6 @@ const useSettingsConfig = () => {
           icon: plugin.icon,
         } as ConfigItem;
       }),
-      Webhooks: {
-        name: t("Webhooks"),
-        path: "/settings/webhooks",
-        component: Webhooks,
-        enabled: can.createWebhookSubscription,
-        group: t("Integrations"),
-        icon: WebhooksIcon,
-      },
       SelfHosted: {
         name: t("Self Hosted"),
         path: integrationSettingsPath("self-hosted"),

package.json

@@ -13,7 +13,7 @@
     "start": "node ./build/server/index.js",
     "dev": "NODE_ENV=development yarn concurrently -n api,collaboration  -c \"blue,magenta\" \"node --inspect=0.0.0.0 build/server/index.js --services=collaboration,websockets,admin,web,worker\"",
     "dev:watch": "nodemon --exec \"yarn build:server && yarn dev\" -e js,ts,tsx --ignore build/ --ignore app/ --ignore shared/editor",
-    "lint": "eslint app server shared",
+    "lint": "eslint app server shared plugins",
     "prepare": "husky install",
     "postinstall": "rimraf node_modules/prosemirror-view/dist/index.d.ts",
     "heroku-postbuild": "yarn build && yarn db:migrate",

plugins/webhooks/client/Icon.tsx

@@ -0,0 +1 @@
+export { WebhooksIcon as default } from "outline-icons";

plugins/webhooks/plugin.json

@@ -0,0 +1,5 @@
+{
+  "name": "Webhooks",
+  "description": "Adds HTTP webhooks for various events.",
+  "requiredEnvVars": []
+}

plugins/webhooks/server/.babelrc

@@ -0,0 +1,3 @@
+{ 
+  "extends": "../../../server/.babelrc"
+}

plugins/webhooks/server/api/webhookSubscriptions.ts

@@ -4,10 +4,10 @@ import { ValidationError } from "@server/errors";
 import auth from "@server/middlewares/authentication";
 import { WebhookSubscription, Event } from "@server/models";
 import { authorize } from "@server/policies";
-import { presentWebhookSubscription } from "@server/presenters";
+import pagination from "@server/routes/api/middlewares/pagination";
 import { WebhookSubscriptionEvent, APIContext } from "@server/types";
 import { assertArray, assertPresent, assertUuid } from "@server/validation";
-import pagination from "./middlewares/pagination";
+import presentWebhookSubscription from "../presenters/webhookSubscription";
 
 const router = new Router();
 

plugins/webhooks/server/processors/WebhookProcessor.ts

@@ -1,7 +1,7 @@
 import { WebhookSubscription } from "@server/models";
+import BaseProcessor from "@server/queues/processors/BaseProcessor";
 import { Event } from "@server/types";
 import DeliverWebhookTask from "../tasks/DeliverWebhookTask";
-import BaseProcessor from "./BaseProcessor";
 
 export default class WebhookProcessor extends BaseProcessor {
   static applicableEvents: ["*"] = ["*"];

plugins/webhooks/server/tasks/CleanupWebhookDeliveriesTask.ts

@@ -2,11 +2,16 @@ import { subDays } from "date-fns";
 import { Op } from "sequelize";
 import Logger from "@server/logging/Logger";
 import { WebhookDelivery } from "@server/models";
-import BaseTask, { TaskPriority } from "./BaseTask";
+import BaseTask, {
+  TaskPriority,
+  TaskSchedule,
+} from "@server/queues/tasks/BaseTask";
 
 type Props = void;
 
 export default class CleanupWebhookDeliveriesTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
   public async perform(_: Props) {
     Logger.info("task", `Deleting WebhookDeliveries older than one week…`);
     const count = await WebhookDelivery.unscoped().destroy({

plugins/webhooks/server/tasks/DeliverWebhookTask.ts

@@ -34,15 +34,13 @@ import {
   presentStar,
   presentTeam,
   presentUser,
-  presentWebhook,
-  presentWebhookSubscription,
   presentView,
   presentShare,
   presentMembership,
   presentGroupMembership,
   presentCollectionGroupMembership,
 } from "@server/presenters";
-import { WebhookPayload } from "@server/presenters/webhook";
+import BaseTask from "@server/queues/tasks/BaseTask";
 import {
   CollectionEvent,
   CollectionGroupEvent,
@@ -62,7 +60,8 @@ import {
   ViewEvent,
   WebhookSubscriptionEvent,
 } from "@server/types";
-import BaseTask from "./BaseTask";
+import presentWebhook, { WebhookPayload } from "../presenters/webhook";
+import presentWebhookSubscription from "../presenters/webhookSubscription";
 
 function assertUnreachable(event: never) {
   Logger.warn(`DeliverWebhookTask did not handle ${(event as any).name}`);

server/presenters/index.ts

@@ -23,8 +23,6 @@ import presentSubscription from "./subscription";
 import presentTeam from "./team";
 import presentUser from "./user";
 import presentView from "./view";
-import presentWebhook from "./webhook";
-import presentWebhookSubscription from "./webhookSubscription";
 
 export {
   presentApiKey,
@@ -52,6 +50,4 @@ export {
   presentTeam,
   presentUser,
   presentView,
-  presentWebhook,
-  presentWebhookSubscription,
 };

server/queues/tasks/BaseTask.ts

@@ -8,7 +8,17 @@ export enum TaskPriority {
   High = 10,
 }
 
+export enum TaskSchedule {
+  Daily = "daily",
+  Hourly = "hourly",
+}
+
 export default abstract class BaseTask<T> {
+  /**
+   * An optional schedule for this task to be run automatically.
+   */
+  static cron: TaskSchedule | undefined;
+
   /**
    * Schedule this task type to be processed asyncronously by a worker.
    *

server/queues/tasks/CleanupDeletedDocumentsTask.ts

@@ -3,13 +3,15 @@ import { Op } from "sequelize";
 import documentPermanentDeleter from "@server/commands/documentPermanentDeleter";
 import Logger from "@server/logging/Logger";
 import { Document } from "@server/models";
-import BaseTask, { TaskPriority } from "./BaseTask";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
 
 type Props = {
   limit: number;
 };
 
 export default class CleanupDeletedDocumentsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
   public async perform({ limit }: Props) {
     Logger.info(
       "task",

server/queues/tasks/CleanupDeletedTeamsTask.ts

@@ -3,13 +3,15 @@ import { Op } from "sequelize";
 import teamPermanentDeleter from "@server/commands/teamPermanentDeleter";
 import Logger from "@server/logging/Logger";
 import { Team } from "@server/models";
-import BaseTask, { TaskPriority } from "./BaseTask";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
 
 type Props = {
   limit: number;
 };
 
 export default class CleanupDeletedTeamsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
   public async perform({ limit }: Props) {
     Logger.info(
       "task",

server/queues/tasks/CleanupExpiredAttachmentsTask.ts

@@ -1,13 +1,15 @@
 import { Op } from "sequelize";
 import Logger from "@server/logging/Logger";
 import { Attachment } from "@server/models";
-import BaseTask, { TaskPriority } from "./BaseTask";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
 
 type Props = {
   limit: number;
 };
 
 export default class CleanupExpiredAttachmentsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
   public async perform({ limit }: Props) {
     Logger.info("task", `Deleting expired attachments…`);
     const attachments = await Attachment.unscoped().findAll({

server/queues/tasks/CleanupExpiredFileOperationsTask.ts

@@ -3,13 +3,15 @@ import { Op } from "sequelize";
 import { FileOperationState } from "@shared/types";
 import Logger from "@server/logging/Logger";
 import { FileOperation } from "@server/models";
-import BaseTask, { TaskPriority } from "./BaseTask";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
 
 type Props = {
   limit: number;
 };
 
 export default class CleanupExpiredFileOperationsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
   public async perform({ limit }: Props) {
     Logger.info("task", `Expiring file operations older than 15 days…`);
     const fileOperations = await FileOperation.unscoped().findAll({

server/queues/tasks/InviteReminderTask.ts

@@ -4,11 +4,13 @@ import { sequelize } from "@server/database/sequelize";
 import InviteReminderEmail from "@server/emails/templates/InviteReminderEmail";
 import { User } from "@server/models";
 import { UserFlag } from "@server/models/User";
-import BaseTask, { TaskPriority } from "./BaseTask";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
 
 type Props = undefined;
 
 export default class InviteReminderTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
   public async perform() {
     const users = await User.scope("invited").findAll({
       attributes: ["id"],

server/routes/api/cron.ts

@@ -3,12 +3,7 @@ import { Context } from "koa";
 import Router from "koa-router";
 import env from "@server/env";
 import { AuthenticationError } from "@server/errors";
-import CleanupDeletedDocumentsTask from "@server/queues/tasks/CleanupDeletedDocumentsTask";
+import tasks from "@server/queues/tasks";
-import CleanupDeletedTeamsTask from "@server/queues/tasks/CleanupDeletedTeamsTask";
-import CleanupExpiredAttachmentsTask from "@server/queues/tasks/CleanupExpiredAttachmentsTask";
-import CleanupExpiredFileOperationsTask from "@server/queues/tasks/CleanupExpiredFileOperationsTask";
-import CleanupWebhookDeliveriesTask from "@server/queues/tasks/CleanupWebhookDeliveriesTask";
-import InviteReminderTask from "@server/queues/tasks/InviteReminderTask";
 
 const router = new Router();
 
@@ -34,17 +29,12 @@ const cronHandler = async (ctx: Context) => {
     throw AuthenticationError("Invalid secret token");
   }
 
-  await CleanupDeletedDocumentsTask.schedule({ limit });
+  for (const name in tasks) {
-
+    const TaskClass = tasks[name];
-  await CleanupExpiredFileOperationsTask.schedule({ limit });
+    if (TaskClass.cron) {
-
+      await TaskClass.schedule({ limit });
-  await CleanupExpiredAttachmentsTask.schedule({ limit });
+    }
-
+  }
-  await CleanupDeletedTeamsTask.schedule({ limit });
-
-  await CleanupWebhookDeliveriesTask.schedule({ limit });
-
-  await InviteReminderTask.schedule();
 
   ctx.body = {
     success: true,

server/routes/api/index.ts

@@ -32,7 +32,6 @@ import subscriptions from "./subscriptions";
 import team from "./team";
 import users from "./users";
 import views from "./views";
-import webhookSubscriptions from "./webhookSubscriptions";
 
 const api = new Koa<AppState, AppContext>();
 const router = new Router();
@@ -82,7 +81,6 @@ router.use("/", attachments.routes());
 router.use("/", utils.routes());
 router.use("/", groups.routes());
 router.use("/", fileOperationsRoute.routes());
-router.use("/", webhookSubscriptions.routes());
 
 if (env.ENVIRONMENT === "development") {
   router.use("/", developer.routes());

shared/i18n/locales/en_US/translation.json

@@ -313,7 +313,6 @@
   "Shared Links": "Shared Links",
   "Import": "Import",
   "Integrations": "Integrations",
-  "Webhooks": "Webhooks",
   "Self Hosted": "Self Hosted",
   "Google Analytics": "Google Analytics",
   "Show path to document": "Show path to document",
@@ -671,22 +670,6 @@
   "Active": "Active",
   "Everyone": "Everyone",
   "Admins": "Admins",
-  "Are you sure you want to delete the {{ name }} webhook?": "Are you sure you want to delete the {{ name }} webhook?",
-  "Webhook updated": "Webhook updated",
-  "Update": "Update",
-  "Updating": "Updating",
-  "Provide a descriptive name for this webhook and the URL we should send a POST request to when matching events are created.": "Provide a descriptive name for this webhook and the URL we should send a POST request to when matching events are created.",
-  "A memorable identifer": "A memorable identifer",
-  "URL": "URL",
-  "Signing secret": "Signing secret",
-  "Subscribe to all events, groups, or individual events. We recommend only subscribing to the minimum amount of events that your application needs to function.": "Subscribe to all events, groups, or individual events. We recommend only subscribing to the minimum amount of events that your application needs to function.",
-  "All events": "All events",
-  "All {{ groupName }} events": "All {{ groupName }} events",
-  "Delete webhook": "Delete webhook",
-  "Disabled": "Disabled",
-  "Subscribed events": "Subscribed events",
-  "Edit webhook": "Edit webhook",
-  "Webhook created": "Webhook created",
   "Settings saved": "Settings saved",
   "Logo updated": "Logo updated",
   "Unable to upload new logo": "Unable to upload new logo",
@@ -770,6 +753,7 @@
   "Settings that impact the access, security, and content of your knowledge base.": "Settings that impact the access, security, and content of your knowledge base.",
   "Allow members to sign-in with {{ authProvider }}": "Allow members to sign-in with {{ authProvider }}",
   "Connected": "Connected",
+  "Disabled": "Disabled",
   "Allow members to sign-in using their email address": "Allow members to sign-in using their email address",
   "The server must have SMTP configured to enable this setting": "The server must have SMTP configured to enable this setting",
   "Access": "Access",
@@ -793,9 +777,6 @@
   "You can create an unlimited amount of personal tokens to authenticate\n          with the API. Tokens have the same permissions as your user account.\n          For more details see the <em>developer documentation</em>.": "You can create an unlimited amount of personal tokens to authenticate\n          with the API. Tokens have the same permissions as your user account.\n          For more details see the <em>developer documentation</em>.",
   "Tokens": "Tokens",
   "Create a token": "Create a token",
-  "New webhook": "New webhook",
-  "Webhooks can be used to notify your application when events happen in {{appName}}. Events are sent as a https request with a JSON payload in near real-time.": "Webhooks can be used to notify your application when events happen in {{appName}}. Events are sent as a https request with a JSON payload in near real-time.",
-  "Create a webhook": "Create a webhook",
   "Zapier is a platform that allows {{appName}} to easily integrate with thousands of other business tools. Automate your workflows, sync data, and more.": "Zapier is a platform that allows {{appName}} to easily integrate with thousands of other business tools. Automate your workflows, sync data, and more.",
   "Your are creating a new workspace using your current account — <em>{{email}}</em>": "Your are creating a new workspace using your current account — <em>{{email}}</em>",
   "Workspace name": "Workspace name",
@@ -838,5 +819,24 @@
   "Post to Channel": "Post to Channel",
   "This is what we found for \"{{ term }}\"": "This is what we found for \"{{ term }}\"",
   "No results for \"{{ term }}\"": "No results for \"{{ term }}\"",
+  "Are you sure you want to delete the {{ name }} webhook?": "Are you sure you want to delete the {{ name }} webhook?",
+  "Webhook updated": "Webhook updated",
+  "Update": "Update",
+  "Updating": "Updating",
+  "Provide a descriptive name for this webhook and the URL we should send a POST request to when matching events are created.": "Provide a descriptive name for this webhook and the URL we should send a POST request to when matching events are created.",
+  "A memorable identifer": "A memorable identifer",
+  "URL": "URL",
+  "Signing secret": "Signing secret",
+  "Subscribe to all events, groups, or individual events. We recommend only subscribing to the minimum amount of events that your application needs to function.": "Subscribe to all events, groups, or individual events. We recommend only subscribing to the minimum amount of events that your application needs to function.",
+  "All events": "All events",
+  "All {{ groupName }} events": "All {{ groupName }} events",
+  "Delete webhook": "Delete webhook",
+  "Subscribed events": "Subscribed events",
+  "Edit webhook": "Edit webhook",
+  "Webhook created": "Webhook created",
+  "Webhooks": "Webhooks",
+  "New webhook": "New webhook",
+  "Webhooks can be used to notify your application when events happen in {{appName}}. Events are sent as a https request with a JSON payload in near real-time.": "Webhooks can be used to notify your application when events happen in {{appName}}. Events are sent as a https request with a JSON payload in near real-time.",
+  "Create a webhook": "Create a webhook",
   "Uploading": "Uploading"
 }