chore: Plugin registration (#6623)

* first pass

* test

* test

* priority

* Reduce boilerplate further

* Update server/utils/PluginManager.ts

Co-authored-by: Apoorv Mishra <apoorvmishra101092@gmail.com>

* fix: matchesNode error in destroyed editor transaction

* fix: Individual imported files do not display source correctly in 'Insights'

* chore: Add sleep before Slack notification

* docs

* fix: Error logged about missing plugin.json

* Remove email template glob

---------

Co-authored-by: Apoorv Mishra <apoorvmishra101092@gmail.com>

.env.test

@@ -3,6 +3,7 @@ DATABASE_URL=postgres://user:pass@127.0.0.1:5432/outline-test
 SECRET_KEY=F0E5AD933D7F6FD8F4DBB3E038C501C052DC0593C686D21ACB30AE205D2F634B
 
 SMTP_HOST=smtp.example.com
+SMTP_USERNAME=test
 SMTP_FROM_EMAIL=hello@example.com
 SMTP_REPLY_EMAIL=hello@example.com
 

app/hooks/useSettingsConfig.ts

@@ -197,7 +197,7 @@ const useSettingsConfig = () => {
     Object.values(PluginLoader.plugins).map((plugin) => {
       const hasSettings = !!plugin.settings;
       const enabledInDeployment =
-        !plugin.config.deployments ||
+        !plugin.config?.deployments ||
         plugin.config.deployments.length === 0 ||
         (plugin.config.deployments.includes("cloud") && isCloudHosted) ||
         (plugin.config.deployments.includes("enterprise") && !isCloudHosted);

app/utils/PluginLoader.ts

@@ -5,7 +5,6 @@ interface Plugin {
   config: {
     name: string;
     description: string;
-    requiredEnvVars?: string[];
     deployments?: string[];
   };
   settings: React.FC;

build.js

@@ -79,7 +79,7 @@ async function build() {
     execAsync("cp package.json ./build"),
     ...d.map(async (plugin) =>
       execAsync(
-        `mkdir -p ./build/plugins/${plugin} && cp ./plugins/${plugin}/plugin.json ./build/plugins/${plugin}/plugin.json`
+        `mkdir -p ./build/plugins/${plugin} && cp ./plugins/${plugin}/plugin.json ./build/plugins/${plugin}/plugin.json 2>/dev/null || :`
       )
     ),
   ]);

plugins/azure/plugin.json

@@ -1,5 +1,6 @@
 {
+  "id": "azure",
   "name": "Microsoft",
-  "description": "Adds a Microsoft Azure authentication provider.",
+  "priority": 20,
-  "requiredEnvVars": ["AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET"]
+  "description": "Adds a Microsoft Azure authentication provider."
 }

plugins/azure/server/auth/azure.ts

@@ -16,10 +16,10 @@ import {
   getTeamFromContext,
   getClientFromContext,
 } from "@server/utils/passport";
+import config from "../../plugin.json";
 import env from "../env";
 
 const router = new Router();
-const providerName = "azure";
 const scopes: string[] = [];
 
 if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
@@ -109,7 +109,7 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
             avatarUrl: profile.picture,
           },
           authenticationProvider: {
-            name: providerName,
+            name: config.id,
             providerId: profile.tid,
           },
           authentication: {
@@ -127,13 +127,11 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
     }
   );
   passport.use(strategy);
-
   router.get(
-    "azure",
+    config.id,
-    passport.authenticate(providerName, { prompt: "select_account" })
+    passport.authenticate(config.id, { prompt: "select_account" })
   );
-
+  router.get(`${config.id}.callback`, passportMiddleware(config.id));
-  router.get("azure.callback", passportMiddleware(providerName));
 }
 
 export default router;

plugins/azure/server/index.ts

@@ -0,0 +1,9 @@
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import router from "./auth/azure";
+import env from "./env";
+
+PluginManager.register(PluginType.AuthProvider, router, {
+  ...config,
+  enabled: !!env.AZURE_CLIENT_ID && !!env.AZURE_CLIENT_SECRET,
+});

plugins/email/plugin.json

@@ -1,4 +1,5 @@
 {
+  "id": "email",
   "name": "Email",
   "description": "Adds an email magic link authentication provider."
 }

plugins/email/server/index.ts

@@ -0,0 +1,9 @@
+import env from "@server/env";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import router from "./auth/email";
+
+PluginManager.register(PluginType.AuthProvider, router, {
+  ...config,
+  enabled: (!!env.SMTP_HOST && !!env.SMTP_USERNAME) || env.isDevelopment,
+});

plugins/google/plugin.json

@@ -1,5 +1,6 @@
 {
+  "id": "google",
   "name": "Google",
-  "description": "Adds a Google authentication provider.",
+  "priority": 10,
-  "requiredEnvVars": ["GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET"]
+  "description": "Adds a Google authentication provider."
 }

plugins/google/server/auth/google.ts

@@ -18,10 +18,10 @@ import {
   getTeamFromContext,
   getClientFromContext,
 } from "@server/utils/passport";
+import config from "../../plugin.json";
 import env from "../env";
 
 const router = new Router();
-const providerName = "google";
 
 const scopes = [
   "https://www.googleapis.com/auth/userinfo.profile",
@@ -42,7 +42,7 @@ if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) {
       {
         clientID: env.GOOGLE_CLIENT_ID,
         clientSecret: env.GOOGLE_CLIENT_SECRET,
-        callbackURL: `${env.URL}/auth/google.callback`,
+        callbackURL: `${env.URL}/auth/${config.id}.callback`,
         passReqToCallback: true,
         // @ts-expect-error StateStore
         store: new StateStore(),
@@ -110,7 +110,7 @@ if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) {
               avatarUrl,
             },
             authenticationProvider: {
-              name: providerName,
+              name: config.id,
               providerId: domain ?? "",
             },
             authentication: {
@@ -131,14 +131,13 @@ if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) {
   );
 
   router.get(
-    "google",
+    config.id,
-    passport.authenticate(providerName, {
+    passport.authenticate(config.id, {
       accessType: "offline",
       prompt: "select_account consent",
     })
   );
-
+  router.get(`${config.id}.callback`, passportMiddleware(config.id));
-  router.get("google.callback", passportMiddleware(providerName));
 }
 
 export default router;

plugins/google/server/index.ts

@@ -0,0 +1,9 @@
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import router from "./auth/google";
+import env from "./env";
+
+PluginManager.register(PluginType.AuthProvider, router, {
+  ...config,
+  enabled: !!env.GOOGLE_CLIENT_ID && !!env.GOOGLE_CLIENT_SECRET,
+});

plugins/iframely/plugin.json

@@ -1,5 +0,0 @@
-{
-  "name": "Iframely",
-  "description": "Integrate Iframely to enable unfurling of arbitrary urls",
-  "requiredEnvVars": ["IFRAMELY_API_KEY"]
-}

plugins/iframely/server/iframely.ts

@@ -1,3 +1,4 @@
+import type { Unfurl } from "@shared/types";
 import { Day } from "@shared/utils/time";
 import { InternalError } from "@server/errors";
 import Logger from "@server/logging/Logger";
@@ -33,7 +34,7 @@ class Iframely {
     }
   }
 
-  private static async fetch(url: string, type = "oembed") {
+  public static async fetch(url: string, type = "oembed") {
     const res = await fetch(
       `${this.apiUrl}/${type}?url=${encodeURIComponent(url)}&api_key=${
         this.apiKey
@@ -55,20 +56,19 @@ class Iframely {
   }
 
   /**
-   * Fetches the preview data for the given url
+   * Fetches the preview data for the given url using Iframely oEmbed API
-   * using Iframely oEmbed API
    *
    * @param url
    * @returns Preview data for the url
    */
-  public static async get(url: string) {
+  public static async get(url: string): Promise<Unfurl | false> {
     try {
-      const cached = await this.cached(url);
+      const cached = await Iframely.cached(url);
       if (cached) {
         return cached;
       }
-      const res = await this.fetch(url);
+      const res = await Iframely.fetch(url);
-      await this.cache(url, res);
+      await Iframely.cache(url, res);
       return res;
     } catch (err) {
       throw InternalError(err);

plugins/iframely/server/index.ts

@@ -0,0 +1,15 @@
+import {
+  PluginManager,
+  PluginPriority,
+  PluginType,
+} from "@server/utils/PluginManager";
+import env from "./env";
+import Iframely from "./iframely";
+
+PluginManager.register(PluginType.UnfurlProvider, Iframely.get, {
+  id: "iframely",
+  enabled: !!env.IFRAMELY_API_KEY && !!env.IFRAMELY_URL,
+
+  // Make sure this is last in the stack to be evaluated after all other unfurl providers
+  priority: PluginPriority.VeryLow,
+});

plugins/iframely/server/unfurl.ts

@@ -1,3 +0,0 @@
-import Iframely from "./iframely";
-
-export const unfurl = async (url: string) => Iframely.get(url);

plugins/oidc/plugin.json

@@ -1,5 +1,6 @@
 {
+  "id": "oidc",
   "name": "OIDC",
-  "description": "Adds an OpenID compatible authentication provider.",
+  "priority": 30,
-  "requiredEnvVars": ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_AUTH_URI", "OIDC_TOKEN_URI", "OIDC_USERINFO_URI"]
+  "description": "Adds an OpenID compatible authentication provider."
 }

plugins/oidc/server/auth/oidc.ts

@@ -18,10 +18,10 @@ import {
   getTeamFromContext,
   getClientFromContext,
 } from "@server/utils/passport";
+import config from "../../plugin.json";
 import env from "../env";
 
 const router = new Router();
-const providerName = "oidc";
 const scopes = env.OIDC_SCOPES.split(" ");
 
 Strategy.prototype.userProfile = async function (accessToken, done) {
@@ -55,14 +55,14 @@ if (
   env.OIDC_USERINFO_URI
 ) {
   passport.use(
-    providerName,
+    config.id,
     new Strategy(
       {
         authorizationURL: env.OIDC_AUTH_URI,
         tokenURL: env.OIDC_TOKEN_URI,
         clientID: env.OIDC_CLIENT_ID,
         clientSecret: env.OIDC_CLIENT_SECRET,
-        callbackURL: `${env.URL}/auth/${providerName}.callback`,
+        callbackURL: `${env.URL}/auth/${config.id}.callback`,
         passReqToCallback: true,
         scope: env.OIDC_SCOPES,
         // @ts-expect-error custom state store
@@ -134,7 +134,7 @@ if (
               avatarUrl: profile.picture,
             },
             authenticationProvider: {
-              name: providerName,
+              name: config.id,
               providerId: domain,
             },
             authentication: {
@@ -153,11 +153,8 @@ if (
     )
   );
 
-  router.get(providerName, passport.authenticate(providerName));
+  router.get(config.id, passport.authenticate(config.id));
-
+  router.get(`${config.id}.callback`, passportMiddleware(config.id));
-  router.get(`${providerName}.callback`, passportMiddleware(providerName));
 }
 
-export const name = env.OIDC_DISPLAY_NAME;
-
 export default router;

plugins/oidc/server/index.ts

@@ -0,0 +1,16 @@
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import router from "./auth/oidc";
+import env from "./env";
+
+PluginManager.register(PluginType.AuthProvider, router, {
+  ...config,
+  name: env.OIDC_DISPLAY_NAME || config.name,
+  enabled: !!(
+    env.OIDC_CLIENT_ID &&
+    env.OIDC_CLIENT_SECRET &&
+    env.OIDC_AUTH_URI &&
+    env.OIDC_TOKEN_URI &&
+    env.OIDC_USERINFO_URI
+  ),
+});

plugins/slack/plugin.json

@@ -1,5 +1,6 @@
 {
+  "id": "slack",
   "name": "Slack",
-  "description": "Adds a Slack authentication provider, support for the /outline slash command, and link unfurling.",
+  "priority": 40,
-  "requiredEnvVars": ["SLACK_CLIENT_ID", "SLACK_CLIENT_SECRET"]
+  "description": "Adds a Slack authentication provider, support for the /outline slash command, and link unfurling."
 }

plugins/slack/server/index.ts

@@ -0,0 +1,20 @@
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import hooks from "./api/hooks";
+import router from "./auth/slack";
+import env from "./env";
+import SlackProcessor from "./processors/SlackProcessor";
+
+const enabled = !!env.SLACK_CLIENT_ID && !!env.SLACK_CLIENT_SECRET;
+
+PluginManager.register(PluginType.AuthProvider, router, {
+  ...config,
+  enabled,
+});
+
+PluginManager.register(PluginType.API, hooks, {
+  ...config,
+  enabled,
+});
+
+PluginManager.registerProcessor(SlackProcessor, { enabled });

plugins/storage/plugin.json

@@ -1,8 +0,0 @@
-{
-  "name": "Storage",
-  "description": "Plugin for storing files on the local file system",
-  "requiredEnvVars": [
-    "FILE_STORAGE_UPLOAD_MAX_SIZE",
-    "FILE_STORAGE_LOCAL_ROOT_DIR"
-  ]
-}

plugins/storage/server/api/files.ts

@@ -18,11 +18,8 @@ import FileStorage from "@server/storage/files";
 import { APIContext } from "@server/types";
 import { RateLimiterStrategy } from "@server/utils/RateLimiter";
 import { getJWTPayload } from "@server/utils/jwt";
-import { createRootDirForLocalStorage } from "../utils";
 import * as T from "./schema";
 
-createRootDirForLocalStorage();
-
 const router = new Router();
 
 router.post(

plugins/storage/server/index.ts

@@ -0,0 +1,31 @@
+import { existsSync, mkdirSync } from "fs";
+import env from "@server/env";
+import Logger from "@server/logging/Logger";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import router from "./api/files";
+
+if (env.FILE_STORAGE === "local") {
+  const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR;
+  try {
+    if (!existsSync(rootDir)) {
+      mkdirSync(rootDir, { recursive: true });
+      Logger.debug("utils", `Created ${rootDir} for local storage`);
+    }
+  } catch (err) {
+    Logger.fatal(
+      `Failed to create directory for local file storage at ${env.FILE_STORAGE_LOCAL_ROOT_DIR}`,
+      err
+    );
+  }
+}
+
+PluginManager.register(PluginType.API, router, {
+  id: "files",
+  name: "Local file storage",
+  description: "Plugin for storing files on the local file system",
+  enabled: !!(
+    env.FILE_STORAGE_UPLOAD_MAX_SIZE &&
+    env.FILE_STORAGE_LOCAL_ROOT_DIR &&
+    env.FILE_STORAGE === "local"
+  ),
+});

plugins/storage/server/utils.ts

@@ -1,20 +0,0 @@
-import { existsSync, mkdirSync } from "fs";
-import env from "@server/env";
-import Logger from "@server/logging/Logger";
-
-export const createRootDirForLocalStorage = () => {
-  if (env.FILE_STORAGE === "local") {
-    const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR;
-    try {
-      if (!existsSync(rootDir)) {
-        mkdirSync(rootDir, { recursive: true });
-        Logger.debug("utils", `Created ${rootDir} for local storage`);
-      }
-    } catch (err) {
-      Logger.fatal(
-        `Failed to create directory for local file storage at ${env.FILE_STORAGE_LOCAL_ROOT_DIR}`,
-        err
-      );
-    }
-  }
-};

plugins/webhooks/plugin.json

@@ -1,5 +1,5 @@
 {
+  "id": "webhooks",
   "name": "Webhooks",
-  "description": "Adds HTTP webhooks for various events.",
+  "description": "Adds HTTP webhooks for various events."
-  "requiredEnvVars": []
 }

plugins/webhooks/server/index.ts

@@ -0,0 +1,11 @@
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import webhookSubscriptions from "./api/webhookSubscriptions";
+import WebhookProcessor from "./processors/WebhookProcessor";
+import CleanupWebhookDeliveriesTask from "./tasks/CleanupWebhookDeliveriesTask";
+import DeliverWebhookTask from "./tasks/DeliverWebhookTask";
+
+PluginManager.register(PluginType.API, webhookSubscriptions, config)
+  .registerProcessor(WebhookProcessor)
+  .registerTask(DeliverWebhookTask)
+  .registerTask(CleanupWebhookDeliveriesTask);

plugins/webhooks/server/tasks/CleanupWebhookDeliveriesTask.ts

@@ -7,12 +7,12 @@ import BaseTask, {
   TaskSchedule,
 } from "@server/queues/tasks/BaseTask";
 
-type Props = void;
+type Props = Record<string, never>;
 
 export default class CleanupWebhookDeliveriesTask extends BaseTask<Props> {
   static cron = TaskSchedule.Daily;
 
-  public async perform(_: Props) {
+  public async perform() {
     Logger.info("task", `Deleting WebhookDeliveries older than one week…`);
     const count = await WebhookDelivery.unscoped().destroy({
       where: {

server/emails/templates/index.ts

@@ -1,7 +1,4 @@
-import path from "path";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
-import { glob } from "glob";
-import env from "@server/env";
-import Logger from "@server/logging/Logger";
 import { requireDirectory } from "@server/utils/fs";
 
 const emails = {};
@@ -17,16 +14,8 @@ requireDirectory(__dirname).forEach(([module, id]) => {
   emails[id] = Email;
 });
 
-const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
+PluginManager.getEnabledPlugins(PluginType.EmailTemplate).forEach((plugin) => {
-glob
+  emails[plugin.id] = plugin.value;
-  .sync(path.join(rootDir, "plugins/*/server/email/templates/!(*.test).[jt]s"))
+});
-  .forEach((filePath: string) => {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const template = require(path.join(process.cwd(), filePath)).default;
-
-    Logger.debug("lifecycle", `Registered email template ${template.name}`);
-
-    emails[template.name] = template;
-  });
 
 export default emails;

server/index.ts

@@ -26,6 +26,7 @@ import ShutdownHelper, { ShutdownOrder } from "./utils/ShutdownHelper";
 import { checkConnection, sequelize } from "./storage/database";
 import RedisAdapter from "./storage/redis";
 import Metrics from "./logging/Metrics";
+import { PluginManager } from "./utils/PluginManager";
 
 // Suppress the AWS maintenance message until upgrade to v3.
 maintenance.suppress = true;
@@ -59,6 +60,9 @@ async function master() {
 
 // This function will only be called in each forked process
 async function start(id: number, disconnect: () => void) {
+  // Ensure plugins are loaded
+  PluginManager.loadPlugins();
+
   // Find if SSL certs are available
   const ssl = getSSLOptions();
   const useHTTPS = !!ssl.key && !!ssl.cert;

server/models/helpers/AuthenticationHelper.ts

@@ -1,23 +1,10 @@
 /* eslint-disable @typescript-eslint/no-var-requires */
-import path from "path";
-import { glob } from "glob";
-import Router from "koa-router";
 import find from "lodash/find";
-import sortBy from "lodash/sortBy";
 import env from "@server/env";
 import Team from "@server/models/Team";
-import environment from "@server/utils/environment";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
-
-export type AuthenticationProviderConfig = {
-  id: string;
-  name: string;
-  enabled: boolean;
-  router: Router;
-};
 
 export default class AuthenticationHelper {
-  private static providersCache: AuthenticationProviderConfig[];
-
   /**
    * Returns the enabled authentication provider configurations for the current
    * installation.
@@ -25,46 +12,7 @@ export default class AuthenticationHelper {
    * @returns A list of authentication providers
    */
   public static get providers() {
-    if (this.providersCache) {
+    return PluginManager.getEnabledPlugins(PluginType.AuthProvider);
-      return this.providersCache;
-    }
-
-    const authenticationProviderConfigs: AuthenticationProviderConfig[] = [];
-    const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
-
-    glob
-      .sync(path.join(rootDir, "plugins/*/server/auth/!(*.test|schema).[jt]s"))
-      .forEach((filePath: string) => {
-        const { default: authProvider, name } = require(path.join(
-          process.cwd(),
-          filePath
-        ));
-        const id = filePath.replace("build/", "").split("/")[1];
-        const config = require(path.join(
-          process.cwd(),
-          rootDir,
-          "plugins",
-          id,
-          "plugin.json"
-        ));
-
-        // Test the all required env vars are set for the auth provider
-        const enabled = (config.requiredEnvVars ?? []).every(
-          (name: string) => !!environment[name]
-        );
-
-        if (enabled) {
-          authenticationProviderConfigs.push({
-            id,
-            name: name ?? config.name,
-            enabled,
-            router: authProvider,
-          });
-        }
-      });
-
-    this.providersCache = sortBy(authenticationProviderConfigs, "id");
-    return this.providersCache;
   }
 
   /**
@@ -78,11 +26,11 @@ export default class AuthenticationHelper {
     const isCloudHosted = env.isCloudHosted;
 
     return AuthenticationHelper.providers
-      .sort((config) => (config.id === "email" ? 1 : -1))
+      .sort((plugin) => (plugin.id === "email" ? 1 : -1))
-      .filter((config) => {
+      .filter((plugin) => {
-        // Guest sign-in is an exception as it does not have an authentication
+        // Email sign-in is an exception as it does not have an authentication
         // provider using passport, instead it exists as a boolean option.
-        if (config.id === "email") {
+        if (plugin.id === "email") {
           return team?.emailSigninEnabled;
         }
 
@@ -92,7 +40,7 @@ export default class AuthenticationHelper {
         }
 
         const authProvider = find(team.authenticationProviders, {
-          name: config.id,
+          name: plugin.id,
         });
 
         // If cloud hosted then the auth provider must be enabled for the team,

server/presenters/providerConfig.ts

@@ -1,8 +1,8 @@
 import { signin } from "@shared/utils/routeHelpers";
-import { AuthenticationProviderConfig } from "@server/models/helpers/AuthenticationHelper";
+import { Plugin, PluginType } from "@server/utils/PluginManager";
 
 export default function presentProviderConfig(
-  config: AuthenticationProviderConfig
+  config: Plugin<PluginType.AuthProvider>
 ) {
   return {
     id: config.id,

server/queues/processors/index.ts

@@ -1,12 +1,8 @@
-import path from "path";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
-import { glob } from "glob";
-import env from "@server/env";
-import Logger from "@server/logging/Logger";
 import { requireDirectory } from "@server/utils/fs";
 import BaseProcessor from "./BaseProcessor";
 
 const processors = {};
-const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
 
 requireDirectory<{ default: BaseProcessor }>(__dirname).forEach(
   ([module, id]) => {
@@ -17,14 +13,8 @@ requireDirectory<{ default: BaseProcessor }>(__dirname).forEach(
   }
 );
 
-glob
+PluginManager.getEnabledPlugins(PluginType.Processor).forEach((plugin) => {
-  .sync(path.join(rootDir, "plugins/*/server/processors/!(*.test).[jt]s"))
+  processors[plugin.id] = plugin.value;
-  .forEach((filePath: string) => {
+});
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const processor = require(path.join(process.cwd(), filePath)).default;
-    const name = path.basename(filePath, ".js");
-    processors[name] = processor;
-    Logger.debug("processor", `Registered processor ${name}`);
-  });
 
 export default processors;

server/queues/tasks/BaseTask.ts

@@ -13,7 +13,7 @@ export enum TaskSchedule {
   Hourly = "hourly",
 }
 
-export default abstract class BaseTask<T> {
+export default abstract class BaseTask<T extends Record<string, any>> {
   /**
    * An optional schedule for this task to be run automatically.
    */

server/queues/tasks/InviteReminderTask.ts

@@ -6,7 +6,7 @@ import { UserFlag } from "@server/models/User";
 import { sequelize } from "@server/storage/database";
 import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
 
-type Props = undefined;
+type Props = Record<string, never>;
 
 export default class InviteReminderTask extends BaseTask<Props> {
   static cron = TaskSchedule.Daily;

server/queues/tasks/index.ts

@@ -1,12 +1,8 @@
-import path from "path";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
-import { glob } from "glob";
-import env from "@server/env";
-import Logger from "@server/logging/Logger";
 import { requireDirectory } from "@server/utils/fs";
 import BaseTask from "./BaseTask";
 
 const tasks = {};
-const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
 
 requireDirectory<{ default: BaseTask<any> }>(__dirname).forEach(
   ([module, id]) => {
@@ -17,14 +13,8 @@ requireDirectory<{ default: BaseTask<any> }>(__dirname).forEach(
   }
 );
 
-glob
+PluginManager.getEnabledPlugins(PluginType.Processor).forEach((plugin) => {
-  .sync(path.join(rootDir, "plugins/*/server/tasks/!(*.test).[jt]s"))
+  tasks[plugin.id] = plugin.value;
-  .forEach((filePath: string) => {
+});
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const task = require(path.join(process.cwd(), filePath)).default;
-    const name = path.basename(filePath, ".js");
-    tasks[name] = task;
-    Logger.debug("task", `Registered task ${name}`);
-  });
 
 export default tasks;

server/routes/api/index.ts

@@ -1,14 +1,12 @@
-import path from "path";
-import glob from "glob";
 import Koa, { BaseContext } from "koa";
 import bodyParser from "koa-body";
 import Router from "koa-router";
 import userAgent, { UserAgentContext } from "koa-useragent";
 import env from "@server/env";
 import { NotFoundError } from "@server/errors";
-import Logger from "@server/logging/Logger";
 import coalesceBody from "@server/middlewares/coaleseBody";
 import { AppState, AppContext } from "@server/types";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
 import apiKeys from "./apiKeys";
 import attachments from "./attachments";
 import auth from "./auth";
@@ -60,19 +58,13 @@ api.use(apiTracer());
 api.use(apiResponse());
 api.use(editor());
 
-// register package API routes before others to allow for overrides
+// Register plugin API routes before others to allow for overrides
-const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
+const plugins = PluginManager.getEnabledPlugins(PluginType.API).map(
-glob
+  (plugin) => plugin.value
-  .sync(path.join(rootDir, "plugins/*/server/api/!(*.test).[jt]s"))
+);
-  .forEach((filePath: string) => {
+for (const router of plugins) {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
+  router.use("/", router.routes());
-    const pkg: Router = require(path.join(process.cwd(), filePath)).default;
+}
-
-    if (pkg && "routes" in pkg) {
-      router.use("/", pkg.routes());
-      Logger.debug("lifecycle", `Registered API routes for ${filePath}`);
-    }
-  });
 
 // routes
 router.use("/", auth.routes());

server/routes/api/urls/urls.test.ts

@@ -2,7 +2,7 @@ import env from "@server/env";
 import { User } from "@server/models";
 import { buildDocument, buildUser } from "@server/test/factories";
 import { getTestServer } from "@server/test/support";
-import resolvers from "@server/utils/unfurl";
+import Iframely from "plugins/iframely/server/iframely";
 
 jest.mock("dns", () => ({
   resolveCname: (
@@ -17,9 +17,7 @@ jest.mock("dns", () => ({
   },
 }));
 
-jest
+jest.spyOn(Iframely, "fetch").mockImplementation(() => Promise.resolve(false));
-  .spyOn(resolvers.Iframely, "unfurl")
-  .mockImplementation(async (_: string) => false);
 
 const server = getTestServer();
 
@@ -158,7 +156,7 @@ describe("#urls.unfurl", () => {
   });
 
   it("should succeed with status 200 ok for a valid external url", async () => {
-    (resolvers.Iframely.unfurl as jest.Mock).mockResolvedValue(
+    (Iframely.fetch as jest.Mock).mockResolvedValue(
       Promise.resolve({
         url: "https://www.flickr.com",
         type: "rich",
@@ -180,9 +178,6 @@ describe("#urls.unfurl", () => {
     expect(res.status).toEqual(200);
     const body = await res.json();
 
-    expect(resolvers.Iframely.unfurl).toHaveBeenCalledWith(
-      "https://www.flickr.com"
-    );
     expect(res.status).toEqual(200);
     expect(body.url).toEqual("https://www.flickr.com");
     expect(body.type).toEqual("rich");
@@ -196,7 +191,7 @@ describe("#urls.unfurl", () => {
   });
 
   it("should succeed with status 204 no content for a non-existing external url", async () => {
-    (resolvers.Iframely.unfurl as jest.Mock).mockResolvedValue(
+    (Iframely.fetch as jest.Mock).mockResolvedValue(
       Promise.resolve({
         status: 404,
         error:
@@ -211,9 +206,6 @@ describe("#urls.unfurl", () => {
       },
     });
 
-    expect(resolvers.Iframely.unfurl).toHaveBeenCalledWith(
-      "https://random.url"
-    );
     expect(res.status).toEqual(204);
   });
 });

server/routes/api/urls/urls.ts

@@ -13,11 +13,12 @@ import { authorize } from "@server/policies";
 import { presentDocument, presentMention } from "@server/presenters/unfurls";
 import presentUnfurl from "@server/presenters/unfurls/unfurl";
 import { APIContext } from "@server/types";
+import { PluginManager, PluginType } from "@server/utils/PluginManager";
 import { RateLimiterStrategy } from "@server/utils/RateLimiter";
-import resolvers from "@server/utils/unfurl";
 import * as T from "./schema";
 
 const router = new Router();
+const plugins = PluginManager.getEnabledPlugins(PluginType.UnfurlProvider);
 
 router.post(
   "urls.unfurl",
@@ -74,11 +75,13 @@ router.post(
     }
 
     // External resources
-    if (resolvers.Iframely) {
+    for (const plugin of plugins) {
-      const data = await resolvers.Iframely.unfurl(url);
+      const data = await plugin.value(url);
-      return data.error
+      if (data) {
-        ? (ctx.response.status = 204)
+        return "error" in data
-        : (ctx.body = presentUnfurl(data));
+          ? (ctx.response.status = 204)
+          : (ctx.body = presentUnfurl(data));
+      }
     }
 
     return (ctx.response.status = 204);

server/routes/auth/index.ts

@@ -17,7 +17,7 @@ router.use(passport.initialize());
 
 // dynamically load available authentication provider routes
 AuthenticationHelper.providers.forEach((provider) => {
-  router.use("/", provider.router.routes());
+  router.use("/", provider.value.routes());
 });
 
 router.get("/redirect", auth(), async (ctx: APIContext) => {

server/utils/PluginManager.ts

@@ -0,0 +1,173 @@
+import path from "path";
+import { glob } from "glob";
+import type Router from "koa-router";
+import sortBy from "lodash/sortBy";
+import { v4 as uuid } from "uuid";
+import { UnfurlSignature } from "@shared/types";
+import type BaseEmail from "@server/emails/templates/BaseEmail";
+import env from "@server/env";
+import Logger from "@server/logging/Logger";
+import type BaseProcessor from "@server/queues/processors/BaseProcessor";
+import type BaseTask from "@server/queues/tasks/BaseTask";
+
+export enum PluginPriority {
+  VeryHigh = 0,
+  High = 100,
+  Normal = 200,
+  Low = 300,
+  VeryLow = 500,
+}
+
+/**
+ * The different types of server plugins that can be registered.
+ */
+export enum PluginType {
+  API = "api",
+  AuthProvider = "authProvider",
+  EmailTemplate = "emailTemplate",
+  Processor = "processor",
+  Task = "task",
+  UnfurlProvider = "unfurl",
+}
+
+/**
+ * A map of plugin types to their values, for example an API plugin would have a value of type
+ * Router. Registering an API plugin causes the router to be mounted.
+ */
+type PluginValueMap = {
+  [PluginType.API]: Router;
+  [PluginType.AuthProvider]: Router;
+  [PluginType.EmailTemplate]: typeof BaseEmail;
+  [PluginType.Processor]: typeof BaseProcessor;
+  [PluginType.Task]: typeof BaseTask<any>;
+  [PluginType.UnfurlProvider]: UnfurlSignature;
+};
+
+export type Plugin<T extends PluginType> = {
+  /** A unique ID for the plugin */
+  id: string;
+  /** The plugin's display name */
+  name?: string;
+  /** A brief description of the plugin */
+  description?: string;
+  /** The plugin content */
+  value: PluginValueMap[T];
+  /** An optional priority, will affect order in menus and execution. Lower is earlier. */
+  priority?: number;
+  /** Whether the plugin is enabled (default: true) */
+  enabled?: boolean;
+};
+
+export class PluginManager {
+  private static plugins = new Map<PluginType, Plugin<PluginType>[]>();
+
+  /**
+   * Register a plugin of a given type.
+   *
+   * @param type The plugin type
+   * @param value The plugin value
+   * @param options Additional options, including whether the plugin is enabled and it's priority.
+   * @returns The PluginManager instance, for chaining.
+   */
+  public static register<T extends PluginType>(
+    type: T,
+    value: PluginValueMap[T],
+    options: Omit<Plugin<T>, "value"> = {
+      id: uuid(),
+    }
+  ) {
+    if (!this.plugins.has(type)) {
+      this.plugins.set(type, []);
+    }
+
+    const plugin = {
+      value,
+      priority: PluginPriority.Normal,
+      ...options,
+    };
+
+    Logger.debug(
+      "plugins",
+      `Plugin ${options.enabled === false ? "disabled" : "enabled"} "${
+        options.id
+      }" ${options.description ? `(${options.description})` : ""}`
+    );
+
+    this.plugins.get(type)!.push(plugin);
+
+    // allow chaining
+    return this;
+  }
+
+  /**
+   * Syntactic sugar for registering a background Task.
+   *
+   * @param value The task class
+   * @param options Additional options
+   */
+  public static registerTask(
+    value: PluginValueMap[PluginType.Task],
+    options?: Omit<Plugin<PluginType.Task>, "id" | "value">
+  ) {
+    return this.register(PluginType.Task, value, {
+      id: value.name,
+      ...options,
+    });
+  }
+
+  /**
+   * Syntactic sugar for registering a background Processor.
+   *
+   * @param value The processor class
+   * @param options Additional options
+   */
+  public static registerProcessor(
+    value: PluginValueMap[PluginType.Processor],
+    options?: Omit<Plugin<PluginType.Processor>, "id" | "value">
+  ) {
+    return this.register(PluginType.Processor, value, {
+      id: value.name,
+      ...options,
+    });
+  }
+
+  /**
+   * Returns all the plugins of a given type in order of priority.
+   *
+   * @param type The type of plugin to filter by
+   * @returns A list of plugins
+   */
+  public static getPlugins<T extends PluginType>(type: T) {
+    this.loadPlugins();
+    return sortBy(this.plugins.get(type) || [], "priority") as Plugin<T>[];
+  }
+
+  /**
+   * Returns all the enabled plugins of a given type in order of priority.
+   *
+   * @param type The type of plugin to filter by
+   * @returns A list of plugins
+   */
+  public static getEnabledPlugins<T extends PluginType>(type: T) {
+    return this.getPlugins(type).filter((plugin) => plugin.enabled !== false);
+  }
+
+  /**
+   * Load plugin server components (anything in the `/server/` directory of a plugin will be loaded)
+   */
+  public static loadPlugins() {
+    if (this.loaded) {
+      return;
+    }
+    const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
+
+    glob
+      .sync(path.join(rootDir, "plugins/*/server/!(*.test|schema).[jt]s"))
+      .forEach((filePath: string) => {
+        require(path.join(process.cwd(), filePath));
+      });
+    this.loaded = true;
+  }
+
+  private static loaded = false;
+}

server/utils/unfurl.ts

@@ -1,43 +0,0 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-import path from "path";
-import glob from "glob";
-import env from "@server/env";
-import Logger from "@server/logging/Logger";
-import { UnfurlResolver } from "@server/types";
-import environment from "./environment";
-
-const rootDir = env.ENVIRONMENT === "test" ? "" : "build";
-
-const plugins = glob.sync(path.join(rootDir, "plugins/*/server/unfurl.[jt]s"));
-const resolvers: Record<string, UnfurlResolver> = plugins.reduce(
-  (resolvers, filePath) => {
-    const resolver: UnfurlResolver = require(path.join(
-      process.cwd(),
-      filePath
-    ));
-    const id = filePath.replace("build/", "").split("/")[1];
-    const config = require(path.join(
-      process.cwd(),
-      rootDir,
-      "plugins",
-      id,
-      "plugin.json"
-    ));
-
-    // Test the all required env vars are set for the resolver
-    const enabled = (config.requiredEnvVars ?? []).every(
-      (name: string) => !!environment[name]
-    );
-    if (!enabled) {
-      return resolvers;
-    }
-
-    resolvers[config.name] = resolver;
-    Logger.debug("utils", `Registered unfurl resolver ${filePath}`);
-
-    return resolvers;
-  },
-  {}
-);
-
-export default resolvers;

shared/types.ts

@@ -270,14 +270,20 @@ export enum QueryNotices {
 
 export type OEmbedType = "photo" | "video" | "rich";
 
-export type Unfurl<T = OEmbedType> = {
+export type Unfurl<T = OEmbedType> =
-  url?: string;
+  | {
-  type: T;
+      url?: string;
-  title: string;
+      type: T;
-  description?: string;
+      title: string;
-  thumbnailUrl?: string | null;
+      description?: string;
-  meta?: Record<string, string>;
+      thumbnailUrl?: string | null;
-};
+      meta?: Record<string, string>;
+    }
+  | {
+      error: string;
+    };
+
+export type UnfurlSignature = (url: string) => Promise<Unfurl | false>;
 
 export type JSONValue =
   | string