From f9a11a28d8fadecda301cc68c51199957af3d213 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Fri, 8 Mar 2024 21:32:05 -0700 Subject: [PATCH] chore: Plugin registration (#6623) * first pass * test * test * priority * Reduce boilerplate further * Update server/utils/PluginManager.ts Co-authored-by: Apoorv Mishra * fix: matchesNode error in destroyed editor transaction * fix: Individual imported files do not display source correctly in 'Insights' * chore: Add sleep before Slack notification * docs * fix: Error logged about missing plugin.json * Remove email template glob --------- Co-authored-by: Apoorv Mishra --- .env.test | 1 + app/hooks/useSettingsConfig.ts | 2 +- app/utils/PluginLoader.ts | 1 - build.js | 2 +- plugins/azure/plugin.json | 5 +- plugins/azure/server/auth/azure.ts | 12 +- plugins/azure/server/index.ts | 9 + plugins/email/plugin.json | 1 + plugins/email/server/index.ts | 9 + plugins/google/plugin.json | 5 +- plugins/google/server/auth/google.ts | 13 +- plugins/google/server/index.ts | 9 + plugins/iframely/plugin.json | 5 - plugins/iframely/server/iframely.ts | 14 +- plugins/iframely/server/index.ts | 15 ++ plugins/iframely/server/unfurl.ts | 3 - plugins/oidc/plugin.json | 5 +- plugins/oidc/server/auth/oidc.ts | 15 +- plugins/oidc/server/index.ts | 16 ++ plugins/slack/plugin.json | 5 +- plugins/slack/server/index.ts | 20 ++ plugins/storage/plugin.json | 8 - plugins/storage/server/api/files.ts | 3 - plugins/storage/server/index.ts | 31 ++++ plugins/storage/server/utils.ts | 20 -- plugins/webhooks/plugin.json | 4 +- plugins/webhooks/server/index.ts | 11 ++ .../tasks/CleanupWebhookDeliveriesTask.ts | 4 +- server/emails/templates/index.ts | 19 +- server/index.ts | 4 + server/models/helpers/AuthenticationHelper.ts | 66 +------ server/presenters/providerConfig.ts | 4 +- server/queues/processors/index.ts | 18 +- server/queues/tasks/BaseTask.ts | 2 +- server/queues/tasks/InviteReminderTask.ts | 2 +- server/queues/tasks/index.ts | 18 +- server/routes/api/index.ts | 24 +-- server/routes/api/urls/urls.test.ts | 16 +- server/routes/api/urls/urls.ts | 15 +- server/routes/auth/index.ts | 2 +- server/utils/PluginManager.ts | 173 ++++++++++++++++++ server/utils/unfurl.ts | 43 ----- shared/types.ts | 22 ++- 43 files changed, 400 insertions(+), 276 deletions(-) create mode 100644 plugins/azure/server/index.ts create mode 100644 plugins/email/server/index.ts create mode 100644 plugins/google/server/index.ts delete mode 100644 plugins/iframely/plugin.json create mode 100644 plugins/iframely/server/index.ts delete mode 100644 plugins/iframely/server/unfurl.ts create mode 100644 plugins/oidc/server/index.ts create mode 100644 plugins/slack/server/index.ts delete mode 100644 plugins/storage/plugin.json create mode 100644 plugins/storage/server/index.ts delete mode 100644 plugins/storage/server/utils.ts create mode 100644 plugins/webhooks/server/index.ts create mode 100644 server/utils/PluginManager.ts delete mode 100644 server/utils/unfurl.ts diff --git a/.env.test b/.env.test index ab2700f2f..3ca155f2e 100644 --- a/.env.test +++ b/.env.test @@ -3,6 +3,7 @@ DATABASE_URL=postgres://user:pass@127.0.0.1:5432/outline-test SECRET_KEY=F0E5AD933D7F6FD8F4DBB3E038C501C052DC0593C686D21ACB30AE205D2F634B SMTP_HOST=smtp.example.com +SMTP_USERNAME=test SMTP_FROM_EMAIL=hello@example.com SMTP_REPLY_EMAIL=hello@example.com diff --git a/app/hooks/useSettingsConfig.ts b/app/hooks/useSettingsConfig.ts index 57859ed74..572f106bb 100644 --- a/app/hooks/useSettingsConfig.ts +++ b/app/hooks/useSettingsConfig.ts @@ -197,7 +197,7 @@ const useSettingsConfig = () => { Object.values(PluginLoader.plugins).map((plugin) => { const hasSettings = !!plugin.settings; const enabledInDeployment = - !plugin.config.deployments || + !plugin.config?.deployments || plugin.config.deployments.length === 0 || (plugin.config.deployments.includes("cloud") && isCloudHosted) || (plugin.config.deployments.includes("enterprise") && !isCloudHosted); diff --git a/app/utils/PluginLoader.ts b/app/utils/PluginLoader.ts index 05bebb2ad..54bb6e133 100644 --- a/app/utils/PluginLoader.ts +++ b/app/utils/PluginLoader.ts @@ -5,7 +5,6 @@ interface Plugin { config: { name: string; description: string; - requiredEnvVars?: string[]; deployments?: string[]; }; settings: React.FC; diff --git a/build.js b/build.js index 998fa4df9..f761f88a2 100755 --- a/build.js +++ b/build.js @@ -79,7 +79,7 @@ async function build() { execAsync("cp package.json ./build"), ...d.map(async (plugin) => execAsync( - `mkdir -p ./build/plugins/${plugin} && cp ./plugins/${plugin}/plugin.json ./build/plugins/${plugin}/plugin.json` + `mkdir -p ./build/plugins/${plugin} && cp ./plugins/${plugin}/plugin.json ./build/plugins/${plugin}/plugin.json 2>/dev/null || :` ) ), ]); diff --git a/plugins/azure/plugin.json b/plugins/azure/plugin.json index 2a33a206a..1979c0afa 100644 --- a/plugins/azure/plugin.json +++ b/plugins/azure/plugin.json @@ -1,5 +1,6 @@ { + "id": "azure", "name": "Microsoft", - "description": "Adds a Microsoft Azure authentication provider.", - "requiredEnvVars": ["AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET"] + "priority": 20, + "description": "Adds a Microsoft Azure authentication provider." } diff --git a/plugins/azure/server/auth/azure.ts b/plugins/azure/server/auth/azure.ts index 5dada618c..77268025b 100644 --- a/plugins/azure/server/auth/azure.ts +++ b/plugins/azure/server/auth/azure.ts @@ -16,10 +16,10 @@ import { getTeamFromContext, getClientFromContext, } from "@server/utils/passport"; +import config from "../../plugin.json"; import env from "../env"; const router = new Router(); -const providerName = "azure"; const scopes: string[] = []; if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) { @@ -109,7 +109,7 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) { avatarUrl: profile.picture, }, authenticationProvider: { - name: providerName, + name: config.id, providerId: profile.tid, }, authentication: { @@ -127,13 +127,11 @@ if (env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) { } ); passport.use(strategy); - router.get( - "azure", - passport.authenticate(providerName, { prompt: "select_account" }) + config.id, + passport.authenticate(config.id, { prompt: "select_account" }) ); - - router.get("azure.callback", passportMiddleware(providerName)); + router.get(`${config.id}.callback`, passportMiddleware(config.id)); } export default router; diff --git a/plugins/azure/server/index.ts b/plugins/azure/server/index.ts new file mode 100644 index 000000000..52f749074 --- /dev/null +++ b/plugins/azure/server/index.ts @@ -0,0 +1,9 @@ +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import config from "../plugin.json"; +import router from "./auth/azure"; +import env from "./env"; + +PluginManager.register(PluginType.AuthProvider, router, { + ...config, + enabled: !!env.AZURE_CLIENT_ID && !!env.AZURE_CLIENT_SECRET, +}); diff --git a/plugins/email/plugin.json b/plugins/email/plugin.json index 8346107b0..617e58868 100644 --- a/plugins/email/plugin.json +++ b/plugins/email/plugin.json @@ -1,4 +1,5 @@ { + "id": "email", "name": "Email", "description": "Adds an email magic link authentication provider." } diff --git a/plugins/email/server/index.ts b/plugins/email/server/index.ts new file mode 100644 index 000000000..179cb689d --- /dev/null +++ b/plugins/email/server/index.ts @@ -0,0 +1,9 @@ +import env from "@server/env"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import config from "../plugin.json"; +import router from "./auth/email"; + +PluginManager.register(PluginType.AuthProvider, router, { + ...config, + enabled: (!!env.SMTP_HOST && !!env.SMTP_USERNAME) || env.isDevelopment, +}); diff --git a/plugins/google/plugin.json b/plugins/google/plugin.json index f2b687c2a..30b732cd2 100644 --- a/plugins/google/plugin.json +++ b/plugins/google/plugin.json @@ -1,5 +1,6 @@ { + "id": "google", "name": "Google", - "description": "Adds a Google authentication provider.", - "requiredEnvVars": ["GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET"] + "priority": 10, + "description": "Adds a Google authentication provider." } diff --git a/plugins/google/server/auth/google.ts b/plugins/google/server/auth/google.ts index 0aefa3564..aca999bdb 100644 --- a/plugins/google/server/auth/google.ts +++ b/plugins/google/server/auth/google.ts @@ -18,10 +18,10 @@ import { getTeamFromContext, getClientFromContext, } from "@server/utils/passport"; +import config from "../../plugin.json"; import env from "../env"; const router = new Router(); -const providerName = "google"; const scopes = [ "https://www.googleapis.com/auth/userinfo.profile", @@ -42,7 +42,7 @@ if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) { { clientID: env.GOOGLE_CLIENT_ID, clientSecret: env.GOOGLE_CLIENT_SECRET, - callbackURL: `${env.URL}/auth/google.callback`, + callbackURL: `${env.URL}/auth/${config.id}.callback`, passReqToCallback: true, // @ts-expect-error StateStore store: new StateStore(), @@ -110,7 +110,7 @@ if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) { avatarUrl, }, authenticationProvider: { - name: providerName, + name: config.id, providerId: domain ?? "", }, authentication: { @@ -131,14 +131,13 @@ if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) { ); router.get( - "google", - passport.authenticate(providerName, { + config.id, + passport.authenticate(config.id, { accessType: "offline", prompt: "select_account consent", }) ); - - router.get("google.callback", passportMiddleware(providerName)); + router.get(`${config.id}.callback`, passportMiddleware(config.id)); } export default router; diff --git a/plugins/google/server/index.ts b/plugins/google/server/index.ts new file mode 100644 index 000000000..7152dd606 --- /dev/null +++ b/plugins/google/server/index.ts @@ -0,0 +1,9 @@ +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import config from "../plugin.json"; +import router from "./auth/google"; +import env from "./env"; + +PluginManager.register(PluginType.AuthProvider, router, { + ...config, + enabled: !!env.GOOGLE_CLIENT_ID && !!env.GOOGLE_CLIENT_SECRET, +}); diff --git a/plugins/iframely/plugin.json b/plugins/iframely/plugin.json deleted file mode 100644 index 8b5767606..000000000 --- a/plugins/iframely/plugin.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "Iframely", - "description": "Integrate Iframely to enable unfurling of arbitrary urls", - "requiredEnvVars": ["IFRAMELY_API_KEY"] -} diff --git a/plugins/iframely/server/iframely.ts b/plugins/iframely/server/iframely.ts index 14700b616..25403e57c 100644 --- a/plugins/iframely/server/iframely.ts +++ b/plugins/iframely/server/iframely.ts @@ -1,3 +1,4 @@ +import type { Unfurl } from "@shared/types"; import { Day } from "@shared/utils/time"; import { InternalError } from "@server/errors"; import Logger from "@server/logging/Logger"; @@ -33,7 +34,7 @@ class Iframely { } } - private static async fetch(url: string, type = "oembed") { + public static async fetch(url: string, type = "oembed") { const res = await fetch( `${this.apiUrl}/${type}?url=${encodeURIComponent(url)}&api_key=${ this.apiKey @@ -55,20 +56,19 @@ class Iframely { } /** - * Fetches the preview data for the given url - * using Iframely oEmbed API + * Fetches the preview data for the given url using Iframely oEmbed API * * @param url * @returns Preview data for the url */ - public static async get(url: string) { + public static async get(url: string): Promise { try { - const cached = await this.cached(url); + const cached = await Iframely.cached(url); if (cached) { return cached; } - const res = await this.fetch(url); - await this.cache(url, res); + const res = await Iframely.fetch(url); + await Iframely.cache(url, res); return res; } catch (err) { throw InternalError(err); diff --git a/plugins/iframely/server/index.ts b/plugins/iframely/server/index.ts new file mode 100644 index 000000000..5d59f9f22 --- /dev/null +++ b/plugins/iframely/server/index.ts @@ -0,0 +1,15 @@ +import { + PluginManager, + PluginPriority, + PluginType, +} from "@server/utils/PluginManager"; +import env from "./env"; +import Iframely from "./iframely"; + +PluginManager.register(PluginType.UnfurlProvider, Iframely.get, { + id: "iframely", + enabled: !!env.IFRAMELY_API_KEY && !!env.IFRAMELY_URL, + + // Make sure this is last in the stack to be evaluated after all other unfurl providers + priority: PluginPriority.VeryLow, +}); diff --git a/plugins/iframely/server/unfurl.ts b/plugins/iframely/server/unfurl.ts deleted file mode 100644 index a70e2212d..000000000 --- a/plugins/iframely/server/unfurl.ts +++ /dev/null @@ -1,3 +0,0 @@ -import Iframely from "./iframely"; - -export const unfurl = async (url: string) => Iframely.get(url); diff --git a/plugins/oidc/plugin.json b/plugins/oidc/plugin.json index fdb7121f0..b693a1d6e 100644 --- a/plugins/oidc/plugin.json +++ b/plugins/oidc/plugin.json @@ -1,5 +1,6 @@ { + "id": "oidc", "name": "OIDC", - "description": "Adds an OpenID compatible authentication provider.", - "requiredEnvVars": ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_AUTH_URI", "OIDC_TOKEN_URI", "OIDC_USERINFO_URI"] + "priority": 30, + "description": "Adds an OpenID compatible authentication provider." } diff --git a/plugins/oidc/server/auth/oidc.ts b/plugins/oidc/server/auth/oidc.ts index 4ca362d7a..9b4f0abdf 100644 --- a/plugins/oidc/server/auth/oidc.ts +++ b/plugins/oidc/server/auth/oidc.ts @@ -18,10 +18,10 @@ import { getTeamFromContext, getClientFromContext, } from "@server/utils/passport"; +import config from "../../plugin.json"; import env from "../env"; const router = new Router(); -const providerName = "oidc"; const scopes = env.OIDC_SCOPES.split(" "); Strategy.prototype.userProfile = async function (accessToken, done) { @@ -55,14 +55,14 @@ if ( env.OIDC_USERINFO_URI ) { passport.use( - providerName, + config.id, new Strategy( { authorizationURL: env.OIDC_AUTH_URI, tokenURL: env.OIDC_TOKEN_URI, clientID: env.OIDC_CLIENT_ID, clientSecret: env.OIDC_CLIENT_SECRET, - callbackURL: `${env.URL}/auth/${providerName}.callback`, + callbackURL: `${env.URL}/auth/${config.id}.callback`, passReqToCallback: true, scope: env.OIDC_SCOPES, // @ts-expect-error custom state store @@ -134,7 +134,7 @@ if ( avatarUrl: profile.picture, }, authenticationProvider: { - name: providerName, + name: config.id, providerId: domain, }, authentication: { @@ -153,11 +153,8 @@ if ( ) ); - router.get(providerName, passport.authenticate(providerName)); - - router.get(`${providerName}.callback`, passportMiddleware(providerName)); + router.get(config.id, passport.authenticate(config.id)); + router.get(`${config.id}.callback`, passportMiddleware(config.id)); } -export const name = env.OIDC_DISPLAY_NAME; - export default router; diff --git a/plugins/oidc/server/index.ts b/plugins/oidc/server/index.ts new file mode 100644 index 000000000..6fa79560b --- /dev/null +++ b/plugins/oidc/server/index.ts @@ -0,0 +1,16 @@ +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import config from "../plugin.json"; +import router from "./auth/oidc"; +import env from "./env"; + +PluginManager.register(PluginType.AuthProvider, router, { + ...config, + name: env.OIDC_DISPLAY_NAME || config.name, + enabled: !!( + env.OIDC_CLIENT_ID && + env.OIDC_CLIENT_SECRET && + env.OIDC_AUTH_URI && + env.OIDC_TOKEN_URI && + env.OIDC_USERINFO_URI + ), +}); diff --git a/plugins/slack/plugin.json b/plugins/slack/plugin.json index 8a97b6c27..0bee50894 100644 --- a/plugins/slack/plugin.json +++ b/plugins/slack/plugin.json @@ -1,5 +1,6 @@ { + "id": "slack", "name": "Slack", - "description": "Adds a Slack authentication provider, support for the /outline slash command, and link unfurling.", - "requiredEnvVars": ["SLACK_CLIENT_ID", "SLACK_CLIENT_SECRET"] + "priority": 40, + "description": "Adds a Slack authentication provider, support for the /outline slash command, and link unfurling." } diff --git a/plugins/slack/server/index.ts b/plugins/slack/server/index.ts new file mode 100644 index 000000000..7e23135f5 --- /dev/null +++ b/plugins/slack/server/index.ts @@ -0,0 +1,20 @@ +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import config from "../plugin.json"; +import hooks from "./api/hooks"; +import router from "./auth/slack"; +import env from "./env"; +import SlackProcessor from "./processors/SlackProcessor"; + +const enabled = !!env.SLACK_CLIENT_ID && !!env.SLACK_CLIENT_SECRET; + +PluginManager.register(PluginType.AuthProvider, router, { + ...config, + enabled, +}); + +PluginManager.register(PluginType.API, hooks, { + ...config, + enabled, +}); + +PluginManager.registerProcessor(SlackProcessor, { enabled }); diff --git a/plugins/storage/plugin.json b/plugins/storage/plugin.json deleted file mode 100644 index b7274ca29..000000000 --- a/plugins/storage/plugin.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "Storage", - "description": "Plugin for storing files on the local file system", - "requiredEnvVars": [ - "FILE_STORAGE_UPLOAD_MAX_SIZE", - "FILE_STORAGE_LOCAL_ROOT_DIR" - ] -} diff --git a/plugins/storage/server/api/files.ts b/plugins/storage/server/api/files.ts index 3a35ad857..0d66e8d5c 100644 --- a/plugins/storage/server/api/files.ts +++ b/plugins/storage/server/api/files.ts @@ -18,11 +18,8 @@ import FileStorage from "@server/storage/files"; import { APIContext } from "@server/types"; import { RateLimiterStrategy } from "@server/utils/RateLimiter"; import { getJWTPayload } from "@server/utils/jwt"; -import { createRootDirForLocalStorage } from "../utils"; import * as T from "./schema"; -createRootDirForLocalStorage(); - const router = new Router(); router.post( diff --git a/plugins/storage/server/index.ts b/plugins/storage/server/index.ts new file mode 100644 index 000000000..de32dea9a --- /dev/null +++ b/plugins/storage/server/index.ts @@ -0,0 +1,31 @@ +import { existsSync, mkdirSync } from "fs"; +import env from "@server/env"; +import Logger from "@server/logging/Logger"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import router from "./api/files"; + +if (env.FILE_STORAGE === "local") { + const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR; + try { + if (!existsSync(rootDir)) { + mkdirSync(rootDir, { recursive: true }); + Logger.debug("utils", `Created ${rootDir} for local storage`); + } + } catch (err) { + Logger.fatal( + `Failed to create directory for local file storage at ${env.FILE_STORAGE_LOCAL_ROOT_DIR}`, + err + ); + } +} + +PluginManager.register(PluginType.API, router, { + id: "files", + name: "Local file storage", + description: "Plugin for storing files on the local file system", + enabled: !!( + env.FILE_STORAGE_UPLOAD_MAX_SIZE && + env.FILE_STORAGE_LOCAL_ROOT_DIR && + env.FILE_STORAGE === "local" + ), +}); diff --git a/plugins/storage/server/utils.ts b/plugins/storage/server/utils.ts deleted file mode 100644 index 13520b9d1..000000000 --- a/plugins/storage/server/utils.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { existsSync, mkdirSync } from "fs"; -import env from "@server/env"; -import Logger from "@server/logging/Logger"; - -export const createRootDirForLocalStorage = () => { - if (env.FILE_STORAGE === "local") { - const rootDir = env.FILE_STORAGE_LOCAL_ROOT_DIR; - try { - if (!existsSync(rootDir)) { - mkdirSync(rootDir, { recursive: true }); - Logger.debug("utils", `Created ${rootDir} for local storage`); - } - } catch (err) { - Logger.fatal( - `Failed to create directory for local file storage at ${env.FILE_STORAGE_LOCAL_ROOT_DIR}`, - err - ); - } - } -}; diff --git a/plugins/webhooks/plugin.json b/plugins/webhooks/plugin.json index 704a16f8e..af34afda2 100644 --- a/plugins/webhooks/plugin.json +++ b/plugins/webhooks/plugin.json @@ -1,5 +1,5 @@ { + "id": "webhooks", "name": "Webhooks", - "description": "Adds HTTP webhooks for various events.", - "requiredEnvVars": [] + "description": "Adds HTTP webhooks for various events." } diff --git a/plugins/webhooks/server/index.ts b/plugins/webhooks/server/index.ts new file mode 100644 index 000000000..832d5ad68 --- /dev/null +++ b/plugins/webhooks/server/index.ts @@ -0,0 +1,11 @@ +import { PluginManager, PluginType } from "@server/utils/PluginManager"; +import config from "../plugin.json"; +import webhookSubscriptions from "./api/webhookSubscriptions"; +import WebhookProcessor from "./processors/WebhookProcessor"; +import CleanupWebhookDeliveriesTask from "./tasks/CleanupWebhookDeliveriesTask"; +import DeliverWebhookTask from "./tasks/DeliverWebhookTask"; + +PluginManager.register(PluginType.API, webhookSubscriptions, config) + .registerProcessor(WebhookProcessor) + .registerTask(DeliverWebhookTask) + .registerTask(CleanupWebhookDeliveriesTask); diff --git a/plugins/webhooks/server/tasks/CleanupWebhookDeliveriesTask.ts b/plugins/webhooks/server/tasks/CleanupWebhookDeliveriesTask.ts index 6be34b13a..13f4cf741 100644 --- a/plugins/webhooks/server/tasks/CleanupWebhookDeliveriesTask.ts +++ b/plugins/webhooks/server/tasks/CleanupWebhookDeliveriesTask.ts @@ -7,12 +7,12 @@ import BaseTask, { TaskSchedule, } from "@server/queues/tasks/BaseTask"; -type Props = void; +type Props = Record; export default class CleanupWebhookDeliveriesTask extends BaseTask { static cron = TaskSchedule.Daily; - public async perform(_: Props) { + public async perform() { Logger.info("task", `Deleting WebhookDeliveries older than one week…`); const count = await WebhookDelivery.unscoped().destroy({ where: { diff --git a/server/emails/templates/index.ts b/server/emails/templates/index.ts index bea83dc52..d90ee5fa8 100644 --- a/server/emails/templates/index.ts +++ b/server/emails/templates/index.ts @@ -1,7 +1,4 @@ -import path from "path"; -import { glob } from "glob"; -import env from "@server/env"; -import Logger from "@server/logging/Logger"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; import { requireDirectory } from "@server/utils/fs"; const emails = {}; @@ -17,16 +14,8 @@ requireDirectory(__dirname).forEach(([module, id]) => { emails[id] = Email; }); -const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; -glob - .sync(path.join(rootDir, "plugins/*/server/email/templates/!(*.test).[jt]s")) - .forEach((filePath: string) => { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const template = require(path.join(process.cwd(), filePath)).default; - - Logger.debug("lifecycle", `Registered email template ${template.name}`); - - emails[template.name] = template; - }); +PluginManager.getEnabledPlugins(PluginType.EmailTemplate).forEach((plugin) => { + emails[plugin.id] = plugin.value; +}); export default emails; diff --git a/server/index.ts b/server/index.ts index 803ef9f93..1d2b1363d 100644 --- a/server/index.ts +++ b/server/index.ts @@ -26,6 +26,7 @@ import ShutdownHelper, { ShutdownOrder } from "./utils/ShutdownHelper"; import { checkConnection, sequelize } from "./storage/database"; import RedisAdapter from "./storage/redis"; import Metrics from "./logging/Metrics"; +import { PluginManager } from "./utils/PluginManager"; // Suppress the AWS maintenance message until upgrade to v3. maintenance.suppress = true; @@ -59,6 +60,9 @@ async function master() { // This function will only be called in each forked process async function start(id: number, disconnect: () => void) { + // Ensure plugins are loaded + PluginManager.loadPlugins(); + // Find if SSL certs are available const ssl = getSSLOptions(); const useHTTPS = !!ssl.key && !!ssl.cert; diff --git a/server/models/helpers/AuthenticationHelper.ts b/server/models/helpers/AuthenticationHelper.ts index b6f51723e..cba423407 100644 --- a/server/models/helpers/AuthenticationHelper.ts +++ b/server/models/helpers/AuthenticationHelper.ts @@ -1,23 +1,10 @@ /* eslint-disable @typescript-eslint/no-var-requires */ -import path from "path"; -import { glob } from "glob"; -import Router from "koa-router"; import find from "lodash/find"; -import sortBy from "lodash/sortBy"; import env from "@server/env"; import Team from "@server/models/Team"; -import environment from "@server/utils/environment"; - -export type AuthenticationProviderConfig = { - id: string; - name: string; - enabled: boolean; - router: Router; -}; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; export default class AuthenticationHelper { - private static providersCache: AuthenticationProviderConfig[]; - /** * Returns the enabled authentication provider configurations for the current * installation. @@ -25,46 +12,7 @@ export default class AuthenticationHelper { * @returns A list of authentication providers */ public static get providers() { - if (this.providersCache) { - return this.providersCache; - } - - const authenticationProviderConfigs: AuthenticationProviderConfig[] = []; - const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; - - glob - .sync(path.join(rootDir, "plugins/*/server/auth/!(*.test|schema).[jt]s")) - .forEach((filePath: string) => { - const { default: authProvider, name } = require(path.join( - process.cwd(), - filePath - )); - const id = filePath.replace("build/", "").split("/")[1]; - const config = require(path.join( - process.cwd(), - rootDir, - "plugins", - id, - "plugin.json" - )); - - // Test the all required env vars are set for the auth provider - const enabled = (config.requiredEnvVars ?? []).every( - (name: string) => !!environment[name] - ); - - if (enabled) { - authenticationProviderConfigs.push({ - id, - name: name ?? config.name, - enabled, - router: authProvider, - }); - } - }); - - this.providersCache = sortBy(authenticationProviderConfigs, "id"); - return this.providersCache; + return PluginManager.getEnabledPlugins(PluginType.AuthProvider); } /** @@ -78,11 +26,11 @@ export default class AuthenticationHelper { const isCloudHosted = env.isCloudHosted; return AuthenticationHelper.providers - .sort((config) => (config.id === "email" ? 1 : -1)) - .filter((config) => { - // Guest sign-in is an exception as it does not have an authentication + .sort((plugin) => (plugin.id === "email" ? 1 : -1)) + .filter((plugin) => { + // Email sign-in is an exception as it does not have an authentication // provider using passport, instead it exists as a boolean option. - if (config.id === "email") { + if (plugin.id === "email") { return team?.emailSigninEnabled; } @@ -92,7 +40,7 @@ export default class AuthenticationHelper { } const authProvider = find(team.authenticationProviders, { - name: config.id, + name: plugin.id, }); // If cloud hosted then the auth provider must be enabled for the team, diff --git a/server/presenters/providerConfig.ts b/server/presenters/providerConfig.ts index 91fdba9db..82080f026 100644 --- a/server/presenters/providerConfig.ts +++ b/server/presenters/providerConfig.ts @@ -1,8 +1,8 @@ import { signin } from "@shared/utils/routeHelpers"; -import { AuthenticationProviderConfig } from "@server/models/helpers/AuthenticationHelper"; +import { Plugin, PluginType } from "@server/utils/PluginManager"; export default function presentProviderConfig( - config: AuthenticationProviderConfig + config: Plugin ) { return { id: config.id, diff --git a/server/queues/processors/index.ts b/server/queues/processors/index.ts index 6afcdd7fe..2523aa1ba 100644 --- a/server/queues/processors/index.ts +++ b/server/queues/processors/index.ts @@ -1,12 +1,8 @@ -import path from "path"; -import { glob } from "glob"; -import env from "@server/env"; -import Logger from "@server/logging/Logger"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; import { requireDirectory } from "@server/utils/fs"; import BaseProcessor from "./BaseProcessor"; const processors = {}; -const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; requireDirectory<{ default: BaseProcessor }>(__dirname).forEach( ([module, id]) => { @@ -17,14 +13,8 @@ requireDirectory<{ default: BaseProcessor }>(__dirname).forEach( } ); -glob - .sync(path.join(rootDir, "plugins/*/server/processors/!(*.test).[jt]s")) - .forEach((filePath: string) => { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const processor = require(path.join(process.cwd(), filePath)).default; - const name = path.basename(filePath, ".js"); - processors[name] = processor; - Logger.debug("processor", `Registered processor ${name}`); - }); +PluginManager.getEnabledPlugins(PluginType.Processor).forEach((plugin) => { + processors[plugin.id] = plugin.value; +}); export default processors; diff --git a/server/queues/tasks/BaseTask.ts b/server/queues/tasks/BaseTask.ts index a914607b1..4bd34fd2f 100644 --- a/server/queues/tasks/BaseTask.ts +++ b/server/queues/tasks/BaseTask.ts @@ -13,7 +13,7 @@ export enum TaskSchedule { Hourly = "hourly", } -export default abstract class BaseTask { +export default abstract class BaseTask> { /** * An optional schedule for this task to be run automatically. */ diff --git a/server/queues/tasks/InviteReminderTask.ts b/server/queues/tasks/InviteReminderTask.ts index 91fb0e64a..3e1d238d3 100644 --- a/server/queues/tasks/InviteReminderTask.ts +++ b/server/queues/tasks/InviteReminderTask.ts @@ -6,7 +6,7 @@ import { UserFlag } from "@server/models/User"; import { sequelize } from "@server/storage/database"; import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask"; -type Props = undefined; +type Props = Record; export default class InviteReminderTask extends BaseTask { static cron = TaskSchedule.Daily; diff --git a/server/queues/tasks/index.ts b/server/queues/tasks/index.ts index 8f892964f..1303fc019 100644 --- a/server/queues/tasks/index.ts +++ b/server/queues/tasks/index.ts @@ -1,12 +1,8 @@ -import path from "path"; -import { glob } from "glob"; -import env from "@server/env"; -import Logger from "@server/logging/Logger"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; import { requireDirectory } from "@server/utils/fs"; import BaseTask from "./BaseTask"; const tasks = {}; -const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; requireDirectory<{ default: BaseTask }>(__dirname).forEach( ([module, id]) => { @@ -17,14 +13,8 @@ requireDirectory<{ default: BaseTask }>(__dirname).forEach( } ); -glob - .sync(path.join(rootDir, "plugins/*/server/tasks/!(*.test).[jt]s")) - .forEach((filePath: string) => { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const task = require(path.join(process.cwd(), filePath)).default; - const name = path.basename(filePath, ".js"); - tasks[name] = task; - Logger.debug("task", `Registered task ${name}`); - }); +PluginManager.getEnabledPlugins(PluginType.Processor).forEach((plugin) => { + tasks[plugin.id] = plugin.value; +}); export default tasks; diff --git a/server/routes/api/index.ts b/server/routes/api/index.ts index 90cce1451..00cc0223e 100644 --- a/server/routes/api/index.ts +++ b/server/routes/api/index.ts @@ -1,14 +1,12 @@ -import path from "path"; -import glob from "glob"; import Koa, { BaseContext } from "koa"; import bodyParser from "koa-body"; import Router from "koa-router"; import userAgent, { UserAgentContext } from "koa-useragent"; import env from "@server/env"; import { NotFoundError } from "@server/errors"; -import Logger from "@server/logging/Logger"; import coalesceBody from "@server/middlewares/coaleseBody"; import { AppState, AppContext } from "@server/types"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; import apiKeys from "./apiKeys"; import attachments from "./attachments"; import auth from "./auth"; @@ -60,19 +58,13 @@ api.use(apiTracer()); api.use(apiResponse()); api.use(editor()); -// register package API routes before others to allow for overrides -const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; -glob - .sync(path.join(rootDir, "plugins/*/server/api/!(*.test).[jt]s")) - .forEach((filePath: string) => { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const pkg: Router = require(path.join(process.cwd(), filePath)).default; - - if (pkg && "routes" in pkg) { - router.use("/", pkg.routes()); - Logger.debug("lifecycle", `Registered API routes for ${filePath}`); - } - }); +// Register plugin API routes before others to allow for overrides +const plugins = PluginManager.getEnabledPlugins(PluginType.API).map( + (plugin) => plugin.value +); +for (const router of plugins) { + router.use("/", router.routes()); +} // routes router.use("/", auth.routes()); diff --git a/server/routes/api/urls/urls.test.ts b/server/routes/api/urls/urls.test.ts index 69d3c9377..a5fff4252 100644 --- a/server/routes/api/urls/urls.test.ts +++ b/server/routes/api/urls/urls.test.ts @@ -2,7 +2,7 @@ import env from "@server/env"; import { User } from "@server/models"; import { buildDocument, buildUser } from "@server/test/factories"; import { getTestServer } from "@server/test/support"; -import resolvers from "@server/utils/unfurl"; +import Iframely from "plugins/iframely/server/iframely"; jest.mock("dns", () => ({ resolveCname: ( @@ -17,9 +17,7 @@ jest.mock("dns", () => ({ }, })); -jest - .spyOn(resolvers.Iframely, "unfurl") - .mockImplementation(async (_: string) => false); +jest.spyOn(Iframely, "fetch").mockImplementation(() => Promise.resolve(false)); const server = getTestServer(); @@ -158,7 +156,7 @@ describe("#urls.unfurl", () => { }); it("should succeed with status 200 ok for a valid external url", async () => { - (resolvers.Iframely.unfurl as jest.Mock).mockResolvedValue( + (Iframely.fetch as jest.Mock).mockResolvedValue( Promise.resolve({ url: "https://www.flickr.com", type: "rich", @@ -180,9 +178,6 @@ describe("#urls.unfurl", () => { expect(res.status).toEqual(200); const body = await res.json(); - expect(resolvers.Iframely.unfurl).toHaveBeenCalledWith( - "https://www.flickr.com" - ); expect(res.status).toEqual(200); expect(body.url).toEqual("https://www.flickr.com"); expect(body.type).toEqual("rich"); @@ -196,7 +191,7 @@ describe("#urls.unfurl", () => { }); it("should succeed with status 204 no content for a non-existing external url", async () => { - (resolvers.Iframely.unfurl as jest.Mock).mockResolvedValue( + (Iframely.fetch as jest.Mock).mockResolvedValue( Promise.resolve({ status: 404, error: @@ -211,9 +206,6 @@ describe("#urls.unfurl", () => { }, }); - expect(resolvers.Iframely.unfurl).toHaveBeenCalledWith( - "https://random.url" - ); expect(res.status).toEqual(204); }); }); diff --git a/server/routes/api/urls/urls.ts b/server/routes/api/urls/urls.ts index 25b7336be..76f12ad56 100644 --- a/server/routes/api/urls/urls.ts +++ b/server/routes/api/urls/urls.ts @@ -13,11 +13,12 @@ import { authorize } from "@server/policies"; import { presentDocument, presentMention } from "@server/presenters/unfurls"; import presentUnfurl from "@server/presenters/unfurls/unfurl"; import { APIContext } from "@server/types"; +import { PluginManager, PluginType } from "@server/utils/PluginManager"; import { RateLimiterStrategy } from "@server/utils/RateLimiter"; -import resolvers from "@server/utils/unfurl"; import * as T from "./schema"; const router = new Router(); +const plugins = PluginManager.getEnabledPlugins(PluginType.UnfurlProvider); router.post( "urls.unfurl", @@ -74,11 +75,13 @@ router.post( } // External resources - if (resolvers.Iframely) { - const data = await resolvers.Iframely.unfurl(url); - return data.error - ? (ctx.response.status = 204) - : (ctx.body = presentUnfurl(data)); + for (const plugin of plugins) { + const data = await plugin.value(url); + if (data) { + return "error" in data + ? (ctx.response.status = 204) + : (ctx.body = presentUnfurl(data)); + } } return (ctx.response.status = 204); diff --git a/server/routes/auth/index.ts b/server/routes/auth/index.ts index dcd903dda..8aec6b5e0 100644 --- a/server/routes/auth/index.ts +++ b/server/routes/auth/index.ts @@ -17,7 +17,7 @@ router.use(passport.initialize()); // dynamically load available authentication provider routes AuthenticationHelper.providers.forEach((provider) => { - router.use("/", provider.router.routes()); + router.use("/", provider.value.routes()); }); router.get("/redirect", auth(), async (ctx: APIContext) => { diff --git a/server/utils/PluginManager.ts b/server/utils/PluginManager.ts new file mode 100644 index 000000000..828d4267c --- /dev/null +++ b/server/utils/PluginManager.ts @@ -0,0 +1,173 @@ +import path from "path"; +import { glob } from "glob"; +import type Router from "koa-router"; +import sortBy from "lodash/sortBy"; +import { v4 as uuid } from "uuid"; +import { UnfurlSignature } from "@shared/types"; +import type BaseEmail from "@server/emails/templates/BaseEmail"; +import env from "@server/env"; +import Logger from "@server/logging/Logger"; +import type BaseProcessor from "@server/queues/processors/BaseProcessor"; +import type BaseTask from "@server/queues/tasks/BaseTask"; + +export enum PluginPriority { + VeryHigh = 0, + High = 100, + Normal = 200, + Low = 300, + VeryLow = 500, +} + +/** + * The different types of server plugins that can be registered. + */ +export enum PluginType { + API = "api", + AuthProvider = "authProvider", + EmailTemplate = "emailTemplate", + Processor = "processor", + Task = "task", + UnfurlProvider = "unfurl", +} + +/** + * A map of plugin types to their values, for example an API plugin would have a value of type + * Router. Registering an API plugin causes the router to be mounted. + */ +type PluginValueMap = { + [PluginType.API]: Router; + [PluginType.AuthProvider]: Router; + [PluginType.EmailTemplate]: typeof BaseEmail; + [PluginType.Processor]: typeof BaseProcessor; + [PluginType.Task]: typeof BaseTask; + [PluginType.UnfurlProvider]: UnfurlSignature; +}; + +export type Plugin = { + /** A unique ID for the plugin */ + id: string; + /** The plugin's display name */ + name?: string; + /** A brief description of the plugin */ + description?: string; + /** The plugin content */ + value: PluginValueMap[T]; + /** An optional priority, will affect order in menus and execution. Lower is earlier. */ + priority?: number; + /** Whether the plugin is enabled (default: true) */ + enabled?: boolean; +}; + +export class PluginManager { + private static plugins = new Map[]>(); + + /** + * Register a plugin of a given type. + * + * @param type The plugin type + * @param value The plugin value + * @param options Additional options, including whether the plugin is enabled and it's priority. + * @returns The PluginManager instance, for chaining. + */ + public static register( + type: T, + value: PluginValueMap[T], + options: Omit, "value"> = { + id: uuid(), + } + ) { + if (!this.plugins.has(type)) { + this.plugins.set(type, []); + } + + const plugin = { + value, + priority: PluginPriority.Normal, + ...options, + }; + + Logger.debug( + "plugins", + `Plugin ${options.enabled === false ? "disabled" : "enabled"} "${ + options.id + }" ${options.description ? `(${options.description})` : ""}` + ); + + this.plugins.get(type)!.push(plugin); + + // allow chaining + return this; + } + + /** + * Syntactic sugar for registering a background Task. + * + * @param value The task class + * @param options Additional options + */ + public static registerTask( + value: PluginValueMap[PluginType.Task], + options?: Omit, "id" | "value"> + ) { + return this.register(PluginType.Task, value, { + id: value.name, + ...options, + }); + } + + /** + * Syntactic sugar for registering a background Processor. + * + * @param value The processor class + * @param options Additional options + */ + public static registerProcessor( + value: PluginValueMap[PluginType.Processor], + options?: Omit, "id" | "value"> + ) { + return this.register(PluginType.Processor, value, { + id: value.name, + ...options, + }); + } + + /** + * Returns all the plugins of a given type in order of priority. + * + * @param type The type of plugin to filter by + * @returns A list of plugins + */ + public static getPlugins(type: T) { + this.loadPlugins(); + return sortBy(this.plugins.get(type) || [], "priority") as Plugin[]; + } + + /** + * Returns all the enabled plugins of a given type in order of priority. + * + * @param type The type of plugin to filter by + * @returns A list of plugins + */ + public static getEnabledPlugins(type: T) { + return this.getPlugins(type).filter((plugin) => plugin.enabled !== false); + } + + /** + * Load plugin server components (anything in the `/server/` directory of a plugin will be loaded) + */ + public static loadPlugins() { + if (this.loaded) { + return; + } + const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; + + glob + .sync(path.join(rootDir, "plugins/*/server/!(*.test|schema).[jt]s")) + .forEach((filePath: string) => { + require(path.join(process.cwd(), filePath)); + }); + this.loaded = true; + } + + private static loaded = false; +} diff --git a/server/utils/unfurl.ts b/server/utils/unfurl.ts deleted file mode 100644 index fcf5624f2..000000000 --- a/server/utils/unfurl.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ -import path from "path"; -import glob from "glob"; -import env from "@server/env"; -import Logger from "@server/logging/Logger"; -import { UnfurlResolver } from "@server/types"; -import environment from "./environment"; - -const rootDir = env.ENVIRONMENT === "test" ? "" : "build"; - -const plugins = glob.sync(path.join(rootDir, "plugins/*/server/unfurl.[jt]s")); -const resolvers: Record = plugins.reduce( - (resolvers, filePath) => { - const resolver: UnfurlResolver = require(path.join( - process.cwd(), - filePath - )); - const id = filePath.replace("build/", "").split("/")[1]; - const config = require(path.join( - process.cwd(), - rootDir, - "plugins", - id, - "plugin.json" - )); - - // Test the all required env vars are set for the resolver - const enabled = (config.requiredEnvVars ?? []).every( - (name: string) => !!environment[name] - ); - if (!enabled) { - return resolvers; - } - - resolvers[config.name] = resolver; - Logger.debug("utils", `Registered unfurl resolver ${filePath}`); - - return resolvers; - }, - {} -); - -export default resolvers; diff --git a/shared/types.ts b/shared/types.ts index d1f9f5ac1..d8692f141 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -270,14 +270,20 @@ export enum QueryNotices { export type OEmbedType = "photo" | "video" | "rich"; -export type Unfurl = { - url?: string; - type: T; - title: string; - description?: string; - thumbnailUrl?: string | null; - meta?: Record; -}; +export type Unfurl = + | { + url?: string; + type: T; + title: string; + description?: string; + thumbnailUrl?: string | null; + meta?: Record; + } + | { + error: string; + }; + +export type UnfurlSignature = (url: string) => Promise; export type JSONValue = | string