Separate environment configs (#6597)

* Separate environment configs

* wip

* wip

* test

* plugins

* test

* test

* .sequelizerc, unfortunately can't go through /utils/environment due to not supporting TS

* docker-compose -> docker compose

* fix: .local wipes .development

* Add custom validation message for invalid SECRET_KEY (often confused)

plugins/azure/server/auth/azure.ts

@@ -6,7 +6,6 @@ import Router from "koa-router";
 import { Profile } from "passport";
 import { slugifyDomain } from "@shared/utils/domains";
 import accountProvisioner from "@server/commands/accountProvisioner";
-import env from "@server/env";
 import { MicrosoftGraphError } from "@server/errors";
 import passportMiddleware from "@server/middlewares/passport";
 import { User } from "@server/models";
@@ -17,6 +16,7 @@ import {
   getTeamFromContext,
   getClientFromContext,
 } from "@server/utils/passport";
+import env from "../env";
 
 const router = new Router();
 const providerName = "azure";

plugins/azure/server/azure.ts

@@ -0,0 +1,44 @@
+import invariant from "invariant";
+import JWT from "jsonwebtoken";
+import OAuthClient from "@server/utils/oauth";
+import env from "./env";
+
+type AzurePayload = {
+  /** A GUID that represents the Azure AD tenant that the user is from */
+  tid: string;
+};
+
+export default class AzureClient extends OAuthClient {
+  endpoints = {
+    authorize: "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
+    token: "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+    userinfo: "https://graph.microsoft.com/v1.0/me",
+  };
+
+  constructor() {
+    invariant(env.AZURE_CLIENT_ID, "AZURE_CLIENT_ID is required");
+    invariant(env.AZURE_CLIENT_SECRET, "AZURE_CLIENT_SECRET is required");
+
+    super(env.AZURE_CLIENT_ID, env.AZURE_CLIENT_SECRET);
+  }
+
+  async rotateToken(
+    accessToken: string,
+    refreshToken: string
+  ): Promise<{
+    accessToken: string;
+    refreshToken?: string;
+    expiresAt: Date;
+  }> {
+    if (env.isCloudHosted) {
+      return super.rotateToken(accessToken, refreshToken);
+    }
+
+    const payload = JWT.decode(accessToken) as AzurePayload;
+    return super.rotateToken(
+      accessToken,
+      refreshToken,
+      `https://login.microsoftonline.com/${payload.tid}/oauth2/v2.0/token`
+    );
+  }
+}

plugins/azure/server/env.ts

@@ -0,0 +1,27 @@
+import { IsOptional } from "class-validator";
+import { Environment } from "@server/env";
+import environment from "@server/utils/environment";
+import { CannotUseWithout } from "@server/utils/validators";
+
+class AzurePluginEnvironment extends Environment {
+  /**
+   * Azure OAuth2 client credentials. To enable authentication with Azure.
+   */
+  @IsOptional()
+  @CannotUseWithout("AZURE_CLIENT_SECRET")
+  public AZURE_CLIENT_ID = this.toOptionalString(environment.AZURE_CLIENT_ID);
+
+  @IsOptional()
+  @CannotUseWithout("AZURE_CLIENT_ID")
+  public AZURE_CLIENT_SECRET = this.toOptionalString(
+    environment.AZURE_CLIENT_SECRET
+  );
+
+  @IsOptional()
+  @CannotUseWithout("AZURE_CLIENT_ID")
+  public AZURE_RESOURCE_APP_ID = this.toOptionalString(
+    environment.AZURE_RESOURCE_APP_ID
+  );
+}
+
+export default new AzurePluginEnvironment();

plugins/google/server/auth/google.ts

@@ -6,7 +6,6 @@ import { Profile } from "passport";
 import { Strategy as GoogleStrategy } from "passport-google-oauth2";
 import { slugifyDomain } from "@shared/utils/domains";
 import accountProvisioner from "@server/commands/accountProvisioner";
-import env from "@server/env";
 import {
   GmailAccountCreationError,
   TeamDomainRequiredError,
@@ -19,6 +18,7 @@ import {
   getTeamFromContext,
   getClientFromContext,
 } from "@server/utils/passport";
+import env from "../env";
 
 const router = new Router();
 const providerName = "google";

plugins/google/server/env.ts

@@ -0,0 +1,21 @@
+import { IsOptional } from "class-validator";
+import { Environment } from "@server/env";
+import environment from "@server/utils/environment";
+import { CannotUseWithout } from "@server/utils/validators";
+
+class GooglePluginEnvironment extends Environment {
+  /**
+   * Google OAuth2 client credentials. To enable authentication with Google.
+   */
+  @IsOptional()
+  @CannotUseWithout("GOOGLE_CLIENT_SECRET")
+  public GOOGLE_CLIENT_ID = this.toOptionalString(environment.GOOGLE_CLIENT_ID);
+
+  @IsOptional()
+  @CannotUseWithout("GOOGLE_CLIENT_ID")
+  public GOOGLE_CLIENT_SECRET = this.toOptionalString(
+    environment.GOOGLE_CLIENT_SECRET
+  );
+}
+
+export default new GooglePluginEnvironment();

plugins/google/server/google.ts

@@ -0,0 +1,18 @@
+import invariant from "invariant";
+import OAuthClient from "@server/utils/oauth";
+import env from "./env";
+
+export default class GoogleClient extends OAuthClient {
+  endpoints = {
+    authorize: "https://accounts.google.com/o/oauth2/auth",
+    token: "https://accounts.google.com/o/oauth2/token",
+    userinfo: "https://www.googleapis.com/oauth2/v3/userinfo",
+  };
+
+  constructor() {
+    invariant(env.GOOGLE_CLIENT_ID, "GOOGLE_CLIENT_ID is required");
+    invariant(env.GOOGLE_CLIENT_SECRET, "GOOGLE_CLIENT_SECRET is required");
+
+    super(env.GOOGLE_CLIENT_ID, env.GOOGLE_CLIENT_SECRET);
+  }
+}

plugins/iframely/plugin.json

@@ -1,5 +1,5 @@
 {
   "name": "Iframely",
   "description": "Integrate Iframely to enable unfurling of arbitrary urls",
-  "requiredEnvVars": ["IFRAMELY_URL", "IFRAMELY_API_KEY"]
+  "requiredEnvVars": ["IFRAMELY_API_KEY"]
 }

plugins/iframely/server/env.ts

@@ -0,0 +1,27 @@
+import { IsOptional, IsUrl } from "class-validator";
+import { Environment } from "@server/env";
+import environment from "@server/utils/environment";
+import { CannotUseWithout } from "@server/utils/validators";
+
+class IframelyPluginEnvironment extends Environment {
+  /**
+   * Iframely url
+   */
+  @IsOptional()
+  @IsUrl({
+    require_tld: false,
+    require_protocol: true,
+    allow_underscores: true,
+    protocols: ["http", "https"],
+  })
+  public IFRAMELY_URL = environment.IFRAMELY_URL ?? "https://iframe.ly";
+
+  /**
+   * Iframely API key
+   */
+  @IsOptional()
+  @CannotUseWithout("IFRAMELY_URL")
+  public IFRAMELY_API_KEY = this.toOptionalString(environment.IFRAMELY_API_KEY);
+}
+
+export default new IframelyPluginEnvironment();

plugins/iframely/server/iframely.ts

@@ -1,9 +1,9 @@
 import { Day } from "@shared/utils/time";
-import env from "@server/env";
 import { InternalError } from "@server/errors";
 import Logger from "@server/logging/Logger";
 import Redis from "@server/storage/redis";
 import fetch from "@server/utils/fetch";
+import env from "./env";
 
 class Iframely {
   private static apiUrl = `${env.IFRAMELY_URL}/api`;

plugins/oidc/plugin.json

@@ -1,12 +1,5 @@
 {
   "name": "OIDC",
   "description": "Adds an OpenID compatible authentication provider.",
-  "requiredEnvVars": [
-    "OIDC_CLIENT_ID",
-    "OIDC_CLIENT_SECRET",
-    "OIDC_AUTH_URI",
-    "OIDC_TOKEN_URI",
-    "OIDC_USERINFO_URI",
-    "OIDC_DISPLAY_NAME"
-  ]
+  "requiredEnvVars": ["OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_AUTH_URI", "OIDC_TOKEN_URI", "OIDC_USERINFO_URI"]
 }

plugins/oidc/server/auth/oidc.ts

@@ -5,7 +5,6 @@ import get from "lodash/get";
 import { Strategy } from "passport-oauth2";
 import { slugifyDomain } from "@shared/utils/domains";
 import accountProvisioner from "@server/commands/accountProvisioner";
-import env from "@server/env";
 import {
   OIDCMalformedUserInfoError,
   AuthenticationError,
@@ -19,6 +18,7 @@ import {
   getTeamFromContext,
   getClientFromContext,
 } from "@server/utils/passport";
+import env from "../env";
 
 const router = new Router();
 const providerName = "oidc";

plugins/oidc/server/env.ts

@@ -0,0 +1,79 @@
+import { IsOptional, IsUrl, MaxLength } from "class-validator";
+import { Environment } from "@server/env";
+import environment from "@server/utils/environment";
+import { CannotUseWithout } from "@server/utils/validators";
+
+class OIDCPluginEnvironment extends Environment {
+  /**
+   * OIDC client credentials. To enable authentication with any
+   * compatible provider.
+   */
+  @IsOptional()
+  @CannotUseWithout("OIDC_CLIENT_SECRET")
+  @CannotUseWithout("OIDC_AUTH_URI")
+  @CannotUseWithout("OIDC_TOKEN_URI")
+  @CannotUseWithout("OIDC_USERINFO_URI")
+  @CannotUseWithout("OIDC_DISPLAY_NAME")
+  public OIDC_CLIENT_ID = this.toOptionalString(environment.OIDC_CLIENT_ID);
+
+  @IsOptional()
+  @CannotUseWithout("OIDC_CLIENT_ID")
+  public OIDC_CLIENT_SECRET = this.toOptionalString(
+    environment.OIDC_CLIENT_SECRET
+  );
+
+  /**
+   * The name of the OIDC provider, eg "GitLab" – this will be displayed on the
+   * sign-in button and other places in the UI. The default value is:
+   * "OpenID Connect".
+   */
+  @MaxLength(50)
+  public OIDC_DISPLAY_NAME = environment.OIDC_DISPLAY_NAME ?? "OpenID Connect";
+
+  /**
+   * The OIDC authorization endpoint.
+   */
+  @IsOptional()
+  @IsUrl({
+    require_tld: false,
+    allow_underscores: true,
+  })
+  public OIDC_AUTH_URI = this.toOptionalString(environment.OIDC_AUTH_URI);
+
+  /**
+   * The OIDC token endpoint.
+   */
+  @IsOptional()
+  @IsUrl({
+    require_tld: false,
+    allow_underscores: true,
+  })
+  public OIDC_TOKEN_URI = this.toOptionalString(environment.OIDC_TOKEN_URI);
+
+  /**
+   * The OIDC userinfo endpoint.
+   */
+  @IsOptional()
+  @IsUrl({
+    require_tld: false,
+    allow_underscores: true,
+  })
+  public OIDC_USERINFO_URI = this.toOptionalString(
+    environment.OIDC_USERINFO_URI
+  );
+
+  /**
+   * The OIDC profile field to use as the username. The default value is
+   * "preferred_username".
+   */
+  public OIDC_USERNAME_CLAIM =
+    environment.OIDC_USERNAME_CLAIM ?? "preferred_username";
+
+  /**
+   * A space separated list of OIDC scopes to request. Defaults to "openid
+   * profile email".
+   */
+  public OIDC_SCOPES = environment.OIDC_SCOPES ?? "openid profile email";
+}
+
+export default new OIDCPluginEnvironment();

plugins/oidc/server/oidc.ts

@@ -0,0 +1,18 @@
+import invariant from "invariant";
+import OAuthClient from "@server/utils/oauth";
+import env from "./env";
+
+export default class OIDCClient extends OAuthClient {
+  endpoints = {
+    authorize: env.OIDC_AUTH_URI || "",
+    token: env.OIDC_TOKEN_URI || "",
+    userinfo: env.OIDC_USERINFO_URI || "",
+  };
+
+  constructor() {
+    invariant(env.OIDC_CLIENT_ID, "OIDC_CLIENT_ID is required");
+    invariant(env.OIDC_CLIENT_SECRET, "OIDC_CLIENT_SECRET is required");
+
+    super(env.OIDC_CLIENT_ID, env.OIDC_CLIENT_SECRET);
+  }
+}

plugins/slack/server/api/hooks.test.ts

@@ -1,6 +1,5 @@
 import randomstring from "randomstring";
 import { IntegrationService } from "@shared/types";
-import env from "@server/env";
 import { IntegrationAuthentication, SearchQuery } from "@server/models";
 import {
   buildDocument,
@@ -9,6 +8,7 @@ import {
   buildUser,
 } from "@server/test/factories";
 import { getTestServer } from "@server/test/support";
+import env from "../env";
 import * as Slack from "../slack";
 
 jest.mock("../slack", () => ({

plugins/slack/server/api/hooks.ts

@@ -4,7 +4,6 @@ import escapeRegExp from "lodash/escapeRegExp";
 import { Op } from "sequelize";
 import { z } from "zod";
 import { IntegrationService } from "@shared/types";
-import env from "@server/env";
 import {
   AuthenticationError,
   InvalidRequestError,
@@ -26,6 +25,7 @@ import SearchHelper from "@server/models/helpers/SearchHelper";
 import { APIContext } from "@server/types";
 import { safeEqual } from "@server/utils/crypto";
 import { opts } from "@server/utils/i18n";
+import env from "../env";
 import presentMessageAttachment from "../presenters/messageAttachment";
 import * as Slack from "../slack";
 import * as T from "./schema";

plugins/slack/server/auth/slack.ts

@@ -6,7 +6,6 @@ import { Strategy as SlackStrategy } from "passport-slack-oauth2";
 import { IntegrationService, IntegrationType } from "@shared/types";
 import { integrationSettingsPath } from "@shared/utils/routeHelpers";
 import accountProvisioner from "@server/commands/accountProvisioner";
-import env from "@server/env";
 import auth from "@server/middlewares/authentication";
 import passportMiddleware from "@server/middlewares/passport";
 import validate from "@server/middlewares/validate";
@@ -23,6 +22,7 @@ import {
   getTeamFromContext,
   StateStore,
 } from "@server/utils/passport";
+import env from "../env";
 import * as Slack from "../slack";
 import * as T from "./schema";
 

plugins/slack/server/env.ts

@@ -0,0 +1,44 @@
+import { IsBoolean, IsOptional } from "class-validator";
+import { Environment } from "@server/env";
+import Deprecated from "@server/models/decorators/Deprecated";
+import environment from "@server/utils/environment";
+import { CannotUseWithout } from "@server/utils/validators";
+
+class SlackPluginEnvironment extends Environment {
+  /**
+   * Slack OAuth2 client credentials. To enable authentication with Slack.
+   */
+  @IsOptional()
+  @Deprecated("Use SLACK_CLIENT_SECRET instead")
+  public SLACK_SECRET = this.toOptionalString(environment.SLACK_SECRET);
+
+  @IsOptional()
+  @Deprecated("Use SLACK_CLIENT_ID instead")
+  public SLACK_KEY = this.toOptionalString(environment.SLACK_KEY);
+
+  @IsOptional()
+  @CannotUseWithout("SLACK_CLIENT_ID")
+  public SLACK_CLIENT_SECRET = this.toOptionalString(
+    environment.SLACK_CLIENT_SECRET ?? environment.SLACK_SECRET
+  );
+
+  /**
+   * Secret to verify webhook requests received from Slack.
+   */
+  @IsOptional()
+  public SLACK_VERIFICATION_TOKEN = this.toOptionalString(
+    environment.SLACK_VERIFICATION_TOKEN
+  );
+
+  /**
+   * If enabled a "Post to Channel" button will be added to search result
+   * messages inside of Slack. This also requires setup in Slack UI.
+   */
+  @IsOptional()
+  @IsBoolean()
+  public SLACK_MESSAGE_ACTIONS = this.toBoolean(
+    environment.SLACK_MESSAGE_ACTIONS ?? "false"
+  );
+}
+
+export default new SlackPluginEnvironment();

plugins/slack/server/processors/SlackProcessor.ts

@@ -2,7 +2,6 @@ import { differenceInMilliseconds } from "date-fns";
 import { Op } from "sequelize";
 import { IntegrationService, IntegrationType } from "@shared/types";
 import { Minute } from "@shared/utils/time";
-import env from "@server/env";
 import { Document, Integration, Collection, Team } from "@server/models";
 import BaseProcessor from "@server/queues/processors/BaseProcessor";
 import {
@@ -12,6 +11,7 @@ import {
   Event,
 } from "@server/types";
 import fetch from "@server/utils/fetch";
+import env from "../env";
 import presentMessageAttachment from "../presenters/messageAttachment";
 
 export default class SlackProcessor extends BaseProcessor {

plugins/slack/server/slack.ts

@@ -1,7 +1,7 @@
 import querystring from "querystring";
-import env from "@server/env";
 import { InvalidRequestError } from "@server/errors";
 import fetch from "@server/utils/fetch";
+import env from "./env";
 
 const SLACK_API_URL = "https://slack.com/api";
 

plugins/storage/server/api/files.test.ts

@@ -5,7 +5,6 @@ import FormData from "form-data";
 import { ensureDirSync } from "fs-extra";
 import { v4 as uuidV4 } from "uuid";
 import env from "@server/env";
-import "@server/test/env";
 import FileStorage from "@server/storage/files";
 import { buildAttachment, buildUser } from "@server/test/factories";
 import { getTestServer } from "@server/test/support";