feat: make ioredis configurable via environment variables (#3365)

* feat: expose ioredis client options

* run linter

* refactor redis client init into class extension

* explicitly handle constructor errors

* rename singletons

.env.sample

@@ -16,7 +16,15 @@ DATABASE_CONNECTION_POOL_MIN=
 DATABASE_CONNECTION_POOL_MAX=
 # Uncomment this to disable SSL for connecting to Postgres
 # PGSSLMODE=disable
-REDIS_URL=redis://localhost:6379
+
+# For redis you can either specify an ioredis compatible url like this
+REDIS_URL=redis://localhost:6479
+# or alternatively, if you would like to provide addtional connection options,
+# use a base64 encoded JSON connection option object. Refer to the ioredis documentation
+# for a list of available options.
+# Example: Use Redis Sentinel for high availability
+# {"sentinels":[{"host":"sentinel-0","port":26379},{"host":"sentinel-1","port":26379}],"name":"mymaster"}
+# REDIS_URL=ioredis://eyJzZW50aW5lbHMiOlt7Imhvc3QiOiJzZW50aW5lbC0wIiwicG9ydCI6MjYzNzl9LHsiaG9zdCI6InNlbnRpbmVsLTEiLCJwb3J0IjoyNjM3OX1dLCJuYW1lIjoibXltYXN0ZXIifQ==
 
 # URL should point to the fully qualified, publicly accessible URL. If using a
 # proxy the port in URL and PORT may be different.

docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: redis
     ports:
       - "127.0.0.1:6479:6379"
+    user: "redis:redis"
   postgres:
     image: postgres
     ports:
@@ -12,6 +13,7 @@ services:
       POSTGRES_USER: user
       POSTGRES_PASSWORD: pass
       POSTGRES_DB: outline
+    user: "postgres:postgres"
   s3:
     image: lphoward/fake-s3
     ports:

server/redis.ts

@@ -1,7 +1,8 @@
 import Redis from "ioredis";
+import { defaults } from "lodash";
 import Logger from "./logging/logger";
 
-const options = {
+const defaultOptions = {
   maxRetriesPerRequest: 20,
 
   retryStrategy(times: number) {
@@ -18,12 +19,45 @@ const options = {
         }
       : undefined,
 };
-const client = new Redis(process.env.REDIS_URL, options);
-const subscriber = new Redis(process.env.REDIS_URL, options);
-// More than the default of 10 listeners is expected for the amount of queues
-// we're running. Increase the max here to prevent a warning in the console:
-// https://github.com/OptimalBits/bull/issues/1192
-client.setMaxListeners(100);
-subscriber.setMaxListeners(100);
 
-export { client, subscriber };
+export default class RedisAdapter extends Redis {
+  constructor(url: string | undefined) {
+    if (!(url || "").startsWith("ioredis://")) {
+      super(process.env.REDIS_URL, defaultOptions);
+    } else {
+      let customOptions = {};
+      try {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const decodedString = Buffer.from(url!.slice(10), "base64").toString();
+        customOptions = JSON.parse(decodedString);
+      } catch (error) {
+        throw new Error(`Failed to decode redis adapter options: ${error}`);
+      }
+
+      try {
+        const mergedOptions = defaults(defaultOptions, customOptions);
+        super(mergedOptions);
+      } catch (error) {
+        throw new Error(`Failed to initialize redis client: ${error}`);
+      }
+    }
+
+    // More than the default of 10 listeners is expected for the amount of queues
+    // we're running. Increase the max here to prevent a warning in the console:
+    // https://github.com/OptimalBits/bull/issues/1192
+    this.setMaxListeners(100);
+  }
+
+  private static _client: RedisAdapter;
+  private static _subscriber: RedisAdapter;
+
+  public static get defaultClient(): RedisAdapter {
+    return this._client || (this._client = new this(process.env.REDIS_URL));
+  }
+
+  public static get defaultSubscriber(): RedisAdapter {
+    return (
+      this._subscriber || (this._subscriber = new this(process.env.REDIS_URL))
+    );
+  }
+}

server/services/websockets.ts

@@ -11,7 +11,7 @@ import { can } from "@server/policies";
 import { getUserForJWT } from "@server/utils/jwt";
 import { websocketQueue } from "../queues";
 import WebsocketsProcessor from "../queues/processors/WebsocketsProcessor";
-import { client, subscriber } from "../redis";
+import Redis from "../redis";
 
 export default function init(app: Koa, server: http.Server) {
   const path = "/realtime";
@@ -49,8 +49,8 @@ export default function init(app: Koa, server: http.Server) {
 
   io.adapter(
     socketRedisAdapter({
-      pubClient: client,
-      subClient: subscriber,
+      pubClient: Redis.defaultClient,
+      subClient: Redis.defaultSubscriber,
     })
   );
 
@@ -92,7 +92,7 @@ export default function init(app: Koa, server: http.Server) {
 
         // store the mapping between socket id and user id in redis
         // so that it is accessible across multiple server nodes
-        await client.hset(socket.id, "userId", user.id);
+        await Redis.defaultClient.hset(socket.id, "userId", user.id);
         return callback(null, true);
       } catch (err) {
         return callback(err, false);
@@ -173,7 +173,10 @@ export default function init(app: Koa, server: http.Server) {
                 const userIds = new Map();
 
                 for (const socketId of sockets) {
-                  const userId = await client.hget(socketId, "userId");
+                  const userId = await Redis.defaultClient.hget(
+                    socketId,
+                    "userId"
+                  );
                   userIds.set(userId, userId);
                 }
 

server/utils/queue.ts

@@ -1,8 +1,7 @@
 import Queue from "bull";
-import Redis from "ioredis";
 import { snakeCase } from "lodash";
 import Metrics from "@server/logging/metrics";
-import { client, subscriber } from "../redis";
+import Redis from "../redis";
 
 export function createQueue(
   name: string,
@@ -13,10 +12,10 @@ export function createQueue(
     createClient(type) {
       switch (type) {
         case "client":
-          return client;
+          return Redis.defaultClient;
 
         case "subscriber":
-          return subscriber;
+          return Redis.defaultSubscriber;
 
         default:
           return new Redis(process.env.REDIS_URL);

server/utils/updates.ts

@@ -6,7 +6,7 @@ import Document from "@server/models/Document";
 import Team from "@server/models/Team";
 import User from "@server/models/User";
 import packageInfo from "../../package.json";
-import { client } from "../redis";
+import Redis from "../redis";
 
 const UPDATES_URL = "https://updates.getoutline.com";
 const UPDATES_KEY = "UPDATES_KEY";
@@ -40,7 +40,7 @@ export async function checkUpdates() {
       documentCount,
     },
   });
-  await client.del(UPDATES_KEY);
+  await Redis.defaultClient.del(UPDATES_KEY);
 
   try {
     const response = await fetch(UPDATES_URL, {
@@ -54,7 +54,7 @@ export async function checkUpdates() {
     const data = await response.json();
 
     if (data.severity) {
-      await client.set(
+      await Redis.defaultClient.set(
         UPDATES_KEY,
         JSON.stringify({
           severity: data.severity,