yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: Rate limiter audit (#3965)

Browse Source 
* chore: Rate limiter audit api/users

* Make requests required

* api/collections

* Remove checkRateLimit on FileOperation (now done at route level through rate limiter)

* auth rate limit

* Add metric logging when rate limit exceeded

* Refactor to shared configs

* test
This commit is contained in:
Tom Moor
2022-08-14 16:04:04 +01:00
committed by GitHub
parent 9338328a82
commit a326e0ee88
14 changed files with 367 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/utils/ApiClient.ts
View File

@@ -12,6 +12,7 @@ import {
NetworkError,
NotFoundError,
OfflineError,
RateLimitExceededError,
RequestError,
ServiceUnavailableError,
UpdateRequiredError,
@@ -181,6 +182,12 @@ class ApiClient {
throw new ServiceUnavailableError(error.message);
}
if (response.status === 429) {
throw new RateLimitExceededError(
`Too many requests, try again in a minute.`
);
}
throw new RequestError(`Error ${response.status}: ${error.message}`);
};

2
app/utils/errors.ts
View File

@@ -12,6 +12,8 @@ export class OfflineError extends ExtendableError {}
export class ServiceUnavailableError extends ExtendableError {}
export class RateLimitExceededError extends ExtendableError {}
export class RequestError extends ExtendableError {}
export class UpdateRequiredError extends ExtendableError {}

1
app/utils/sentry.ts
View File

@@ -22,6 +22,7 @@ export function initSentry(history: History) {
"NetworkError",
"NotFoundError",
"OfflineError",
"RateLimitExceededError",
"ServiceUnavailableError",
"UpdateRequiredError",
"ChunkLoadError",

34
server/RateLimiter.ts
View File

@@ -1,7 +1,9 @@
import { RateLimiterRedis } from "rate-limiter-flexible";
import {
IRateLimiterStoreOptions,
RateLimiterRedis,
} from "rate-limiter-flexible";
import env from "@server/env";
import Redis from "@server/redis";
import { RateLimiterConfig } from "@server/types";
export default class RateLimiter {
constructor() {
@@ -22,7 +24,7 @@ export default class RateLimiter {
return this.rateLimiterMap.get(path) || this.defaultRateLimiter;
}
static setRateLimiter(path: string, config: RateLimiterConfig): void {
static setRateLimiter(path: string, config: IRateLimiterStoreOptions): void {
const rateLimiter = new RateLimiterRedis(config);
this.rateLimiterMap.set(path, rateLimiter);
}
@@ -31,3 +33,29 @@ export default class RateLimiter {
return this.rateLimiterMap.has(path);
}
}
/**
* Re-useable configuration for rate limiter middleware.
*/
export const RateLimiterStrategy = {
/** Allows five requests per minute, per IP address */
FivePerMinute: {
duration: 60,
requests: 5,
},
/** Allows ten requests per minute, per IP address */
TenPerMinute: {
duration: 60,
requests: 10,
},
/** Allows ten requests per hour, per IP address */
TenPerHour: {
duration: 3600,
requests: 10,
},
/** Allows five requests per hour, per IP address */
FivePerHour: {
duration: 3600,
requests: 5,
},
};

12
server/__mocks__/RateLimiter.ts
View File

@@ -14,3 +14,15 @@ export default class MockRateLimiter {
return false;
}
}
export const RateLimiterStrategy = new Proxy(
{},
{
get() {
return {
duration: 60,
requests: 10,
};
},
}
);

4
server/logging/Logger.ts
View File

@@ -110,7 +110,9 @@ class Logger {
extra?: Extra,
request?: IncomingMessage
) {
Metrics.increment("logger.error");
Metrics.increment("logger.error", {
name: error.name,
});
Tracing.setError(error);
if (env.SENTRY_DSN) {

47
server/middlewares/rateLimiter.ts
View File

@@ -3,10 +3,17 @@ import { defaults } from "lodash";
import RateLimiter from "@server/RateLimiter";
import env from "@server/env";
import { RateLimitExceededError } from "@server/errors";
import Metrics from "@server/logging/metrics";
import Redis from "@server/redis";
import { RateLimiterConfig } from "@server/types";
export function rateLimiter() {
/**
* Middleware that limits the number of requests per IP address that are allowed
* within a window. Should only be applied once to a server – do not use on
* individual routes.
*
* @returns The middleware function.
*/
export function defaultRateLimiter() {
return async function rateLimiterMiddleware(ctx: Context, next: Next) {
if (!env.RATE_LIMITER_ENABLED) {
return next();
@@ -28,6 +35,10 @@ export function rateLimiter() {
`${new Date(Date.now() + rateLimiterRes.msBeforeNext)}`
);
Metrics.increment("rate_limit.exceeded", {
path: ctx.path,
});
throw RateLimitExceededError();
}
@@ -35,7 +46,20 @@ export function rateLimiter() {
};
}
export function registerRateLimiter(config: RateLimiterConfig) {
type RateLimiterConfig = {
/** The window for which this rate limiter is considered (defaults to 60s) */
duration?: number;
/** The number of requests per IP address that are allowed within the window */
requests: number;
};
/**
* Middleware that limits the number of requests per IP address that are allowed
* within a window, overrides default middleware when used on a route.
*
* @returns The middleware function.
*/
export function rateLimiter(config: RateLimiterConfig) {
return async function registerRateLimiterMiddleware(
ctx: Context,
next: Next
@@ -47,11 +71,18 @@ export function registerRateLimiter(config: RateLimiterConfig) {
if (!RateLimiter.hasRateLimiter(ctx.path)) {
RateLimiter.setRateLimiter(
ctx.path,
defaults(config, {
duration: env.RATE_LIMITER_DURATION_WINDOW,
keyPrefix: RateLimiter.RATE_LIMITER_REDIS_KEY_PREFIX,
storeClient: Redis.defaultClient,
})
defaults(
{
...config,
points: config.requests,
},
{
duration: 60,
points: env.RATE_LIMITER_REQUESTS,
keyPrefix: RateLimiter.RATE_LIMITER_REDIS_KEY_PREFIX,
storeClient: Redis.defaultClient,
}
)
);
}

24
server/models/FileOperation.ts
View File

@@ -1,4 +1,3 @@
import { subHours } from "date-fns";
import { Op, WhereOptions } from "sequelize";
import {
ForeignKey,
@@ -8,9 +7,7 @@ import {
BelongsTo,
Table,
DataType,
AfterValidate,
} from "sequelize-typescript";
import { RateLimitExceededError } from "@server/errors";
import { deleteFromS3, getFileByKey } from "@server/utils/s3";
import Collection from "./Collection";
import Team from "./Team";
@@ -53,15 +50,13 @@ export enum FileOperationState {
@Table({ tableName: "file_operations", modelName: "file_operation" })
@Fix
class FileOperation extends IdModel {
@Column(DataType.ENUM("import", "export"))
@Column(DataType.ENUM(...Object.values(FileOperationType)))
type: FileOperationType;
@Column(DataType.STRING)
format: FileOperationFormat;
@Column(
DataType.ENUM("creating", "uploading", "complete", "error", "expired")
)
@Column(DataType.ENUM(...Object.values(FileOperationState)))
state: FileOperationState;
@Column
@@ -93,21 +88,6 @@ class FileOperation extends IdModel {
await deleteFromS3(model.key);
}
@AfterValidate
static async checkRateLimit(model: FileOperation) {
const count = await this.countExportsAfterDateTime(
model.teamId,
subHours(new Date(), 12),
{
type: model.type,
}
);
if (count >= 12) {
throw RateLimitExceededError();
}
}
// associations
@BelongsTo(() => User, "userId")

186
server/routes/api/collections.ts
View File

@@ -4,12 +4,13 @@ import Router from "koa-router";
import { Sequelize, Op, WhereOptions } from "sequelize";
import { randomElement } from "@shared/random";
import { colorPalette } from "@shared/utils/collections";
import { RateLimiterStrategy } from "@server/RateLimiter";
import collectionExporter from "@server/commands/collectionExporter";
import teamUpdater from "@server/commands/teamUpdater";
import { sequelize } from "@server/database/sequelize";
import { ValidationError } from "@server/errors";
import auth from "@server/middlewares/authentication";
import { rateLimiter } from "@server/middlewares/rateLimiter";
import {
Collection,
CollectionUser,
@@ -143,54 +144,59 @@ router.post("collections.info", auth(), async (ctx) => {
};
});
router.post("collections.import", auth(), async (ctx) => {
const { attachmentId, format = FileOperationFormat.MarkdownZip } = ctx.body;
assertUuid(attachmentId, "attachmentId is required");
router.post(
"collections.import",
auth(),
rateLimiter(RateLimiterStrategy.TenPerHour),
async (ctx) => {
const { attachmentId, format = FileOperationFormat.MarkdownZip } = ctx.body;
assertUuid(attachmentId, "attachmentId is required");
const { user } = ctx.state;
authorize(user, "importCollection", user.team);
const { user } = ctx.state;
authorize(user, "importCollection", user.team);
const attachment = await Attachment.findByPk(attachmentId);
authorize(user, "read", attachment);
const attachment = await Attachment.findByPk(attachmentId);
authorize(user, "read", attachment);
assertIn(format, Object.values(FileOperationFormat), "Invalid format");
assertIn(format, Object.values(FileOperationFormat), "Invalid format");
await sequelize.transaction(async (transaction) => {
const fileOperation = await FileOperation.create(
{
type: FileOperationType.Import,
state: FileOperationState.Creating,
format,
size: attachment.size,
key: attachment.key,
userId: user.id,
teamId: user.teamId,
},
{
transaction,
}
);
await Event.create(
{
name: "fileOperations.create",
teamId: user.teamId,
actorId: user.id,
modelId: fileOperation.id,
data: {
await sequelize.transaction(async (transaction) => {
const fileOperation = await FileOperation.create(
{
type: FileOperationType.Import,
state: FileOperationState.Creating,
format,
size: attachment.size,
key: attachment.key,
userId: user.id,
teamId: user.teamId,
},
},
{
transaction,
}
);
});
{
transaction,
}
);
ctx.body = {
success: true,
};
});
await Event.create(
{
name: "fileOperations.create",
teamId: user.teamId,
actorId: user.id,
modelId: fileOperation.id,
data: {
type: FileOperationType.Import,
},
},
{
transaction,
}
);
});
ctx.body = {
success: true,
};
}
);
router.post("collections.add_group", auth(), async (ctx) => {
const { id, groupId, permission = "read_write" } = ctx.body;
@@ -485,57 +491,67 @@ router.post("collections.memberships", auth(), pagination(), async (ctx) => {
};
});
router.post("collections.export", auth(), async (ctx) => {
const { id } = ctx.body;
assertUuid(id, "id is required");
const { user } = ctx.state;
const team = await Team.findByPk(user.teamId);
authorize(user, "createExport", team);
router.post(
"collections.export",
auth(),
rateLimiter(RateLimiterStrategy.TenPerHour),
async (ctx) => {
const { id } = ctx.body;
assertUuid(id, "id is required");
const { user } = ctx.state;
const team = await Team.findByPk(user.teamId);
authorize(user, "createExport", team);
const collection = await Collection.scope({
method: ["withMembership", user.id],
}).findByPk(id);
authorize(user, "read", collection);
const collection = await Collection.scope({
method: ["withMembership", user.id],
}).findByPk(id);
authorize(user, "read", collection);
const fileOperation = await sequelize.transaction(async (transaction) => {
return collectionExporter({
collection,
user,
team,
ip: ctx.request.ip,
transaction,
const fileOperation = await sequelize.transaction(async (transaction) => {
return collectionExporter({
collection,
user,
team,
ip: ctx.request.ip,
transaction,
});
});
});
ctx.body = {
success: true,
data: {
fileOperation: presentFileOperation(fileOperation),
},
};
});
ctx.body = {
success: true,
data: {
fileOperation: presentFileOperation(fileOperation),
},
};
}
);
router.post("collections.export_all", auth(), async (ctx) => {
const { user } = ctx.state;
const team = await Team.findByPk(user.teamId);
authorize(user, "createExport", team);
router.post(
"collections.export_all",
auth(),
rateLimiter(RateLimiterStrategy.TenPerHour),
async (ctx) => {
const { user } = ctx.state;
const team = await Team.findByPk(user.teamId);
authorize(user, "createExport", team);
const fileOperation = await sequelize.transaction(async (transaction) => {
return collectionExporter({
user,
team,
ip: ctx.request.ip,
transaction,
const fileOperation = await sequelize.transaction(async (transaction) => {
return collectionExporter({
user,
team,
ip: ctx.request.ip,
transaction,
});
});
});
ctx.body = {
success: true,
data: {
fileOperation: presentFileOperation(fileOperation),
},
};
});
ctx.body = {
success: true,
data: {
fileOperation: presentFileOperation(fileOperation),
},
};
}
);
router.post("collections.update", auth(), async (ctx) => {
const {

4
server/routes/api/index.ts
View File

@@ -5,7 +5,7 @@ import env from "@server/env";
import { NotFoundError } from "@server/errors";
import errorHandling from "@server/middlewares/errorHandling";
import methodOverride from "@server/middlewares/methodOverride";
import { rateLimiter } from "@server/middlewares/rateLimiter";
import { defaultRateLimiter } from "@server/middlewares/rateLimiter";
import apiKeys from "./apiKeys";
import attachments from "./attachments";
import auth from "./auth";
@@ -81,7 +81,7 @@ router.post("*", (ctx) => {
ctx.throw(NotFoundError("Endpoint not found"));
});
api.use(rateLimiter());
api.use(defaultRateLimiter());
// Router is embedded in a Koa application wrapper, because koa-router does not
// allow middleware to catch any routes which were not explicitly defined.

135
server/routes/api/users.ts
View File

@@ -2,6 +2,7 @@ import crypto from "crypto";
import Router from "koa-router";
import { Op, WhereOptions } from "sequelize";
import { UserValidation } from "@shared/validations";
import { RateLimiterStrategy } from "@server/RateLimiter";
import userDemoter from "@server/commands/userDemoter";
import userDestroyer from "@server/commands/userDestroyer";
import userInviter from "@server/commands/userInviter";
@@ -13,6 +14,7 @@ import env from "@server/env";
import { ValidationError } from "@server/errors";
import logger from "@server/logging/Logger";
import auth from "@server/middlewares/authentication";
import { rateLimiter } from "@server/middlewares/rateLimiter";
import { Event, User, Team } from "@server/models";
import { UserFlag, UserRole } from "@server/models/User";
import { can, authorize } from "@server/policies";
@@ -308,26 +310,31 @@ router.post("users.activate", auth(), async (ctx) => {
};
});
router.post("users.invite", auth(), async (ctx) => {
const { invites } = ctx.body;
assertArray(invites, "invites must be an array");
const { user } = ctx.state;
const team = await Team.findByPk(user.teamId);
authorize(user, "inviteUser", team);
router.post(
"users.invite",
auth(),
rateLimiter(RateLimiterStrategy.TenPerHour),
async (ctx) => {
const { invites } = ctx.body;
assertArray(invites, "invites must be an array");
const { user } = ctx.state;
const team = await Team.findByPk(user.teamId);
authorize(user, "inviteUser", team);
const response = await userInviter({
user,
invites: invites.slice(0, UserValidation.maxInvitesPerRequest),
ip: ctx.request.ip,
});
const response = await userInviter({
user,
invites: invites.slice(0, UserValidation.maxInvitesPerRequest),
ip: ctx.request.ip,
});
ctx.body = {
data: {
sent: response.sent,
users: response.users.map((user) => presentUser(user)),
},
};
});
ctx.body = {
data: {
sent: response.sent,
users: response.users.map((user) => presentUser(user)),
},
};
}
);
router.post("users.resendInvite", auth(), async (ctx) => {
const { id } = ctx.body;
@@ -371,49 +378,59 @@ router.post("users.resendInvite", auth(), async (ctx) => {
};
});
router.post("users.requestDelete", auth(), async (ctx) => {
const { user } = ctx.state;
authorize(user, "delete", user);
router.post(
"users.requestDelete",
auth(),
rateLimiter(RateLimiterStrategy.FivePerHour),
async (ctx) => {
const { user } = ctx.state;
authorize(user, "delete", user);
if (emailEnabled) {
await ConfirmUserDeleteEmail.schedule({
to: user.email,
deleteConfirmationCode: user.deleteConfirmationCode,
if (emailEnabled) {
await ConfirmUserDeleteEmail.schedule({
to: user.email,
deleteConfirmationCode: user.deleteConfirmationCode,
});
}
ctx.body = {
success: true,
};
}
);
router.post(
"users.delete",
auth(),
rateLimiter(RateLimiterStrategy.FivePerHour),
async (ctx) => {
const { code = "" } = ctx.body;
const { user } = ctx.state;
authorize(user, "delete", user);
const deleteConfirmationCode = user.deleteConfirmationCode;
if (
emailEnabled &&
(code.length !== deleteConfirmationCode.length ||
!crypto.timingSafeEqual(
Buffer.from(code),
Buffer.from(deleteConfirmationCode)
))
) {
throw ValidationError("The confirmation code was incorrect");
}
await userDestroyer({
user,
actor: user,
ip: ctx.request.ip,
});
ctx.body = {
success: true,
};
}
ctx.body = {
success: true,
};
});
router.post("users.delete", auth(), async (ctx) => {
const { code = "" } = ctx.body;
const { user } = ctx.state;
authorize(user, "delete", user);
const deleteConfirmationCode = user.deleteConfirmationCode;
if (
emailEnabled &&
(code.length !== deleteConfirmationCode.length ||
!crypto.timingSafeEqual(
Buffer.from(code),
Buffer.from(deleteConfirmationCode)
))
) {
throw ValidationError("The confirmation code was incorrect");
}
await userDestroyer({
user,
actor: user,
ip: ctx.request.ip,
});
ctx.body = {
success: true,
};
});
);
export default router;

3
server/routes/auth/index.ts
View File

@@ -5,12 +5,15 @@ import bodyParser from "koa-body";
import Router from "koa-router";
import { AuthenticationError } from "@server/errors";
import auth from "@server/middlewares/authentication";
import { defaultRateLimiter } from "@server/middlewares/rateLimiter";
import { Collection, Team, View } from "@server/models";
import providers from "./providers";
const app = new Koa();
const router = new Router();
router.use(passport.initialize());
router.use(defaultRateLimiter());
// dynamically load available authentication provider routes
providers.forEach((provider) => {

183
server/routes/auth/providers/email.ts
View File

@@ -1,7 +1,7 @@
import { subMinutes } from "date-fns";
import Router from "koa-router";
import { find } from "lodash";
import { parseDomain } from "@shared/utils/domains";
import { RateLimiterStrategy } from "@server/RateLimiter";
import InviteAcceptedEmail from "@server/emails/templates/InviteAcceptedEmail";
import SigninEmail from "@server/emails/templates/SigninEmail";
import WelcomeEmail from "@server/emails/templates/WelcomeEmail";
@@ -9,6 +9,7 @@ import env from "@server/env";
import { AuthorizationError } from "@server/errors";
import errorHandling from "@server/middlewares/errorHandling";
import methodOverride from "@server/middlewares/methodOverride";
import { rateLimiter } from "@server/middlewares/rateLimiter";
import { User, Team } from "@server/models";
import { signIn } from "@server/utils/authentication";
import { getUserForEmailSigninToken } from "@server/utils/jwt";
@@ -23,102 +24,94 @@ export const config = {
router.use(methodOverride());
router.post("email", errorHandling(), async (ctx) => {
const { email } = ctx.body;
assertEmail(email, "email is required");
const users = await User.scope("withAuthentications").findAll({
where: {
email: email.toLowerCase(),
},
});
if (users.length) {
let team!: Team | null;
const domain = parseDomain(ctx.request.hostname);
if (domain.custom) {
team = await Team.scope("withAuthenticationProviders").findOne({
where: {
domain: ctx.request.hostname,
},
});
} else if (env.SUBDOMAINS_ENABLED && domain.teamSubdomain) {
team = await Team.scope("withAuthenticationProviders").findOne({
where: {
subdomain: domain.teamSubdomain,
},
});
}
// If there are multiple users with this email address then give precedence
// to the one that is active on this subdomain/domain (if any)
let user = users.find((user) => team && user.teamId === team.id);
// A user was found for the email address, but they don't belong to the team
// that this subdomain belongs to, we load their team and allow the logic to
// continue
if (!user) {
user = users[0];
team = await Team.scope("withAuthenticationProviders").findByPk(
user.teamId
);
}
if (!team) {
team = await Team.scope("withAuthenticationProviders").findByPk(
user.teamId
);
}
if (!team) {
ctx.redirect(`/?notice=auth-error`);
return;
}
// If the user matches an email address associated with an SSO
// provider then just forward them directly to that sign-in page
if (user.authentications.length) {
const authProvider = find(team.authenticationProviders, {
id: user.authentications[0].authenticationProviderId,
});
ctx.body = {
redirect: `${team.url}/auth/${authProvider?.name}`,
};
return;
}
if (!team.emailSigninEnabled) {
throw AuthorizationError();
}
// basic rate limit of endpoint to prevent send email abuse
if (
user.lastSigninEmailSentAt &&
user.lastSigninEmailSentAt > subMinutes(new Date(), 2)
) {
ctx.body = {
redirect: `${team.url}?notice=email-auth-ratelimit`,
message: "Rate limit exceeded",
success: false,
};
return;
}
// send email to users registered address with a short-lived token
await SigninEmail.schedule({
to: user.email,
token: user.getEmailSigninToken(),
teamUrl: team.url,
router.post(
"email",
errorHandling(),
rateLimiter(RateLimiterStrategy.TenPerHour),
async (ctx) => {
const { email } = ctx.body;
assertEmail(email, "email is required");
const users = await User.scope("withAuthentications").findAll({
where: {
email: email.toLowerCase(),
},
});
user.lastSigninEmailSentAt = new Date();
await user.save();
}
// respond with success regardless of whether an email was sent
ctx.body = {
success: true,
};
});
if (users.length) {
let team!: Team | null;
const domain = parseDomain(ctx.request.hostname);
if (domain.custom) {
team = await Team.scope("withAuthenticationProviders").findOne({
where: {
domain: ctx.request.hostname,
},
});
} else if (env.SUBDOMAINS_ENABLED && domain.teamSubdomain) {
team = await Team.scope("withAuthenticationProviders").findOne({
where: {
subdomain: domain.teamSubdomain,
},
});
}
// If there are multiple users with this email address then give precedence
// to the one that is active on this subdomain/domain (if any)
let user = users.find((user) => team && user.teamId === team.id);
// A user was found for the email address, but they don't belong to the team
// that this subdomain belongs to, we load their team and allow the logic to
// continue
if (!user) {
user = users[0];
team = await Team.scope("withAuthenticationProviders").findByPk(
user.teamId
);
}
if (!team) {
team = await Team.scope("withAuthenticationProviders").findByPk(
user.teamId
);
}
if (!team) {
ctx.redirect(`/?notice=auth-error`);
return;
}
// If the user matches an email address associated with an SSO
// provider then just forward them directly to that sign-in page
if (user.authentications.length) {
const authProvider = find(team.authenticationProviders, {
id: user.authentications[0].authenticationProviderId,
});
ctx.body = {
redirect: `${team.url}/auth/${authProvider?.name}`,
};
return;
}
if (!team.emailSigninEnabled) {
throw AuthorizationError();
}
// send email to users registered address with a short-lived token
await SigninEmail.schedule({
to: user.email,
token: user.getEmailSigninToken(),
teamUrl: team.url,
});
user.lastSigninEmailSentAt = new Date();
await user.save();
}
// respond with success regardless of whether an email was sent
ctx.body = {
success: true,
};
}
);
router.get("email.callback", async (ctx) => {
const { token } = ctx.request.query;

7
server/types.ts
View File

@@ -1,5 +1,4 @@
import { Context } from "koa";
import Redis from "@server/redis";
import { FileOperation, Team, User } from "./models";
export enum AuthenticationTypes {
@@ -298,9 +297,3 @@ export type Event =
| UserEvent
| ViewEvent
| WebhookSubscriptionEvent;
export type RateLimiterConfig = {
points: number;
duration: number;
storeClient: Redis;
};