Move bulk of webhook logic to plugin (#4866)

* Move bulk of webhook logic to plugin

* Re-enable cleanup task

* cron tasks
This commit is contained in:
Tom Moor
2023-02-12 19:28:11 -05:00
committed by GitHub
parent 7895ee207c
commit 60101c507a
30 changed files with 74 additions and 67 deletions

View File

@@ -0,0 +1 @@
export { WebhooksIcon as default } from "outline-icons";

View File

@@ -0,0 +1,75 @@
import { observer } from "mobx-react";
import { WebhooksIcon } from "outline-icons";
import * as React from "react";
import { useTranslation, Trans } from "react-i18next";
import WebhookSubscription from "~/models/WebhookSubscription";
import { Action } from "~/components/Actions";
import Button from "~/components/Button";
import Heading from "~/components/Heading";
import Modal from "~/components/Modal";
import PaginatedList from "~/components/PaginatedList";
import Scene from "~/components/Scene";
import Subheading from "~/components/Subheading";
import Text from "~/components/Text";
import env from "~/env";
import useBoolean from "~/hooks/useBoolean";
import useCurrentTeam from "~/hooks/useCurrentTeam";
import usePolicy from "~/hooks/usePolicy";
import useStores from "~/hooks/useStores";
import WebhookSubscriptionListItem from "./components/WebhookSubscriptionListItem";
import WebhookSubscriptionNew from "./components/WebhookSubscriptionNew";
function Webhooks() {
const team = useCurrentTeam();
const { t } = useTranslation();
const { webhookSubscriptions } = useStores();
const [newModalOpen, handleNewModalOpen, handleNewModalClose] = useBoolean();
const can = usePolicy(team);
const appName = env.APP_NAME;
return (
<Scene
title={t("Webhooks")}
icon={<WebhooksIcon color="currentColor" />}
actions={
<>
{can.createWebhookSubscription && (
<Action>
<Button
type="submit"
value={`${t("New webhook")}`}
onClick={handleNewModalOpen}
/>
</Action>
)}
</>
}
>
<Heading>{t("Webhooks")}</Heading>
<Text type="secondary">
<Trans>
Webhooks can be used to notify your application when events happen in{" "}
{{ appName }}. Events are sent as a https request with a JSON payload
in near real-time.
</Trans>
</Text>
<PaginatedList
fetch={webhookSubscriptions.fetchPage}
items={webhookSubscriptions.orderedData}
heading={<Subheading sticky>{t("Webhooks")}</Subheading>}
renderItem={(webhook: WebhookSubscription) => (
<WebhookSubscriptionListItem key={webhook.id} webhook={webhook} />
)}
/>
<Modal
title={t("Create a webhook")}
onRequestClose={handleNewModalClose}
isOpen={newModalOpen}
>
<WebhookSubscriptionNew onSubmit={handleNewModalClose} />
</Modal>
</Scene>
);
}
export default observer(Webhooks);

View File

@@ -0,0 +1,34 @@
import * as React from "react";
import { useTranslation } from "react-i18next";
import WebhookSubscription from "~/models/WebhookSubscription";
import ConfirmationDialog from "~/components/ConfirmationDialog";
type Props = {
webhook: WebhookSubscription;
onSubmit: () => void;
};
export default function WebhookSubscriptionRevokeDialog({
webhook,
onSubmit,
}: Props) {
const { t } = useTranslation();
const handleSubmit = async () => {
await webhook.delete();
onSubmit();
};
return (
<ConfirmationDialog
onSubmit={handleSubmit}
submitText={t("Delete")}
savingText={`${t("Deleting")}`}
danger
>
{t("Are you sure you want to delete the {{ name }} webhook?", {
name: webhook.name,
})}
</ConfirmationDialog>
);
}

View File

@@ -0,0 +1,57 @@
import * as React from "react";
import { useTranslation } from "react-i18next";
import WebhookSubscription from "~/models/WebhookSubscription";
import useToasts from "~/hooks/useToasts";
import WebhookSubscriptionForm from "./WebhookSubscriptionForm";
type Props = {
onSubmit: () => void;
webhookSubscription: WebhookSubscription;
};
interface FormData {
name: string;
url: string;
events: string[];
}
function WebhookSubscriptionEdit({ onSubmit, webhookSubscription }: Props) {
const { showToast } = useToasts();
const { t } = useTranslation();
const handleSubmit = React.useCallback(
async (data: FormData) => {
try {
const events = Array.isArray(data.events) ? data.events : [data.events];
const toSend = {
...data,
events,
};
await webhookSubscription.save(toSend);
showToast(
t("Webhook updated", {
type: "success",
})
);
onSubmit();
} catch (err) {
showToast(err.message, {
type: "error",
});
}
},
[t, showToast, onSubmit, webhookSubscription]
);
return (
<WebhookSubscriptionForm
handleSubmit={handleSubmit}
webhookSubscription={webhookSubscription}
/>
);
}
export default WebhookSubscriptionEdit;

View File

@@ -0,0 +1,298 @@
import { isEqual, filter, includes } from "lodash";
import randomstring from "randomstring";
import * as React from "react";
import { useEffect } from "react";
import { useForm } from "react-hook-form";
import { useTranslation, Trans } from "react-i18next";
import styled from "styled-components";
import WebhookSubscription from "~/models/WebhookSubscription";
import Button from "~/components/Button";
import Input from "~/components/Input";
import Text from "~/components/Text";
import useMobile from "~/hooks/useMobile";
const WEBHOOK_EVENTS = {
user: [
"users.create",
"users.signin",
"users.update",
"users.suspend",
"users.activate",
"users.delete",
"users.invite",
"users.promote",
"users.demote",
],
document: [
"documents.create",
"documents.publish",
"documents.unpublish",
"documents.delete",
"documents.permanent_delete",
"documents.archive",
"documents.unarchive",
"documents.restore",
"documents.move",
"documents.update",
"documents.title_change",
],
revision: ["revisions.create"],
fileOperation: [
"fileOperations.create",
"fileOperations.update",
"fileOperations.delete",
],
collection: [
"collections.create",
"collections.update",
"collections.delete",
"collections.add_user",
"collections.remove_user",
"collections.add_group",
"collections.remove_group",
"collections.move",
"collections.permission_changed",
],
group: [
"groups.create",
"groups.update",
"groups.delete",
"groups.add_user",
"groups.remove_user",
],
integration: ["integrations.create", "integrations.update"],
share: ["shares.create", "shares.update", "shares.revoke"],
team: ["teams.update"],
pin: ["pins.create", "pins.update", "pins.delete"],
webhookSubscription: [
"webhookSubscriptions.create",
"webhookSubscriptions.delete",
"webhookSubscriptions.update",
],
view: ["views.create"],
};
const EventCheckboxLabel = styled.label`
display: flex;
align-items: center;
font-size: 15px;
padding: 0.2em 0;
`;
const GroupEventCheckboxLabel = styled(EventCheckboxLabel)`
font-weight: 500;
font-size: 1.2em;
`;
const AllEventCheckboxLabel = styled(GroupEventCheckboxLabel)`
font-size: 1.4em;
`;
const EventCheckboxText = styled.span`
margin-left: 0.5rem;
`;
interface FieldProps {
disabled?: boolean;
}
const FieldSet = styled.fieldset<FieldProps>`
padding: 0;
margin: 0;
border: none;
${({ disabled }) =>
disabled &&
`
opacity: 0.5;
`}
`;
interface MobileProps {
isMobile?: boolean;
}
const GroupGrid = styled.div<MobileProps>`
display: grid;
grid-template-columns: 1fr 1fr;
${({ isMobile }) =>
isMobile &&
`
grid-template-columns: 1fr;
`}
`;
const GroupWrapper = styled.div<MobileProps>`
padding-bottom: 2rem;
${({ isMobile }) =>
isMobile &&
`
padding-bottom: 1rem;
`}
`;
const TextFields = styled.div`
display: flex;
flex-direction: column;
margin-bottom: 1em;
`;
type Props = {
handleSubmit: (data: FormData) => void;
webhookSubscription?: WebhookSubscription;
};
interface FormData {
name: string;
url: string;
secret: string;
events: string[];
}
function generateSigningSecret() {
return `ol_whs_${randomstring.generate(32)}`;
}
function WebhookSubscriptionForm({ handleSubmit, webhookSubscription }: Props) {
const { t } = useTranslation();
const {
register,
handleSubmit: formHandleSubmit,
formState,
watch,
setValue,
} = useForm<FormData>({
mode: "all",
defaultValues: {
events: webhookSubscription ? [...webhookSubscription.events] : [],
name: webhookSubscription?.name,
url: webhookSubscription?.url,
secret: webhookSubscription?.secret ?? generateSigningSecret(),
},
});
const events = watch("events");
const selectedGroups = filter(events, (e) => !e.includes("."));
const isAllEventSelected = includes(events, "*");
const filteredEvents = filter(events, (e) => {
const [beforePeriod] = e.split(".");
return (
selectedGroups.length === 0 ||
e === beforePeriod ||
!selectedGroups.includes(beforePeriod)
);
});
const isMobile = useMobile();
useEffect(() => {
if (isAllEventSelected) {
setValue("events", ["*"]);
}
}, [isAllEventSelected, setValue]);
useEffect(() => {
if (!isEqual(events, filteredEvents)) {
setValue("events", filteredEvents);
}
}, [events, filteredEvents, setValue]);
const verb = webhookSubscription ? t("Update") : t("Create");
const inProgressVerb = webhookSubscription ? t("Updating") : t("Creating");
function EventCheckbox({ label, value }: { label: string; value: string }) {
const LabelComponent =
value === "*"
? AllEventCheckboxLabel
: Object.keys(WEBHOOK_EVENTS).includes(value)
? GroupEventCheckboxLabel
: EventCheckboxLabel;
return (
<LabelComponent>
<input
type="checkbox"
defaultValue={value}
{...register("events", {})}
/>
<EventCheckboxText>{label}</EventCheckboxText>
</LabelComponent>
);
}
return (
<form onSubmit={formHandleSubmit(handleSubmit)}>
<Text type="secondary">
<Trans>
Provide a descriptive name for this webhook and the URL we should send
a POST request to when matching events are created.
</Trans>
</Text>
<TextFields>
<Input
required
autoFocus
flex
label={t("Name")}
placeholder={t("A memorable identifer")}
{...register("name", {
required: true,
})}
/>
<Input
required
autoFocus
flex
pattern="https://.*"
placeholder="https://…"
label={t("URL")}
{...register("url", { required: true })}
/>
<Input
flex
spellCheck={false}
label={t("Signing secret")}
{...register("secret", {
required: false,
})}
/>
</TextFields>
<Text type="secondary">
<Trans>
Subscribe to all events, groups, or individual events. We recommend
only subscribing to the minimum amount of events that your application
needs to function.
</Trans>
</Text>
<EventCheckbox label={t("All events")} value="*" />
<FieldSet disabled={isAllEventSelected}>
<GroupGrid isMobile={isMobile}>
{Object.entries(WEBHOOK_EVENTS).map(([group, events], i) => (
<GroupWrapper key={i} isMobile={isMobile}>
<EventCheckbox
label={t(`All {{ groupName }} events`, { groupName: group })}
value={group}
/>
<FieldSet disabled={selectedGroups.includes(group)}>
{events.map((event) => (
<EventCheckbox label={event} value={event} key={event} />
))}
</FieldSet>
</GroupWrapper>
))}
</GroupGrid>
</FieldSet>
<Button
type="submit"
disabled={formState.isSubmitting || !formState.isValid}
>
{formState.isSubmitting ? `${inProgressVerb}` : verb}
</Button>
</form>
);
}
export default WebhookSubscriptionForm;

View File

@@ -0,0 +1,89 @@
import { EditIcon, TrashIcon } from "outline-icons";
import * as React from "react";
import { useTranslation } from "react-i18next";
import styled from "styled-components";
import WebhookSubscription from "~/models/WebhookSubscription";
import Badge from "~/components/Badge";
import Button from "~/components/Button";
import ListItem from "~/components/List/Item";
import Modal from "~/components/Modal";
import useBoolean from "~/hooks/useBoolean";
import useStores from "~/hooks/useStores";
import WebhookSubscriptionRevokeDialog from "./WebhookSubscriptionDeleteDialog";
import WebhookSubscriptionEdit from "./WebhookSubscriptionEdit";
type Props = {
webhook: WebhookSubscription;
};
const WebhookSubscriptionListItem = ({ webhook }: Props) => {
const { t } = useTranslation();
const { dialogs } = useStores();
const [
editModalOpen,
handleEditModalOpen,
handleEditModalClose,
] = useBoolean();
const showDeletionConfirmation = React.useCallback(() => {
dialogs.openModal({
title: t("Delete webhook"),
isCentered: true,
content: (
<WebhookSubscriptionRevokeDialog
onSubmit={dialogs.closeAllModals}
webhook={webhook}
/>
),
});
}, [t, dialogs, webhook]);
return (
<ListItem
key={webhook.id}
title={
<>
{webhook.name}
{!webhook.enabled && (
<StyledBadge yellow={true}>{t("Disabled")}</StyledBadge>
)}
</>
}
subtitle={
<>
{t("Subscribed events")}: <code>{webhook.events.join(", ")}</code>
</>
}
actions={
<>
<Button
onClick={showDeletionConfirmation}
icon={<TrashIcon />}
neutral
>
{t("Delete")}
</Button>
<Button icon={<EditIcon />} onClick={handleEditModalOpen} neutral>
{t("Edit")}
</Button>
<Modal
title={t("Edit webhook")}
onRequestClose={handleEditModalClose}
isOpen={editModalOpen}
>
<WebhookSubscriptionEdit
onSubmit={handleEditModalClose}
webhookSubscription={webhook}
/>
</Modal>
</>
}
/>
);
};
const StyledBadge = styled(Badge)`
position: absolute;
`;
export default WebhookSubscriptionListItem;

View File

@@ -0,0 +1,51 @@
import * as React from "react";
import { useTranslation } from "react-i18next";
import useStores from "~/hooks/useStores";
import useToasts from "~/hooks/useToasts";
import WebhookSubscriptionForm from "./WebhookSubscriptionForm";
type Props = {
onSubmit: () => void;
};
interface FormData {
name: string;
url: string;
events: string[];
}
function WebhookSubscriptionNew({ onSubmit }: Props) {
const { webhookSubscriptions } = useStores();
const { showToast } = useToasts();
const { t } = useTranslation();
const handleSubmit = React.useCallback(
async (data: FormData) => {
try {
const events = Array.isArray(data.events) ? data.events : [data.events];
const toSend = {
...data,
events,
};
await webhookSubscriptions.create(toSend);
showToast(
t("Webhook created", {
type: "success",
})
);
onSubmit();
} catch (err) {
showToast(err.message, {
type: "error",
});
}
},
[t, showToast, onSubmit, webhookSubscriptions]
);
return <WebhookSubscriptionForm handleSubmit={handleSubmit} />;
}
export default WebhookSubscriptionNew;

View File

@@ -0,0 +1,5 @@
{
"name": "Webhooks",
"description": "Adds HTTP webhooks for various events.",
"requiredEnvVars": []
}

View File

@@ -0,0 +1,3 @@
{
"extends": "../../../server/.babelrc"
}

View File

@@ -0,0 +1,161 @@
import Router from "koa-router";
import { compact, isEmpty } from "lodash";
import { ValidationError } from "@server/errors";
import auth from "@server/middlewares/authentication";
import { WebhookSubscription, Event } from "@server/models";
import { authorize } from "@server/policies";
import pagination from "@server/routes/api/middlewares/pagination";
import { WebhookSubscriptionEvent, APIContext } from "@server/types";
import { assertArray, assertPresent, assertUuid } from "@server/validation";
import presentWebhookSubscription from "../presenters/webhookSubscription";
const router = new Router();
router.post(
"webhookSubscriptions.list",
auth({ admin: true }),
pagination(),
async (ctx: APIContext) => {
const { user } = ctx.state.auth;
authorize(user, "listWebhookSubscription", user.team);
const webhooks = await WebhookSubscription.findAll({
where: {
teamId: user.teamId,
},
order: [["createdAt", "DESC"]],
offset: ctx.state.pagination.offset,
limit: ctx.state.pagination.limit,
});
ctx.body = {
pagination: ctx.state.pagination,
data: webhooks.map(presentWebhookSubscription),
};
}
);
router.post(
"webhookSubscriptions.create",
auth({ admin: true }),
async (ctx: APIContext) => {
const { user } = ctx.state.auth;
authorize(user, "createWebhookSubscription", user.team);
const { name, url, secret } = ctx.request.body;
const events: string[] = compact(ctx.request.body.events);
assertPresent(name, "name is required");
assertPresent(url, "url is required");
assertArray(events, "events is required");
if (events.length === 0) {
throw ValidationError("events are required");
}
const webhookSubscription = await WebhookSubscription.create({
name,
events,
createdById: user.id,
teamId: user.teamId,
url,
enabled: true,
secret: isEmpty(secret) ? undefined : secret,
});
const event: WebhookSubscriptionEvent = {
name: "webhookSubscriptions.create",
modelId: webhookSubscription.id,
teamId: user.teamId,
actorId: user.id,
data: {
name,
url,
events,
},
ip: ctx.request.ip,
};
await Event.create(event);
ctx.body = {
data: presentWebhookSubscription(webhookSubscription),
};
}
);
router.post(
"webhookSubscriptions.delete",
auth({ admin: true }),
async (ctx: APIContext) => {
const { id } = ctx.request.body;
assertUuid(id, "id is required");
const { user } = ctx.state.auth;
const webhookSubscription = await WebhookSubscription.findByPk(id);
authorize(user, "delete", webhookSubscription);
await webhookSubscription.destroy();
const event: WebhookSubscriptionEvent = {
name: "webhookSubscriptions.delete",
modelId: webhookSubscription.id,
teamId: user.teamId,
actorId: user.id,
data: {
name: webhookSubscription.name,
url: webhookSubscription.url,
events: webhookSubscription.events,
},
ip: ctx.request.ip,
};
await Event.create(event);
}
);
router.post(
"webhookSubscriptions.update",
auth({ admin: true }),
async (ctx: APIContext) => {
const { id } = ctx.request.body;
assertUuid(id, "id is required");
const { user } = ctx.state.auth;
const { name, url, secret } = ctx.request.body;
const events: string[] = compact(ctx.request.body.events);
assertPresent(name, "name is required");
assertPresent(url, "url is required");
assertArray(events, "events is required");
if (events.length === 0) {
throw ValidationError("events are required");
}
const webhookSubscription = await WebhookSubscription.findByPk(id);
authorize(user, "update", webhookSubscription);
await webhookSubscription.update({
name,
url,
events,
enabled: true,
secret: isEmpty(secret) ? undefined : secret,
});
const event: WebhookSubscriptionEvent = {
name: "webhookSubscriptions.update",
modelId: webhookSubscription.id,
teamId: user.teamId,
actorId: user.id,
data: {
name: webhookSubscription.name,
url: webhookSubscription.url,
events: webhookSubscription.events,
},
ip: ctx.request.ip,
};
await Event.create(event);
ctx.body = {
data: presentWebhookSubscription(webhookSubscription),
};
}
);
export default router;

View File

@@ -0,0 +1,38 @@
import { WebhookDelivery } from "@server/models";
import { Event } from "@server/types";
export interface WebhookPayload {
model: Record<string, unknown> | null;
id: string;
[key: string]: unknown;
}
interface WebhookProps {
event: Event;
delivery: WebhookDelivery;
payload: WebhookPayload;
}
export interface WebhookPresentation {
id: string;
actorId: string;
webhookSubscriptionId: string;
event: string;
payload: WebhookPayload;
createdAt: Date;
}
export default function presentWebhook({
event,
delivery,
payload,
}: WebhookProps): WebhookPresentation {
return {
id: delivery.id,
actorId: event.actorId,
webhookSubscriptionId: delivery.webhookSubscriptionId,
createdAt: delivery.createdAt,
event: event.name,
payload,
};
}

View File

@@ -0,0 +1,16 @@
import { WebhookSubscription } from "@server/models";
export default function presentWebhookSubscription(
webhook: WebhookSubscription
) {
return {
id: webhook.id,
name: webhook.name,
url: webhook.url,
secret: webhook.secret,
events: webhook.events,
enabled: webhook.enabled,
createdAt: webhook.createdAt,
updatedAt: webhook.updatedAt,
};
}

View File

@@ -0,0 +1,96 @@
import { buildUser, buildWebhookSubscription } from "@server/test/factories";
import { setupTestDatabase } from "@server/test/support";
import { UserEvent } from "@server/types";
import DeliverWebhookTask from "../tasks/DeliverWebhookTask";
import WebhookProcessor from "./WebhookProcessor";
jest.mock("@server/queues/tasks/DeliverWebhookTask");
const ip = "127.0.0.1";
setupTestDatabase();
beforeEach(async () => {
jest.resetAllMocks();
});
describe("WebhookProcessor", () => {
test("it schedules a delivery for the event", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
});
const signedInUser = await buildUser({ teamId: subscription.teamId });
const processor = new WebhookProcessor();
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await processor.perform(event);
expect(DeliverWebhookTask.schedule).toHaveBeenCalled();
expect(DeliverWebhookTask.schedule).toHaveBeenCalledWith({
event,
subscriptionId: subscription.id,
});
});
test("not schedule a delivery when not subscribed to event", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["users.create"],
});
const signedInUser = await buildUser({ teamId: subscription.teamId });
const processor = new WebhookProcessor();
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await processor.perform(event);
expect(DeliverWebhookTask.schedule).toHaveBeenCalledTimes(0);
});
test("it schedules a delivery for the event for each subscription", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
});
const subscriptionTwo = await buildWebhookSubscription({
url: "http://example.com",
teamId: subscription.teamId,
events: ["*"],
});
const signedInUser = await buildUser({ teamId: subscription.teamId });
const processor = new WebhookProcessor();
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await processor.perform(event);
expect(DeliverWebhookTask.schedule).toHaveBeenCalled();
expect(DeliverWebhookTask.schedule).toHaveBeenCalledTimes(2);
expect(DeliverWebhookTask.schedule).toHaveBeenCalledWith({
event,
subscriptionId: subscription.id,
});
expect(DeliverWebhookTask.schedule).toHaveBeenCalledWith({
event,
subscriptionId: subscriptionTwo.id,
});
});
});

View File

@@ -0,0 +1,31 @@
import { WebhookSubscription } from "@server/models";
import BaseProcessor from "@server/queues/processors/BaseProcessor";
import { Event } from "@server/types";
import DeliverWebhookTask from "../tasks/DeliverWebhookTask";
export default class WebhookProcessor extends BaseProcessor {
static applicableEvents: ["*"] = ["*"];
async perform(event: Event) {
if (!event.teamId) {
return;
}
const webhookSubscriptions = await WebhookSubscription.findAll({
where: {
enabled: true,
teamId: event.teamId,
},
});
const applicableSubscriptions = webhookSubscriptions.filter((webhook) =>
webhook.validForEvent(event)
);
await Promise.all(
applicableSubscriptions.map((subscription) =>
DeliverWebhookTask.schedule({ event, subscriptionId: subscription.id })
)
);
}
}

View File

@@ -0,0 +1,33 @@
import { subDays } from "date-fns";
import { WebhookDelivery } from "@server/models";
import { buildWebhookDelivery } from "@server/test/factories";
import { setupTestDatabase } from "@server/test/support";
import CleanupWebhookDeliveriesTask from "./CleanupWebhookDeliveriesTask";
setupTestDatabase();
const deliveryExists = async (delivery: WebhookDelivery) => {
const results = await WebhookDelivery.findOne({ where: { id: delivery.id } });
return !!results;
};
describe("CleanupWebookDeliveriesTask", () => {
it("should delete Webhook Deliveries older than 1 week", async () => {
const brandNewWebhookDelivery = await buildWebhookDelivery({
createdAt: new Date(),
});
const newishWebhookDelivery = await buildWebhookDelivery({
createdAt: subDays(new Date(), 5),
});
const oldWebhookDelivery = await buildWebhookDelivery({
createdAt: subDays(new Date(), 8),
});
const task = new CleanupWebhookDeliveriesTask();
await task.perform();
expect(await deliveryExists(brandNewWebhookDelivery)).toBe(true);
expect(await deliveryExists(newishWebhookDelivery)).toBe(true);
expect(await deliveryExists(oldWebhookDelivery)).toBe(false);
});
});

View File

@@ -0,0 +1,33 @@
import { subDays } from "date-fns";
import { Op } from "sequelize";
import Logger from "@server/logging/Logger";
import { WebhookDelivery } from "@server/models";
import BaseTask, {
TaskPriority,
TaskSchedule,
} from "@server/queues/tasks/BaseTask";
type Props = void;
export default class CleanupWebhookDeliveriesTask extends BaseTask<Props> {
static cron = TaskSchedule.Daily;
public async perform(_: Props) {
Logger.info("task", `Deleting WebhookDeliveries older than one week…`);
const count = await WebhookDelivery.unscoped().destroy({
where: {
createdAt: {
[Op.lt]: subDays(new Date(), 7),
},
},
});
Logger.info("task", `${count} old WebhookDeliveries deleted.`);
}
public get options() {
return {
attempts: 1,
priority: TaskPriority.Background,
};
}
}

View File

@@ -0,0 +1,229 @@
import fetchMock from "jest-fetch-mock";
import { v4 as uuidv4 } from "uuid";
import { WebhookDelivery } from "@server/models";
import {
buildUser,
buildWebhookDelivery,
buildWebhookSubscription,
} from "@server/test/factories";
import { setupTestDatabase } from "@server/test/support";
import { UserEvent } from "@server/types";
import DeliverWebhookTask from "./DeliverWebhookTask";
setupTestDatabase();
beforeEach(async () => {
jest.resetAllMocks();
fetchMock.resetMocks();
fetchMock.doMock();
});
const ip = "127.0.0.1";
describe("DeliverWebhookTask", () => {
test("should hit the subscription url and record a delivery", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
});
const signedInUser = await buildUser({ teamId: subscription.teamId });
const processor = new DeliverWebhookTask();
fetchMock.mockResponse("SUCCESS", { status: 200 });
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await processor.perform({
subscriptionId: subscription.id,
event,
});
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"http://example.com",
expect.anything()
);
const parsedBody = JSON.parse(
fetchMock.mock.calls[0]![1]!.body!.toString()
);
expect(parsedBody.webhookSubscriptionId).toBe(subscription.id);
expect(parsedBody.event).toBe("users.signin");
expect(parsedBody.payload.id).toBe(signedInUser.id);
expect(parsedBody.payload.model).toBeDefined();
const deliveries = await WebhookDelivery.findAll({
where: { webhookSubscriptionId: subscription.id },
});
expect(deliveries.length).toBe(1);
const delivery = deliveries[0];
expect(delivery.status).toBe("success");
expect(delivery.statusCode).toBe(200);
expect(delivery.responseBody).toEqual("SUCCESS");
});
test("should hit the subscription url with signature header", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
secret: "secret",
});
const signedInUser = await buildUser({ teamId: subscription.teamId });
const processor = new DeliverWebhookTask();
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await processor.perform({
subscriptionId: subscription.id,
event,
});
const headers = fetchMock.mock.calls[0]![1]!.headers!;
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(headers["Outline-Signature"]).toMatch(/^t=[0-9]+,s=[a-z0-9]+$/);
});
test("should hit the subscription url when the eventing model doesn't exist", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
});
const deletedUserId = uuidv4();
const signedInUser = await buildUser({ teamId: subscription.teamId });
const task = new DeliverWebhookTask();
const event: UserEvent = {
name: "users.delete",
userId: deletedUserId,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await task.perform({
event,
subscriptionId: subscription.id,
});
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"http://example.com",
expect.anything()
);
const parsedBody = JSON.parse(
fetchMock.mock.calls[0]![1]!.body!.toString()
);
expect(parsedBody.webhookSubscriptionId).toBe(subscription.id);
expect(parsedBody.event).toBe("users.delete");
expect(parsedBody.payload.id).toBe(deletedUserId);
const deliveries = await WebhookDelivery.findAll({
where: { webhookSubscriptionId: subscription.id },
});
expect(deliveries.length).toBe(1);
const delivery = deliveries[0];
expect(delivery.status).toBe("success");
expect(delivery.statusCode).toBe(200);
expect(delivery.responseBody).toBeDefined();
});
test("should mark delivery as failed if post fails", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
});
fetchMock.mockResponse("FAILED", { status: 500 });
const signedInUser = await buildUser({ teamId: subscription.teamId });
const task = new DeliverWebhookTask();
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await task.perform({
event,
subscriptionId: subscription.id,
});
await subscription.reload();
expect(subscription.enabled).toBe(true);
const deliveries = await WebhookDelivery.findAll({
where: { webhookSubscriptionId: subscription.id },
});
expect(deliveries.length).toBe(1);
const delivery = deliveries[0];
expect(delivery.status).toBe("failed");
expect(delivery.statusCode).toBe(500);
expect(delivery.responseBody).toBeDefined();
expect(delivery.responseBody).toEqual("FAILED");
});
test("should disable the subscription if past deliveries failed", async () => {
const subscription = await buildWebhookSubscription({
url: "http://example.com",
events: ["*"],
});
for (let i = 0; i < 25; i++) {
await buildWebhookDelivery({
webhookSubscriptionId: subscription.id,
status: "failed",
});
}
fetchMock.mockResponse(JSON.stringify({ message: "Failure" }), {
status: 500,
});
const signedInUser = await buildUser({ teamId: subscription.teamId });
const task = new DeliverWebhookTask();
const event: UserEvent = {
name: "users.signin",
userId: signedInUser.id,
teamId: subscription.teamId,
actorId: signedInUser.id,
ip,
};
await task.perform({
event,
subscriptionId: subscription.id,
});
await subscription.reload();
expect(subscription.enabled).toBe(false);
const deliveries = await WebhookDelivery.findAll({
where: { webhookSubscriptionId: subscription.id },
order: [["createdAt", "DESC"]],
});
expect(deliveries.length).toBe(26);
const delivery = deliveries[0];
expect(delivery.status).toBe("failed");
expect(delivery.statusCode).toBe(500);
expect(delivery.responseBody).toEqual('{"message":"Failure"}');
});
});

View File

@@ -0,0 +1,633 @@
import fetch from "fetch-with-proxy";
import { useAgent } from "request-filtering-agent";
import { Op } from "sequelize";
import WebhookDisabledEmail from "@server/emails/templates/WebhookDisabledEmail";
import env from "@server/env";
import Logger from "@server/logging/Logger";
import {
Collection,
FileOperation,
Group,
Integration,
Pin,
Star,
Team,
WebhookDelivery,
WebhookSubscription,
Document,
User,
Revision,
View,
Share,
CollectionUser,
CollectionGroup,
GroupUser,
} from "@server/models";
import {
presentCollection,
presentDocument,
presentRevision,
presentFileOperation,
presentGroup,
presentIntegration,
presentPin,
presentStar,
presentTeam,
presentUser,
presentView,
presentShare,
presentMembership,
presentGroupMembership,
presentCollectionGroupMembership,
} from "@server/presenters";
import BaseTask from "@server/queues/tasks/BaseTask";
import {
CollectionEvent,
CollectionGroupEvent,
CollectionUserEvent,
DocumentEvent,
Event,
FileOperationEvent,
GroupEvent,
GroupUserEvent,
IntegrationEvent,
PinEvent,
RevisionEvent,
ShareEvent,
StarEvent,
TeamEvent,
UserEvent,
ViewEvent,
WebhookSubscriptionEvent,
} from "@server/types";
import presentWebhook, { WebhookPayload } from "../presenters/webhook";
import presentWebhookSubscription from "../presenters/webhookSubscription";
function assertUnreachable(event: never) {
Logger.warn(`DeliverWebhookTask did not handle ${(event as any).name}`);
}
type Props = {
subscriptionId: string;
event: Event;
};
export default class DeliverWebhookTask extends BaseTask<Props> {
public async perform({ subscriptionId, event }: Props) {
const subscription = await WebhookSubscription.findByPk(subscriptionId, {
rejectOnEmpty: true,
});
if (!subscription.enabled) {
Logger.info("task", `WebhookSubscription was disabled before delivery`, {
event: event.name,
subscriptionId: subscription.id,
});
return;
}
Logger.info("task", `DeliverWebhookTask: ${event.name}`, {
event: event.name,
subscriptionId: subscription.id,
});
switch (event.name) {
case "api_keys.create":
case "api_keys.delete":
case "attachments.create":
case "attachments.delete":
case "subscriptions.create":
case "subscriptions.delete":
case "authenticationProviders.update":
// Ignored
return;
case "users.create":
case "users.signin":
case "users.signout":
case "users.update":
case "users.suspend":
case "users.activate":
case "users.delete":
case "users.invite":
case "users.promote":
case "users.demote":
await this.handleUserEvent(subscription, event);
return;
case "documents.create":
case "documents.publish":
case "documents.unpublish":
case "documents.delete":
case "documents.permanent_delete":
case "documents.archive":
case "documents.unarchive":
case "documents.restore":
case "documents.move":
case "documents.update":
case "documents.title_change":
await this.handleDocumentEvent(subscription, event);
return;
case "documents.update.delayed":
case "documents.update.debounced":
// Ignored
return;
case "revisions.create":
await this.handleRevisionEvent(subscription, event);
return;
case "fileOperations.create":
case "fileOperations.update":
case "fileOperations.delete":
await this.handleFileOperationEvent(subscription, event);
return;
case "collections.create":
case "collections.update":
case "collections.delete":
case "collections.move":
case "collections.permission_changed":
await this.handleCollectionEvent(subscription, event);
return;
case "collections.add_user":
case "collections.remove_user":
await this.handleCollectionUserEvent(subscription, event);
return;
case "collections.add_group":
case "collections.remove_group":
await this.handleCollectionGroupEvent(subscription, event);
return;
case "groups.create":
case "groups.update":
case "groups.delete":
await this.handleGroupEvent(subscription, event);
return;
case "groups.add_user":
case "groups.remove_user":
await this.handleGroupUserEvent(subscription, event);
return;
case "integrations.create":
case "integrations.update":
await this.handleIntegrationEvent(subscription, event);
return;
case "teams.create":
// Ignored
return;
case "teams.update":
await this.handleTeamEvent(subscription, event);
return;
case "pins.create":
case "pins.update":
case "pins.delete":
await this.handlePinEvent(subscription, event);
return;
case "stars.create":
case "stars.update":
case "stars.delete":
await this.handleStarEvent(subscription, event);
return;
case "shares.create":
case "shares.update":
case "shares.revoke":
await this.handleShareEvent(subscription, event);
return;
case "webhookSubscriptions.create":
case "webhookSubscriptions.delete":
case "webhookSubscriptions.update":
await this.handleWebhookSubscriptionEvent(subscription, event);
return;
case "views.create":
await this.handleViewEvent(subscription, event);
return;
default:
assertUnreachable(event);
}
}
private async handleWebhookSubscriptionEvent(
subscription: WebhookSubscription,
event: WebhookSubscriptionEvent
): Promise<void> {
const model = await WebhookSubscription.findByPk(event.modelId, {
paranoid: false,
});
let data = null;
if (model) {
data = {
...presentWebhookSubscription(model),
secret: undefined,
};
}
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: data,
},
});
}
private async handleViewEvent(
subscription: WebhookSubscription,
event: ViewEvent
): Promise<void> {
const model = await View.scope("withUser").findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentView(model),
},
});
}
private async handleStarEvent(
subscription: WebhookSubscription,
event: StarEvent
): Promise<void> {
const model = await Star.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentStar(model),
},
});
}
private async handleShareEvent(
subscription: WebhookSubscription,
event: ShareEvent
): Promise<void> {
const model = await Share.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentShare(model),
},
});
}
private async handlePinEvent(
subscription: WebhookSubscription,
event: PinEvent
): Promise<void> {
const model = await Pin.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentPin(model),
},
});
}
private async handleTeamEvent(
subscription: WebhookSubscription,
event: TeamEvent
): Promise<void> {
const model = await Team.scope("withDomains").findByPk(event.teamId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.teamId,
model: model && presentTeam(model),
},
});
}
private async handleIntegrationEvent(
subscription: WebhookSubscription,
event: IntegrationEvent
): Promise<void> {
const model = await Integration.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentIntegration(model),
},
});
}
private async handleGroupEvent(
subscription: WebhookSubscription,
event: GroupEvent
): Promise<void> {
const model = await Group.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentGroup(model),
},
});
}
private async handleGroupUserEvent(
subscription: WebhookSubscription,
event: GroupUserEvent
): Promise<void> {
const model = await GroupUser.scope(["withUser", "withGroup"]).findOne({
where: {
groupId: event.modelId,
userId: event.userId,
},
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: `${event.userId}-${event.modelId}`,
model: model && presentGroupMembership(model),
group: model && presentGroup(model.group),
user: model && presentUser(model.user),
},
});
}
private async handleCollectionEvent(
subscription: WebhookSubscription,
event: CollectionEvent
): Promise<void> {
const model = await Collection.findByPk(event.collectionId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.collectionId,
model: model && presentCollection(model),
},
});
}
private async handleCollectionUserEvent(
subscription: WebhookSubscription,
event: CollectionUserEvent
): Promise<void> {
const model = await CollectionUser.scope([
"withUser",
"withCollection",
]).findOne({
where: {
collectionId: event.collectionId,
userId: event.userId,
},
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: `${event.userId}-${event.collectionId}`,
model: model && presentMembership(model),
collection: model && presentCollection(model.collection),
user: model && presentUser(model.user),
},
});
}
private async handleCollectionGroupEvent(
subscription: WebhookSubscription,
event: CollectionGroupEvent
): Promise<void> {
const model = await CollectionGroup.scope([
"withGroup",
"withCollection",
]).findOne({
where: {
collectionId: event.collectionId,
groupId: event.modelId,
},
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: `${event.modelId}-${event.collectionId}`,
model: model && presentCollectionGroupMembership(model),
collection: model && presentCollection(model.collection),
group: model && presentGroup(model.group),
},
});
}
private async handleFileOperationEvent(
subscription: WebhookSubscription,
event: FileOperationEvent
): Promise<void> {
const model = await FileOperation.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && presentFileOperation(model),
},
});
}
private async handleDocumentEvent(
subscription: WebhookSubscription,
event: DocumentEvent
): Promise<void> {
const model = await Document.findByPk(event.documentId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.documentId,
model: model && (await presentDocument(model)),
},
});
}
private async handleRevisionEvent(
subscription: WebhookSubscription,
event: RevisionEvent
): Promise<void> {
const model = await Revision.findByPk(event.modelId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.modelId,
model: model && (await presentRevision(model)),
},
});
}
private async handleUserEvent(
subscription: WebhookSubscription,
event: UserEvent
): Promise<void> {
const model = await User.findByPk(event.userId, {
paranoid: false,
});
await this.sendWebhook({
event,
subscription,
payload: {
id: event.userId,
model: model && presentUser(model),
},
});
}
private async sendWebhook({
event,
subscription,
payload,
}: {
event: Event;
subscription: WebhookSubscription;
payload: WebhookPayload;
}) {
const delivery = await WebhookDelivery.create({
webhookSubscriptionId: subscription.id,
status: "pending",
});
let response, requestBody, requestHeaders, status;
try {
requestBody = presentWebhook({
event,
delivery,
payload,
});
requestHeaders = {
"Content-Type": "application/json",
"user-agent": `Outline-Webhooks${
env.VERSION ? `/${env.VERSION.slice(0, 7)}` : ""
}`,
};
const signature = subscription.signature(JSON.stringify(requestBody));
if (signature) {
requestHeaders["Outline-Signature"] = signature;
}
response = await fetch(subscription.url, {
method: "POST",
headers: requestHeaders,
body: JSON.stringify(requestBody),
redirect: "error",
timeout: 5000,
agent: useAgent(subscription.url),
});
status = response.ok ? "success" : "failed";
} catch (err) {
Logger.error("Failed to send webhook", err, {
event,
deliveryId: delivery.id,
});
status = "failed";
}
await delivery.update({
status,
statusCode: response ? response.status : null,
requestBody,
requestHeaders,
responseBody: response ? await response.text() : "",
responseHeaders: response
? Object.fromEntries(response.headers.entries())
: {},
});
if (status === "failed") {
try {
await this.checkAndDisableSubscription(subscription);
} catch (err) {
Logger.error("Failed to check and disable recent deliveries", err, {
event,
deliveryId: delivery.id,
});
}
}
}
private async checkAndDisableSubscription(subscription: WebhookSubscription) {
const recentDeliveries = await WebhookDelivery.findAll({
where: {
webhookSubscriptionId: subscription.id,
},
order: [["createdAt", "DESC"]],
limit: 25,
});
const allFailed = recentDeliveries.every(
(delivery) => delivery.status === "failed"
);
if (recentDeliveries.length === 25 && allFailed) {
// If the last 25 deliveries failed, disable the subscription
await subscription.disable();
// Send an email to the creator of the webhook to let them know
const [createdBy, team] = await Promise.all([
User.findOne({
where: {
id: subscription.createdById,
suspendedAt: { [Op.is]: null },
},
}),
subscription.$get("team"),
]);
if (createdBy && team) {
await WebhookDisabledEmail.schedule({
to: createdBy.email,
teamUrl: team.url,
webhookName: subscription.name,
});
}
}
}
}