diff --git a/server/models/FileOperation.ts b/server/models/FileOperation.ts index e4cd63b56..4d2aa88c2 100644 --- a/server/models/FileOperation.ts +++ b/server/models/FileOperation.ts @@ -62,7 +62,7 @@ class FileOperation extends IdModel { * Mark the current file operation as expired and remove the file from storage. */ expire = async function () { - this.state = "expired"; + this.state = FileOperationState.Expired; try { await deleteFromS3(this.key); } catch (err) { diff --git a/server/queues/tasks/ErrorTimedOutFileOperationsTask.test.ts b/server/queues/tasks/ErrorTimedOutFileOperationsTask.test.ts new file mode 100644 index 000000000..f413658e0 --- /dev/null +++ b/server/queues/tasks/ErrorTimedOutFileOperationsTask.test.ts @@ -0,0 +1,52 @@ +import { subDays } from "date-fns"; +import { FileOperationState, FileOperationType } from "@shared/types"; +import { FileOperation } from "@server/models"; +import { buildFileOperation } from "@server/test/factories"; +import { setupTestDatabase } from "@server/test/support"; +import ErrorTimedOutFileOperationsTask from "./ErrorTimedOutFileOperationsTask"; + +setupTestDatabase(); + +describe("ErrorTimedOutFileOperationsTask", () => { + it("should error exports older than 12 hours", async () => { + await buildFileOperation({ + type: FileOperationType.Export, + state: FileOperationState.Creating, + createdAt: subDays(new Date(), 15), + }); + await buildFileOperation({ + type: FileOperationType.Export, + state: FileOperationState.Complete, + }); + + /* This is a test helper that creates a new task and runs it. */ + const task = new ErrorTimedOutFileOperationsTask(); + await task.perform({ limit: 100 }); + + const data = await FileOperation.count({ + where: { + type: FileOperationType.Export, + state: FileOperationState.Error, + }, + }); + expect(data).toEqual(1); + }); + + it("should not error exports created less than 12 hours ago", async () => { + await buildFileOperation({ + type: FileOperationType.Export, + state: FileOperationState.Creating, + }); + + const task = new ErrorTimedOutFileOperationsTask(); + await task.perform({ limit: 100 }); + + const data = await FileOperation.count({ + where: { + type: FileOperationType.Export, + state: FileOperationState.Error, + }, + }); + expect(data).toEqual(0); + }); +}); diff --git a/server/queues/tasks/ErrorTimedOutFileOperationsTask.ts b/server/queues/tasks/ErrorTimedOutFileOperationsTask.ts new file mode 100644 index 000000000..90ad50e25 --- /dev/null +++ b/server/queues/tasks/ErrorTimedOutFileOperationsTask.ts @@ -0,0 +1,49 @@ +import { subHours } from "date-fns"; +import { Op } from "sequelize"; +import { FileOperationState } from "@shared/types"; +import Logger from "@server/logging/Logger"; +import { FileOperation } from "@server/models"; +import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask"; + +type Props = { + limit: number; +}; + +export default class ErrorTimedOutFileOperationsTask extends BaseTask { + static cron = TaskSchedule.Daily; + + public async perform({ limit }: Props) { + Logger.info("task", `Error file operations running longer than 12 hours…`); + const fileOperations = await FileOperation.unscoped().findAll({ + where: { + createdAt: { + [Op.lt]: subHours(new Date(), 12), + }, + [Op.or]: [ + { + state: FileOperationState.Creating, + }, + { + state: FileOperationState.Uploading, + }, + ], + }, + limit, + }); + await Promise.all( + fileOperations.map(async (fileOperation) => { + fileOperation.state = FileOperationState.Error; + fileOperation.error = "Timed out"; + await fileOperation.save(); + }) + ); + Logger.info("task", `Updated ${fileOperations.length} file operations`); + } + + public get options() { + return { + attempts: 1, + priority: TaskPriority.Background, + }; + } +}