Automatically error file operations running longer than 12 hours

server/models/FileOperation.ts

@@ -62,7 +62,7 @@ class FileOperation extends IdModel {
    * Mark the current file operation as expired and remove the file from storage.
    */
   expire = async function () {
-    this.state = "expired";
+    this.state = FileOperationState.Expired;
     try {
       await deleteFromS3(this.key);
     } catch (err) {

server/queues/tasks/ErrorTimedOutFileOperationsTask.test.ts

@@ -0,0 +1,52 @@
+import { subDays } from "date-fns";
+import { FileOperationState, FileOperationType } from "@shared/types";
+import { FileOperation } from "@server/models";
+import { buildFileOperation } from "@server/test/factories";
+import { setupTestDatabase } from "@server/test/support";
+import ErrorTimedOutFileOperationsTask from "./ErrorTimedOutFileOperationsTask";
+
+setupTestDatabase();
+
+describe("ErrorTimedOutFileOperationsTask", () => {
+  it("should error exports older than 12 hours", async () => {
+    await buildFileOperation({
+      type: FileOperationType.Export,
+      state: FileOperationState.Creating,
+      createdAt: subDays(new Date(), 15),
+    });
+    await buildFileOperation({
+      type: FileOperationType.Export,
+      state: FileOperationState.Complete,
+    });
+
+    /* This is a test helper that creates a new task and runs it. */
+    const task = new ErrorTimedOutFileOperationsTask();
+    await task.perform({ limit: 100 });
+
+    const data = await FileOperation.count({
+      where: {
+        type: FileOperationType.Export,
+        state: FileOperationState.Error,
+      },
+    });
+    expect(data).toEqual(1);
+  });
+
+  it("should not error exports created less than 12 hours ago", async () => {
+    await buildFileOperation({
+      type: FileOperationType.Export,
+      state: FileOperationState.Creating,
+    });
+
+    const task = new ErrorTimedOutFileOperationsTask();
+    await task.perform({ limit: 100 });
+
+    const data = await FileOperation.count({
+      where: {
+        type: FileOperationType.Export,
+        state: FileOperationState.Error,
+      },
+    });
+    expect(data).toEqual(0);
+  });
+});

server/queues/tasks/ErrorTimedOutFileOperationsTask.ts

@@ -0,0 +1,49 @@
+import { subHours } from "date-fns";
+import { Op } from "sequelize";
+import { FileOperationState } from "@shared/types";
+import Logger from "@server/logging/Logger";
+import { FileOperation } from "@server/models";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
+
+type Props = {
+  limit: number;
+};
+
+export default class ErrorTimedOutFileOperationsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Daily;
+
+  public async perform({ limit }: Props) {
+    Logger.info("task", `Error file operations running longer than 12 hours…`);
+    const fileOperations = await FileOperation.unscoped().findAll({
+      where: {
+        createdAt: {
+          [Op.lt]: subHours(new Date(), 12),
+        },
+        [Op.or]: [
+          {
+            state: FileOperationState.Creating,
+          },
+          {
+            state: FileOperationState.Uploading,
+          },
+        ],
+      },
+      limit,
+    });
+    await Promise.all(
+      fileOperations.map(async (fileOperation) => {
+        fileOperation.state = FileOperationState.Error;
+        fileOperation.error = "Timed out";
+        await fileOperation.save();
+      })
+    );
+    Logger.info("task", `Updated ${fileOperations.length} file operations`);
+  }
+
+  public get options() {
+    return {
+      attempts: 1,
+      priority: TaskPriority.Background,
+    };
+  }
+}