Refactor 'uploadFromUrl' to base storage implementation

Add safety around using fetch implementation

server/storage/files/BaseStorage.ts

@@ -1,5 +1,7 @@
 import { Readable } from "stream";
 import { PresignedPost } from "aws-sdk/clients/s3";
+import Logger from "@server/logging/Logger";
+import fetch from "@server/utils/fetch";
 
 export default abstract class BaseStorage {
   /**
@@ -83,11 +85,31 @@ export default abstract class BaseStorage {
    * @param acl The ACL to use
    * @returns The URL of the file
    */
-  public abstract uploadFromUrl(
-    url: string,
-    key: string,
-    acl: string
-  ): Promise<string | undefined>;
+  public async uploadFromUrl(url: string, key: string, acl: string) {
+    const endpoint = this.getPublicEndpoint(true);
+    if (url.startsWith("/api") || url.startsWith(endpoint)) {
+      return;
+    }
+
+    try {
+      const res = await fetch(url);
+      const buffer = await res.buffer();
+      return this.upload({
+        body: buffer,
+        contentLength: res.headers["content-length"],
+        contentType: res.headers["content-type"],
+        key,
+        acl,
+      });
+    } catch (err) {
+      Logger.error("Error uploading to S3 from URL", err, {
+        url,
+        key,
+        acl,
+      });
+      return;
+    }
+  }
 
   /**
    * Delete a file from the storage provider.

server/storage/files/S3Storage.ts

@@ -1,9 +1,7 @@
 import util from "util";
 import AWS, { S3 } from "aws-sdk";
-import fetch from "fetch-with-proxy";
 import invariant from "invariant";
 import compact from "lodash/compact";
-import { useAgent } from "request-filtering-agent";
 import env from "@server/env";
 import Logger from "@server/logging/Logger";
 import BaseStorage from "./BaseStorage";
@@ -113,44 +111,6 @@ export default class S3Storage extends BaseStorage {
     return `${endpoint}/${key}`;
   };
 
-  public async uploadFromUrl(url: string, key: string, acl: string) {
-    invariant(
-      env.AWS_S3_UPLOAD_BUCKET_NAME,
-      "AWS_S3_UPLOAD_BUCKET_NAME is required"
-    );
-
-    const endpoint = this.getPublicEndpoint(true);
-    if (url.startsWith("/api") || url.startsWith(endpoint)) {
-      return;
-    }
-
-    try {
-      const res = await fetch(url, {
-        agent: useAgent(url),
-      });
-      const buffer = await res.buffer();
-      await this.client
-        .putObject({
-          ACL: acl,
-          Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
-          Key: key,
-          ContentType: res.headers["content-type"],
-          ContentLength: res.headers["content-length"],
-          ContentDisposition: "attachment",
-          Body: buffer,
-        })
-        .promise();
-      return `${endpoint}/${key}`;
-    } catch (err) {
-      Logger.error("Error uploading to S3 from URL", err, {
-        url,
-        key,
-        acl,
-      });
-      return;
-    }
-  }
-
   public async deleteFile(key: string) {
     invariant(
       env.AWS_S3_UPLOAD_BUCKET_NAME,