refactor: Upload file to storage, and then pass attachmentId to collections.import

This avoids having large file uploads going directly to the server and allows us to fetch it async into a worker process

server/api/attachments.js

@@ -38,7 +38,7 @@ router.post("attachments.create", auth(), async (ctx) => {
   const key = `${bucket}/${user.id}/${s3Key}/${name}`;
   const credential = makeCredential();
   const longDate = format(new Date(), "YYYYMMDDTHHmmss\\Z");
-  const policy = makePolicy(credential, longDate, acl);
+  const policy = makePolicy(credential, longDate, acl, contentType);
   const endpoint = publicS3Endpoint();
   const url = `${endpoint}/${key}`;
 
@@ -85,6 +85,7 @@ router.post("attachments.create", auth(), async (ctx) => {
         documentId,
         contentType,
         name,
+        id: attachment.id,
         url: attachment.redirectUrl,
         size,
       },

server/api/collections.js

@@ -1,8 +1,10 @@
 // @flow
 import fs from "fs";
+import os from "os";
+import File from "formidable/lib/file";
 import Router from "koa-router";
 import collectionImporter from "../commands/collectionImporter";
-import { ValidationError, InvalidRequestError } from "../errors";
+import { ValidationError } from "../errors";
 import { exportCollections } from "../logistics";
 import auth from "../middlewares/authentication";
 import {
@@ -13,6 +15,7 @@ import {
   Event,
   User,
   Group,
+  Attachment,
 } from "../models";
 import policy from "../policies";
 import {
@@ -100,23 +103,27 @@ router.post("collections.info", auth(), async (ctx) => {
 });
 
 router.post("collections.import", auth(), async (ctx) => {
-  const { type } = ctx.body;
+  const { type, attachmentId } = ctx.body;
   ctx.assertIn(type, ["outline"], "type must be one of 'outline'");
-
-  if (!ctx.is("multipart/form-data")) {
-    throw new InvalidRequestError("Request type must be multipart/form-data");
-  }
-
-  const file: any = Object.values(ctx.request.files)[0];
-  ctx.assertPresent(file, "file is required");
-
-  if (file.type !== "application/zip") {
-    throw new InvalidRequestError("File type must be a zip");
-  }
+  ctx.assertUuid(attachmentId, "attachmentId is required");
 
   const user = ctx.state.user;
   authorize(user, "import", Collection);
 
+  const attachment = await Attachment.findByPk(attachmentId);
+  authorize(user, "read", attachment);
+
+  const buffer = await attachment.buffer;
+  const tmpDir = os.tmpdir();
+  const tmpFilePath = `${tmpDir}/upload-${attachmentId}`;
+
+  await fs.promises.writeFile(tmpFilePath, buffer);
+  const file = new File({
+    name: attachment.name,
+    type: attachment.type,
+    path: tmpFilePath,
+  });
+
   const { documents, attachments, collections } = await collectionImporter({
     file,
     user,

server/api/collections.test.js

@@ -111,7 +111,7 @@ describe("#collections.list", () => {
 });
 
 describe("#collections.import", () => {
-  it("should error if no file is passed", async () => {
+  it("should error if no attachmentId is passed", async () => {
     const user = await buildUser();
     const res = await server.post("/api/collections.import", {
       body: {

server/commands/collectionImporter.js

@@ -9,7 +9,7 @@ import { values, keys } from "lodash";
 import uuid from "uuid";
 import { parseOutlineExport } from "../../shared/utils/zip";
 import { FileImportError } from "../errors";
-import { Attachment, Document, Collection, User } from "../models";
+import { Attachment, Event, Document, Collection, User } from "../models";
 import attachmentCreator from "./attachmentCreator";
 import documentCreator from "./documentCreator";
 import documentImporter from "./documentImporter";
@@ -66,12 +66,21 @@ export default async function collectionImporter({
       // there is also a "Name (Imported)" but this is a case not worth dealing
       // with right now
       if (!isCreated) {
+        const name = `${item.name} (Imported)`;
         collection = await Collection.create({
           teamId: user.teamId,
           creatorId: user.id,
-          name: `${item.name} (Imported)`,
+          name,
           private: false,
         });
+        await Event.create({
+          name: "collections.create",
+          collectionId: collection.id,
+          teamId: collection.teamId,
+          actorId: user.id,
+          data: { name },
+          ip,
+        });
       }
 
       collections[item.path] = collection;

server/models/Attachment.js

@@ -1,7 +1,7 @@
 // @flow
 import path from "path";
 import { DataTypes, sequelize } from "../sequelize";
-import { deleteFromS3 } from "../utils/s3";
+import { deleteFromS3, getFileByKey } from "../utils/s3";
 
 const Attachment = sequelize.define(
   "attachment",
@@ -47,6 +47,9 @@ const Attachment = sequelize.define(
       isPrivate: function () {
         return this.acl === "private";
       },
+      buffer: function () {
+        return getFileByKey(this.key);
+      },
     },
   }
 );

server/policies/attachment.js

@@ -6,7 +6,7 @@ const { allow } = policy;
 
 allow(User, "create", Attachment);
 
-allow(User, "delete", Attachment, (actor, attachment) => {
+allow(User, ["read", "delete"], Attachment, (actor, attachment) => {
   if (!attachment || attachment.teamId !== actor.teamId) return false;
   if (actor.isAdmin) return true;
   if (actor.id === attachment.userId) return true;

server/utils/s3.js

@@ -46,7 +46,8 @@ export const makeCredential = () => {
 export const makePolicy = (
   credential: string,
   longDate: string,
-  acl: string
+  acl: string,
+  contentType: string = "image"
 ) => {
   const tomorrow = addHours(new Date(), 24);
   const policy = {
@@ -55,7 +56,7 @@ export const makePolicy = (
       ["starts-with", "$key", ""],
       { acl },
       ["content-length-range", 0, +process.env.AWS_S3_UPLOAD_MAX_SIZE],
-      ["starts-with", "$Content-Type", "image"],
+      ["starts-with", "$Content-Type", contentType],
       ["starts-with", "$Cache-Control", ""],
       { "x-amz-algorithm": "AWS4-HMAC-SHA256" },
       { "x-amz-credential": credential },
@@ -177,7 +178,7 @@ export const getSignedImageUrl = async (key: string) => {
     : s3.getSignedUrl("getObject", params);
 };
 
-export const getImageByKey = async (key: string) => {
+export const getFileByKey = async (key: string) => {
   const params = {
     Bucket: AWS_S3_UPLOAD_BUCKET_NAME,
     Key: key,

server/utils/zip.js

@@ -4,7 +4,7 @@ import * as Sentry from "@sentry/node";
 import JSZip from "jszip";
 import tmp from "tmp";
 import { Attachment, Collection, Document } from "../models";
-import { getImageByKey } from "./s3";
+import { getFileByKey } from "./s3";
 
 async function addToArchive(zip, documents) {
   for (const doc of documents) {
@@ -38,7 +38,7 @@ async function addToArchive(zip, documents) {
 
 async function addImageToArchive(zip, key) {
   try {
-    const img = await getImageByKey(key);
+    const img = await getFileByKey(key);
     zip.file(key, img, { createFolders: true });
   } catch (err) {
     if (process.env.SENTRY_DSN) {