yuuza/outline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bulk export (#684)

Browse Source 
* First pass (working) collection export to zip

* Add export confirmation screen

* 👕

* Refactor

* Job for team export, move to tmp file, settings UI

* Export all collections job

* 👕

* Add specs

* Clarify UI
This commit is contained in:
Tom Moor
2018-06-20 21:33:21 -07:00
committed by GitHub
parent cedd31c9ea
commit b9e0668d7d
26 changed files with 543 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/__snapshots__/mailer.test.js.snap
View File

@@ -2,6 +2,7 @@
exports[`Mailer #welcome 1`] = `
Object {
"attachments": undefined,
"from": "hello@example.com",
"html": "
<!DOCTYPE html PUBLIC \\"-//W3C//DTD XHTML 1.0 Strict//EN\\" \\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\\">

18
server/api/__snapshots__/collections.test.js.snap
View File

@@ -18,6 +18,24 @@ Object {
}
`;
exports[`#collections.export should require authentication 1`] = `
Object {
"error": "authentication_required",
"message": "Authentication required",
"ok": false,
"status": 401,
}
`;
exports[`#collections.exportAll should require authentication 1`] = `
Object {
"error": "authentication_required",
"message": "Authentication required",
"ok": false,
"status": 401,
}
`;
exports[`#collections.info should require authentication 1`] = `
Object {
"error": "authentication_required",

32
server/api/collections.js
View File

@@ -4,8 +4,9 @@ import Router from 'koa-router';
import auth from '../middlewares/authentication';
import pagination from './middlewares/pagination';
import { presentCollection } from '../presenters';
import { Collection } from '../models';
import { Collection, Team } from '../models';
import { ValidationError } from '../errors';
import { exportCollection, exportCollections } from '../logistics';
import policy from '../policies';
const { authorize } = policy;
@@ -46,6 +47,35 @@ router.post('collections.info', auth(), async ctx => {
};
});
router.post('collections.export', auth(), async ctx => {
const { id } = ctx.body;
ctx.assertPresent(id, 'id is required');
const user = ctx.state.user;
const collection = await Collection.findById(id);
authorize(user, 'export', collection);
// async operation to create zip archive and email user
exportCollection(id, user.email);
ctx.body = {
success: true,
};
});
router.post('collections.exportAll', auth(), async ctx => {
const user = ctx.state.user;
const team = await Team.findById(user.teamId);
authorize(user, 'export', team);
// async operation to create zip archive and email user
exportCollections(user.teamId, user.email);
ctx.body = {
success: true,
};
});
router.post('collections.update', auth(), async ctx => {
const { id, name, color } = ctx.body;
ctx.assertPresent(name, 'name is required');

46
server/api/collections.test.js
View File

@@ -31,6 +31,52 @@ describe('#collections.list', async () => {
});
});
describe('#collections.export', async () => {
it('should require authentication', async () => {
const res = await server.post('/api/collections.export');
const body = await res.json();
expect(res.status).toEqual(401);
expect(body).toMatchSnapshot();
});
it('should return success', async () => {
const { user, collection } = await seed();
const res = await server.post('/api/collections.export', {
body: { token: user.getJwtToken(), id: collection.id },
});
expect(res.status).toEqual(200);
});
});
describe('#collections.exportAll', async () => {
it('should require authentication', async () => {
const res = await server.post('/api/collections.exportAll');
const body = await res.json();
expect(res.status).toEqual(401);
expect(body).toMatchSnapshot();
});
it('should require authorization', async () => {
const user = await buildUser();
const res = await server.post('/api/collections.exportAll', {
body: { token: user.getJwtToken() },
});
expect(res.status).toEqual(403);
});
it('should return success', async () => {
const { admin } = await seed();
const res = await server.post('/api/collections.exportAll', {
body: { token: admin.getJwtToken() },
});
expect(res.status).toEqual(200);
});
});
describe('#collections.info', async () => {
it('should return collection', async () => {
const { user, collection } = await seed();

36
server/emails/ExportEmail.js Normal file
View File

@@ -0,0 +1,36 @@
// @flow
import * as React from 'react';
import EmailTemplate from './components/EmailLayout';
import Body from './components/Body';
import Button from './components/Button';
import Heading from './components/Heading';
import Header from './components/Header';
import Footer from './components/Footer';
import EmptySpace from './components/EmptySpace';
export const exportEmailText = `
Your Data Export
Your requested data export is attached as a zip file to this email.
`;
export const ExportEmail = () => {
return (
<EmailTemplate>
<Header />
<Body>
<Heading>Your Data Export</Heading>
<p>
Your requested data export is attached as a zip file to this email.
</p>
<EmptySpace height={10} />
<p>
<Button href={`${process.env.URL}/dashboard`}>Go to dashboard</Button>
</p>
</Body>
<Footer />
</EmailTemplate>
);
};

3
server/emails/WelcomeEmail.js
View File

@@ -27,7 +27,6 @@ export const WelcomeEmail = () => {
<Body>
<Heading>Welcome to Outline!</Heading>
<p>Outline is a place for your team to build and share knowledge.</p>
<p>
To get started, head to your dashboard and try creating a collection
@@ -38,9 +37,7 @@ export const WelcomeEmail = () => {
You can also import existing Markdown document by drag and dropping
them to your collections
</p>
<EmptySpace height={10} />
<p>
<Button href={`${process.env.URL}/dashboard`}>
View my dashboard

2
server/emails/index.js
View File

@@ -2,7 +2,7 @@
import Koa from 'koa';
import Router from 'koa-router';
import { NotFoundError } from '../errors';
import { Mailer } from '../mailer';
import Mailer from '../mailer';
const emailPreviews = new Koa();
const router = new Router();

94
server/logistics.js Normal file
View File

@@ -0,0 +1,94 @@
// @flow
import Queue from 'bull';
import debug from 'debug';
import Mailer from './mailer';
import { Collection, Team } from './models';
import { archiveCollection, archiveCollections } from './utils/zip';
const log = debug('logistics');
const logisticsQueue = new Queue('logistics', process.env.REDIS_URL);
const mailer = new Mailer();
const queueOptions = {
attempts: 2,
backoff: {
type: 'exponential',
delay: 60 * 1000,
},
};
async function exportAndEmailCollection(collectionId: string, email: string) {
log('Archiving collection', collectionId);
const collection = await Collection.findById(collectionId);
const filePath = await archiveCollection(collection);
log('Archive path', filePath);
mailer.export({
to: email,
attachments: [
{
filename: `${collection.name} Export.zip`,
path: filePath,
},
],
});
}
async function exportAndEmailCollections(teamId: string, email: string) {
log('Archiving team', teamId);
const team = await Team.findById(teamId);
const collections = await Collection.findAll({
where: { teamId },
order: [['name', 'ASC']],
});
const filePath = await archiveCollections(collections);
log('Archive path', filePath);
mailer.export({
to: email,
attachments: [
{
filename: `${team.name} Export.zip`,
path: filePath,
},
],
});
}
logisticsQueue.process(async job => {
log('Process', job.data);
switch (job.data.type) {
case 'export-collection':
return await exportAndEmailCollection(
job.data.collectionId,
job.data.email
);
case 'export-collections':
return await exportAndEmailCollections(job.data.teamId, job.data.email);
default:
}
});
export const exportCollection = (collectionId: string, email: string) => {
logisticsQueue.add(
{
type: 'export-collection',
collectionId,
email,
},
queueOptions
);
};
export const exportCollections = (teamId: string, email: string) => {
logisticsQueue.add(
{
type: 'export-collections',
teamId,
email,
},
queueOptions
);
};

56
server/mailer.js
View File

@@ -1,10 +1,17 @@
// @flow
import * as React from 'react';
import debug from 'debug';
import bugsnag from 'bugsnag';
import nodemailer from 'nodemailer';
import Oy from 'oy-vey';
import Queue from 'bull';
import { baseStyles } from './emails/components/EmailLayout';
import { WelcomeEmail, welcomeEmailText } from './emails/WelcomeEmail';
import { ExportEmail, exportEmailText } from './emails/ExportEmail';
const log = debug('emails');
type Emails = 'welcome' | 'export';
type SendMailType = {
to: string,
@@ -14,6 +21,14 @@ type SendMailType = {
text: string,
html: React.Node,
headCSS?: string,
attachments?: Object[],
};
type EmailJob = {
data: {
type: Emails,
opts: SendMailType,
},
};
/**
@@ -27,7 +42,7 @@ type SendMailType = {
* HTML: http://localhost:3000/email/:email_type/html
* TEXT: http://localhost:3000/email/:email_type/text
*/
class Mailer {
export default class Mailer {
transporter: ?any;
/**
@@ -44,6 +59,7 @@ class Mailer {
});
try {
log(`Sending email "${data.title}" to ${data.to}`);
await transporter.sendMail({
from: process.env.SMTP_FROM_EMAIL,
replyTo: process.env.SMTP_REPLY_EMAIL || process.env.SMTP_FROM_EMAIL,
@@ -51,10 +67,11 @@ class Mailer {
subject: data.title,
html: html,
text: data.text,
attachments: data.attachments,
});
} catch (e) {
Bugsnag.notifyException(e);
throw e; // Re-throw for queue to re-try
} catch (err) {
bugsnag.notify(err);
throw err; // Re-throw for queue to re-try
}
}
};
@@ -70,17 +87,32 @@ class Mailer {
});
};
export = async (opts: { to: string, attachments: Object[] }) => {
this.sendMail({
to: opts.to,
attachments: opts.attachments,
title: 'Your requested export',
previewText: "Here's your request data export from Outline",
html: <ExportEmail />,
text: exportEmailText,
});
};
constructor() {
if (process.env.SMTP_HOST) {
let smtpConfig = {
host: process.env.SMTP_HOST,
port: process.env.SMTP_PORT,
secure: true,
auth: {
secure: process.env.NODE_ENV === 'production',
auth: undefined,
};
if (process.env.SMTP_USERNAME) {
smtpConfig.auth = {
user: process.env.SMTP_USERNAME,
pass: process.env.SMTP_PASSWORD,
},
};
};
}
this.transporter = nodemailer.createTransport(smtpConfig);
}
@@ -88,14 +120,14 @@ class Mailer {
}
const mailer = new Mailer();
const mailerQueue = new Queue('email', process.env.REDIS_URL);
export const mailerQueue = new Queue('email', process.env.REDIS_URL);
mailerQueue.process(async function(job) {
mailerQueue.process(async (job: EmailJob) => {
// $FlowIssue flow doesn't like dynamic values
await mailer[job.data.type](job.data.opts);
});
const sendEmail = (type: string, to: string, options?: Object = {}) => {
export const sendEmail = (type: Emails, to: string, options?: Object = {}) => {
mailerQueue.add(
{
type,
@@ -113,5 +145,3 @@ const sendEmail = (type: string, to: string, options?: Object = {}) => {
}
);
};
export { Mailer, mailerQueue, sendEmail };

2
server/mailer.test.js
View File

@@ -1,5 +1,5 @@
/* eslint-disable flowtype/require-valid-file-annotation */
import { Mailer } from './mailer';
import Mailer from './mailer';
describe('Mailer', () => {
let fakeMailer;

2
server/policies/collection.js
View File

@@ -9,7 +9,7 @@ allow(User, 'create', Collection);
allow(
User,
['read', 'publish', 'update'],
['read', 'publish', 'update', 'export'],
Collection,
(user, collection) => collection && user.teamId === collection.teamId
);

2
server/policies/team.js
View File

@@ -7,7 +7,7 @@ const { allow } = policy;
allow(User, 'read', Team, (user, team) => team && user.teamId === team.id);
allow(User, 'update', Team, (user, team) => {
allow(User, ['update', 'export'], Team, (user, team) => {
if (!team || user.teamId !== team.id) return false;
if (user.isAdmin) return true;
throw new AdminRequiredError();

49
server/utils/zip.js Normal file
View File

@@ -0,0 +1,49 @@
// @flow
import fs from 'fs';
import JSZip from 'jszip';
import tmp from 'tmp';
import unescape from '../../shared/utils/unescape';
import { Collection, Document } from '../models';
async function addToArchive(zip, documents) {
for (const doc of documents) {
const document = await Document.findById(doc.id);
zip.file(`${document.title}.md`, unescape(document.text));
if (doc.children && doc.children.length) {
const folder = zip.folder(document.title);
await addToArchive(folder, doc.children);
}
}
}
async function archiveToPath(zip) {
return new Promise((resolve, reject) => {
tmp.file({ prefix: 'export-', postfix: '.zip' }, (err, path) => {
if (err) return reject(err);
zip
.generateNodeStream({ type: 'nodebuffer', streamFiles: true })
.pipe(fs.createWriteStream(path))
.on('finish', () => resolve(path))
.on('error', reject);
});
});
}
export async function archiveCollection(collection: Collection) {
const zip = new JSZip();
await addToArchive(zip, collection.documentStructure);
return archiveToPath(zip);
}
export async function archiveCollections(collections: Collection[]) {
const zip = new JSZip();
for (const collection of collections) {
const folder = zip.folder(collection.name);
await addToArchive(folder, collection.documentStructure);
}
return archiveToPath(zip);
}