From 61c32ee1757f8d3c9805090f7bd888e8a4a37b44 Mon Sep 17 00:00:00 2001 From: Saso Matejina Date: Tue, 5 Dec 2017 10:42:52 -0800 Subject: [PATCH 1/3] support local development with docker --- Dockerfile | 10 ++++++++++ Makefile | 13 +++++++++++++ docker-compose.yml | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..fdcc4a2a2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM node:latest + +ENV APP_PATH /opt/outline +RUN mkdir -p $APP_PATH + +WORKDIR $APP_PATH +COPY . $APP_PATH +RUN yarn +RUN cp -r /opt/outline/node_modules /opt/node_modules + diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..6a8d4b860 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +up: + docker-compose up -d redis postgres s3 + docker-compose run --rm outline yarn sequelize db:migrate + docker-compose up outline + +build: + docker-compose build --pull outline + +destroy: + docker-compose stop + docker-compose rm -f + +.PHONY: up build destroy # let's go to reserve rules names diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..9a95b9f27 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: "3" +services: + redis: + image: redis + ports: + - "6379:6379" + postgres: + image: postgres + ports: + - "5432:5432" + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: pass + POSTGRES_DB: outline + s3: + image: lphoward/fake-s3 + ports: + - "4569:4569" + outline: + image: outline:v001 + command: yarn dev + build: + context: . + dockerfile: Dockerfile + args: + pull: 1 + ports: + - "3000:3000" + volumes: + - .:/opt/outline + depends_on: + - postgres + - redis + - s3 + environment: + NODE_PATH: "/opt/outline/node_modules:/opt/node_modules" + PATH: "/opt/outline/node_modules/.bin:/opt/node_modules/.bin:$PATH" From ec86b9fe8c9d3ed0f2892697a8ddd06c47ac9a53 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Sun, 10 Dec 2017 17:38:56 -0800 Subject: [PATCH 2/3] Update sample .env Update README to reflect new process --- .env.sample | 14 ++++++++++---- README.md | 28 +++++++++------------------- server/utils/s3.js | 5 +++-- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/.env.sample b/.env.sample index 4f290470e..620dce54c 100644 --- a/.env.sample +++ b/.env.sample @@ -2,11 +2,11 @@ # # Please use `openssl rand -hex 32` to create SECRET_KEY -DATABASE_URL=postgres://user:pass@example.com:5432/outline -DATABASE_URL_TEST=postgres://user:pass@example.com:5432/outline-test +DATABASE_URL=postgres://user:pass@localhost:5432/outline +DATABASE_URL_TEST=postgres://user:pass@localhost:5432/outline-test SECRET_KEY=F0E5AD933D7F6FD8F4DBB3E038C501C052DC0593C686D21ACB30AE205D2F634B PORT=3000 -REDIS_URL=redis://localhost:6379 +REDIS_URL=redis://redis:6379 SLACK_KEY=71315967491.XXXXXXXXXX SLACK_SECRET=d2dc414f9953226bad0a356c794XXXXX URL=http://localhost:3000 @@ -14,9 +14,15 @@ DEPLOYMENT=hosted ENABLE_UPDATES=true GOOGLE_ANALYTICS_ID= +AWS_ACCESS_KEY_ID=notcheckedindev +AWS_SECRET_ACCESS_KEY=notcheckedindev +AWS_S3_UPLOAD_BUCKET_URL=http://localhost:4569 +AWS_S3_UPLOAD_BUCKET_NAME=outline-dev +AWS_S3_UPLOAD_MAX_SIZE=26214400 + SMTP_HOST= SMTP_PORT= SMTP_USERNAME= SMTP_PASSWORD= SMTP_FROM_EMAIL= -SMTP_REPLY_EMAIL= \ No newline at end of file +SMTP_REPLY_EMAIL= diff --git a/README.md b/README.md index 28b25299a..5d530ad66 100644 --- a/README.md +++ b/README.md @@ -8,37 +8,27 @@ An open, extensible, knowledge base for your team built using React and Node.js. ## Installation -Outline requires following dependencies to work: +Outline requires the following dependencies: - Postgres >=9.5 - Redis -- S3 bucket configured to support CORS uploads - Slack developer application -To install and run the application: +In development you can quickly can an environment running using Docker by +following these steps: - 1. Install dependencies with `yarn` - 1. Register a Slack app at https://api.slack.com/apps - 1. Copy the file `.env.sample` to `.env` and fill out the keys - 1. Run DB migrations `yarn sequelize db:migrate` - - To run Outline in development mode with server and frontend code reloading: +1. Install [Docker for Desktop](https://www.docker.com) if you don't already have it. +1. Register a Slack app at https://api.slack.com/apps +1. Copy the file `.env.sample` to `.env` and fill out the Slack keys, everything + else should work well for development. +1. Run `make up`. This will download dependencies, build and launch a development version of Outline. -```shell -yarn dev -``` - -To run Outline in production mode: - -```shell -yarn start -``` ## Development ### Server -To enable debugging statements, set the following env vars: +To enable debugging statements, add the following to your `.env` file: ``` DEBUG=sql,cache,presenters diff --git a/server/utils/s3.js b/server/utils/s3.js index 14072b5a5..d1ef21cee 100644 --- a/server/utils/s3.js +++ b/server/utils/s3.js @@ -1,6 +1,7 @@ // @flow import crypto from 'crypto'; import moment from 'moment'; +import path from 'path'; import AWS from 'aws-sdk'; import invariant from 'invariant'; import fetch from 'isomorphic-fetch'; @@ -47,7 +48,7 @@ const uploadToS3FromUrl = async (url: string, key: string) => { invariant(AWS_S3_UPLOAD_BUCKET_NAME, 'AWS_S3_UPLOAD_BUCKET_NAME not set'); try { - // $FlowIssue dunno it's fine + // $FlowIssue https://github.com/facebook/flow/issues/2171 const res = await fetch(url); const buffer = await res.buffer(); await s3 @@ -59,7 +60,7 @@ const uploadToS3FromUrl = async (url: string, key: string) => { Body: buffer, }) .promise(); - return `https://s3.amazonaws.com/${AWS_S3_UPLOAD_BUCKET_NAME}/${key}`; + return path.join(process.env.AWS_S3_UPLOAD_BUCKET_URL, key); } catch (e) { bugsnag.notify(e); } From ced80b6723d51a54be907040429d3fb33fb61fb9 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Sun, 10 Dec 2017 22:58:52 -0800 Subject: [PATCH 3/3] FakeS3 support --- .env.sample | 2 +- .gitignore | 1 + docker-compose.yml | 2 ++ server/api/user.js | 8 ++++---- server/utils/s3.js | 45 ++++++++++++++++++++++++++++++--------------- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.env.sample b/.env.sample index 620dce54c..a140297ae 100644 --- a/.env.sample +++ b/.env.sample @@ -16,7 +16,7 @@ GOOGLE_ANALYTICS_ID= AWS_ACCESS_KEY_ID=notcheckedindev AWS_SECRET_ACCESS_KEY=notcheckedindev -AWS_S3_UPLOAD_BUCKET_URL=http://localhost:4569 +AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569 AWS_S3_UPLOAD_BUCKET_NAME=outline-dev AWS_S3_UPLOAD_MAX_SIZE=26214400 diff --git a/.gitignore b/.gitignore index 4f57b31d1..ceb5122b2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ node_modules/* .env npm-debug.log .DS_Store +fakes3/* diff --git a/docker-compose.yml b/docker-compose.yml index 9a95b9f27..9e4b8910d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: image: lphoward/fake-s3 ports: - "4569:4569" + volumes: + - ./fakes3:/fakes3_root outline: image: outline:v001 command: yarn dev diff --git a/server/api/user.js b/server/api/user.js index 4ed8db81b..b50b0bd1b 100644 --- a/server/api/user.js +++ b/server/api/user.js @@ -1,8 +1,7 @@ // @flow import uuid from 'uuid'; import Router from 'koa-router'; - -import { makePolicy, signPolicy } from '../utils/s3'; +import { makePolicy, signPolicy, publicS3Endpoint } from '../utils/s3'; import auth from './middlewares/authentication'; import { presentUser } from '../presenters'; @@ -21,11 +20,12 @@ router.post('user.s3Upload', auth(), async ctx => { const s3Key = uuid.v4(); const key = `uploads/${ctx.state.user.id}/${s3Key}/${filename}`; const policy = makePolicy(); + const endpoint = publicS3Endpoint(); ctx.body = { data: { maxUploadSize: process.env.AWS_S3_UPLOAD_MAX_SIZE, - uploadUrl: process.env.AWS_S3_UPLOAD_BUCKET_URL, + uploadUrl: endpoint, form: { AWSAccessKeyId: process.env.AWS_ACCESS_KEY_ID, 'Cache-Control': 'max-age=31557600', @@ -37,7 +37,7 @@ router.post('user.s3Upload', auth(), async ctx => { }, asset: { contentType: kind, - url: `${process.env.AWS_S3_UPLOAD_BUCKET_URL}${key}`, + url: `${endpoint}/${key}`, name: filename, size, }, diff --git a/server/utils/s3.js b/server/utils/s3.js index d1ef21cee..bbfc12f17 100644 --- a/server/utils/s3.js +++ b/server/utils/s3.js @@ -1,21 +1,15 @@ // @flow import crypto from 'crypto'; import moment from 'moment'; -import path from 'path'; import AWS from 'aws-sdk'; import invariant from 'invariant'; import fetch from 'isomorphic-fetch'; import bugsnag from 'bugsnag'; -AWS.config.update({ - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, -}); - const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY; const AWS_S3_UPLOAD_BUCKET_NAME = process.env.AWS_S3_UPLOAD_BUCKET_NAME; -const makePolicy = () => { +export const makePolicy = () => { const policy = { conditions: [ { bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME }, @@ -33,7 +27,7 @@ const makePolicy = () => { return new Buffer(JSON.stringify(policy)).toString('base64'); }; -const signPolicy = (policy: any) => { +export const signPolicy = (policy: any) => { invariant(AWS_SECRET_ACCESS_KEY, 'AWS_SECRET_ACCESS_KEY not set'); const signature = crypto .createHmac('sha1', AWS_SECRET_ACCESS_KEY) @@ -43,8 +37,24 @@ const signPolicy = (policy: any) => { return signature; }; -const uploadToS3FromUrl = async (url: string, key: string) => { - const s3 = new AWS.S3(); +export const publicS3Endpoint = () => { + // lose trailing slash if there is one and convert fake-s3 url to localhost + // for access outside of docker containers in local development + const host = process.env.AWS_S3_UPLOAD_BUCKET_URL.replace( + 's3:', + 'localhost:' + ).replace(/\/$/, ''); + + return `${host}/${process.env.AWS_S3_UPLOAD_BUCKET_NAME}`; +}; + +export const uploadToS3FromUrl = async (url: string, key: string) => { + const s3 = new AWS.S3({ + s3ForcePathStyle: true, + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + endpoint: new AWS.Endpoint(process.env.AWS_S3_UPLOAD_BUCKET_URL), + }); invariant(AWS_S3_UPLOAD_BUCKET_NAME, 'AWS_S3_UPLOAD_BUCKET_NAME not set'); try { @@ -53,6 +63,7 @@ const uploadToS3FromUrl = async (url: string, key: string) => { const buffer = await res.buffer(); await s3 .putObject({ + ACL: 'public-read', Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME, Key: key, ContentType: res.headers['content-type'], @@ -60,10 +71,14 @@ const uploadToS3FromUrl = async (url: string, key: string) => { Body: buffer, }) .promise(); - return path.join(process.env.AWS_S3_UPLOAD_BUCKET_URL, key); - } catch (e) { - bugsnag.notify(e); + + const endpoint = publicS3Endpoint(); + return `${endpoint}/${key}`; + } catch (err) { + if (process.env.NODE_ENV === 'production') { + bugsnag.notify(err); + } else { + throw err; + } } }; - -export { makePolicy, signPolicy, uploadToS3FromUrl };