Not Found
diff --git a/app/scenes/Starred.js b/app/scenes/Starred.js
index 25000f7ac..8ca71611c 100644
--- a/app/scenes/Starred.js
+++ b/app/scenes/Starred.js
@@ -49,7 +49,7 @@ class Starred extends React.Component
{
-
+
diff --git a/app/scenes/UserDelete.js b/app/scenes/UserDelete.js
index 8f5128eda..c03e004c2 100644
--- a/app/scenes/UserDelete.js
+++ b/app/scenes/UserDelete.js
@@ -3,6 +3,7 @@ import { observable } from "mobx";
import { inject, observer } from "mobx-react";
import * as React from "react";
import AuthStore from "stores/AuthStore";
+import UiStore from "stores/UiStore";
import Button from "components/Button";
import Flex from "components/Flex";
import HelpText from "components/HelpText";
@@ -10,6 +11,7 @@ import Modal from "components/Modal";
type Props = {
auth: AuthStore,
+ ui: UiStore,
onRequestClose: () => void,
};
@@ -24,6 +26,9 @@ class UserDelete extends React.Component {
try {
await this.props.auth.deleteUser();
this.props.auth.logout();
+ } catch (error) {
+ this.props.ui.showToast(error.message);
+ throw error;
} finally {
this.isDeleting = false;
}
@@ -56,4 +61,4 @@ class UserDelete extends React.Component {
}
}
-export default inject("auth")(UserDelete);
+export default inject("auth", "ui")(UserDelete);
diff --git a/app/stores/BaseStore.js b/app/stores/BaseStore.js
index eab617e52..a8f97e929 100644
--- a/app/stores/BaseStore.js
+++ b/app/stores/BaseStore.js
@@ -114,7 +114,7 @@ export default class BaseStore {
}
@action
- async delete(item: T, options?: Object = {}) {
+ async delete(item: T, options: Object = {}) {
if (!this.actions.includes("delete")) {
throw new Error(`Cannot delete ${this.modelName}`);
}
@@ -132,7 +132,7 @@ export default class BaseStore {
}
@action
- async fetch(id: string, options?: Object = {}): Promise<*> {
+ async fetch(id: string, options: Object = {}): Promise<*> {
if (!this.actions.includes("info")) {
throw new Error(`Cannot fetch ${this.modelName}`);
}
diff --git a/app/stores/DocumentsStore.js b/app/stores/DocumentsStore.js
index ff1aa120e..1ddada7ec 100644
--- a/app/stores/DocumentsStore.js
+++ b/app/stores/DocumentsStore.js
@@ -1,15 +1,6 @@
// @flow
import invariant from "invariant";
-import {
- without,
- map,
- find,
- orderBy,
- filter,
- compact,
- omitBy,
- uniq,
-} from "lodash";
+import { find, orderBy, filter, compact, omitBy } from "lodash";
import { observable, action, computed, runInAction } from "mobx";
import naturalSort from "shared/utils/naturalSort";
import BaseStore from "stores/BaseStore";
@@ -23,7 +14,6 @@ type ImportOptions = {
};
export default class DocumentsStore extends BaseStore {
- @observable recentlyViewedIds: string[] = [];
@observable searchCache: Map = new Map();
@observable starredIds: Map = new Map();
@observable backlinks: Map = new Map();
@@ -50,8 +40,8 @@ export default class DocumentsStore extends BaseStore {
@computed
get recentlyViewed(): Document[] {
return orderBy(
- compact(this.recentlyViewedIds.map((id) => this.data.get(id))),
- "updatedAt",
+ filter(this.all, (d) => d.lastViewedAt),
+ "lastViewedAt",
"desc"
);
}
@@ -299,15 +289,7 @@ export default class DocumentsStore extends BaseStore {
@action
fetchRecentlyViewed = async (options: ?PaginationParams): Promise<*> => {
- const data = await this.fetchNamedPage("viewed", options);
-
- runInAction("DocumentsStore#fetchRecentlyViewed", () => {
- // $FlowFixMe
- this.recentlyViewedIds.replace(
- uniq(this.recentlyViewedIds.concat(map(data, "id")))
- );
- });
- return data;
+ return this.fetchNamedPage("viewed", options);
};
@action
@@ -330,12 +312,25 @@ export default class DocumentsStore extends BaseStore {
return this.fetchNamedPage("list", options);
};
+ @action
+ searchTitles = async (query: string, options: PaginationParams = {}) => {
+ const res = await client.get("/documents.search_titles", {
+ query,
+ ...options,
+ });
+ invariant(res && res.data, "Search response should be available");
+
+ // add the documents and associated policies to the store
+ res.data.forEach(this.add);
+ this.addPolicies(res.policies);
+ return res.data;
+ };
+
@action
search = async (
query: string,
options: PaginationParams = {}
): Promise => {
- // $FlowFixMe
const compactedOptions = omitBy(options, (o) => !o);
const res = await client.get("/documents.search", {
...compactedOptions,
@@ -399,7 +394,7 @@ export default class DocumentsStore extends BaseStore {
@action
fetch = async (
id: string,
- options?: FetchOptions = {}
+ options: FetchOptions = {}
): Promise => {
if (!options.prefetch) this.isFetching = true;
@@ -482,7 +477,7 @@ export default class DocumentsStore extends BaseStore {
{ key: "title", value: title },
{ key: "publish", value: options.publish },
{ key: "file", value: file },
- ].map((info) => {
+ ].forEach((info) => {
if (typeof info.value === "string" && info.value) {
formData.append(info.key, info.value);
}
@@ -541,10 +536,6 @@ export default class DocumentsStore extends BaseStore {
async delete(document: Document) {
await super.delete(document);
- runInAction(() => {
- this.recentlyViewedIds = without(this.recentlyViewedIds, document.id);
- });
-
// check to see if we have any shares related to this document already
// loaded in local state. If so we can go ahead and remove those too.
const share = this.rootStore.shares.getByDocumentId(document.id);
diff --git a/app/utils/routeHelpers.js b/app/utils/routeHelpers.js
index 1d7c402a2..145bb2834 100644
--- a/app/utils/routeHelpers.js
+++ b/app/utils/routeHelpers.js
@@ -63,14 +63,22 @@ export function newDocumentUrl(
return `/collections/${collectionId}/new?${queryString.stringify(params)}`;
}
-export function searchUrl(query?: string, collectionId?: string): string {
- let route = "/search";
- if (query) route += `/${encodeURIComponent(query)}`;
-
- if (collectionId) {
- route += `?collectionId=${collectionId}`;
+export function searchUrl(
+ query?: string,
+ params?: {
+ collectionId?: string,
+ ref?: string,
}
- return route;
+): string {
+ let search = queryString.stringify(params);
+ let route = "/search";
+
+ if (query) {
+ route += `/${encodeURIComponent(query)}`;
+ }
+
+ search = search ? `?${search}` : "";
+ return `${route}${search}`;
}
export function notFoundUrl(): string {
diff --git a/package.json b/package.json
index 6ef02a6d0..3947fb109 100644
--- a/package.json
+++ b/package.json
@@ -140,7 +140,7 @@
"react-portal": "^4.0.0",
"react-router-dom": "^5.1.2",
"react-waypoint": "^9.0.2",
- "rich-markdown-editor": "^11.0.0-4",
+ "rich-markdown-editor": "^11.0.0-9",
"semver": "^7.3.2",
"sequelize": "^6.3.4",
"sequelize-cli": "^6.2.0",
@@ -195,4 +195,4 @@
"js-yaml": "^3.13.1"
},
"version": "0.47.1"
-}
+}
\ No newline at end of file
diff --git a/server/api/documents.js b/server/api/documents.js
index 6795d9164..44e54b2a2 100644
--- a/server/api/documents.js
+++ b/server/api/documents.js
@@ -11,6 +11,7 @@ import {
Document,
Event,
Revision,
+ SearchQuery,
Share,
Star,
User,
@@ -98,10 +99,12 @@ router.post("documents.list", auth(), pagination(), async (ctx) => {
// add the users starred state to the response by default
const starredScope = { method: ["withStarred", user.id] };
const collectionScope = { method: ["withCollection", user.id] };
+ const viewScope = { method: ["withViews", user.id] };
const documents = await Document.scope(
"defaultScope",
starredScope,
- collectionScope
+ collectionScope,
+ viewScope
).findAll({
where,
order: [[sort, direction]],
@@ -137,10 +140,12 @@ router.post("documents.pinned", auth(), pagination(), async (ctx) => {
const starredScope = { method: ["withStarred", user.id] };
const collectionScope = { method: ["withCollection", user.id] };
+ const viewScope = { method: ["withViews", user.id] };
const documents = await Document.scope(
"defaultScope",
starredScope,
- collectionScope
+ collectionScope,
+ viewScope
).findAll({
where: {
teamId: user.teamId,
@@ -176,9 +181,11 @@ router.post("documents.archived", auth(), pagination(), async (ctx) => {
const collectionIds = await user.collectionIds();
const collectionScope = { method: ["withCollection", user.id] };
+ const viewScope = { method: ["withViews", user.id] };
const documents = await Document.scope(
"defaultScope",
- collectionScope
+ collectionScope,
+ viewScope
).findAll({
where: {
teamId: user.teamId,
@@ -214,7 +221,8 @@ router.post("documents.deleted", auth(), pagination(), async (ctx) => {
const collectionIds = await user.collectionIds({ paranoid: false });
const collectionScope = { method: ["withCollection", user.id] };
- const documents = await Document.scope(collectionScope).findAll({
+ const viewScope = { method: ["withViews", user.id] };
+ const documents = await Document.scope(collectionScope, viewScope).findAll({
where: {
teamId: user.teamId,
collectionId: collectionIds,
@@ -276,7 +284,11 @@ router.post("documents.viewed", auth(), pagination(), async (ctx) => {
limit: ctx.state.pagination.limit,
});
- const documents = views.map((view) => view.document);
+ const documents = views.map((view) => {
+ const document = view.document;
+ document.views = [view];
+ return document;
+ });
const data = await Promise.all(
documents.map((document) => presentDocument(document))
);
@@ -349,9 +361,11 @@ router.post("documents.drafts", auth(), pagination(), async (ctx) => {
const collectionIds = await user.collectionIds();
const collectionScope = { method: ["withCollection", user.id] };
+ const viewScope = { method: ["withViews", user.id] };
const documents = await Document.scope(
"defaultScope",
- collectionScope
+ collectionScope,
+ viewScope
).findAll({
where: {
userId: user.id,
@@ -537,6 +551,52 @@ router.post("documents.restore", auth(), async (ctx) => {
};
});
+router.post("documents.search_titles", auth(), pagination(), async (ctx) => {
+ const { query } = ctx.body;
+ const { offset, limit } = ctx.state.pagination;
+ const user = ctx.state.user;
+ ctx.assertPresent(query, "query is required");
+
+ const collectionIds = await user.collectionIds();
+
+ const documents = await Document.scope(
+ {
+ method: ["withViews", user.id],
+ },
+ {
+ method: ["withCollection", user.id],
+ }
+ ).findAll({
+ where: {
+ title: {
+ [Op.iLike]: `%${query}%`,
+ },
+ collectionId: collectionIds,
+ archivedAt: {
+ [Op.eq]: null,
+ },
+ },
+ order: [["updatedAt", "DESC"]],
+ include: [
+ { model: User, as: "createdBy", paranoid: false },
+ { model: User, as: "updatedBy", paranoid: false },
+ ],
+ offset,
+ limit,
+ });
+
+ const policies = presentPolicies(user, documents);
+ const data = await Promise.all(
+ documents.map((document) => presentDocument(document))
+ );
+
+ ctx.body = {
+ pagination: ctx.state.pagination,
+ data,
+ policies,
+ };
+});
+
router.post("documents.search", auth(), pagination(), async (ctx) => {
const {
query,
@@ -573,7 +633,7 @@ router.post("documents.search", auth(), pagination(), async (ctx) => {
);
}
- const results = await Document.searchForUser(user, query, {
+ const { results, totalCount } = await Document.searchForUser(user, query, {
includeArchived: includeArchived === "true",
includeDrafts: includeDrafts === "true",
collaboratorIds,
@@ -591,6 +651,18 @@ router.post("documents.search", auth(), pagination(), async (ctx) => {
})
);
+ // When requesting subsequent pages of search results we don't want to record
+ // duplicate search query records
+ if (offset === 0) {
+ SearchQuery.create({
+ userId: user.id,
+ teamId: user.teamId,
+ source: ctx.state.authType,
+ query,
+ results: totalCount,
+ });
+ }
+
const policies = presentPolicies(user, documents);
ctx.body = {
diff --git a/server/api/documents.test.js b/server/api/documents.test.js
index 4d0792537..176215d8e 100644
--- a/server/api/documents.test.js
+++ b/server/api/documents.test.js
@@ -8,6 +8,7 @@ import {
Revision,
Backlink,
CollectionUser,
+ SearchQuery,
} from "../models";
import {
buildShare,
@@ -627,6 +628,56 @@ describe("#documents.drafts", () => {
});
});
+describe("#documents.search_titles", () => {
+ it("should return case insensitive results for partial query", async () => {
+ const user = await buildUser();
+ const document = await buildDocument({
+ userId: user.id,
+ teamId: user.teamId,
+ title: "Super secret",
+ });
+
+ const res = await server.post("/api/documents.search_titles", {
+ body: { token: user.getJwtToken(), query: "SECRET" },
+ });
+ const body = await res.json();
+
+ expect(res.status).toEqual(200);
+ expect(body.data.length).toEqual(1);
+ expect(body.data[0].id).toEqual(document.id);
+ });
+
+ it("should not include archived or deleted documents", async () => {
+ const user = await buildUser();
+ await buildDocument({
+ userId: user.id,
+ teamId: user.teamId,
+ title: "Super secret",
+ archivedAt: new Date(),
+ });
+
+ await buildDocument({
+ userId: user.id,
+ teamId: user.teamId,
+ title: "Super secret",
+ deletedAt: new Date(),
+ });
+
+ const res = await server.post("/api/documents.search_titles", {
+ body: { token: user.getJwtToken(), query: "SECRET" },
+ });
+ const body = await res.json();
+
+ expect(res.status).toEqual(200);
+ expect(body.data.length).toEqual(0);
+ });
+
+ it("should require authentication", async () => {
+ const res = await server.post("/api/documents.search_titles");
+ expect(res.status).toEqual(401);
+ });
+});
+
describe("#documents.search", () => {
it("should return results", async () => {
const { user } = await seed();
@@ -954,6 +1005,25 @@ describe("#documents.search", () => {
expect(res.status).toEqual(401);
expect(body).toMatchSnapshot();
});
+
+ it("should save search term, hits and source", async (done) => {
+ const { user } = await seed();
+ await server.post("/api/documents.search", {
+ body: { token: user.getJwtToken(), query: "my term" },
+ });
+
+ // setTimeout is needed here because SearchQuery is saved asynchronously
+ // in order to not slow down the response time.
+ setTimeout(async () => {
+ const searchQuery = await SearchQuery.findAll({
+ where: { query: "my term" },
+ });
+ expect(searchQuery.length).toBe(1);
+ expect(searchQuery[0].results).toBe(0);
+ expect(searchQuery[0].source).toBe("app");
+ done();
+ }, 100);
+ });
});
describe("#documents.archived", () => {
diff --git a/server/api/hooks.js b/server/api/hooks.js
index 7dacc2550..0616b920d 100644
--- a/server/api/hooks.js
+++ b/server/api/hooks.js
@@ -2,7 +2,14 @@
import Router from "koa-router";
import { escapeRegExp } from "lodash";
import { AuthenticationError, InvalidRequestError } from "../errors";
-import { Authentication, Document, User, Team, Collection } from "../models";
+import {
+ Authentication,
+ Document,
+ User,
+ Team,
+ Collection,
+ SearchQuery,
+} from "../models";
import { presentSlackAttachment } from "../presenters";
import * as Slack from "../slack";
const router = new Router();
@@ -146,10 +153,18 @@ router.post("hooks.slack", async (ctx) => {
const options = {
limit: 5,
};
- const results = user
+ const { results, totalCount } = user
? await Document.searchForUser(user, text, options)
: await Document.searchForTeam(team, text, options);
+ SearchQuery.create({
+ userId: user ? user.id : null,
+ teamId: team.id,
+ source: "slack",
+ query: text,
+ results: totalCount,
+ });
+
if (results.length) {
const attachments = [];
for (const result of results) {
diff --git a/server/api/hooks.test.js b/server/api/hooks.test.js
index 7e643b1c4..10873f5f7 100644
--- a/server/api/hooks.test.js
+++ b/server/api/hooks.test.js
@@ -1,7 +1,7 @@
/* eslint-disable flowtype/require-valid-file-annotation */
import TestServer from "fetch-test-server";
import app from "../app";
-import { Authentication } from "../models";
+import { Authentication, SearchQuery } from "../models";
import * as Slack from "../slack";
import { buildDocument } from "../test/factories";
import { flushdb, seed } from "../test/support";
@@ -132,6 +132,30 @@ describe("#hooks.slack", () => {
);
});
+ it("should save search term, hits and source", async (done) => {
+ const { user, team } = await seed();
+ await server.post("/api/hooks.slack", {
+ body: {
+ token: process.env.SLACK_VERIFICATION_TOKEN,
+ user_id: user.serviceId,
+ team_id: team.slackId,
+ text: "contains",
+ },
+ });
+
+ // setTimeout is needed here because SearchQuery is saved asynchronously
+ // in order to not slow down the response time.
+ setTimeout(async () => {
+ const searchQuery = await SearchQuery.findAll({
+ where: { query: "contains" },
+ });
+ expect(searchQuery.length).toBe(1);
+ expect(searchQuery[0].results).toBe(0);
+ expect(searchQuery[0].source).toBe("slack");
+ done();
+ }, 100);
+ });
+
it("should respond with help content for help keyword", async () => {
const { user, team } = await seed();
const res = await server.post("/api/hooks.slack", {
diff --git a/server/commands/documentImporter.js b/server/commands/documentImporter.js
index efa615c48..4410914f7 100644
--- a/server/commands/documentImporter.js
+++ b/server/commands/documentImporter.js
@@ -5,6 +5,7 @@ import mammoth from "mammoth";
import TurndownService from "turndown";
import uuid from "uuid";
import parseTitle from "../../shared/utils/parseTitle";
+import { InvalidRequestError } from "../errors";
import { Attachment, Event, User } from "../models";
import dataURItoBuffer from "../utils/dataURItoBuffer";
import parseImages from "../utils/parseImages";
@@ -66,6 +67,9 @@ export default async function documentImporter({
ip: string,
}): Promise<{ text: string, title: string }> {
const fileInfo = importMapping.filter((item) => item.type === file.type)[0];
+ if (!fileInfo) {
+ throw new InvalidRequestError(`File type ${file.type} not supported`);
+ }
let title = file.name.replace(/\.[^/.]+$/, "");
let text = await fileInfo.getMarkdown(file);
diff --git a/server/commands/documentImporter.test.js b/server/commands/documentImporter.test.js
index 2cac52725..66714eeea 100644
--- a/server/commands/documentImporter.test.js
+++ b/server/commands/documentImporter.test.js
@@ -74,4 +74,27 @@ describe("documentImporter", () => {
expect(response.text).toContain("This is a test paragraph");
expect(response.title).toEqual("Heading 1");
});
+
+ it("should error with unknown file type", async () => {
+ const user = await buildUser();
+ const name = "markdown.md";
+ const file = new File({
+ name,
+ type: "executable/zip",
+ path: path.resolve(__dirname, "..", "test", "fixtures", name),
+ });
+
+ let error;
+ try {
+ await documentImporter({
+ user,
+ file,
+ ip,
+ });
+ } catch (err) {
+ error = err.message;
+ }
+
+ expect(error).toEqual("File type executable/zip not supported");
+ });
});
diff --git a/server/middlewares/authentication.js b/server/middlewares/authentication.js
index 8f923faeb..b3fc3635d 100644
--- a/server/middlewares/authentication.js
+++ b/server/middlewares/authentication.js
@@ -2,14 +2,17 @@
import addMinutes from "date-fns/add_minutes";
import addMonths from "date-fns/add_months";
import JWT from "jsonwebtoken";
-import { type Context } from "koa";
import { AuthenticationError, UserSuspendedError } from "../errors";
-import { User, ApiKey } from "../models";
+import { User, Team, ApiKey } from "../models";
+import type { ContextWithState } from "../types";
import { getCookieDomain } from "../utils/domains";
import { getUserForJWT } from "../utils/jwt";
export default function auth(options?: { required?: boolean } = {}) {
- return async function authMiddleware(ctx: Context, next: () => Promise<*>) {
+ return async function authMiddleware(
+ ctx: ContextWithState,
+ next: () => Promise
+ ) {
let token;
const authorizationHeader = ctx.request.get("authorization");
@@ -27,7 +30,6 @@ export default function auth(options?: { required?: boolean } = {}) {
`Bad Authorization header format. Format is "Authorization: Bearer "`
);
}
- // $FlowFixMe
} else if (ctx.body && ctx.body.token) {
token = ctx.body.token;
} else if (ctx.request.query.token) {
@@ -43,7 +45,8 @@ export default function auth(options?: { required?: boolean } = {}) {
let user;
if (token) {
if (String(token).match(/^[\w]{38}$/)) {
- // API key
+ ctx.state.authType = "api";
+
let apiKey;
try {
apiKey = await ApiKey.findOne({
@@ -51,18 +54,22 @@ export default function auth(options?: { required?: boolean } = {}) {
secret: token,
},
});
- } catch (e) {
+ } catch (err) {
throw new AuthenticationError("Invalid API key");
}
- if (!apiKey) throw new AuthenticationError("Invalid API key");
+ if (!apiKey) {
+ throw new AuthenticationError("Invalid API key");
+ }
user = await User.findByPk(apiKey.userId);
- if (!user) throw new AuthenticationError("Invalid API key");
+ if (!user) {
+ throw new AuthenticationError("Invalid API key");
+ }
} else {
- /* $FlowFixMeNowPlease This comment suppresses an error found when upgrading
- * flow-bin@0.104.0. To view the error, delete this comment and run Flow. */
- user = await getUserForJWT(token);
+ ctx.state.authType = "app";
+
+ user = await getUserForJWT(String(token));
}
if (user.isSuspended) {
@@ -76,21 +83,16 @@ export default function auth(options?: { required?: boolean } = {}) {
// not awaiting the promise here so that the request is not blocked
user.updateActiveAt(ctx.request.ip);
- /* $FlowFixMeNowPlease This comment suppresses an error found when upgrading
- * flow-bin@0.104.0. To view the error, delete this comment and run Flow. */
- ctx.state.token = token;
-
- /* $FlowFixMeNowPlease This comment suppresses an error found when upgrading
- * flow-bin@0.104.0. To view the error, delete this comment and run Flow. */
+ ctx.state.token = String(token);
ctx.state.user = user;
- if (!ctx.cache) ctx.cache = {};
-
- /* $FlowFixMeNowPlease This comment suppresses an error found when upgrading
- * flow-bin@0.104.0. To view the error, delete this comment and run Flow. */
- ctx.cache[user.id] = user;
}
- ctx.signIn = async (user, team, service, isFirstSignin = false) => {
+ ctx.signIn = async (
+ user: User,
+ team: Team,
+ service,
+ isFirstSignin = false
+ ) => {
if (user.isSuspended) {
return ctx.redirect("/?notice=suspended");
}
diff --git a/server/migrations/20200915010511-create-search-queries.js b/server/migrations/20200915010511-create-search-queries.js
new file mode 100644
index 000000000..ac5e29d09
--- /dev/null
+++ b/server/migrations/20200915010511-create-search-queries.js
@@ -0,0 +1,43 @@
+"use strict";
+module.exports = {
+ up: async (queryInterface, Sequelize) => {
+ await queryInterface.createTable("search_queries", {
+ id: {
+ allowNull: false,
+ primaryKey: true,
+ type: Sequelize.UUID,
+ },
+ userId: {
+ type: Sequelize.UUID,
+ references: {
+ model: "users",
+ },
+ },
+ teamId: {
+ type: Sequelize.UUID,
+ references: {
+ model: "teams",
+ },
+ },
+ source: {
+ type: Sequelize.ENUM("slack", "app", "api"),
+ allowNull: false,
+ },
+ query: {
+ type: Sequelize.STRING,
+ allowNull: false,
+ },
+ results: {
+ type: Sequelize.INTEGER,
+ allowNull: false,
+ },
+ createdAt: {
+ allowNull: false,
+ type: Sequelize.DATE,
+ },
+ });
+ },
+ down: async (queryInterface, Sequelize) => {
+ await queryInterface.dropTable("search_queries");
+ },
+};
diff --git a/server/migrations/20200926204620-add-missing-indexes.js b/server/migrations/20200926204620-add-missing-indexes.js
new file mode 100644
index 000000000..3e730d266
--- /dev/null
+++ b/server/migrations/20200926204620-add-missing-indexes.js
@@ -0,0 +1,19 @@
+'use strict';
+
+module.exports = {
+ up: async (queryInterface, Sequelize) => {
+ await queryInterface.addIndex("search_queries", ["teamId"]);
+ await queryInterface.addIndex("search_queries", ["userId"]);
+ await queryInterface.addIndex("search_queries", ["createdAt"]);
+
+ await queryInterface.addIndex("users", ["teamId"]);
+ },
+
+ down: async (queryInterface, Sequelize) => {
+ await queryInterface.removeIndex("search_queries", ["teamId"]);
+ await queryInterface.removeIndex("search_queries", ["userId"]);
+ await queryInterface.removeIndex("search_queries", ["createdAt"]);
+
+ await queryInterface.removeIndex("users", ["teamId"]);
+ }
+};
diff --git a/server/models/Document.js b/server/models/Document.js
index e6f222e20..a49328712 100644
--- a/server/models/Document.js
+++ b/server/models/Document.js
@@ -207,11 +207,15 @@ Document.associate = (models) => {
{ model: models.User, as: "updatedBy", paranoid: false },
],
});
- Document.addScope("withViews", (userId) => ({
- include: [
- { model: models.View, as: "views", where: { userId }, required: false },
- ],
- }));
+ Document.addScope("withViews", (userId) => {
+ if (!userId) return {};
+
+ return {
+ include: [
+ { model: models.View, as: "views", where: { userId }, required: false },
+ ],
+ };
+ });
Document.addScope("withStarred", (userId) => ({
include: [
{ model: models.Star, as: "starred", where: { userId }, required: false },
@@ -222,9 +226,15 @@ Document.associate = (models) => {
Document.findByPk = async function (id, options = {}) {
// allow default preloading of collection membership if `userId` is passed in find options
// almost every endpoint needs the collection membership to determine policy permissions.
- const scope = this.scope("withUnpublished", {
- method: ["withCollection", options.userId],
- });
+ const scope = this.scope(
+ "withUnpublished",
+ {
+ method: ["withCollection", options.userId],
+ },
+ {
+ method: ["withViews", options.userId],
+ }
+ );
if (isUUID(id)) {
return scope.findOne({
@@ -241,10 +251,13 @@ Document.findByPk = async function (id, options = {}) {
}
};
-type SearchResult = {
- ranking: number,
- context: string,
- document: Document,
+type SearchResponse = {
+ results: {
+ ranking: number,
+ context: string,
+ document: Document,
+ }[],
+ totalCount: number,
};
type SearchOptions = {
@@ -267,7 +280,7 @@ Document.searchForTeam = async (
team,
query,
options: SearchOptions = {}
-): Promise => {
+): Promise => {
const limit = options.limit || 15;
const offset = options.offset || 0;
const wildcardQuery = `${escape(query)}:*`;
@@ -275,21 +288,25 @@ Document.searchForTeam = async (
// If the team has access no public collections then shortcircuit the rest of this
if (!collectionIds.length) {
- return [];
+ return { results: [], totalCount: 0 };
}
// Build the SQL query to get documentIds, ranking, and search term context
- const sql = `
+ const whereClause = `
+ "searchVector" @@ to_tsquery('english', :query) AND
+ "teamId" = :teamId AND
+ "collectionId" IN(:collectionIds) AND
+ "deletedAt" IS NULL AND
+ "publishedAt" IS NOT NULL
+ `;
+
+ const selectSql = `
SELECT
id,
ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
FROM documents
- WHERE "searchVector" @@ to_tsquery('english', :query) AND
- "teamId" = :teamId AND
- "collectionId" IN(:collectionIds) AND
- "deletedAt" IS NULL AND
- "publishedAt" IS NOT NULL
+ WHERE ${whereClause}
ORDER BY
"searchRanking" DESC,
"updatedAt" DESC
@@ -297,17 +314,34 @@ Document.searchForTeam = async (
OFFSET :offset;
`;
- const results = await sequelize.query(sql, {
+ const countSql = `
+ SELECT COUNT(id)
+ FROM documents
+ WHERE ${whereClause}
+ `;
+
+ const queryReplacements = {
+ teamId: team.id,
+ query: wildcardQuery,
+ collectionIds,
+ };
+
+ const resultsQuery = sequelize.query(selectSql, {
type: sequelize.QueryTypes.SELECT,
replacements: {
- teamId: team.id,
- query: wildcardQuery,
+ ...queryReplacements,
limit,
offset,
- collectionIds,
},
});
+ const countQuery = sequelize.query(countSql, {
+ type: sequelize.QueryTypes.SELECT,
+ replacements: queryReplacements,
+ });
+
+ const [results, [{ count }]] = await Promise.all([resultsQuery, countQuery]);
+
// Final query to get associated document data
const documents = await Document.findAll({
where: {
@@ -320,20 +354,23 @@ Document.searchForTeam = async (
],
});
- return map(results, (result) => ({
- ranking: result.searchRanking,
- context: removeMarkdown(unescape(result.searchContext), {
- stripHTML: false,
- }),
- document: find(documents, { id: result.id }),
- }));
+ return {
+ results: map(results, (result) => ({
+ ranking: result.searchRanking,
+ context: removeMarkdown(unescape(result.searchContext), {
+ stripHTML: false,
+ }),
+ document: find(documents, { id: result.id }),
+ })),
+ totalCount: count,
+ };
};
Document.searchForUser = async (
user,
query,
options: SearchOptions = {}
-): Promise => {
+): Promise => {
const limit = options.limit || 15;
const offset = options.offset || 0;
const wildcardQuery = `${escape(query)}:*`;
@@ -350,7 +387,7 @@ Document.searchForUser = async (
// If the user has access to no collections then shortcircuit the rest of this
if (!collectionIds.length) {
- return [];
+ return { results: [], totalCount: 0 };
}
let dateFilter;
@@ -359,13 +396,8 @@ Document.searchForUser = async (
}
// Build the SQL query to get documentIds, ranking, and search term context
- const sql = `
- SELECT
- id,
- ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
- ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
- FROM documents
- WHERE "searchVector" @@ to_tsquery('english', :query) AND
+ const whereClause = `
+ "searchVector" @@ to_tsquery('english', :query) AND
"teamId" = :teamId AND
"collectionId" IN(:collectionIds) AND
${
@@ -383,27 +415,52 @@ Document.searchForUser = async (
? '("publishedAt" IS NOT NULL OR "createdById" = :userId)'
: '"publishedAt" IS NOT NULL'
}
+ `;
+
+ const selectSql = `
+ SELECT
+ id,
+ ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
+ ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
+ FROM documents
+ WHERE ${whereClause}
ORDER BY
"searchRanking" DESC,
"updatedAt" DESC
LIMIT :limit
OFFSET :offset;
-`;
+ `;
- const results = await sequelize.query(sql, {
+ const countSql = `
+ SELECT COUNT(id)
+ FROM documents
+ WHERE ${whereClause}
+ `;
+
+ const queryReplacements = {
+ teamId: user.teamId,
+ userId: user.id,
+ collaboratorIds: options.collaboratorIds,
+ query: wildcardQuery,
+ collectionIds,
+ dateFilter,
+ };
+
+ const resultsQuery = sequelize.query(selectSql, {
type: sequelize.QueryTypes.SELECT,
replacements: {
- teamId: user.teamId,
- userId: user.id,
- collaboratorIds: options.collaboratorIds,
- query: wildcardQuery,
+ ...queryReplacements,
limit,
offset,
- collectionIds,
- dateFilter,
},
});
+ const countQuery = sequelize.query(countSql, {
+ type: sequelize.QueryTypes.SELECT,
+ replacements: queryReplacements,
+ });
+
+ const [results, [{ count }]] = await Promise.all([resultsQuery, countQuery]);
// Final query to get associated document data
const documents = await Document.scope(
{
@@ -422,13 +479,16 @@ Document.searchForUser = async (
],
});
- return map(results, (result) => ({
- ranking: result.searchRanking,
- context: removeMarkdown(unescape(result.searchContext), {
- stripHTML: false,
- }),
- document: find(documents, { id: result.id }),
- }));
+ return {
+ results: map(results, (result) => ({
+ ranking: result.searchRanking,
+ context: removeMarkdown(unescape(result.searchContext), {
+ stripHTML: false,
+ }),
+ document: find(documents, { id: result.id }),
+ })),
+ totalCount: count,
+ };
};
// Hooks
diff --git a/server/models/Document.test.js b/server/models/Document.test.js
index ae48995e4..3befac7c5 100644
--- a/server/models/Document.test.js
+++ b/server/models/Document.test.js
@@ -201,7 +201,7 @@ describe("#searchForTeam", () => {
title: "test",
});
- const results = await Document.searchForTeam(team, "test");
+ const { results } = await Document.searchForTeam(team, "test");
expect(results.length).toBe(1);
expect(results[0].document.id).toBe(document.id);
});
@@ -218,15 +218,85 @@ describe("#searchForTeam", () => {
title: "test",
});
- const results = await Document.searchForTeam(team, "test");
+ const { results } = await Document.searchForTeam(team, "test");
expect(results.length).toBe(0);
});
test("should handle no collections", async () => {
const team = await buildTeam();
- const results = await Document.searchForTeam(team, "test");
+ const { results } = await Document.searchForTeam(team, "test");
expect(results.length).toBe(0);
});
+
+ test("should return the total count of search results", async () => {
+ const team = await buildTeam();
+ const collection = await buildCollection({ teamId: team.id });
+ await buildDocument({
+ teamId: team.id,
+ collectionId: collection.id,
+ title: "test number 1",
+ });
+ await buildDocument({
+ teamId: team.id,
+ collectionId: collection.id,
+ title: "test number 2",
+ });
+
+ const { totalCount } = await Document.searchForTeam(team, "test");
+ expect(totalCount).toBe("2");
+ });
+});
+
+describe("#searchForUser", () => {
+ test("should return search results from collections", async () => {
+ const team = await buildTeam();
+ const user = await buildUser({ teamId: team.id });
+ const collection = await buildCollection({
+ userId: user.id,
+ teamId: team.id,
+ });
+ const document = await buildDocument({
+ userId: user.id,
+ teamId: team.id,
+ collectionId: collection.id,
+ title: "test",
+ });
+
+ const { results } = await Document.searchForUser(user, "test");
+ expect(results.length).toBe(1);
+ expect(results[0].document.id).toBe(document.id);
+ });
+
+ test("should handle no collections", async () => {
+ const team = await buildTeam();
+ const user = await buildUser({ teamId: team.id });
+ const { results } = await Document.searchForUser(user, "test");
+ expect(results.length).toBe(0);
+ });
+
+ test("should return the total count of search results", async () => {
+ const team = await buildTeam();
+ const user = await buildUser({ teamId: team.id });
+ const collection = await buildCollection({
+ userId: user.id,
+ teamId: team.id,
+ });
+ await buildDocument({
+ userId: user.id,
+ teamId: team.id,
+ collectionId: collection.id,
+ title: "test number 1",
+ });
+ await buildDocument({
+ userId: user.id,
+ teamId: team.id,
+ collectionId: collection.id,
+ title: "test number 2",
+ });
+
+ const { totalCount } = await Document.searchForUser(user, "test");
+ expect(totalCount).toBe("2");
+ });
});
describe("#delete", () => {
diff --git a/server/models/SearchQuery.js b/server/models/SearchQuery.js
new file mode 100644
index 000000000..cd7258d35
--- /dev/null
+++ b/server/models/SearchQuery.js
@@ -0,0 +1,42 @@
+// @flow
+import { DataTypes, sequelize } from "../sequelize";
+
+const SearchQuery = sequelize.define(
+ "search_queries",
+ {
+ id: {
+ type: DataTypes.UUID,
+ defaultValue: DataTypes.UUIDV4,
+ primaryKey: true,
+ },
+ source: {
+ type: DataTypes.ENUM("slack", "app", "api"),
+ allowNull: false,
+ },
+ query: {
+ type: DataTypes.STRING,
+ allowNull: false,
+ },
+ results: {
+ type: DataTypes.NUMBER,
+ allowNull: false,
+ },
+ },
+ {
+ timestamps: true,
+ updatedAt: false,
+ }
+);
+
+SearchQuery.associate = (models) => {
+ SearchQuery.belongsTo(models.User, {
+ as: "user",
+ foreignKey: "userId",
+ });
+ SearchQuery.belongsTo(models.Team, {
+ as: "team",
+ foreignKey: "teamId",
+ });
+};
+
+export default SearchQuery;
diff --git a/server/models/View.js b/server/models/View.js
index b0d88e485..7607aebc8 100644
--- a/server/models/View.js
+++ b/server/models/View.js
@@ -2,7 +2,7 @@
import subMilliseconds from "date-fns/sub_milliseconds";
import { USER_PRESENCE_INTERVAL } from "../../shared/constants";
import { User } from "../models";
-import { Op, DataTypes, sequelize } from "../sequelize";
+import { DataTypes, Op, sequelize } from "../sequelize";
const View = sequelize.define(
"view",
diff --git a/server/models/index.js b/server/models/index.js
index 16e8fe854..c5f2da6ac 100644
--- a/server/models/index.js
+++ b/server/models/index.js
@@ -14,6 +14,7 @@ import Integration from "./Integration";
import Notification from "./Notification";
import NotificationSetting from "./NotificationSetting";
import Revision from "./Revision";
+import SearchQuery from "./SearchQuery";
import Share from "./Share";
import Star from "./Star";
import Team from "./Team";
@@ -36,6 +37,7 @@ const models = {
Notification,
NotificationSetting,
Revision,
+ SearchQuery,
Share,
Star,
Team,
@@ -66,6 +68,7 @@ export {
Notification,
NotificationSetting,
Revision,
+ SearchQuery,
Share,
Star,
Team,
diff --git a/server/presenters/document.js b/server/presenters/document.js
index 969555abe..c459db85f 100644
--- a/server/presenters/document.js
+++ b/server/presenters/document.js
@@ -1,6 +1,6 @@
// @flow
import { takeRight } from "lodash";
-import { User, Document, Attachment } from "../models";
+import { Attachment, Document, User } from "../models";
import { getSignedImageUrl } from "../utils/s3";
import presentUser from "./user";
@@ -62,8 +62,13 @@ export default async function present(document: Document, options: ?Options) {
pinned: undefined,
collectionId: undefined,
parentDocumentId: undefined,
+ lastViewedAt: undefined,
};
+ if (!!document.views && document.views.length > 0) {
+ data.lastViewedAt = document.views[0].updatedAt;
+ }
+
if (!options.isPublic) {
data.pinned = !!document.pinnedById;
data.collectionId = document.collectionId;
diff --git a/server/routes.js b/server/routes.js
index ae634a949..537b5712c 100644
--- a/server/routes.js
+++ b/server/routes.js
@@ -37,6 +37,22 @@ const readIndexFile = async (ctx) => {
});
};
+const renderApp = async (ctx, next) => {
+ if (ctx.request.path === "/realtime/") {
+ return next();
+ }
+
+ const page = await readIndexFile(ctx);
+ const env = `
+ window.env = ${JSON.stringify(environment)};
+ `;
+ ctx.body = page
+ .toString()
+ .replace(/\/\/inject-env\/\//g, env)
+ .replace(/\/\/inject-sentry-dsn\/\//g, process.env.SENTRY_DSN || "")
+ .replace(/\/\/inject-slack-app-id\/\//g, process.env.SLACK_APP_ID || "");
+};
+
// serve static assets
koa.use(
serve(path.resolve(__dirname, "../../public"), {
@@ -65,23 +81,14 @@ router.get("/opensearch.xml", (ctx) => {
ctx.body = opensearchResponse();
});
-// catch all for application
-router.get("*", async (ctx, next) => {
- if (ctx.request.path === "/realtime/") {
- return next();
- }
-
- const page = await readIndexFile(ctx);
- const env = `
- window.env = ${JSON.stringify(environment)};
- `;
- ctx.body = page
- .toString()
- .replace(/\/\/inject-env\/\//g, env)
- .replace(/\/\/inject-sentry-dsn\/\//g, process.env.SENTRY_DSN || "")
- .replace(/\/\/inject-slack-app-id\/\//g, process.env.SLACK_APP_ID || "");
+router.get("/share/*", (ctx, next) => {
+ ctx.remove("X-Frame-Options");
+ return renderApp(ctx, next);
});
+// catch all for application
+router.get("*", renderApp);
+
// middleware
koa.use(apexRedirect());
koa.use(router.routes());
diff --git a/server/static/index.html b/server/static/index.html
index a0982d8f4..d5ef94d9e 100644
--- a/server/static/index.html
+++ b/server/static/index.html
@@ -62,7 +62,7 @@
});
}
- if (window.localStorage.getItem("theme") === "dark") {
+ if (window.localStorage && window.localStorage.getItem("theme") === "dark") {
window.document.querySelector("#root").style.background = "#111319";
}
diff --git a/server/types.js b/server/types.js
new file mode 100644
index 000000000..5fde708a1
--- /dev/null
+++ b/server/types.js
@@ -0,0 +1,12 @@
+// @flow
+import { type Context } from "koa";
+import { User } from "./models";
+
+export type ContextWithState = {|
+ ...$Exact,
+ state: {
+ user: User,
+ token: string,
+ authType: "app" | "api",
+ },
+|};
diff --git a/server/utils/jwt.js b/server/utils/jwt.js
index 925fdaff6..d1bbfa2bf 100644
--- a/server/utils/jwt.js
+++ b/server/utils/jwt.js
@@ -18,7 +18,7 @@ function getJWTPayload(token) {
return payload;
}
-export async function getUserForJWT(token: string) {
+export async function getUserForJWT(token: string): Promise {
const payload = getJWTPayload(token);
const user = await User.findByPk(payload.id);
@@ -31,7 +31,7 @@ export async function getUserForJWT(token: string) {
return user;
}
-export async function getUserForEmailSigninToken(token: string) {
+export async function getUserForEmailSigninToken(token: string): Promise {
const payload = getJWTPayload(token);
// check the token is within it's expiration time
diff --git a/shared/styles/theme.js b/shared/styles/theme.js
index 5252f49be..fb0c458a8 100644
--- a/shared/styles/theme.js
+++ b/shared/styles/theme.js
@@ -26,6 +26,7 @@ const colors = {
yellow: "#FBCA04",
warmGrey: "#EDF2F7",
+ searchHighlight: "#FDEA9B",
danger: "#ff476f",
warning: "#f08a24",
success: "#2f3336",
@@ -138,6 +139,7 @@ export const light = {
listItemHoverBackground: colors.warmGrey,
+ toolbarHoverBackground: colors.black,
toolbarBackground: colors.lightBlack,
toolbarInput: colors.white10,
toolbarItem: colors.white,
@@ -192,6 +194,7 @@ export const dark = {
listItemHoverBackground: colors.black50,
+ toolbarHoverBackground: colors.slate,
toolbarBackground: colors.white,
toolbarInput: colors.black10,
toolbarItem: colors.lightBlack,
diff --git a/yarn.lock b/yarn.lock
index 601b87b60..510194f28 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -9887,10 +9887,10 @@ retry-as-promised@^3.2.0:
dependencies:
any-promise "^1.3.0"
-rich-markdown-editor@^11.0.0-4:
- version "11.0.0-4"
- resolved "https://registry.yarnpkg.com/rich-markdown-editor/-/rich-markdown-editor-11.0.0-4.tgz#b65f5b03502d70a2b2bbea5c916c23b071f4bab6"
- integrity sha512-+llzd8Plxzsc/jJ8RwtMSV5QIpxpZdM5nQejG/SLe/lfqHNOFNnIiOszSPERIcULLxsLdMT5Ajz+Yr5PXPicOQ==
+rich-markdown-editor@^11.0.0-9:
+ version "11.0.0-9"
+ resolved "https://registry.yarnpkg.com/rich-markdown-editor/-/rich-markdown-editor-11.0.0-9.tgz#a7a4bfa09fca3cdf3168027e11fd9af46c708680"
+ integrity sha512-B1q6VbRF/6yjHsYMQEXjgjwPJCSU3mNEmLGsJOF+PZACII5ojg8bV51jGd4W1rTvbIzqnLK4iPWlAbn+hrMtXw==
dependencies:
copy-to-clipboard "^3.0.8"
lodash "^4.17.11"