Refactor document search
This commit is contained in:
Tom Moor
@@ -5,7 +5,6 @@ import {
|
||||
buildCollection,
|
||||
buildTeam,
|
||||
buildUser,
|
||||
buildShare,
|
||||
} from "@server/test/factories";
|
||||
import { setupTestDatabase, seed } from "@server/test/support";
|
||||
import slugify from "@server/utils/slugify";
|
||||
@@ -163,319 +162,6 @@ paragraph`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#searchForTeam", () => {
|
||||
test("should return search results from public collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await Document.searchForTeam(team, "test");
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].document?.id).toBe(document.id);
|
||||
});
|
||||
|
||||
test("should not return results from private collections without providing collectionId", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
permission: null,
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await Document.searchForTeam(team, "test");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should return results from private collections when collectionId is provided", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
permission: null,
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await Document.searchForTeam(team, "test", {
|
||||
collectionId: collection.id,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should return results from document tree of shared document", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
permission: null,
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test 1",
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test 2",
|
||||
});
|
||||
|
||||
const share = await buildShare({
|
||||
documentId: document.id,
|
||||
includeChildDocuments: true,
|
||||
});
|
||||
|
||||
const { results } = await Document.searchForTeam(team, "test", {
|
||||
collectionId: collection.id,
|
||||
share,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const { results } = await Document.searchForTeam(team, "test");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should handle backslashes in search term", async () => {
|
||||
const team = await buildTeam();
|
||||
const { results } = await Document.searchForTeam(team, "\\\\");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should return the total count of search results", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 2",
|
||||
});
|
||||
const { totalCount } = await Document.searchForTeam(team, "test");
|
||||
expect(totalCount).toBe("2");
|
||||
});
|
||||
|
||||
test("should return the document when searched with their previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await Document.searchForTeam(team, "test number");
|
||||
expect(totalCount).toBe("1");
|
||||
});
|
||||
|
||||
test("should not return the document when searched with neither the titles nor the previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await Document.searchForTeam(
|
||||
team,
|
||||
"title doesn't exist"
|
||||
);
|
||||
expect(totalCount).toBe("0");
|
||||
});
|
||||
});
|
||||
|
||||
describe("#searchForUser", () => {
|
||||
test("should return search results from collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await Document.searchForUser(user, "test");
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].document?.id).toBe(document.id);
|
||||
});
|
||||
|
||||
test("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const { results } = await Document.searchForUser(user, "test");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should search only drafts created by user", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await Document.searchForUser(user, "test", {
|
||||
includeDrafts: true,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should not include drafts", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await Document.searchForUser(user, "test", {
|
||||
includeDrafts: false,
|
||||
});
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should include results from drafts as well", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: user.teamId,
|
||||
createdById: user.id,
|
||||
title: "not draft",
|
||||
});
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "draft",
|
||||
});
|
||||
const { results } = await Document.searchForUser(user, "draft", {
|
||||
includeDrafts: true,
|
||||
});
|
||||
expect(results.length).toBe(2);
|
||||
});
|
||||
|
||||
test("should not include results from drafts", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: user.teamId,
|
||||
createdById: user.id,
|
||||
title: "not draft",
|
||||
});
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "draft",
|
||||
});
|
||||
const { results } = await Document.searchForUser(user, "draft", {
|
||||
includeDrafts: false,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should return the total count of search results", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 2",
|
||||
});
|
||||
const { totalCount } = await Document.searchForUser(user, "test");
|
||||
expect(totalCount).toBe("2");
|
||||
});
|
||||
|
||||
test("should return the document when searched with their previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await Document.searchForUser(user, "test number");
|
||||
expect(totalCount).toBe("1");
|
||||
});
|
||||
|
||||
test("should not return the document when searched with neither the titles nor the previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await Document.searchForUser(
|
||||
user,
|
||||
"title doesn't exist"
|
||||
);
|
||||
expect(totalCount).toBe("0");
|
||||
});
|
||||
});
|
||||
|
||||
describe("#delete", () => {
|
||||
test("should soft delete and set last modified", async () => {
|
||||
const document = await buildDocument();
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import removeMarkdown from "@tommoor/remove-markdown";
|
||||
import invariant from "invariant";
|
||||
import { compact, find, map, uniq } from "lodash";
|
||||
import { compact, uniq } from "lodash";
|
||||
import randomstring from "randomstring";
|
||||
import type { SaveOptions } from "sequelize";
|
||||
import {
|
||||
Transaction,
|
||||
Op,
|
||||
QueryTypes,
|
||||
FindOptions,
|
||||
ScopeOptions,
|
||||
WhereOptions,
|
||||
@@ -33,7 +31,6 @@ import {
|
||||
} from "sequelize-typescript";
|
||||
import MarkdownSerializer from "slate-md-serializer";
|
||||
import isUUID from "validator/lib/isUUID";
|
||||
import { DateFilter } from "@shared/types";
|
||||
import getTasks from "@shared/utils/getTasks";
|
||||
import parseTitle from "@shared/utils/parseTitle";
|
||||
import unescape from "@shared/utils/unescape";
|
||||
@@ -43,7 +40,6 @@ import slugify from "@server/utils/slugify";
|
||||
import Backlink from "./Backlink";
|
||||
import Collection from "./Collection";
|
||||
import Revision from "./Revision";
|
||||
import Share from "./Share";
|
||||
import Star from "./Star";
|
||||
import Team from "./Team";
|
||||
import User from "./User";
|
||||
@@ -52,28 +48,6 @@ import ParanoidModel from "./base/ParanoidModel";
|
||||
import Fix from "./decorators/Fix";
|
||||
import Length from "./validators/Length";
|
||||
|
||||
export type SearchResponse = {
|
||||
results: {
|
||||
ranking: number;
|
||||
context: string;
|
||||
document: Document;
|
||||
}[];
|
||||
totalCount: number;
|
||||
};
|
||||
|
||||
type SearchOptions = {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
collectionId?: string;
|
||||
share?: Share;
|
||||
dateFilter?: DateFilter;
|
||||
collaboratorIds?: string[];
|
||||
includeArchived?: boolean;
|
||||
includeDrafts?: boolean;
|
||||
snippetMinWords?: number;
|
||||
snippetMaxWords?: number;
|
||||
};
|
||||
|
||||
const serializer = new MarkdownSerializer();
|
||||
|
||||
export const DOCUMENT_VERSION = 2;
|
||||
@@ -474,257 +448,6 @@ class Document extends ParanoidModel {
|
||||
return null;
|
||||
}
|
||||
|
||||
static async searchForTeam(
|
||||
team: Team,
|
||||
query: string,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResponse> {
|
||||
const wildcardQuery = `${escapeQuery(query)}:*`;
|
||||
const {
|
||||
snippetMinWords = 20,
|
||||
snippetMaxWords = 30,
|
||||
limit = 15,
|
||||
offset = 0,
|
||||
} = options;
|
||||
|
||||
// restrict to specific collection if provided
|
||||
// enables search in private collections if specified
|
||||
let collectionIds;
|
||||
if (options.collectionId) {
|
||||
collectionIds = [options.collectionId];
|
||||
} else {
|
||||
collectionIds = await team.collectionIds();
|
||||
}
|
||||
|
||||
// short circuit if no relevant collections
|
||||
if (!collectionIds.length) {
|
||||
return {
|
||||
results: [],
|
||||
totalCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// restrict to documents in the tree of a shared document when one is provided
|
||||
let documentIds;
|
||||
|
||||
if (options.share?.includeChildDocuments) {
|
||||
const sharedDocument = await options.share.$get("document");
|
||||
invariant(sharedDocument, "Cannot find document for share");
|
||||
|
||||
const childDocumentIds = await sharedDocument.getChildDocumentIds({
|
||||
archivedAt: {
|
||||
[Op.is]: null,
|
||||
},
|
||||
});
|
||||
documentIds = [sharedDocument.id, ...childDocumentIds];
|
||||
}
|
||||
|
||||
const documentClause = documentIds ? `"id" IN(:documentIds) AND` : "";
|
||||
|
||||
// Build the SQL query to get result documentIds, ranking, and search term context
|
||||
const whereClause = `
|
||||
"searchVector" @@ to_tsquery('english', :query) AND
|
||||
"teamId" = :teamId AND
|
||||
"collectionId" IN(:collectionIds) AND
|
||||
${documentClause}
|
||||
"deletedAt" IS NULL AND
|
||||
"publishedAt" IS NOT NULL
|
||||
`;
|
||||
const selectSql = `
|
||||
SELECT
|
||||
id,
|
||||
ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
|
||||
ts_headline('english', "text", to_tsquery('english', :query), :headlineOptions) as "searchContext"
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
ORDER BY
|
||||
"searchRanking" DESC,
|
||||
"updatedAt" DESC
|
||||
LIMIT :limit
|
||||
OFFSET :offset;
|
||||
`;
|
||||
const countSql = `
|
||||
SELECT COUNT(id)
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
`;
|
||||
const queryReplacements = {
|
||||
teamId: team.id,
|
||||
query: wildcardQuery,
|
||||
collectionIds,
|
||||
documentIds,
|
||||
headlineOptions: `MaxFragments=1, MinWords=${snippetMinWords}, MaxWords=${snippetMaxWords}`,
|
||||
};
|
||||
const resultsQuery = this.sequelize!.query(selectSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { ...queryReplacements, limit, offset },
|
||||
});
|
||||
const countQuery = this.sequelize!.query(countSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: queryReplacements,
|
||||
});
|
||||
const [results, [{ count }]]: [any, any] = await Promise.all([
|
||||
resultsQuery,
|
||||
countQuery,
|
||||
]);
|
||||
|
||||
// Final query to get associated document data
|
||||
const documents = await this.findAll({
|
||||
where: {
|
||||
id: map(results, "id"),
|
||||
teamId: team.id,
|
||||
},
|
||||
include: [
|
||||
{
|
||||
model: Collection,
|
||||
as: "collection",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
results: map(results, (result: any) => ({
|
||||
ranking: result.searchRanking,
|
||||
context: removeMarkdown(unescape(result.searchContext), {
|
||||
stripHTML: false,
|
||||
}),
|
||||
document: find(documents, {
|
||||
id: result.id,
|
||||
}) as Document,
|
||||
})),
|
||||
totalCount: count,
|
||||
};
|
||||
}
|
||||
|
||||
static async searchForUser(
|
||||
user: User,
|
||||
query: string,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResponse> {
|
||||
const {
|
||||
snippetMinWords = 20,
|
||||
snippetMaxWords = 30,
|
||||
limit = 15,
|
||||
offset = 0,
|
||||
} = options;
|
||||
const wildcardQuery = `${escapeQuery(query)}:*`;
|
||||
|
||||
// Ensure we're filtering by the users accessible collections. If
|
||||
// collectionId is passed as an option it is assumed that the authorization
|
||||
// has already been done in the router
|
||||
let collectionIds;
|
||||
|
||||
if (options.collectionId) {
|
||||
collectionIds = [options.collectionId];
|
||||
} else {
|
||||
collectionIds = await user.collectionIds();
|
||||
}
|
||||
|
||||
let dateFilter;
|
||||
|
||||
if (options.dateFilter) {
|
||||
dateFilter = `1 ${options.dateFilter}`;
|
||||
}
|
||||
|
||||
// Build the SQL query to get documentIds, ranking, and search term context
|
||||
const whereClause = `
|
||||
"searchVector" @@ to_tsquery('english', :query) AND
|
||||
"teamId" = :teamId AND
|
||||
${
|
||||
collectionIds.length
|
||||
? `(
|
||||
"collectionId" IN(:collectionIds) OR
|
||||
("collectionId" IS NULL AND "createdById" = :userId)
|
||||
) AND`
|
||||
: '"collectionId" IS NULL AND "createdById" = :userId AND'
|
||||
}
|
||||
${
|
||||
options.dateFilter ? '"updatedAt" > now() - interval :dateFilter AND' : ""
|
||||
}
|
||||
${
|
||||
options.collaboratorIds
|
||||
? '"collaboratorIds" @> ARRAY[:collaboratorIds]::uuid[] AND'
|
||||
: ""
|
||||
}
|
||||
${options.includeArchived ? "" : '"archivedAt" IS NULL AND'}
|
||||
"deletedAt" IS NULL AND
|
||||
${
|
||||
options.includeDrafts
|
||||
? '("publishedAt" IS NOT NULL OR "createdById" = :userId)'
|
||||
: '"publishedAt" IS NOT NULL'
|
||||
}
|
||||
`;
|
||||
const selectSql = `
|
||||
SELECT
|
||||
id,
|
||||
ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
|
||||
ts_headline('english', "text", to_tsquery('english', :query), :headlineOptions) as "searchContext"
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
ORDER BY
|
||||
"searchRanking" DESC,
|
||||
"updatedAt" DESC
|
||||
LIMIT :limit
|
||||
OFFSET :offset;
|
||||
`;
|
||||
const countSql = `
|
||||
SELECT COUNT(id)
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
`;
|
||||
const queryReplacements = {
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
collaboratorIds: options.collaboratorIds,
|
||||
query: wildcardQuery,
|
||||
collectionIds,
|
||||
dateFilter,
|
||||
headlineOptions: `MaxFragments=1, MinWords=${snippetMinWords}, MaxWords=${snippetMaxWords}`,
|
||||
};
|
||||
const resultsQuery = this.sequelize!.query(selectSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { ...queryReplacements, limit, offset },
|
||||
});
|
||||
const countQuery = this.sequelize!.query(countSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: queryReplacements,
|
||||
});
|
||||
const [results, [{ count }]]: [any, any] = await Promise.all([
|
||||
resultsQuery,
|
||||
countQuery,
|
||||
]);
|
||||
|
||||
// Final query to get associated document data
|
||||
const documents = await this.scope([
|
||||
"withoutState",
|
||||
"withDrafts",
|
||||
{
|
||||
method: ["withViews", user.id],
|
||||
},
|
||||
{
|
||||
method: ["withCollectionPermissions", user.id],
|
||||
},
|
||||
]).findAll({
|
||||
where: {
|
||||
teamId: user.teamId,
|
||||
id: map(results, "id"),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
results: map(results, (result: any) => ({
|
||||
ranking: result.searchRanking,
|
||||
context: removeMarkdown(unescape(result.searchContext), {
|
||||
stripHTML: false,
|
||||
}),
|
||||
document: find(documents, {
|
||||
id: result.id,
|
||||
}) as Document,
|
||||
})),
|
||||
totalCount: count,
|
||||
};
|
||||
}
|
||||
|
||||
// instance methods
|
||||
|
||||
migrateVersion = () => {
|
||||
@@ -1022,10 +745,4 @@ class Document extends ParanoidModel {
|
||||
};
|
||||
}
|
||||
|
||||
function escapeQuery(query: string): string {
|
||||
// replace "\" with escaped "\\" because sequelize.escape doesn't do it
|
||||
// https://github.com/sequelize/sequelize/issues/2950
|
||||
return Document.sequelize!.escape(query).replace(/\\/g, "\\\\");
|
||||
}
|
||||
|
||||
export default Document;
|
||||
|
||||
335
server/models/helpers/SearchHelper.test.ts
Normal file
335
server/models/helpers/SearchHelper.test.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import {
|
||||
buildDocument,
|
||||
buildDraftDocument,
|
||||
buildCollection,
|
||||
buildTeam,
|
||||
buildUser,
|
||||
buildShare,
|
||||
} from "@server/test/factories";
|
||||
import { setupTestDatabase } from "@server/test/support";
|
||||
|
||||
setupTestDatabase();
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
describe("#searchForTeam", () => {
|
||||
test("should return search results from public collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, "test");
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].document?.id).toBe(document.id);
|
||||
});
|
||||
|
||||
test("should not return results from private collections without providing collectionId", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
permission: null,
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, "test");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should return results from private collections when collectionId is provided", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
permission: null,
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForTeam(team, "test", {
|
||||
collectionId: collection.id,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should return results from document tree of shared document", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
permission: null,
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test 1",
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test 2",
|
||||
});
|
||||
|
||||
const share = await buildShare({
|
||||
documentId: document.id,
|
||||
includeChildDocuments: true,
|
||||
});
|
||||
|
||||
const { results } = await SearchHelper.searchForTeam(team, "test", {
|
||||
collectionId: collection.id,
|
||||
share,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const { results } = await SearchHelper.searchForTeam(team, "test");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should handle backslashes in search term", async () => {
|
||||
const team = await buildTeam();
|
||||
const { results } = await SearchHelper.searchForTeam(team, "\\\\");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should return the total count of search results", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 2",
|
||||
});
|
||||
const { totalCount } = await SearchHelper.searchForTeam(team, "test");
|
||||
expect(totalCount).toBe("2");
|
||||
});
|
||||
|
||||
test("should return the document when searched with their previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await SearchHelper.searchForTeam(
|
||||
team,
|
||||
"test number"
|
||||
);
|
||||
expect(totalCount).toBe("1");
|
||||
});
|
||||
|
||||
test("should not return the document when searched with neither the titles nor the previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await SearchHelper.searchForTeam(
|
||||
team,
|
||||
"title doesn't exist"
|
||||
);
|
||||
expect(totalCount).toBe("0");
|
||||
});
|
||||
});
|
||||
|
||||
describe("#searchForUser", () => {
|
||||
test("should return search results from collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, "test");
|
||||
expect(results.length).toBe(1);
|
||||
expect(results[0].document?.id).toBe(document.id);
|
||||
});
|
||||
|
||||
test("should handle no collections", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, "test");
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should search only drafts created by user", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, "test", {
|
||||
includeDrafts: true,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should not include drafts", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "test",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, "test", {
|
||||
includeDrafts: false,
|
||||
});
|
||||
expect(results.length).toBe(0);
|
||||
});
|
||||
|
||||
test("should include results from drafts as well", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: user.teamId,
|
||||
createdById: user.id,
|
||||
title: "not draft",
|
||||
});
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "draft",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, "draft", {
|
||||
includeDrafts: true,
|
||||
});
|
||||
expect(results.length).toBe(2);
|
||||
});
|
||||
|
||||
test("should not include results from drafts", async () => {
|
||||
const user = await buildUser();
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: user.teamId,
|
||||
createdById: user.id,
|
||||
title: "not draft",
|
||||
});
|
||||
await buildDraftDocument({
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
createdById: user.id,
|
||||
title: "draft",
|
||||
});
|
||||
const { results } = await SearchHelper.searchForUser(user, "draft", {
|
||||
includeDrafts: false,
|
||||
});
|
||||
expect(results.length).toBe(1);
|
||||
});
|
||||
|
||||
test("should return the total count of search results", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
});
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
await buildDocument({
|
||||
userId: user.id,
|
||||
teamId: team.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 2",
|
||||
});
|
||||
const { totalCount } = await SearchHelper.searchForUser(user, "test");
|
||||
expect(totalCount).toBe("2");
|
||||
});
|
||||
|
||||
test("should return the document when searched with their previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await SearchHelper.searchForUser(
|
||||
user,
|
||||
"test number"
|
||||
);
|
||||
expect(totalCount).toBe("1");
|
||||
});
|
||||
|
||||
test("should not return the document when searched with neither the titles nor the previous titles", async () => {
|
||||
const team = await buildTeam();
|
||||
const user = await buildUser({
|
||||
teamId: team.id,
|
||||
});
|
||||
const collection = await buildCollection({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
});
|
||||
const document = await buildDocument({
|
||||
teamId: team.id,
|
||||
userId: user.id,
|
||||
collectionId: collection.id,
|
||||
title: "test number 1",
|
||||
});
|
||||
document.title = "change";
|
||||
await document.save();
|
||||
const { totalCount } = await SearchHelper.searchForUser(
|
||||
user,
|
||||
"title doesn't exist"
|
||||
);
|
||||
expect(totalCount).toBe("0");
|
||||
});
|
||||
});
|
||||
310
server/models/helpers/SearchHelper.ts
Normal file
310
server/models/helpers/SearchHelper.ts
Normal file
@@ -0,0 +1,310 @@
|
||||
import removeMarkdown from "@tommoor/remove-markdown";
|
||||
import invariant from "invariant";
|
||||
import { find, map } from "lodash";
|
||||
import { Op, QueryTypes } from "sequelize";
|
||||
import { DateFilter } from "@shared/types";
|
||||
import unescape from "@shared/utils/unescape";
|
||||
import { sequelize } from "@server/database/sequelize";
|
||||
import Collection from "@server/models/Collection";
|
||||
import Document from "@server/models/Document";
|
||||
import Share from "@server/models/Share";
|
||||
import Team from "@server/models/Team";
|
||||
import User from "@server/models/User";
|
||||
|
||||
type SearchResponse = {
|
||||
results: {
|
||||
/** The search ranking, for sorting results */
|
||||
ranking: number;
|
||||
/** A snippet of contextual text around the search result */
|
||||
context: string;
|
||||
/** The document result */
|
||||
document: Document;
|
||||
}[];
|
||||
/** The total number of results for the search query without pagination */
|
||||
totalCount: number;
|
||||
};
|
||||
|
||||
type SearchOptions = {
|
||||
/** The query limit for pagination */
|
||||
limit?: number;
|
||||
/** The query offset for pagination */
|
||||
offset?: number;
|
||||
/** Limit results to a collection. Authorization is presumed to have been done before passing to this helper. */
|
||||
collectionId?: string;
|
||||
/** Limit results to a shared document. */
|
||||
share?: Share;
|
||||
/** Limit results to a date range. */
|
||||
dateFilter?: DateFilter;
|
||||
/** Limit results to a list of users that collaborated on the document. */
|
||||
collaboratorIds?: string[];
|
||||
/** Include archived documents in the results */
|
||||
includeArchived?: boolean;
|
||||
/** Include draft documents in the results (will only ever return your own) */
|
||||
includeDrafts?: boolean;
|
||||
/** The minimum number of words to be returned in the contextual snippet */
|
||||
snippetMinWords?: number;
|
||||
/** The maximum number of words to be returned in the contextual snippet */
|
||||
snippetMaxWords?: number;
|
||||
};
|
||||
|
||||
type Results = {
|
||||
searchRanking: number;
|
||||
searchContext: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
export default class SearchHelper {
|
||||
public static async searchForTeam(
|
||||
team: Team,
|
||||
query: string,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResponse> {
|
||||
const wildcardQuery = `${this.escapeQuery(query)}:*`;
|
||||
const {
|
||||
snippetMinWords = 20,
|
||||
snippetMaxWords = 30,
|
||||
limit = 15,
|
||||
offset = 0,
|
||||
} = options;
|
||||
|
||||
// restrict to specific collection if provided
|
||||
// enables search in private collections if specified
|
||||
let collectionIds: string[];
|
||||
if (options.collectionId) {
|
||||
collectionIds = [options.collectionId];
|
||||
} else {
|
||||
collectionIds = await team.collectionIds();
|
||||
}
|
||||
|
||||
// short circuit if no relevant collections
|
||||
if (!collectionIds.length) {
|
||||
return {
|
||||
results: [],
|
||||
totalCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// restrict to documents in the tree of a shared document when one is provided
|
||||
let documentIds: string[] | undefined;
|
||||
|
||||
if (options.share?.includeChildDocuments) {
|
||||
const sharedDocument = await options.share.$get("document");
|
||||
invariant(sharedDocument, "Cannot find document for share");
|
||||
|
||||
const childDocumentIds = await sharedDocument.getChildDocumentIds({
|
||||
archivedAt: {
|
||||
[Op.is]: null,
|
||||
},
|
||||
});
|
||||
documentIds = [sharedDocument.id, ...childDocumentIds];
|
||||
}
|
||||
|
||||
const documentClause = documentIds ? `"id" IN(:documentIds) AND` : "";
|
||||
|
||||
// Build the SQL query to get result documentIds, ranking, and search term context
|
||||
const whereClause = `
|
||||
"searchVector" @@ to_tsquery('english', :query) AND
|
||||
"teamId" = :teamId AND
|
||||
"collectionId" IN(:collectionIds) AND
|
||||
${documentClause}
|
||||
"deletedAt" IS NULL AND
|
||||
"publishedAt" IS NOT NULL
|
||||
`;
|
||||
const selectSql = `
|
||||
SELECT
|
||||
id,
|
||||
ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
|
||||
ts_headline('english', "text", to_tsquery('english', :query), :headlineOptions) as "searchContext"
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
ORDER BY
|
||||
"searchRanking" DESC,
|
||||
"updatedAt" DESC
|
||||
LIMIT :limit
|
||||
OFFSET :offset;
|
||||
`;
|
||||
const countSql = `
|
||||
SELECT COUNT(id)
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
`;
|
||||
const queryReplacements = {
|
||||
teamId: team.id,
|
||||
query: wildcardQuery,
|
||||
collectionIds,
|
||||
documentIds,
|
||||
headlineOptions: `MaxFragments=1, MinWords=${snippetMinWords}, MaxWords=${snippetMaxWords}`,
|
||||
};
|
||||
const resultsQuery = sequelize.query<Results>(selectSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { ...queryReplacements, limit, offset },
|
||||
});
|
||||
const countQuery = sequelize.query<{ count: number }>(countSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: queryReplacements,
|
||||
});
|
||||
const [results, [{ count }]] = await Promise.all([
|
||||
resultsQuery,
|
||||
countQuery,
|
||||
]);
|
||||
|
||||
// Final query to get associated document data
|
||||
const documents = await Document.findAll({
|
||||
where: {
|
||||
id: map(results, "id"),
|
||||
teamId: team.id,
|
||||
},
|
||||
include: [
|
||||
{
|
||||
model: Collection,
|
||||
as: "collection",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return SearchHelper.buildResponse(results, documents, count);
|
||||
}
|
||||
|
||||
public static async searchForUser(
|
||||
user: User,
|
||||
query: string,
|
||||
options: SearchOptions = {}
|
||||
): Promise<SearchResponse> {
|
||||
const {
|
||||
snippetMinWords = 20,
|
||||
snippetMaxWords = 30,
|
||||
limit = 15,
|
||||
offset = 0,
|
||||
} = options;
|
||||
const wildcardQuery = `${SearchHelper.escapeQuery(query)}:*`;
|
||||
|
||||
// Ensure we're filtering by the users accessible collections. If
|
||||
// collectionId is passed as an option it is assumed that the authorization
|
||||
// has already been done in the router
|
||||
let collectionIds;
|
||||
|
||||
if (options.collectionId) {
|
||||
collectionIds = [options.collectionId];
|
||||
} else {
|
||||
collectionIds = await user.collectionIds();
|
||||
}
|
||||
|
||||
let dateFilter;
|
||||
|
||||
if (options.dateFilter) {
|
||||
dateFilter = `1 ${options.dateFilter}`;
|
||||
}
|
||||
|
||||
// Build the SQL query to get documentIds, ranking, and search term context
|
||||
const whereClause = `
|
||||
"searchVector" @@ to_tsquery('english', :query) AND
|
||||
"teamId" = :teamId AND
|
||||
${
|
||||
collectionIds.length
|
||||
? `(
|
||||
"collectionId" IN(:collectionIds) OR
|
||||
("collectionId" IS NULL AND "createdById" = :userId)
|
||||
) AND`
|
||||
: '"collectionId" IS NULL AND "createdById" = :userId AND'
|
||||
}
|
||||
${
|
||||
options.dateFilter ? '"updatedAt" > now() - interval :dateFilter AND' : ""
|
||||
}
|
||||
${
|
||||
options.collaboratorIds
|
||||
? '"collaboratorIds" @> ARRAY[:collaboratorIds]::uuid[] AND'
|
||||
: ""
|
||||
}
|
||||
${options.includeArchived ? "" : '"archivedAt" IS NULL AND'}
|
||||
"deletedAt" IS NULL AND
|
||||
${
|
||||
options.includeDrafts
|
||||
? '("publishedAt" IS NOT NULL OR "createdById" = :userId)'
|
||||
: '"publishedAt" IS NOT NULL'
|
||||
}
|
||||
`;
|
||||
const selectSql = `
|
||||
SELECT
|
||||
id,
|
||||
ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
|
||||
ts_headline('english', "text", to_tsquery('english', :query), :headlineOptions) as "searchContext"
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
ORDER BY
|
||||
"searchRanking" DESC,
|
||||
"updatedAt" DESC
|
||||
LIMIT :limit
|
||||
OFFSET :offset;
|
||||
`;
|
||||
const countSql = `
|
||||
SELECT COUNT(id)
|
||||
FROM documents
|
||||
WHERE ${whereClause}
|
||||
`;
|
||||
const queryReplacements = {
|
||||
teamId: user.teamId,
|
||||
userId: user.id,
|
||||
collaboratorIds: options.collaboratorIds,
|
||||
query: wildcardQuery,
|
||||
collectionIds,
|
||||
dateFilter,
|
||||
headlineOptions: `MaxFragments=1, MinWords=${snippetMinWords}, MaxWords=${snippetMaxWords}`,
|
||||
};
|
||||
const resultsQuery = sequelize.query<Results>(selectSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: { ...queryReplacements, limit, offset },
|
||||
});
|
||||
const countQuery = sequelize.query<{ count: number }>(countSql, {
|
||||
type: QueryTypes.SELECT,
|
||||
replacements: queryReplacements,
|
||||
});
|
||||
const [results, [{ count }]] = await Promise.all([
|
||||
resultsQuery,
|
||||
countQuery,
|
||||
]);
|
||||
|
||||
// Final query to get associated document data
|
||||
const documents = await Document.scope([
|
||||
"withoutState",
|
||||
"withDrafts",
|
||||
{
|
||||
method: ["withViews", user.id],
|
||||
},
|
||||
{
|
||||
method: ["withCollectionPermissions", user.id],
|
||||
},
|
||||
]).findAll({
|
||||
where: {
|
||||
teamId: user.teamId,
|
||||
id: map(results, "id"),
|
||||
},
|
||||
});
|
||||
|
||||
return SearchHelper.buildResponse(results, documents, count);
|
||||
}
|
||||
|
||||
private static buildResponse(
|
||||
results: Results[],
|
||||
documents: Document[],
|
||||
count: number
|
||||
): SearchResponse {
|
||||
return {
|
||||
results: map(results, (result) => ({
|
||||
ranking: result.searchRanking,
|
||||
context: removeMarkdown(unescape(result.searchContext), {
|
||||
stripHTML: false,
|
||||
}),
|
||||
document: find(documents, {
|
||||
id: result.id,
|
||||
}) as Document,
|
||||
})),
|
||||
totalCount: count,
|
||||
};
|
||||
}
|
||||
|
||||
private static escapeQuery(query: string): string {
|
||||
// replace "\" with escaped "\\" because sequelize.escape doesn't do it
|
||||
// https://github.com/sequelize/sequelize/issues/2950
|
||||
return sequelize.escape(query).replace(/\\/g, "\\\\");
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
View,
|
||||
} from "@server/models";
|
||||
import DocumentHelper from "@server/models/helpers/DocumentHelper";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import { authorize, cannot } from "@server/policies";
|
||||
import {
|
||||
presentCollection,
|
||||
@@ -701,7 +702,7 @@ router.post(
|
||||
const team = await share.$get("team");
|
||||
invariant(team, "Share must belong to a team");
|
||||
|
||||
response = await Document.searchForTeam(team, query, {
|
||||
response = await SearchHelper.searchForTeam(team, query, {
|
||||
includeArchived,
|
||||
includeDrafts,
|
||||
collectionId: document.collectionId,
|
||||
@@ -742,7 +743,7 @@ router.post(
|
||||
);
|
||||
}
|
||||
|
||||
response = await Document.searchForUser(user, query, {
|
||||
response = await SearchHelper.searchForUser(user, query, {
|
||||
includeArchived,
|
||||
includeDrafts,
|
||||
collaboratorIds,
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
Integration,
|
||||
IntegrationAuthentication,
|
||||
} from "@server/models";
|
||||
import SearchHelper from "@server/models/helpers/SearchHelper";
|
||||
import { presentSlackAttachment } from "@server/presenters";
|
||||
import * as Slack from "@server/utils/slack";
|
||||
import { assertPresent } from "@server/validation";
|
||||
@@ -281,8 +282,8 @@ router.post("hooks.slack", async (ctx) => {
|
||||
// to load more documents based on the collections they have access to. Otherwise
|
||||
// just a generic search against team-visible documents is allowed.
|
||||
const { results, totalCount } = user
|
||||
? await Document.searchForUser(user, text, options)
|
||||
: await Document.searchForTeam(team, text, options);
|
||||
? await SearchHelper.searchForUser(user, text, options)
|
||||
: await SearchHelper.searchForTeam(team, text, options);
|
||||
SearchQuery.create({
|
||||
userId: user ? user.id : null,
|
||||
teamId: team.id,
|
||||
|
||||
Reference in New Issue
Block a user