Improved search filtering (#940)

* Filter search by collectionId

* Improve spec, remove recursive import

* Add userId filter for documents.search

* 💚

* Search filter UI

* WIP UI

* Date filtering
Prevent dupe menu

* Refactor

* button

* Added year option, improved hover states

* Add new indexes

* Remove manual string interpolation in SQL construction

* Move dateFilter validation to controller

* Fixes: Double query when changing filter
Fixes: Visual jump between filters in dropdown

* Add option to clear filters

* More clearly define dropdowns in dark mode

* Checkbox -> Checkmark

server/api/documents.js

@@ -378,13 +378,37 @@ router.post('documents.restore', auth(), async ctx => {
 });
 
 router.post('documents.search', auth(), pagination(), async ctx => {
-  const { query, includeArchived } = ctx.body;
+  const { query, includeArchived, collectionId, userId, dateFilter } = ctx.body;
   const { offset, limit } = ctx.state.pagination;
+  const user = ctx.state.user;
   ctx.assertPresent(query, 'query is required');
 
-  const user = ctx.state.user;
+  if (collectionId) {
+    ctx.assertUuid(collectionId, 'collectionId must be a UUID');
+
+    const collection = await Collection.findById(collectionId);
+    authorize(user, 'read', collection);
+  }
+
+  let collaboratorIds = undefined;
+  if (userId) {
+    ctx.assertUuid(userId, 'userId must be a UUID');
+    collaboratorIds = [userId];
+  }
+
+  if (dateFilter) {
+    ctx.assertIn(
+      dateFilter,
+      ['day', 'week', 'month', 'year'],
+      'dateFilter must be one of day,week,month,year'
+    );
+  }
+
   const results = await Document.searchForUser(user, query, {
     includeArchived: includeArchived === 'true',
+    collaboratorIds,
+    collectionId,
+    dateFilter,
     offset,
     limit,
   });

server/api/documents.test.js

@@ -410,7 +410,7 @@ describe('#documents.search', async () => {
 
   it('should return draft documents created by user', async () => {
     const { user } = await seed();
-    await buildDocument({
+    const document = await buildDocument({
       title: 'search term',
       text: 'search term',
       publishedAt: null,
@@ -424,7 +424,7 @@ describe('#documents.search', async () => {
 
     expect(res.status).toEqual(200);
     expect(body.data.length).toEqual(1);
-    expect(body.data[0].document.text).toEqual('search term');
+    expect(body.data[0].document.id).toEqual(document.id);
   });
 
   it('should not return draft documents created by other users', async () => {
@@ -482,7 +482,70 @@ describe('#documents.search', async () => {
 
     expect(res.status).toEqual(200);
     expect(body.data.length).toEqual(1);
-    expect(body.data[0].document.text).toEqual('search term');
+    expect(body.data[0].document.id).toEqual(document.id);
+  });
+
+  it('should return documents for a specific user', async () => {
+    const { user } = await seed();
+
+    const document = await buildDocument({
+      title: 'search term',
+      text: 'search term',
+      teamId: user.teamId,
+      userId: user.id,
+    });
+
+    // This one will be filtered out
+    await buildDocument({
+      title: 'search term',
+      text: 'search term',
+      teamId: user.teamId,
+    });
+
+    const res = await server.post('/api/documents.search', {
+      body: {
+        token: user.getJwtToken(),
+        query: 'search term',
+        userId: user.id,
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toEqual(1);
+    expect(body.data[0].document.id).toEqual(document.id);
+  });
+
+  it('should return documents for a specific collection', async () => {
+    const { user } = await seed();
+    const collection = await buildCollection();
+
+    const document = await buildDocument({
+      title: 'search term',
+      text: 'search term',
+      teamId: user.teamId,
+    });
+
+    // This one will be filtered out
+    await buildDocument({
+      title: 'search term',
+      text: 'search term',
+      teamId: user.teamId,
+      collectionId: collection.id,
+    });
+
+    const res = await server.post('/api/documents.search', {
+      body: {
+        token: user.getJwtToken(),
+        query: 'search term',
+        collectionId: document.collectionId,
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toEqual(1);
+    expect(body.data[0].document.id).toEqual(document.id);
   });
 
   it('should not return documents in private collections not a member of', async () => {
@@ -505,6 +568,20 @@ describe('#documents.search', async () => {
     expect(body.data.length).toEqual(0);
   });
 
+  it('should not allow unknown dateFilter values', async () => {
+    const { user } = await seed();
+
+    const res = await server.post('/api/documents.search', {
+      body: {
+        token: user.getJwtToken(),
+        query: 'search term',
+        dateFilter: 'DROP TABLE students;',
+      },
+    });
+
+    expect(res.status).toEqual(400);
+  });
+
   it('should require authentication', async () => {
     const res = await server.post('/api/documents.search');
     const body = await res.json();

server/middlewares/validation.js

@@ -12,6 +12,12 @@ export default function validation() {
       }
     };
 
+    ctx.assertIn = (value, options, message) => {
+      if (!options.includes(value)) {
+        throw new ValidationError(message);
+      }
+    };
+
     ctx.assertNotEmpty = (value, message) => {
       if (value === '') {
         throw new ValidationError(message);

server/migrations/20190423051708-add-search-indexes.js

@@ -0,0 +1,13 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addIndex('documents', ['updatedAt']);
+    await queryInterface.addIndex('documents', ['archivedAt']);
+    await queryInterface.addIndex('documents', ['collaboratorIds']);
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    await queryInterface.removeIndex('documents', ['updatedAt']);
+    await queryInterface.removeIndex('documents', ['archivedAt']);
+    await queryInterface.removeIndex('documents', ['collaboratorIds']);
+  },
+};

server/models/Document.js

@@ -215,32 +215,60 @@ Document.searchForUser = async (
   const offset = options.offset || 0;
   const wildcardQuery = `${sequelize.escape(query)}:*`;
 
-  const sql = `
-    SELECT
-      id,
-      ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
-      ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
-    FROM documents
-    WHERE "searchVector" @@ to_tsquery('english', :query) AND
-      "collectionId" IN(:collectionIds) AND
-      ${options.includeArchived ? '' : '"archivedAt" IS NULL AND'}
-      "deletedAt" IS NULL AND
-      ("publishedAt" IS NOT NULL OR "createdById" = '${user.id}')
-    ORDER BY 
-      "searchRanking" DESC,
-      "updatedAt" DESC
-    LIMIT :limit
-    OFFSET :offset;
-  `;
+  // Ensure we're filtering by the users accessible collections. If
+  // collectionId is passed as an option it is assumed that the authorization
+  // has already been done in the router
+  let collectionIds;
+  if (options.collectionId) {
+    collectionIds = [options.collectionId];
+  } else {
+    collectionIds = await user.collectionIds();
+  }
+
+  let dateFilter;
+  if (options.dateFilter) {
+    dateFilter = `1 ${options.dateFilter}`;
+  }
+
+  // Build the SQL query to get documentIds, ranking, and search term context
+  const sql = `
+  SELECT
+    id,
+    ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
+    ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
+  FROM documents
+  WHERE "searchVector" @@ to_tsquery('english', :query) AND
+    "teamId" = :teamId AND
+    "collectionId" IN(:collectionIds) AND
+    ${
+      options.dateFilter ? '"updatedAt" > now() - interval :dateFilter AND' : ''
+    }
+    ${
+      options.collaboratorIds
+        ? '"collaboratorIds" @> ARRAY[:collaboratorIds]::uuid[] AND'
+        : ''
+    }
+    ${options.includeArchived ? '' : '"archivedAt" IS NULL AND'}
+    "deletedAt" IS NULL AND
+    ("publishedAt" IS NOT NULL OR "createdById" = :userId)
+  ORDER BY 
+    "searchRanking" DESC,
+    "updatedAt" DESC
+  LIMIT :limit
+  OFFSET :offset;
+`;
 
-  const collectionIds = await user.collectionIds();
   const results = await sequelize.query(sql, {
     type: sequelize.QueryTypes.SELECT,
     replacements: {
+      teamId: user.teamId,
+      userId: user.id,
+      collaboratorIds: options.collaboratorIds,
       query: wildcardQuery,
       limit,
       offset,
       collectionIds,
+      dateFilter,
     },
   });
 

server/pages/developers/Api.js

@@ -256,7 +256,13 @@ export default function Api() {
             </Description>
             <Arguments>
               <Argument id="query" description="Search query" required />
+              <Argument id="userId" description="User ID" />
+              <Argument id="collectionId" description="Collection ID" />
               <Argument id="includeArchived" description="Boolean" />
+              <Argument
+                id="dateFilter"
+                description="Date range to consider (day, week, month or year)"
+              />
             </Arguments>
           </Method>