feat: Record search queries (#1554)

* Record search queries * feat: add totalCount to the search response * feat: add comments to explain why we use setTimeout
2020-09-22 03:05:42 -03:00
parent 0fa8a6ed2e
commit 98626ebbaf
9 changed files with 330 additions and 54 deletions
--- a/server/models/Document.js
+++ b/server/models/Document.js
@@ -251,10 +251,13 @@ Document.findByPk = async function (id, options = {}) {
  }
 };

-type SearchResult = {
-  ranking: number,
-  context: string,
-  document: Document,
+type SearchResponse = {
+  results: {
+    ranking: number,
+    context: string,
+    document: Document,
+  }[],
+  totalCount: number,
 };

 type SearchOptions = {
@@ -277,7 +280,7 @@ Document.searchForTeam = async (
  team,
  query,
  options: SearchOptions = {}
-): Promise<SearchResult[]> => {
+): Promise<SearchResponse> => {
  const limit = options.limit || 15;
  const offset = options.offset || 0;
  const wildcardQuery = `${escape(query)}:*`;
@@ -285,21 +288,25 @@ Document.searchForTeam = async (

  // If the team has access no public collections then shortcircuit the rest of this
  if (!collectionIds.length) {
-    return [];
+    return { results: [], totalCount: 0 };
  }

  // Build the SQL query to get documentIds, ranking, and search term context
-  const sql = `
+  const whereClause = `
+  "searchVector" @@ to_tsquery('english', :query) AND
+    "teamId" = :teamId AND
+    "collectionId" IN(:collectionIds) AND
+    "deletedAt" IS NULL AND
+    "publishedAt" IS NOT NULL
+  `;
+
+  const selectSql = `
    SELECT
      id,
      ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
      ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
    FROM documents
-    WHERE "searchVector" @@ to_tsquery('english', :query) AND
-      "teamId" = :teamId AND
-      "collectionId" IN(:collectionIds) AND
-      "deletedAt" IS NULL AND
-      "publishedAt" IS NOT NULL
+    WHERE ${whereClause}
    ORDER BY
      "searchRanking" DESC,
      "updatedAt" DESC
@@ -307,17 +314,34 @@ Document.searchForTeam = async (
    OFFSET :offset;
  `;

-  const results = await sequelize.query(sql, {
+  const countSql = `
+    SELECT COUNT(id)
+    FROM documents
+    WHERE ${whereClause}
+  `;
+
+  const queryReplacements = {
+    teamId: team.id,
+    query: wildcardQuery,
+    collectionIds,
+  };
+
+  const resultsQuery = sequelize.query(selectSql, {
    type: sequelize.QueryTypes.SELECT,
    replacements: {
-      teamId: team.id,
-      query: wildcardQuery,
+      ...queryReplacements,
      limit,
      offset,
-      collectionIds,
    },
  });

+  const countQuery = sequelize.query(countSql, {
+    type: sequelize.QueryTypes.SELECT,
+    replacements: queryReplacements,
+  });
+
+  const [results, [{ count }]] = await Promise.all([resultsQuery, countQuery]);
+
  // Final query to get associated document data
  const documents = await Document.findAll({
    where: {
@@ -330,20 +354,23 @@ Document.searchForTeam = async (
    ],
  });

-  return map(results, (result) => ({
-    ranking: result.searchRanking,
-    context: removeMarkdown(unescape(result.searchContext), {
-      stripHTML: false,
-    }),
-    document: find(documents, { id: result.id }),
-  }));
+  return {
+    results: map(results, (result) => ({
+      ranking: result.searchRanking,
+      context: removeMarkdown(unescape(result.searchContext), {
+        stripHTML: false,
+      }),
+      document: find(documents, { id: result.id }),
+    })),
+    totalCount: count,
+  };
 };

 Document.searchForUser = async (
  user,
  query,
  options: SearchOptions = {}
-): Promise<SearchResult[]> => {
+): Promise<SearchResponse> => {
  const limit = options.limit || 15;
  const offset = options.offset || 0;
  const wildcardQuery = `${escape(query)}:*`;
@@ -360,7 +387,7 @@ Document.searchForUser = async (

  // If the user has access to no collections then shortcircuit the rest of this
  if (!collectionIds.length) {
-    return [];
+    return { results: [], totalCount: 0 };
  }

  let dateFilter;
@@ -369,13 +396,8 @@ Document.searchForUser = async (
  }

  // Build the SQL query to get documentIds, ranking, and search term context
-  const sql = `
-  SELECT
-    id,
-    ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
-    ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
-  FROM documents
-  WHERE "searchVector" @@ to_tsquery('english', :query) AND
+  const whereClause = `
+  "searchVector" @@ to_tsquery('english', :query) AND
    "teamId" = :teamId AND
    "collectionId" IN(:collectionIds) AND
    ${
@@ -393,27 +415,52 @@ Document.searchForUser = async (
        ? '("publishedAt" IS NOT NULL OR "createdById" = :userId)'
        : '"publishedAt" IS NOT NULL'
    }
+  `;
+
+  const selectSql = `
+  SELECT
+    id,
+    ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking",
+    ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext"
+  FROM documents
+  WHERE ${whereClause}
  ORDER BY
    "searchRanking" DESC,
    "updatedAt" DESC
  LIMIT :limit
  OFFSET :offset;
-`;
+  `;

-  const results = await sequelize.query(sql, {
+  const countSql = `
+    SELECT COUNT(id)
+    FROM documents
+    WHERE ${whereClause}
+  `;
+
+  const queryReplacements = {
+    teamId: user.teamId,
+    userId: user.id,
+    collaboratorIds: options.collaboratorIds,
+    query: wildcardQuery,
+    collectionIds,
+    dateFilter,
+  };
+
+  const resultsQuery = sequelize.query(selectSql, {
    type: sequelize.QueryTypes.SELECT,
    replacements: {
-      teamId: user.teamId,
-      userId: user.id,
-      collaboratorIds: options.collaboratorIds,
-      query: wildcardQuery,
+      ...queryReplacements,
      limit,
      offset,
-      collectionIds,
-      dateFilter,
    },
  });

+  const countQuery = sequelize.query(countSql, {
+    type: sequelize.QueryTypes.SELECT,
+    replacements: queryReplacements,
+  });
+
+  const [results, [{ count }]] = await Promise.all([resultsQuery, countQuery]);
  // Final query to get associated document data
  const documents = await Document.scope(
    {
@@ -432,13 +479,16 @@ Document.searchForUser = async (
    ],
  });

-  return map(results, (result) => ({
-    ranking: result.searchRanking,
-    context: removeMarkdown(unescape(result.searchContext), {
-      stripHTML: false,
-    }),
-    document: find(documents, { id: result.id }),
-  }));
+  return {
+    results: map(results, (result) => ({
+      ranking: result.searchRanking,
+      context: removeMarkdown(unescape(result.searchContext), {
+        stripHTML: false,
+      }),
+      document: find(documents, { id: result.id }),
+    })),
+    totalCount: count,
+  };
 };

 // Hooks
--- a/server/models/Document.test.js
+++ b/server/models/Document.test.js
@@ -201,7 +201,7 @@ describe("#searchForTeam", () => {
      title: "test",
    });

-    const results = await Document.searchForTeam(team, "test");
+    const { results } = await Document.searchForTeam(team, "test");
    expect(results.length).toBe(1);
    expect(results[0].document.id).toBe(document.id);
  });
@@ -218,15 +218,85 @@ describe("#searchForTeam", () => {
      title: "test",
    });

-    const results = await Document.searchForTeam(team, "test");
+    const { results } = await Document.searchForTeam(team, "test");
    expect(results.length).toBe(0);
  });

  test("should handle no collections", async () => {
    const team = await buildTeam();
-    const results = await Document.searchForTeam(team, "test");
+    const { results } = await Document.searchForTeam(team, "test");
    expect(results.length).toBe(0);
  });
+
+  test("should return the total count of search results", async () => {
+    const team = await buildTeam();
+    const collection = await buildCollection({ teamId: team.id });
+    await buildDocument({
+      teamId: team.id,
+      collectionId: collection.id,
+      title: "test number 1",
+    });
+    await buildDocument({
+      teamId: team.id,
+      collectionId: collection.id,
+      title: "test number 2",
+    });
+
+    const { totalCount } = await Document.searchForTeam(team, "test");
+    expect(totalCount).toBe("2");
+  });
+});
+
+describe("#searchForUser", () => {
+  test("should return search results from collections", async () => {
+    const team = await buildTeam();
+    const user = await buildUser({ teamId: team.id });
+    const collection = await buildCollection({
+      userId: user.id,
+      teamId: team.id,
+    });
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: team.id,
+      collectionId: collection.id,
+      title: "test",
+    });
+
+    const { results } = await Document.searchForUser(user, "test");
+    expect(results.length).toBe(1);
+    expect(results[0].document.id).toBe(document.id);
+  });
+
+  test("should handle no collections", async () => {
+    const team = await buildTeam();
+    const user = await buildUser({ teamId: team.id });
+    const { results } = await Document.searchForUser(user, "test");
+    expect(results.length).toBe(0);
+  });
+
+  test("should return the total count of search results", async () => {
+    const team = await buildTeam();
+    const user = await buildUser({ teamId: team.id });
+    const collection = await buildCollection({
+      userId: user.id,
+      teamId: team.id,
+    });
+    await buildDocument({
+      userId: user.id,
+      teamId: team.id,
+      collectionId: collection.id,
+      title: "test number 1",
+    });
+    await buildDocument({
+      userId: user.id,
+      teamId: team.id,
+      collectionId: collection.id,
+      title: "test number 2",
+    });
+
+    const { totalCount } = await Document.searchForUser(user, "test");
+    expect(totalCount).toBe("2");
+  });
 });

 describe("#delete", () => {
--- a/server/models/SearchQuery.js
+++ b/server/models/SearchQuery.js
@@ -0,0 +1,42 @@
+// @flow
+import { DataTypes, sequelize } from "../sequelize";
+
+const SearchQuery = sequelize.define(
+  "search_queries",
+  {
+    id: {
+      type: DataTypes.UUID,
+      defaultValue: DataTypes.UUIDV4,
+      primaryKey: true,
+    },
+    source: {
+      type: DataTypes.ENUM("slack", "app"),
+      allowNull: false,
+    },
+    query: {
+      type: DataTypes.STRING,
+      allowNull: false,
+    },
+    results: {
+      type: DataTypes.NUMBER,
+      allowNull: false,
+    },
+  },
+  {
+    timestamps: true,
+    updatedAt: false,
+  }
+);
+
+SearchQuery.associate = (models) => {
+  SearchQuery.belongsTo(models.User, {
+    as: "user",
+    foreignKey: "userId",
+  });
+  SearchQuery.belongsTo(models.Team, {
+    as: "team",
+    foreignKey: "teamId",
+  });
+};
+
+export default SearchQuery;
--- a/server/models/index.js
+++ b/server/models/index.js
@@ -14,6 +14,7 @@ import Integration from "./Integration";
 import Notification from "./Notification";
 import NotificationSetting from "./NotificationSetting";
 import Revision from "./Revision";
+import SearchQuery from "./SearchQuery";
 import Share from "./Share";
 import Star from "./Star";
 import Team from "./Team";
@@ -36,6 +37,7 @@ const models = {
  Notification,
  NotificationSetting,
  Revision,
+  SearchQuery,
  Share,
  Star,
  Team,
@@ -66,6 +68,7 @@ export {
  Notification,
  NotificationSetting,
  Revision,
+  SearchQuery,
  Share,
  Star,
  Team,