Improves ordering of search results

Modifies documents.search to return a context snippet and search ranking
Displays context snipped on search results screen

app/components/DocumentPreview/DocumentPreview.js

@@ -13,6 +13,7 @@ import DocumentMenu from 'menus/DocumentMenu';
 type Props = {
   document: Document,
   highlight?: ?string,
+  context?: ?string,
   showCollection?: boolean,
   innerRef?: *,
 };
@@ -96,6 +97,15 @@ const Title = styled(Highlight)`
   text-overflow: ellipsis;
 `;
 
+const ResultContext = styled(Highlight)`
+  color: ${props => props.theme.slateDark};
+  font-size: 14px;
+  margin-top: 0;
+  margin-bottom: 0.25em;
+`;
+
+const SEARCH_RESULT_REGEX = /<b\b[^>]*>(.*?)<\/b>/gi;
+
 @observer
 class DocumentPreview extends React.Component<Props> {
   star = (ev: SyntheticEvent<*>) => {
@@ -110,15 +120,26 @@ class DocumentPreview extends React.Component<Props> {
     this.props.document.unstar();
   };
 
+  replaceResultMarks = (tag: string) => {
+    // don't use SEARCH_RESULT_REGEX here as it causes
+    // an infinite loop to trigger a regex inside it's own callback
+    return tag.replace(/<b\b[^>]*>(.*?)<\/b>/gi, '$1');
+  };
+
   render() {
     const {
       document,
       showCollection,
       innerRef,
       highlight,
+      context,
       ...rest
     } = this.props;
 
+    const queryIsInTitle =
+      !!highlight &&
+      !!document.title.toLowerCase().match(highlight.toLowerCase());
+
     return (
       <DocumentLink
         to={{
@@ -141,6 +162,13 @@ class DocumentPreview extends React.Component<Props> {
           )}
           <StyledDocumentMenu document={document} />
         </Heading>
+        {!queryIsInTitle && (
+          <ResultContext
+            text={context}
+            highlight={highlight ? SEARCH_RESULT_REGEX : undefined}
+            processResult={this.replaceResultMarks}
+          />
+        )}
         <PublishingInfo
           document={document}
           collection={showCollection ? document.collection : undefined}

app/components/Highlight/Highlight.js

@@ -4,23 +4,34 @@ import replace from 'string-replace-to-array';
 import styled from 'styled-components';
 
 type Props = {
-  highlight: ?string,
+  highlight: ?string | RegExp,
+  processResult?: (tag: string) => string,
   text: string,
   caseSensitive?: boolean,
 };
 
-function Highlight({ highlight, caseSensitive, text, ...rest }: Props) {
+function Highlight({
+  highlight,
+  processResult,
+  caseSensitive,
+  text,
+  ...rest
+}: Props) {
+  let regex;
+  if (highlight instanceof RegExp) {
+    regex = highlight;
+  } else {
+    regex = new RegExp(
+      (highlight || '').replace(/[-\\^$*+?.()|[\]{}]/g, '\\$&'),
+      caseSensitive ? 'g' : 'gi'
+    );
+  }
   return (
     <span {...rest}>
       {highlight
-        ? replace(
-            text,
-            new RegExp(
-              (highlight || '').replace(/[-\\^$*+?.()|[\]{}]/g, '\\$&'),
-              caseSensitive ? 'g' : 'gi'
-            ),
-            (tag, index) => <Mark key={index}>{tag}</Mark>
-          )
+        ? replace(text, regex, (tag, index) => (
+            <Mark key={index}>{processResult ? processResult(tag) : tag}</Mark>
+          ))
         : text}
     </span>
   );

app/scenes/Document/Document.js

@@ -222,17 +222,12 @@ class DocumentScene extends React.Component<Props> {
   };
 
   onSearchLink = async (term: string) => {
-    const resultIds = await this.props.documents.search(term);
+    const results = await this.props.documents.search(term);
 
-    return resultIds.map((id, index) => {
-      const document = this.props.documents.getById(id);
-      if (!document) return {};
-
-      return {
-        title: document.title,
-        url: document.url,
-      };
-    });
+    return results.map((result, index) => ({
+      title: result.document.title,
+      url: result.document.url,
+    }));
   };
 
   onClickLink = (href: string) => {

app/scenes/Search/Search.js

@@ -5,6 +5,7 @@ import keydown from 'react-keydown';
 import Waypoint from 'react-waypoint';
 import { observable, action } from 'mobx';
 import { observer, inject } from 'mobx-react';
+import { SearchResult } from 'types';
 import _ from 'lodash';
 import DocumentsStore, {
   DEFAULT_PAGINATION_LIMIT,
@@ -62,7 +63,7 @@ const StyledArrowKeyNavigation = styled(ArrowKeyNavigation)`
 class Search extends React.Component<Props> {
   firstDocument: HTMLElement;
 
-  @observable resultIds: string[] = []; // Document IDs
+  @observable results: SearchResult[] = [];
   @observable query: string = '';
   @observable offset: number = 0;
   @observable allowLoadMore: boolean = true;
@@ -104,7 +105,7 @@ class Search extends React.Component<Props> {
   handleQueryChange = () => {
     const query = this.props.match.params.query;
     this.query = query ? query : '';
-    this.resultIds = [];
+    this.results = [];
     this.offset = 0;
     this.allowLoadMore = true;
 
@@ -134,16 +135,14 @@ class Search extends React.Component<Props> {
 
     if (this.query) {
       try {
-        const newResults = await this.props.documents.search(this.query, {
+        const results = await this.props.documents.search(this.query, {
           offset: this.offset,
           limit: DEFAULT_PAGINATION_LIMIT,
         });
-        this.resultIds = this.resultIds.concat(newResults);
-        if (this.resultIds.length > 0) this.pinToTop = true;
-        if (
-          newResults.length === 0 ||
-          newResults.length < DEFAULT_PAGINATION_LIMIT
-        ) {
+        this.results = this.results.concat(results);
+
+        if (this.results.length > 0) this.pinToTop = true;
+        if (results.length === 0 || results.length < DEFAULT_PAGINATION_LIMIT) {
           this.allowLoadMore = false;
         } else {
           this.offset += DEFAULT_PAGINATION_LIMIT;
@@ -152,7 +151,7 @@ class Search extends React.Component<Props> {
         console.error('Something went wrong');
       }
     } else {
-      this.resultIds = [];
+      this.results = [];
       this.pinToTop = false;
     }
 
@@ -177,7 +176,7 @@ class Search extends React.Component<Props> {
   render() {
     const { documents, notFound } = this.props;
     const showEmpty =
-      !this.isFetching && this.query && this.resultIds.length === 0;
+      !this.isFetching && this.query && this.results.length === 0;
 
     return (
       <Container auto>
@@ -201,17 +200,19 @@ class Search extends React.Component<Props> {
               mode={ArrowKeyNavigation.mode.VERTICAL}
               defaultActiveChildIndex={0}
             >
-              {this.resultIds.map((documentId, index) => {
-                const document = documents.getById(documentId);
+              {this.results.map((result, index) => {
+                const document = documents.getById(result.document.id);
                 if (!document) return null;
+
                 return (
                   <DocumentPreview
                     innerRef={ref =>
                       index === 0 && this.setFirstDocumentRef(ref)
                     }
-                    key={documentId}
+                    key={document.id}
                     document={document}
                     highlight={this.query}
+                    context={result.context}
                     showCollection
                   />
                 );

app/stores/DocumentsStore.js

@@ -7,7 +7,7 @@ import invariant from 'invariant';
 import BaseStore from 'stores/BaseStore';
 import Document from 'models/Document';
 import UiStore from 'stores/UiStore';
-import type { PaginationParams } from 'types';
+import type { PaginationParams, SearchResult } from 'types';
 
 export const DEFAULT_PAGINATION_LIMIT = 25;
 
@@ -156,15 +156,15 @@ class DocumentsStore extends BaseStore {
   search = async (
     query: string,
     options: ?PaginationParams
-  ): Promise<string[]> => {
+  ): Promise<SearchResult[]> => {
     const res = await client.get('/documents.search', {
       ...options,
       query,
     });
-    invariant(res && res.data, 'res or res.data missing');
+    invariant(res && res.data, 'Search API response should be available');
     const { data } = res;
-    data.forEach(documentData => this.add(new Document(documentData)));
-    return data.map(documentData => documentData.id);
+    data.forEach(result => this.add(new Document(result.document)));
+    return data;
   };
 
   @action

app/types/index.js

@@ -78,3 +78,9 @@ export type ApiKey = {
   name: string,
   secret: string,
 };
+
+export type SearchResult = {
+  ranking: number,
+  context: string,
+  document: Document,
+};

server/api/documents.js

@@ -226,13 +226,16 @@ router.post('documents.search', auth(), pagination(), async ctx => {
   ctx.assertPresent(query, 'query is required');
 
   const user = ctx.state.user;
-  const documents = await Document.searchForUser(user, query, {
+  const results = await Document.searchForUser(user, query, {
     offset,
     limit,
   });
 
   const data = await Promise.all(
-    documents.map(async document => await presentDocument(ctx, document))
+    results.map(async result => {
+      const document = await presentDocument(ctx, result.document);
+      return { ...result, document };
+    })
   );
 
   ctx.body = {

server/api/documents.test.js

@@ -228,7 +228,7 @@ describe('#documents.search', async () => {
 
     expect(res.status).toEqual(200);
     expect(body.data.length).toEqual(1);
-    expect(body.data[0].text).toEqual('# Much guidance');
+    expect(body.data[0].document.text).toEqual('# Much guidance');
   });
 
   it('should require authentication', async () => {

server/api/hooks.js

@@ -64,14 +64,14 @@ router.post('hooks.slack', async ctx => {
 
   if (!user) throw new InvalidRequestError('Invalid user');
 
-  const documents = await Document.searchForUser(user, text, {
+  const results = await Document.searchForUser(user, text, {
     limit: 5,
   });
 
-  if (documents.length) {
+  if (results.length) {
     const attachments = [];
-    for (const document of documents) {
-      attachments.push(presentSlackAttachment(document));
+    for (const result of results) {
+      attachments.push(presentSlackAttachment(result.document));
     }
 
     ctx.body = {

server/models/Document.js

@@ -1,6 +1,6 @@
 // @flow
 import slug from 'slug';
-import _ from 'lodash';
+import { map, find, compact, uniq } from 'lodash';
 import randomstring from 'randomstring';
 import MarkdownSerializer from 'slate-md-serializer';
 import Plain from 'slate-plain-serializer';
@@ -11,6 +11,7 @@ import { Collection } from '../models';
 import { DataTypes, sequelize } from '../sequelize';
 import events from '../events';
 import parseTitle from '../../shared/utils/parseTitle';
+import unescape from '../../shared/utils/unescape';
 import Revision from './Revision';
 
 const Op = Sequelize.Op;
@@ -65,7 +66,7 @@ const beforeSave = async doc => {
 
   // add the current user as revision hasn't been generated yet
   ids.push(doc.lastModifiedById);
-  doc.collaboratorIds = _.uniq(ids);
+  doc.collaboratorIds = uniq(ids);
 
   // increment revision
   doc.revisionCount += 1;
@@ -188,44 +189,57 @@ Document.findById = async id => {
   }
 };
 
+type SearchResult = {
+  ranking: number,
+  context: string,
+  document: Document,
+};
+
 Document.searchForUser = async (
   user,
   query,
   options = {}
-): Promise<Document[]> => {
+): Promise<SearchResult[]> => {
   const limit = options.limit || 15;
   const offset = options.offset || 0;
 
   const sql = `
-        SELECT *, ts_rank(documents."searchVector", plainto_tsquery('english', :query)) as "searchRanking" FROM documents
+    SELECT
+      id,
+      ts_rank(documents."searchVector", plainto_tsquery('english', :query)) as "searchRanking",
+      ts_headline('english', "text", plainto_tsquery('english', :query), 'MaxFragments=0, MinWords=10, MaxWords=30') as "searchContext"
+    FROM documents
     WHERE "searchVector" @@ plainto_tsquery('english', :query) AND
       "teamId" = '${user.teamId}'::uuid AND
       "deletedAt" IS NULL
-        ORDER BY "searchRanking" DESC
-        LIMIT :limit OFFSET :offset;
+    ORDER BY "searchRanking", "updatedAt" DESC
+    LIMIT :limit
+    OFFSET :offset;
   `;
 
   const results = await sequelize.query(sql, {
+    type: sequelize.QueryTypes.SELECT,
     replacements: {
       query,
       limit,
       offset,
     },
-    model: Document,
   });
-  const ids = results.map(document => document.id);
 
-  // Second query to get views for the data
+  // Second query to get associated document data
   const withViewsScope = { method: ['withViews', user.id] };
   const documents = await Document.scope(
     'defaultScope',
     withViewsScope
   ).findAll({
-    where: { id: ids },
+    where: { id: map(results, 'id') },
   });
 
-  // Order the documents in the same order as the first query
-  return _.sortBy(documents, doc => ids.indexOf(doc.id));
+  return map(results, result => ({
+    ranking: result.searchRanking,
+    context: unescape(result.searchContext),
+    document: find(documents, { id: result.id }),
+  }));
 };
 
 // Hooks
@@ -282,7 +296,7 @@ Document.prototype.getTimestamp = function() {
 Document.prototype.getSummary = function() {
   const value = Markdown.deserialize(this.text);
   const plain = Plain.serialize(value);
-  const lines = _.compact(plain.split('\n'));
+  const lines = compact(plain.split('\n'));
   return lines.length >= 1 ? lines[1] : '';
 };