// @flow import removeMarkdown from "@tommoor/remove-markdown"; import { compact, find, map, uniq } from "lodash"; import randomstring from "randomstring"; import Sequelize, { Transaction } from "sequelize"; import MarkdownSerializer from "slate-md-serializer"; import isUUID from "validator/lib/isUUID"; import { MAX_TITLE_LENGTH } from "../../shared/constants"; import getTasks from "../../shared/utils/getTasks"; import parseTitle from "../../shared/utils/parseTitle"; import { SLUG_URL_REGEX } from "../../shared/utils/routeHelpers"; import unescape from "../../shared/utils/unescape"; import { Collection, User } from "../models"; import { DataTypes, sequelize } from "../sequelize"; import slugify from "../utils/slugify"; import Revision from "./Revision"; const Op = Sequelize.Op; const serializer = new MarkdownSerializer(); export const DOCUMENT_VERSION = 2; const createUrlId = (doc) => { return (doc.urlId = doc.urlId || randomstring.generate(10)); }; const beforeCreate = async (doc) => { if (doc.version === undefined) { doc.version = DOCUMENT_VERSION; } return beforeSave(doc); }; const beforeSave = async (doc) => { const { emoji } = parseTitle(doc.text); // emoji in the title is split out for easier display doc.emoji = emoji; // ensure documents have a title doc.title = doc.title || ""; if (doc.previous("title") && doc.previous("title") !== doc.title) { if (!doc.previousTitles) doc.previousTitles = []; doc.previousTitles = uniq(doc.previousTitles.concat(doc.previous("title"))); } // add the current user as a collaborator on this doc if (!doc.collaboratorIds) doc.collaboratorIds = []; doc.collaboratorIds = uniq(doc.collaboratorIds.concat(doc.lastModifiedById)); // increment revision doc.revisionCount += 1; return doc; }; const Document = sequelize.define( "document", { id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true, }, urlId: { type: DataTypes.STRING, primaryKey: true, }, title: { type: DataTypes.STRING, validate: { len: { args: [0, MAX_TITLE_LENGTH], msg: `Document title must be less than ${MAX_TITLE_LENGTH} characters`, }, }, }, previousTitles: DataTypes.ARRAY(DataTypes.STRING), version: DataTypes.SMALLINT, template: DataTypes.BOOLEAN, editorVersion: DataTypes.STRING, text: DataTypes.TEXT, state: DataTypes.BLOB, isWelcome: { type: DataTypes.BOOLEAN, defaultValue: false }, revisionCount: { type: DataTypes.INTEGER, defaultValue: 0 }, archivedAt: DataTypes.DATE, publishedAt: DataTypes.DATE, parentDocumentId: DataTypes.UUID, collaboratorIds: DataTypes.ARRAY(DataTypes.UUID), }, { paranoid: true, hooks: { beforeValidate: createUrlId, beforeCreate: beforeCreate, beforeUpdate: beforeSave, }, getterMethods: { url: function () { if (!this.title) return `/doc/untitled-${this.urlId}`; const slugifiedTitle = slugify(this.title); return `/doc/${slugifiedTitle}-${this.urlId}`; }, tasks: function () { return getTasks(this.text || ""); }, }, } ); // Class methods Document.associate = (models) => { Document.belongsTo(models.Collection, { as: "collection", foreignKey: "collectionId", onDelete: "cascade", }); Document.belongsTo(models.Team, { as: "team", foreignKey: "teamId", }); Document.belongsTo(models.Document, { as: "document", foreignKey: "templateId", }); Document.belongsTo(models.User, { as: "createdBy", foreignKey: "createdById", }); Document.belongsTo(models.User, { as: "updatedBy", foreignKey: "lastModifiedById", }); Document.belongsTo(models.User, { as: "pinnedBy", foreignKey: "pinnedById", }); Document.hasMany(models.Revision, { as: "revisions", onDelete: "cascade", }); Document.hasMany(models.Backlink, { as: "backlinks", onDelete: "cascade", }); Document.hasMany(models.Star, { as: "starred", onDelete: "cascade", }); Document.hasMany(models.View, { as: "views", }); Document.addScope("defaultScope", { include: [ { model: models.User, as: "createdBy", paranoid: false }, { model: models.User, as: "updatedBy", paranoid: false }, ], where: { publishedAt: { [Op.ne]: null, }, }, }); Document.addScope("withCollection", (userId, paranoid = true) => { if (userId) { return { include: [ { model: models.Collection.scope({ method: ["withMembership", userId], }), as: "collection", paranoid, }, ], }; } return { include: [{ model: models.Collection, as: "collection" }], }; }); Document.addScope("withUnpublished", { include: [ { model: models.User, as: "createdBy", paranoid: false }, { model: models.User, as: "updatedBy", paranoid: false }, ], }); Document.addScope("withViews", (userId) => { if (!userId) return {}; return { include: [ { model: models.View, as: "views", where: { userId }, required: false, separate: true, }, ], }; }); Document.addScope("withStarred", (userId) => ({ include: [ { model: models.Star, as: "starred", where: { userId }, required: false, separate: true, }, ], })); }; Document.findByPk = async function (id, options = {}) { // allow default preloading of collection membership if `userId` is passed in find options // almost every endpoint needs the collection membership to determine policy permissions. const scope = this.scope( "withUnpublished", { method: ["withCollection", options.userId, options.paranoid], }, { method: ["withViews", options.userId], } ); if (isUUID(id)) { return scope.findOne({ where: { id }, ...options, }); } else if (id.match(SLUG_URL_REGEX)) { return scope.findOne({ where: { urlId: id.match(SLUG_URL_REGEX)[1], }, ...options, }); } }; type SearchResponse = { results: { ranking: number, context: string, document: Document, }[], totalCount: number, }; type SearchOptions = { limit?: number, offset?: number, collectionId?: string, dateFilter?: "day" | "week" | "month" | "year", collaboratorIds?: string[], includeArchived?: boolean, includeDrafts?: boolean, }; function escape(query: string): string { // replace "\" with escaped "\\" because sequelize.escape doesn't do it // https://github.com/sequelize/sequelize/issues/2950 return sequelize.escape(query).replace(/\\/g, "\\\\"); } Document.searchForTeam = async ( team, query, options: SearchOptions = {} ): Promise => { const limit = options.limit || 15; const offset = options.offset || 0; const wildcardQuery = `${escape(query)}:*`; const collectionIds = await team.collectionIds(); // If the team has access no public collections then shortcircuit the rest of this if (!collectionIds.length) { return { results: [], totalCount: 0 }; } // Build the SQL query to get documentIds, ranking, and search term context const whereClause = ` "searchVector" @@ to_tsquery('english', :query) AND "teamId" = :teamId AND "collectionId" IN(:collectionIds) AND "deletedAt" IS NULL AND "publishedAt" IS NOT NULL `; const selectSql = ` SELECT id, ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking", ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext" FROM documents WHERE ${whereClause} ORDER BY "searchRanking" DESC, "updatedAt" DESC LIMIT :limit OFFSET :offset; `; const countSql = ` SELECT COUNT(id) FROM documents WHERE ${whereClause} `; const queryReplacements = { teamId: team.id, query: wildcardQuery, collectionIds, }; const resultsQuery = sequelize.query(selectSql, { type: sequelize.QueryTypes.SELECT, replacements: { ...queryReplacements, limit, offset, }, }); const countQuery = sequelize.query(countSql, { type: sequelize.QueryTypes.SELECT, replacements: queryReplacements, }); const [results, [{ count }]] = await Promise.all([resultsQuery, countQuery]); // Final query to get associated document data const documents = await Document.findAll({ where: { id: map(results, "id"), }, include: [ { model: Collection, as: "collection" }, { model: User, as: "createdBy", paranoid: false }, { model: User, as: "updatedBy", paranoid: false }, ], }); return { results: map(results, (result) => ({ ranking: result.searchRanking, context: removeMarkdown(unescape(result.searchContext), { stripHTML: false, }), document: find(documents, { id: result.id }), })), totalCount: count, }; }; Document.searchForUser = async ( user, query, options: SearchOptions = {} ): Promise => { const limit = options.limit || 15; const offset = options.offset || 0; const wildcardQuery = `${escape(query)}:*`; // Ensure we're filtering by the users accessible collections. If // collectionId is passed as an option it is assumed that the authorization // has already been done in the router let collectionIds; if (options.collectionId) { collectionIds = [options.collectionId]; } else { collectionIds = await user.collectionIds(); } // If the user has access to no collections then shortcircuit the rest of this if (!collectionIds.length) { return { results: [], totalCount: 0 }; } let dateFilter; if (options.dateFilter) { dateFilter = `1 ${options.dateFilter}`; } // Build the SQL query to get documentIds, ranking, and search term context const whereClause = ` "searchVector" @@ to_tsquery('english', :query) AND "teamId" = :teamId AND "collectionId" IN(:collectionIds) AND ${ options.dateFilter ? '"updatedAt" > now() - interval :dateFilter AND' : "" } ${ options.collaboratorIds ? '"collaboratorIds" @> ARRAY[:collaboratorIds]::uuid[] AND' : "" } ${options.includeArchived ? "" : '"archivedAt" IS NULL AND'} "deletedAt" IS NULL AND ${ options.includeDrafts ? '("publishedAt" IS NOT NULL OR "createdById" = :userId)' : '"publishedAt" IS NOT NULL' } `; const selectSql = ` SELECT id, ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking", ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext" FROM documents WHERE ${whereClause} ORDER BY "searchRanking" DESC, "updatedAt" DESC LIMIT :limit OFFSET :offset; `; const countSql = ` SELECT COUNT(id) FROM documents WHERE ${whereClause} `; const queryReplacements = { teamId: user.teamId, userId: user.id, collaboratorIds: options.collaboratorIds, query: wildcardQuery, collectionIds, dateFilter, }; const resultsQuery = sequelize.query(selectSql, { type: sequelize.QueryTypes.SELECT, replacements: { ...queryReplacements, limit, offset, }, }); const countQuery = sequelize.query(countSql, { type: sequelize.QueryTypes.SELECT, replacements: queryReplacements, }); const [results, [{ count }]] = await Promise.all([resultsQuery, countQuery]); // Final query to get associated document data const documents = await Document.scope( { method: ["withViews", user.id], }, { method: ["withCollection", user.id], } ).findAll({ where: { id: map(results, "id"), }, include: [ { model: User, as: "createdBy", paranoid: false }, { model: User, as: "updatedBy", paranoid: false }, ], }); return { results: map(results, (result) => ({ ranking: result.searchRanking, context: removeMarkdown(unescape(result.searchContext), { stripHTML: false, }), document: find(documents, { id: result.id }), })), totalCount: count, }; }; // Hooks Document.addHook("beforeSave", async (model) => { if (!model.publishedAt || model.template) { return; } const collection = await Collection.findByPk(model.collectionId); if (!collection) { return; } await collection.updateDocument(model); model.collection = collection; }); Document.addHook("afterCreate", async (model) => { if (!model.publishedAt || model.template) { return; } const collection = await Collection.findByPk(model.collectionId); if (!collection) { return; } await collection.addDocumentToStructure(model, 0); model.collection = collection; return model; }); // Instance methods Document.prototype.toMarkdown = function () { const text = unescape(this.text); if (this.version) { return `# ${this.title}\n\n${text}`; } return text; }; Document.prototype.migrateVersion = function () { let migrated = false; // migrate from document version 0 -> 1 if (!this.version) { // removing the title from the document text attribute this.text = this.text.replace(/^#\s(.*)\n/, ""); this.version = 1; migrated = true; } // migrate from document version 1 -> 2 if (this.version === 1) { const nodes = serializer.deserialize(this.text); this.text = serializer.serialize(nodes, { version: 2 }); this.version = 2; migrated = true; } if (migrated) { return this.save({ silent: true, hooks: false }); } }; // Note: This method marks the document and it's children as deleted // in the database, it does not permanently delete them OR remove // from the collection structure. Document.prototype.deleteWithChildren = async function (options) { // Helper to destroy all child documents for a document const loopChildren = async (documentId, opts) => { const childDocuments = await Document.findAll({ where: { parentDocumentId: documentId }, }); childDocuments.forEach(async (child) => { await loopChildren(child.id, opts); await child.destroy(opts); }); }; await loopChildren(this.id, options); await this.destroy(options); }; Document.prototype.archiveWithChildren = async function (userId, options) { const archivedAt = new Date(); // Helper to archive all child documents for a document const archiveChildren = async (parentDocumentId) => { const childDocuments = await Document.findAll({ where: { parentDocumentId }, }); childDocuments.forEach(async (child) => { await archiveChildren(child.id); child.archivedAt = archivedAt; child.lastModifiedById = userId; await child.save(options); }); }; await archiveChildren(this.id); this.archivedAt = archivedAt; this.lastModifiedById = userId; return this.save(options); }; Document.prototype.publish = async function (userId: string, options) { if (this.publishedAt) return this.save(options); if (!this.template) { const collection = await Collection.findByPk(this.collectionId); await collection.addDocumentToStructure(this, 0); } this.lastModifiedById = userId; this.publishedAt = new Date(); await this.save(options); return this; }; Document.prototype.unpublish = async function (userId: string, options) { if (!this.publishedAt) return this; const collection = await this.getCollection(); await collection.removeDocumentInStructure(this); // unpublishing a document converts the "ownership" to yourself, so that it // can appear in your drafts rather than the original creators this.userId = userId; this.lastModifiedById = userId; this.publishedAt = null; await this.save(options); return this; }; // Moves a document from being visible to the team within a collection // to the archived area, where it can be subsequently restored. Document.prototype.archive = async function (userId) { // archive any children and remove from the document structure const collection = await this.getCollection(); await collection.removeDocumentInStructure(this); this.collection = collection; await this.archiveWithChildren(userId); return this; }; // Restore an archived document back to being visible to the team Document.prototype.unarchive = async function (userId: string) { const collection = await this.getCollection(); // check to see if the documents parent hasn't been archived also // If it has then restore the document to the collection root. if (this.parentDocumentId) { const parent = await Document.findOne({ where: { id: this.parentDocumentId, archivedAt: { [Op.eq]: null, }, }, }); if (!parent) this.parentDocumentId = null; } if (!this.template) { await collection.addDocumentToStructure(this); this.collection = collection; } if (this.deletedAt) { await this.restore(); } this.archivedAt = null; this.lastModifiedById = userId; await this.save(); return this; }; // Delete a document, archived or otherwise. Document.prototype.delete = function (userId: string) { return sequelize.transaction( async (transaction: Transaction): Promise => { if (!this.archivedAt && !this.template) { // delete any children and remove from the document structure const collection = await this.getCollection({ transaction }); if (collection) await collection.deleteDocument(this, { transaction }); } else { await this.destroy({ transaction }); } await Revision.destroy({ where: { documentId: this.id }, transaction, }); await this.update( { lastModifiedById: userId }, { transaction, } ); return this; } ); }; Document.prototype.getTimestamp = function () { return Math.round(new Date(this.updatedAt).getTime() / 1000); }; Document.prototype.getSummary = function () { const plain = removeMarkdown(unescape(this.text), { stripHTML: false, }); const lines = compact(plain.split("\n")); const notEmpty = lines.length >= 1; if (this.version) { return notEmpty ? lines[0] : ""; } return notEmpty ? lines[1] : ""; }; Document.prototype.toJSON = function () { // Warning: only use for new documents as order of children is // handled in the collection's documentStructure return { id: this.id, title: this.title, url: this.url, children: [], }; }; export default Document;