// @flow import { map, find, compact, uniq } from "lodash"; import MarkdownSerializer from "slate-md-serializer"; import randomstring from "randomstring"; import Sequelize, { type Transaction } from "sequelize"; import removeMarkdown from "@tommoor/remove-markdown"; import isUUID from "validator/lib/isUUID"; import { Collection, User } from "../models"; import { DataTypes, sequelize } from "../sequelize"; import parseTitle from "../../shared/utils/parseTitle"; import unescape from "../../shared/utils/unescape"; import slugify from "../utils/slugify"; import Revision from "./Revision"; const Op = Sequelize.Op; const URL_REGEX = /^[0-9a-zA-Z-_~]*-([a-zA-Z0-9]{10,15})$/; const serializer = new MarkdownSerializer(); export const DOCUMENT_VERSION = 2; const createRevision = (doc, options = {}) => { // we don't create revisions for autosaves if (options.autosave) return; // we don't create revisions if identical to previous if ( doc.text === doc.previous("text") && doc.title === doc.previous("title") ) { return; } return Revision.create( { title: doc.title, text: doc.text, userId: doc.lastModifiedById, editorVersion: doc.editorVersion, version: doc.version, documentId: doc.id, }, { transaction: options.transaction, } ); }; const createUrlId = doc => { return (doc.urlId = doc.urlId || randomstring.generate(10)); }; const beforeCreate = async doc => { if (doc.version === undefined) { doc.version = DOCUMENT_VERSION; } return beforeSave(doc); }; const beforeSave = async doc => { const { emoji } = parseTitle(doc.text); // emoji in the title is split out for easier display doc.emoji = emoji; // ensure documents have a title doc.title = doc.title || ""; // add the current user as a collaborator on this doc if (!doc.collaboratorIds) doc.collaboratorIds = []; doc.collaboratorIds = uniq(doc.collaboratorIds.concat(doc.lastModifiedById)); // increment revision doc.revisionCount += 1; return doc; }; const Document = sequelize.define( "document", { id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, primaryKey: true, }, urlId: { type: DataTypes.STRING, primaryKey: true, }, title: { type: DataTypes.STRING, validate: { len: { args: [0, 100], msg: "Document title must be less than 100 characters", }, }, }, version: DataTypes.SMALLINT, editorVersion: DataTypes.STRING, text: DataTypes.TEXT, // backup contains a record of text at the moment it was converted to v2 // this is a safety measure during deployment of new editor and will be // dropped in a future update backup: DataTypes.TEXT, isWelcome: { type: DataTypes.BOOLEAN, defaultValue: false }, revisionCount: { type: DataTypes.INTEGER, defaultValue: 0 }, archivedAt: DataTypes.DATE, publishedAt: DataTypes.DATE, parentDocumentId: DataTypes.UUID, collaboratorIds: DataTypes.ARRAY(DataTypes.UUID), }, { paranoid: true, hooks: { beforeValidate: createUrlId, beforeCreate: beforeCreate, beforeUpdate: beforeSave, afterCreate: createRevision, afterUpdate: createRevision, }, getterMethods: { url: function() { const slugifiedTitle = slugify(this.title); return `/doc/${slugifiedTitle}-${this.urlId}`; }, }, } ); // Class methods Document.associate = models => { Document.belongsTo(models.Collection, { as: "collection", foreignKey: "collectionId", onDelete: "cascade", }); Document.belongsTo(models.Team, { as: "team", foreignKey: "teamId", }); Document.belongsTo(models.User, { as: "createdBy", foreignKey: "createdById", }); Document.belongsTo(models.User, { as: "updatedBy", foreignKey: "lastModifiedById", }); Document.belongsTo(models.User, { as: "pinnedBy", foreignKey: "pinnedById", }); Document.hasMany(models.Revision, { as: "revisions", onDelete: "cascade", }); Document.hasMany(models.Backlink, { as: "backlinks", onDelete: "cascade", }); Document.hasMany(models.Star, { as: "starred", onDelete: "cascade", }); Document.hasMany(models.View, { as: "views", }); Document.addScope("defaultScope", { include: [ { model: models.User, as: "createdBy", paranoid: false }, { model: models.User, as: "updatedBy", paranoid: false }, ], where: { publishedAt: { [Op.ne]: null, }, }, }); Document.addScope("withCollection", userId => { if (userId) { return { include: [ { model: models.Collection.scope({ method: ["withMembership", userId], }), as: "collection", }, ], }; } return { include: [{ model: models.Collection, as: "collection" }], }; }); Document.addScope("withUnpublished", { include: [ { model: models.User, as: "createdBy", paranoid: false }, { model: models.User, as: "updatedBy", paranoid: false }, ], }); Document.addScope("withViews", userId => ({ include: [ { model: models.View, as: "views", where: { userId }, required: false }, ], })); Document.addScope("withStarred", userId => ({ include: [ { model: models.Star, as: "starred", where: { userId }, required: false }, ], })); }; Document.findByPk = async function(id, options = {}) { // allow default preloading of collection membership if `userId` is passed in find options // almost every endpoint needs the collection membership to determine policy permissions. const scope = this.scope("withUnpublished", { method: ["withCollection", options.userId], }); if (isUUID(id)) { return scope.findOne({ where: { id }, ...options, }); } else if (id.match(URL_REGEX)) { return scope.findOne({ where: { urlId: id.match(URL_REGEX)[1], }, ...options, }); } }; type SearchResult = { ranking: number, context: string, document: Document, }; type SearchOptions = { limit?: number, offset?: number, collectionId?: string, dateFilter?: "day" | "week" | "month" | "year", collaboratorIds?: string[], includeArchived?: boolean, includeDrafts?: boolean, }; function escape(query: string): string { // replace "\" with escaped "\\" because sequelize.escape doesn't do it // https://github.com/sequelize/sequelize/issues/2950 return sequelize.escape(query).replace("\\", "\\\\"); } Document.searchForTeam = async ( team, query, options: SearchOptions = {} ): Promise => { const limit = options.limit || 15; const offset = options.offset || 0; const wildcardQuery = `${escape(query)}:*`; const collectionIds = await team.collectionIds(); // If the team has access no public collections then shortcircuit the rest of this if (!collectionIds.length) { return []; } // Build the SQL query to get documentIds, ranking, and search term context const sql = ` SELECT id, ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking", ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext" FROM documents WHERE "searchVector" @@ to_tsquery('english', :query) AND "teamId" = :teamId AND "collectionId" IN(:collectionIds) AND "deletedAt" IS NULL AND "publishedAt" IS NOT NULL ORDER BY "searchRanking" DESC, "updatedAt" DESC LIMIT :limit OFFSET :offset; `; const results = await sequelize.query(sql, { type: sequelize.QueryTypes.SELECT, replacements: { teamId: team.id, query: wildcardQuery, limit, offset, collectionIds, }, }); // Final query to get associated document data const documents = await Document.findAll({ where: { id: map(results, "id"), }, include: [ { model: Collection, as: "collection" }, { model: User, as: "createdBy", paranoid: false }, { model: User, as: "updatedBy", paranoid: false }, ], }); return map(results, result => ({ ranking: result.searchRanking, context: removeMarkdown(unescape(result.searchContext), { stripHTML: false, }), document: find(documents, { id: result.id }), })); }; Document.searchForUser = async ( user, query, options: SearchOptions = {} ): Promise => { const limit = options.limit || 15; const offset = options.offset || 0; const wildcardQuery = `${escape(query)}:*`; // Ensure we're filtering by the users accessible collections. If // collectionId is passed as an option it is assumed that the authorization // has already been done in the router let collectionIds; if (options.collectionId) { collectionIds = [options.collectionId]; } else { collectionIds = await user.collectionIds(); } // If the user has access to no collections then shortcircuit the rest of this if (!collectionIds.length) { return []; } let dateFilter; if (options.dateFilter) { dateFilter = `1 ${options.dateFilter}`; } // Build the SQL query to get documentIds, ranking, and search term context const sql = ` SELECT id, ts_rank(documents."searchVector", to_tsquery('english', :query)) as "searchRanking", ts_headline('english', "text", to_tsquery('english', :query), 'MaxFragments=1, MinWords=20, MaxWords=30') as "searchContext" FROM documents WHERE "searchVector" @@ to_tsquery('english', :query) AND "teamId" = :teamId AND "collectionId" IN(:collectionIds) AND ${ options.dateFilter ? '"updatedAt" > now() - interval :dateFilter AND' : "" } ${ options.collaboratorIds ? '"collaboratorIds" @> ARRAY[:collaboratorIds]::uuid[] AND' : "" } ${options.includeArchived ? "" : '"archivedAt" IS NULL AND'} "deletedAt" IS NULL AND ${ options.includeDrafts ? '("publishedAt" IS NOT NULL OR "createdById" = :userId)' : '"publishedAt" IS NOT NULL' } ORDER BY "searchRanking" DESC, "updatedAt" DESC LIMIT :limit OFFSET :offset; `; const results = await sequelize.query(sql, { type: sequelize.QueryTypes.SELECT, replacements: { teamId: user.teamId, userId: user.id, collaboratorIds: options.collaboratorIds, query: wildcardQuery, limit, offset, collectionIds, dateFilter, }, }); // Final query to get associated document data const documents = await Document.scope( { method: ["withViews", user.id], }, { method: ["withCollection", user.id], } ).findAll({ where: { id: map(results, "id"), }, include: [ { model: User, as: "createdBy", paranoid: false }, { model: User, as: "updatedBy", paranoid: false }, ], }); return map(results, result => ({ ranking: result.searchRanking, context: removeMarkdown(unescape(result.searchContext), { stripHTML: false, }), document: find(documents, { id: result.id }), })); }; // Hooks Document.addHook("beforeSave", async model => { if (!model.publishedAt) return; const collection = await Collection.findByPk(model.collectionId); if (!collection || collection.type !== "atlas") return; await collection.updateDocument(model); model.collection = collection; }); Document.addHook("afterCreate", async model => { if (!model.publishedAt) return; const collection = await Collection.findByPk(model.collectionId); if (!collection || collection.type !== "atlas") return; await collection.addDocumentToStructure(model); model.collection = collection; return model; }); // Instance methods Document.prototype.toMarkdown = function() { const text = unescape(this.text); if (this.version) { return `# ${this.title}\n\n${text}`; } return text; }; Document.prototype.migrateVersion = function() { let migrated = false; // migrate from document version 0 -> 1 if (!this.version) { // removing the title from the document text attribute this.text = this.text.replace(/^#\s(.*)\n/, ""); this.version = 1; migrated = true; } // migrate from document version 1 -> 2 if (this.version === 1) { const nodes = serializer.deserialize(this.text); this.backup = this.text; this.text = serializer.serialize(nodes, { version: 2 }); this.version = 2; migrated = true; } if (migrated) { return this.save({ silent: true, hooks: false }); } }; // Note: This method marks the document and it's children as deleted // in the database, it does not permanantly delete them OR remove // from the collection structure. Document.prototype.deleteWithChildren = async function(options) { // Helper to destroy all child documents for a document const loopChildren = async (documentId, opts) => { const childDocuments = await Document.findAll({ where: { parentDocumentId: documentId }, }); childDocuments.forEach(async child => { await loopChildren(child.id, opts); await child.destroy(opts); }); }; await loopChildren(this.id, options); await this.destroy(options); }; Document.prototype.archiveWithChildren = async function(userId, options) { const archivedAt = new Date(); // Helper to archive all child documents for a document const archiveChildren = async parentDocumentId => { const childDocuments = await Document.findAll({ where: { parentDocumentId }, }); childDocuments.forEach(async child => { await archiveChildren(child.id); child.archivedAt = archivedAt; child.lastModifiedById = userId; await child.save(options); }); }; await archiveChildren(this.id); this.archivedAt = archivedAt; this.lastModifiedById = userId; return this.save(options); }; Document.prototype.publish = async function(options) { if (this.publishedAt) return this.save(options); const collection = await Collection.findByPk(this.collectionId); if (collection.type !== "atlas") return this.save(options); await collection.addDocumentToStructure(this); this.publishedAt = new Date(); await this.save(options); return this; }; // Moves a document from being visible to the team within a collection // to the archived area, where it can be subsequently restored. Document.prototype.archive = async function(userId) { // archive any children and remove from the document structure const collection = await this.getCollection(); await collection.removeDocumentInStructure(this); this.collection = collection; await this.archiveWithChildren(userId); return this; }; // Restore an archived document back to being visible to the team Document.prototype.unarchive = async function(userId) { const collection = await this.getCollection(); // check to see if the documents parent hasn't been archived also // If it has then restore the document to the collection root. if (this.parentDocumentId) { const parent = await Document.findOne({ where: { id: this.parentDocumentId, archivedAt: { [Op.eq]: null, }, }, }); if (!parent) this.parentDocumentId = undefined; } await collection.addDocumentToStructure(this); this.collection = collection; if (this.deletedAt) { await this.restore(); } this.archivedAt = null; this.lastModifiedById = userId; await this.save(); return this; }; // Delete a document, archived or otherwise. Document.prototype.delete = function(options) { return sequelize.transaction(async (transaction: Transaction): Promise<*> => { if (!this.archivedAt) { // delete any children and remove from the document structure const collection = await this.getCollection(); if (collection) await collection.deleteDocument(this, { transaction }); } await Revision.destroy({ where: { documentId: this.id }, transaction, }); await this.destroy({ transaction, ...options }); return this; }); }; Document.prototype.getTimestamp = function() { return Math.round(new Date(this.updatedAt).getTime() / 1000); }; Document.prototype.getSummary = function() { const plain = removeMarkdown(unescape(this.text), { stripHTML: false, }); const lines = compact(plain.split("\n")); const notEmpty = lines.length >= 1; if (this.version) { return notEmpty ? lines[0] : ""; } return notEmpty ? lines[1] : ""; }; Document.prototype.toJSON = function() { // Warning: only use for new documents as order of children is // handled in the collection's documentStructure return { id: this.id, title: this.title, url: this.url, children: [], }; }; export default Document;