diff --git a/app/components/DropToImport.js b/app/components/DropToImport.js index ecb8c78c..54b936f6 100644 --- a/app/components/DropToImport.js +++ b/app/components/DropToImport.js @@ -8,7 +8,6 @@ import { withRouter, type RouterHistory, type Match } from "react-router-dom"; import { createGlobalStyle } from "styled-components"; import DocumentsStore from "stores/DocumentsStore"; import LoadingIndicator from "components/LoadingIndicator"; -import importFile from "utils/importFile"; const EMPTY_OBJECT = {}; let importingLock = false; @@ -61,12 +60,12 @@ class DropToImport extends React.Component { } for (const file of files) { - const doc = await importFile({ - documents: this.props.documents, + const doc = await this.props.documents.import( file, documentId, collectionId, - }); + { publish: true } + ); if (redirect) { this.props.history.push(doc.url); @@ -95,7 +94,7 @@ class DropToImport extends React.Component { return ( { const files = getDataTransferFiles(ev); try { - const document = await importFile({ - file: files[0], - documents: this.props.documents, - collectionId: this.props.collection.id, - }); + const file = files[0]; + const document = await this.props.documents.import( + file, + null, + this.props.collection.id, + { publish: true } + ); this.props.history.push(document.url); } catch (err) { this.props.ui.showToast(err.message); @@ -103,7 +104,14 @@ class CollectionMenu extends React.Component { }; render() { - const { policies, collection, position, onOpen, onClose } = this.props; + const { + policies, + documents, + collection, + position, + onOpen, + onClose, + } = this.props; const can = policies.abilities(collection.id); return ( @@ -114,7 +122,7 @@ class CollectionMenu extends React.Component { ref={(ref) => (this.file = ref)} onChange={this.onFilePicked} onClick={(ev) => ev.stopPropagation()} - accept="text/markdown, text/plain" + accept={documents.importFileTypes.join(", ")} /> diff --git a/app/stores/DocumentsStore.js b/app/stores/DocumentsStore.js index d1a76959..ff1aa120 100644 --- a/app/stores/DocumentsStore.js +++ b/app/stores/DocumentsStore.js @@ -18,12 +18,23 @@ import Document from "models/Document"; import type { FetchOptions, PaginationParams, SearchResult } from "types"; import { client } from "utils/ApiClient"; +type ImportOptions = { + publish?: boolean, +}; + export default class DocumentsStore extends BaseStore { @observable recentlyViewedIds: string[] = []; @observable searchCache: Map = new Map(); @observable starredIds: Map = new Map(); @observable backlinks: Map = new Map(); + importFileTypes: string[] = [ + "text/markdown", + "text/plain", + "text/html", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]; + constructor(rootStore: RootStore) { super(rootStore, Document); } @@ -455,6 +466,41 @@ export default class DocumentsStore extends BaseStore { return this.add(res.data); }; + @action + import = async ( + file: File, + parentDocumentId: string, + collectionId: string, + options: ImportOptions + ) => { + const title = file.name.replace(/\.[^/.]+$/, ""); + const formData = new FormData(); + + [ + { key: "parentDocumentId", value: parentDocumentId }, + { key: "collectionId", value: collectionId }, + { key: "title", value: title }, + { key: "publish", value: options.publish }, + { key: "file", value: file }, + ].map((info) => { + if (typeof info.value === "string" && info.value) { + formData.append(info.key, info.value); + } + if (typeof info.value === "boolean") { + formData.append(info.key, info.value.toString()); + } + if (info.value instanceof File) { + formData.append(info.key, info.value); + } + }); + + const res = await client.post("/documents.import", formData); + invariant(res && res.data, "Data should be available"); + + this.addPolicies(res.policies); + return this.add(res.data); + }; + _add = this.add; @action diff --git a/app/utils/ApiClient.js b/app/utils/ApiClient.js index 545f0578..1787d9e4 100644 --- a/app/utils/ApiClient.js +++ b/app/utils/ApiClient.js @@ -28,12 +28,13 @@ class ApiClient { fetch = async ( path: string, method: string, - data: ?Object, + data: ?Object | FormData | void, options: Object = {} ) => { let body; let modifiedPath; let urlToFetch; + let isJson; if (method === "GET") { if (data) { @@ -42,7 +43,18 @@ class ApiClient { modifiedPath = path; } } else if (method === "POST" || method === "PUT") { - body = data ? JSON.stringify(data) : undefined; + body = data || undefined; + + // Only stringify data if its a normal object and + // not if it's [object FormData], in addition to + // toggling Content-Type to application/json + if ( + typeof data === "object" && + (data || "").toString() === "[object Object]" + ) { + isJson = true; + body = JSON.stringify(data); + } } if (path.match(/^http/)) { @@ -51,14 +63,20 @@ class ApiClient { urlToFetch = this.baseUrl + (modifiedPath || path); } - // Construct headers - const headers = new Headers({ + let headerOptions: any = { Accept: "application/json", - "Content-Type": "application/json", "cache-control": "no-cache", "x-editor-version": EDITOR_VERSION, pragma: "no-cache", - }); + }; + // for multipart forms or other non JSON requests fetch + // populates the Content-Type without needing to explicitly + // set it. + if (isJson) { + headerOptions["Content-Type"] = "application/json"; + } + const headers = new Headers(headerOptions); + if (stores.auth.authenticated) { invariant(stores.auth.token, "JWT token not set properly"); headers.set("Authorization", `Bearer ${stores.auth.token}`); diff --git a/app/utils/importFile.js b/app/utils/importFile.js deleted file mode 100644 index c7ef1297..00000000 --- a/app/utils/importFile.js +++ /dev/null @@ -1,58 +0,0 @@ -// @flow -import parseTitle from "shared/utils/parseTitle"; -import DocumentsStore from "stores/DocumentsStore"; -import Document from "models/Document"; - -type Options = { - file: File, - documents: DocumentsStore, - collectionId: string, - documentId?: string, -}; - -const importFile = async ({ - documents, - file, - documentId, - collectionId, -}: Options): Promise => { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - - reader.onload = async (ev) => { - let text = ev.target.result; - let title; - - // If the first line of the imported file looks like a markdown heading - // then we can use this as the document title - if (text.trim().startsWith("# ")) { - const result = parseTitle(text); - title = result.title; - text = text.replace(`# ${title}\n`, ""); - - // otherwise, just use the filename without the extension as our best guess - } else { - title = file.name.replace(/\.[^/.]+$/, ""); - } - - let document = new Document( - { - parentDocumentId: documentId, - collectionId, - text, - title, - }, - documents - ); - try { - document = await document.save({ publish: true }); - resolve(document); - } catch (err) { - reject(err); - } - }; - reader.readAsText(file); - }); -}; - -export default importFile; diff --git a/package.json b/package.json index 76f8dc8e..6ef02a6d 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "jsonwebtoken": "^8.5.0", "jszip": "^3.5.0", "koa": "^2.10.0", - "koa-bodyparser": "4.2.0", + "koa-body": "^4.2.0", "koa-compress": "2.0.0", "koa-convert": "1.2.0", "koa-helmet": "5.2.0", @@ -115,6 +115,7 @@ "koa-sslify": "2.1.2", "koa-static": "^4.0.1", "lodash": "^4.17.19", + "mammoth": "^1.4.11", "mobx": "4.6.0", "mobx-react": "^6.2.5", "natural-sort": "^1.0.0", @@ -156,6 +157,7 @@ "styled-normalize": "^8.0.4", "tiny-cookie": "^2.3.1", "tmp": "0.0.33", + "turndown": "^6.0.0", "uuid": "2.0.2", "validator": "5.2.0" }, @@ -193,4 +195,4 @@ "js-yaml": "^3.13.1" }, "version": "0.47.1" -} \ No newline at end of file +} diff --git a/server/api/documents.js b/server/api/documents.js index d570d037..6795d916 100644 --- a/server/api/documents.js +++ b/server/api/documents.js @@ -1,6 +1,7 @@ // @flow import Router from "koa-router"; import Sequelize from "sequelize"; +import documentImporter from "../commands/documentImporter"; import documentMover from "../commands/documentMover"; import { NotFoundError, InvalidRequestError } from "../errors"; import auth from "../middlewares/authentication"; @@ -707,106 +708,23 @@ router.post("documents.unstar", auth(), async (ctx) => { }; }); -router.post("documents.create", auth(), async (ctx) => { - const { - title = "", - text = "", - publish, - collectionId, - parentDocumentId, - templateId, - template, - index, - } = ctx.body; - const editorVersion = ctx.headers["x-editor-version"]; - - ctx.assertUuid(collectionId, "collectionId must be an uuid"); - if (parentDocumentId) { - ctx.assertUuid(parentDocumentId, "parentDocumentId must be an uuid"); - } - - if (index) ctx.assertPositiveInteger(index, "index must be an integer (>=0)"); +router.post("documents.create", auth(), createDocumentFromContext); +router.post("documents.import", auth(), async (ctx) => { + const file: any = Object.values(ctx.request.files)[0]; const user = ctx.state.user; authorize(user, "create", Document); - const collection = await Collection.scope({ - method: ["withMembership", user.id], - }).findOne({ - where: { - id: collectionId, - teamId: user.teamId, - }, - }); - authorize(user, "publish", collection); - - let parentDocument; - if (parentDocumentId) { - parentDocument = await Document.findOne({ - where: { - id: parentDocumentId, - collectionId: collection.id, - }, - }); - authorize(user, "read", parentDocument, { collection }); - } - - let templateDocument; - if (templateId) { - templateDocument = await Document.findByPk(templateId, { userId: user.id }); - authorize(user, "read", templateDocument); - } - - let document = await Document.create({ - parentDocumentId, - editorVersion, - collectionId: collection.id, - teamId: user.teamId, - userId: user.id, - lastModifiedById: user.id, - createdById: user.id, - template, - templateId: templateDocument ? templateDocument.id : undefined, - title: templateDocument ? templateDocument.title : title, - text: templateDocument ? templateDocument.text : text, - }); - - await Event.create({ - name: "documents.create", - documentId: document.id, - collectionId: document.collectionId, - teamId: document.teamId, - actorId: user.id, - data: { title: document.title, templateId }, + const { text, title } = await documentImporter({ + user, + file, ip: ctx.request.ip, }); - if (publish) { - await document.publish(); + ctx.body.text = text; + ctx.body.title = title; - await Event.create({ - name: "documents.publish", - documentId: document.id, - collectionId: document.collectionId, - teamId: document.teamId, - actorId: user.id, - data: { title: document.title }, - ip: ctx.request.ip, - }); - } - - // reload to get all of the data needed to present (user, collection etc) - // we need to specify publishedAt to bypass default scope that only returns - // published documents - document = await Document.findOne({ - where: { id: document.id, publishedAt: document.publishedAt }, - }); - document.collection = collection; - - ctx.body = { - data: await presentDocument(document), - policies: presentPolicies(user, [document]), - }; + await createDocumentFromContext(ctx); }); router.post("documents.templatize", auth(), async (ctx) => { @@ -1073,4 +991,107 @@ router.post("documents.unpublish", auth(), async (ctx) => { }; }); +// TODO: update to actual `ctx` type +export async function createDocumentFromContext(ctx: any) { + const { + title = "", + text = "", + publish, + collectionId, + parentDocumentId, + templateId, + template, + index, + } = ctx.body; + const editorVersion = ctx.headers["x-editor-version"]; + + ctx.assertUuid(collectionId, "collectionId must be an uuid"); + if (parentDocumentId) { + ctx.assertUuid(parentDocumentId, "parentDocumentId must be an uuid"); + } + + if (index) ctx.assertPositiveInteger(index, "index must be an integer (>=0)"); + + const user = ctx.state.user; + authorize(user, "create", Document); + + const collection = await Collection.scope({ + method: ["withMembership", user.id], + }).findOne({ + where: { + id: collectionId, + teamId: user.teamId, + }, + }); + authorize(user, "publish", collection); + + let parentDocument; + if (parentDocumentId) { + parentDocument = await Document.findOne({ + where: { + id: parentDocumentId, + collectionId: collection.id, + }, + }); + authorize(user, "read", parentDocument, { collection }); + } + + let templateDocument; + if (templateId) { + templateDocument = await Document.findByPk(templateId, { userId: user.id }); + authorize(user, "read", templateDocument); + } + + let document = await Document.create({ + parentDocumentId, + editorVersion, + collectionId: collection.id, + teamId: user.teamId, + userId: user.id, + lastModifiedById: user.id, + createdById: user.id, + template, + templateId: templateDocument ? templateDocument.id : undefined, + title: templateDocument ? templateDocument.title : title, + text: templateDocument ? templateDocument.text : text, + }); + + await Event.create({ + name: "documents.create", + documentId: document.id, + collectionId: document.collectionId, + teamId: document.teamId, + actorId: user.id, + data: { title: document.title, templateId }, + ip: ctx.request.ip, + }); + + if (publish) { + await document.publish(); + + await Event.create({ + name: "documents.publish", + documentId: document.id, + collectionId: document.collectionId, + teamId: document.teamId, + actorId: user.id, + data: { title: document.title }, + ip: ctx.request.ip, + }); + } + + // reload to get all of the data needed to present (user, collection etc) + // we need to specify publishedAt to bypass default scope that only returns + // published documents + document = await Document.findOne({ + where: { id: document.id, publishedAt: document.publishedAt }, + }); + document.collection = collection; + + return (ctx.body = { + data: await presentDocument(document), + policies: presentPolicies(user, [document]), + }); +} + export default router; diff --git a/server/api/index.js b/server/api/index.js index c7023163..8fd0f930 100644 --- a/server/api/index.js +++ b/server/api/index.js @@ -1,6 +1,6 @@ // @flow import Koa from "koa"; -import bodyParser from "koa-bodyparser"; +import bodyParser from "koa-body"; import Router from "koa-router"; import { NotFoundError } from "../errors"; @@ -31,8 +31,13 @@ const api = new Koa(); const router = new Router(); // middlewares +api.use( + bodyParser({ + multipart: true, + formidable: { maxFieldsSize: 10 * 1024 * 1024 }, + }) +); api.use(errorHandling()); -api.use(bodyParser()); api.use(methodOverride()); api.use(validation()); api.use(apiWrapper()); diff --git a/server/auth/index.js b/server/auth/index.js index 7aeb7fb0..11095f49 100644 --- a/server/auth/index.js +++ b/server/auth/index.js @@ -1,7 +1,7 @@ // @flow import addMonths from "date-fns/add_months"; import Koa from "koa"; -import bodyParser from "koa-bodyparser"; +import bodyParser from "koa-body"; import Router from "koa-router"; import auth from "../middlewares/authentication"; import validation from "../middlewares/validation"; diff --git a/server/commands/documentImporter.js b/server/commands/documentImporter.js new file mode 100644 index 00000000..efa615c4 --- /dev/null +++ b/server/commands/documentImporter.js @@ -0,0 +1,113 @@ +// @flow +import fs from "fs"; +import File from "formidable/lib/file"; +import mammoth from "mammoth"; +import TurndownService from "turndown"; +import uuid from "uuid"; +import parseTitle from "../../shared/utils/parseTitle"; +import { Attachment, Event, User } from "../models"; +import dataURItoBuffer from "../utils/dataURItoBuffer"; +import parseImages from "../utils/parseImages"; +import { uploadToS3FromBuffer } from "../utils/s3"; + +// https://github.com/domchristie/turndown#options +const turndownService = new TurndownService({ + hr: "---", + bulletListMarker: "-", + headingStyle: "atx", +}); + +interface ImportableFile { + type: string; + getMarkdown: (file: any) => Promise; +} + +const importMapping: ImportableFile[] = [ + { + type: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + getMarkdown: docxToMarkdown, + }, + { + type: "text/html", + getMarkdown: htmlToMarkdown, + }, + { + type: "text/plain", + getMarkdown: fileToMarkdown, + }, + { + type: "text/markdown", + getMarkdown: fileToMarkdown, + }, +]; + +async function fileToMarkdown(file): Promise { + return fs.promises.readFile(file.path, "utf8"); +} + +async function docxToMarkdown(file): Promise { + const { value } = await mammoth.convertToHtml(file); + return turndownService.turndown(value); +} + +async function htmlToMarkdown(file): Promise { + const value = await fs.promises.readFile(file.path, "utf8"); + return turndownService.turndown(value); +} + +export default async function documentImporter({ + file, + user, + ip, +}: { + user: User, + file: File, + ip: string, +}): Promise<{ text: string, title: string }> { + const fileInfo = importMapping.filter((item) => item.type === file.type)[0]; + let title = file.name.replace(/\.[^/.]+$/, ""); + let text = await fileInfo.getMarkdown(file); + + // If the first line of the imported text looks like a markdown heading + // then we can use this as the document title + if (text.trim().startsWith("# ")) { + const result = parseTitle(text); + title = result.title; + text = text.replace(`# ${title}\n`, ""); + } + + // find data urls, convert to blobs, upload and write attachments + const images = parseImages(text); + const dataURIs = images.filter((href) => href.startsWith("data:")); + + for (const uri of dataURIs) { + const name = "imported"; + const key = `uploads/${user.id}/${uuid.v4()}/${name}`; + const acl = process.env.AWS_S3_ACL || "private"; + const { buffer, type } = dataURItoBuffer(uri); + const url = await uploadToS3FromBuffer(buffer, type, key, acl); + + const attachment = await Attachment.create({ + key, + acl, + url, + size: buffer.length, + contentType: type, + teamId: user.teamId, + userId: user.id, + }); + + await Event.create({ + name: "attachments.create", + data: { name }, + teamId: user.teamId, + userId: user.id, + ip, + }); + + text = text.replace(uri, attachment.redirectUrl); + } + + return { text, title }; +} diff --git a/server/commands/documentImporter.test.js b/server/commands/documentImporter.test.js new file mode 100644 index 00000000..2cac5272 --- /dev/null +++ b/server/commands/documentImporter.test.js @@ -0,0 +1,77 @@ +// @flow +import path from "path"; +import File from "formidable/lib/file"; +import { Attachment } from "../models"; +import { buildUser } from "../test/factories"; +import { flushdb } from "../test/support"; +import documentImporter from "./documentImporter"; + +jest.mock("../utils/s3"); + +beforeEach(() => flushdb()); + +describe("documentImporter", () => { + const ip = "127.0.0.1"; + + it("should convert Word Document to markdown", async () => { + const user = await buildUser(); + const name = "images.docx"; + const file = new File({ + name, + type: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + const response = await documentImporter({ + user, + file, + ip, + }); + + const attachments = await Attachment.count(); + expect(attachments).toEqual(1); + + expect(response.text).toContain("This is a test document for images"); + expect(response.text).toContain("![](/api/attachments.redirect?id="); + expect(response.title).toEqual("images"); + }); + + it("should convert HTML Document to markdown", async () => { + const user = await buildUser(); + const name = "webpage.html"; + const file = new File({ + name, + type: "text/html", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + const response = await documentImporter({ + user, + file, + ip, + }); + + expect(response.text).toContain("Text paragraph"); + expect(response.title).toEqual("Heading 1"); + }); + + it("should load markdown", async () => { + const user = await buildUser(); + const name = "markdown.md"; + const file = new File({ + name, + type: "text/plain", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + const response = await documentImporter({ + user, + file, + ip, + }); + + expect(response.text).toContain("This is a test paragraph"); + expect(response.title).toEqual("Heading 1"); + }); +}); diff --git a/server/commands/documentMover.js b/server/commands/documentMover.js index cdae6d91..353bd455 100644 --- a/server/commands/documentMover.js +++ b/server/commands/documentMover.js @@ -1,6 +1,5 @@ // @flow -import { type Context } from "koa"; -import { Document, Collection, Event } from "../models"; +import { Document, Collection, User, Event } from "../models"; import { sequelize } from "../sequelize"; export default async function documentMover({ @@ -11,7 +10,7 @@ export default async function documentMover({ index, ip, }: { - user: Context, + user: User, document: Document, collectionId: string, parentDocumentId?: string, diff --git a/server/commands/documentMover.test.js b/server/commands/documentMover.test.js index 04dd1020..bee447d6 100644 --- a/server/commands/documentMover.test.js +++ b/server/commands/documentMover.test.js @@ -1,7 +1,7 @@ /* eslint-disable flowtype/require-valid-file-annotation */ -import documentMover from "../commands/documentMover"; import { buildDocument, buildCollection } from "../test/factories"; import { flushdb, seed } from "../test/support"; +import documentMover from "./documentMover"; beforeEach(() => flushdb()); diff --git a/server/commands/userInviter.test.js b/server/commands/userInviter.test.js index 8517a666..327ad367 100644 --- a/server/commands/userInviter.test.js +++ b/server/commands/userInviter.test.js @@ -1,7 +1,7 @@ /* eslint-disable flowtype/require-valid-file-annotation */ -import userInviter from "../commands/userInviter"; import { buildUser } from "../test/factories"; import { flushdb } from "../test/support"; +import userInviter from "./userInviter"; beforeEach(() => flushdb()); diff --git a/server/test/fixtures/images.docx b/server/test/fixtures/images.docx new file mode 100644 index 00000000..25a9156a Binary files /dev/null and b/server/test/fixtures/images.docx differ diff --git a/server/test/fixtures/markdown.md b/server/test/fixtures/markdown.md new file mode 100644 index 00000000..59499fce --- /dev/null +++ b/server/test/fixtures/markdown.md @@ -0,0 +1,8 @@ +# Heading 1 + +## Heading 2 + +This is a test paragraph + +- list item 1 +- list item 2 diff --git a/server/test/fixtures/webpage.html b/server/test/fixtures/webpage.html new file mode 100644 index 00000000..a85c8026 --- /dev/null +++ b/server/test/fixtures/webpage.html @@ -0,0 +1,8 @@ + + + +

Heading 1

+

Text paragraph

+ + + \ No newline at end of file diff --git a/server/utils/__mocks__/s3.js b/server/utils/__mocks__/s3.js new file mode 100644 index 00000000..e0c4056f --- /dev/null +++ b/server/utils/__mocks__/s3.js @@ -0,0 +1,5 @@ +/* eslint-disable flowtype/require-valid-file-annotation */ + +export const uploadToS3FromBuffer = jest.fn().mockReturnValue("/endpoint/key"); + +export const publicS3Endpoint = jest.fn().mockReturnValue("http://mock"); diff --git a/server/utils/dataURItoBuffer.js b/server/utils/dataURItoBuffer.js new file mode 100644 index 00000000..098506ef --- /dev/null +++ b/server/utils/dataURItoBuffer.js @@ -0,0 +1,20 @@ +// @flow + +export default function dataURItoBuffer(dataURI: string) { + const split = dataURI.split(","); + + if (!dataURI.startsWith("data") || split.length <= 1) { + throw new Error("Not a dataURI"); + } + + // separate out the mime component + const type = split[0].split(":")[1].split(";")[0]; + + // convert base64 to buffer + const buffer = Buffer.from(split[1], "base64"); + + return { + buffer, + type, + }; +} diff --git a/server/utils/dataURItoBuffer.test.js b/server/utils/dataURItoBuffer.test.js new file mode 100644 index 00000000..eb23704d --- /dev/null +++ b/server/utils/dataURItoBuffer.test.js @@ -0,0 +1,20 @@ +// @flow +import dataURItoBuffer from "./dataURItoBuffer"; + +it("should parse value data URI", () => { + const response = dataURItoBuffer( + `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAMAAADXqc3KAAAB+FBMVEUAAAA/mUPidDHiLi5Cn0XkNTPmeUrkdUg/m0Q0pEfcpSbwaVdKskg+lUP4zA/iLi3msSHkOjVAmETdJSjtYFE/lkPnRj3sWUs8kkLeqCVIq0fxvhXqUkbVmSjwa1n1yBLepyX1xxP0xRXqUkboST9KukpHpUbuvRrzrhF/ljbwaljuZFM4jELaoSdLtElJrUj1xxP6zwzfqSU4i0HYnydMtUlIqUfywxb60AxZqEXaoifgMCXptR9MtklHpEY2iUHWnSjvvRr70QujkC+pUC/90glMuEnlOjVMt0j70QriLS1LtEnnRj3qUUXfIidOjsxAhcZFo0bjNDH0xxNLr0dIrUdmntVTkMoyfL8jcLBRuErhJyrgKyb4zA/5zg3tYFBBmUTmQTnhMinruBzvvhnxwxZ/st+Ktt5zp9hqota2vtK6y9FemNBblc9HiMiTtMbFtsM6gcPV2r6dwroseLrMrbQrdLGdyKoobKbo3Zh+ynrgVllZulTsXE3rV0pIqUf42UVUo0JyjEHoS0HmsiHRGR/lmRz/1hjqnxjvpRWfwtOhusaz0LRGf7FEfbDVmqHXlJeW0pbXq5bec3fX0nTnzmuJuWvhoFFhm0FtrziBsjaAaDCYWC+uSi6jQS3FsSfLJiTirCOkuCG1KiG+wSC+GBvgyhTszQ64Z77KAAAARXRSTlMAIQRDLyUgCwsE6ebm5ubg2dLR0byXl4FDQzU1NDEuLSUgC+vr6urq6ubb29vb2tra2tG8vLu7u7uXl5eXgYGBgYGBLiUALabIAAABsElEQVQoz12S9VPjQBxHt8VaOA6HE+AOzv1wd7pJk5I2adpCC7RUcHd3d3fXf5PvLkxheD++z+yb7GSRlwD/+Hj/APQCZWxM5M+goF+RMbHK594v+tPoiN1uHxkt+xzt9+R9wnRTZZQpXQ0T5uP1IQxToyOAZiQu5HEpjeA4SWIoksRxNiGC1tRZJ4LNxgHgnU5nJZBDvuDdl8lzQRBsQ+s9PZt7s7Pz8wsL39/DkIfZ4xlB2Gqsq62ta9oxVlVrNZpihFRpGO9fzQw1ms0NDWZz07iGkJmIFH8xxkc3a/WWlubmFkv9AB2SEpDvKxbjidN2faseaNV3zoHXvv7wMODJdkOHAegweAfFPx4G67KluxzottCU9n8CUqXzcIQdXOytAHqXxomvykhEKN9EFutG22p//0rbNvHVxiJywa8yS2KDfV1dfbu31H8jF1RHiTKtWYeHxUvq3bn0pyjCRaiRU6aDO+gb3aEfEeVNsDgm8zzLy9egPa7Qt8TSJdwhjplk06HH43ZNJ3s91KKCHQ5x4sw1fRGYDZ0n1L4FKb9/BP5JLYxToheoFCVxz57PPS8UhhEpLBVeAAAAAElFTkSuQmCC` + ); + expect(response.buffer).toBeTruthy(); + expect(response.type).toBe("image/png"); +}); + +it("should throw an error with junk input", () => { + let err; + try { + dataURItoBuffer("what"); + } catch (error) { + err = error; + } + expect(err).toBeTruthy(); +}); diff --git a/server/utils/parseDocumentIds.test.js b/server/utils/parseDocumentIds.test.js index eb938748..75c210e9 100644 --- a/server/utils/parseDocumentIds.test.js +++ b/server/utils/parseDocumentIds.test.js @@ -1,4 +1,4 @@ -/* eslint-disable flowtype/require-valid-file-annotation */ +// @flow import parseDocumentIds from "./parseDocumentIds"; it("should not return non links", () => { diff --git a/server/utils/parseImages.js b/server/utils/parseImages.js new file mode 100644 index 00000000..d62188d8 --- /dev/null +++ b/server/utils/parseImages.js @@ -0,0 +1,26 @@ +// @flow +import { parser } from "rich-markdown-editor"; + +export default function parseImages(text: string): string[] { + const value = parser.parse(text); + const images = []; + + function findImages(node) { + if (node.type.name === "image") { + if (!images.includes(node.attrs.src)) { + images.push(node.attrs.src); + } + + return; + } + + if (!node.content.size) { + return; + } + + node.content.descendants(findImages); + } + + findImages(value); + return images; +} diff --git a/server/utils/parseImages.test.js b/server/utils/parseImages.test.js new file mode 100644 index 00000000..24cd1a3a --- /dev/null +++ b/server/utils/parseImages.test.js @@ -0,0 +1,24 @@ +// @flow +import parseImages from "./parseImages"; + +it("should not return non images", () => { + expect(parseImages(`# Header`).length).toBe(0); +}); + +it("should return an array of images", () => { + const result = parseImages(`# Header + + ![internal](/attachments/image.png) + `); + + expect(result.length).toBe(1); + expect(result[0]).toBe("/attachments/image.png"); +}); + +it("should not return non document links", () => { + expect(parseImages(`[google](http://www.google.com)`).length).toBe(0); +}); + +it("should not return non document relative links", () => { + expect(parseImages(`[relative](/developers)`).length).toBe(0); +}); diff --git a/server/utils/s3.js b/server/utils/s3.js index 05ed6d9e..4cf11ee2 100644 --- a/server/utils/s3.js +++ b/server/utils/s3.js @@ -89,6 +89,28 @@ export const publicS3Endpoint = (isServerUpload?: boolean) => { }${AWS_S3_UPLOAD_BUCKET_NAME}`; }; +export const uploadToS3FromBuffer = async ( + buffer: Buffer, + contentType: string, + key: string, + acl: string +) => { + await s3 + .putObject({ + ACL: acl, + Bucket: AWS_S3_UPLOAD_BUCKET_NAME, + Key: key, + ContentType: contentType, + ContentLength: buffer.length, + ServerSideEncryption: "AES256", + Body: buffer, + }) + .promise(); + + const endpoint = publicS3Endpoint(true); + return `${endpoint}/${key}`; +}; + export const uploadToS3FromUrl = async ( url: string, key: string, diff --git a/yarn.lock b/yarn.lock index 0d9839a6..601b87b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1510,6 +1510,19 @@ resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ== +"@types/events@*": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== + +"@types/formidable@^1.0.31": + version "1.0.31" + resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-1.0.31.tgz#274f9dc2d0a1a9ce1feef48c24ca0859e7ec947b" + integrity sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q== + dependencies: + "@types/events" "*" + "@types/node" "*" + "@types/graceful-fs@^4.1.2": version "4.1.3" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.3.tgz#039af35fe26bec35003e8d86d2ee9c586354348f" @@ -1945,7 +1958,7 @@ aproba@^1.1.1: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== -argparse@^1.0.7: +argparse@^1.0.7, argparse@~1.0.3: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== @@ -2490,7 +2503,7 @@ bluebird@^3.5.5, bluebird@^3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -bluebird@~3.4.1: +bluebird@~3.4.0, bluebird@~3.4.1: version "3.4.7" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM= @@ -3184,7 +3197,7 @@ cluster-key-slot@^1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== -co-body@^5.1.0: +co-body@^5.1.1: version "5.2.0" resolved "https://registry.yarnpkg.com/co-body/-/co-body-5.2.0.tgz#5a0a658c46029131e0e3a306f67647302f71c124" integrity sha512-sX/LQ7LqUhgyaxzbe7IqwPeTr2yfpfUIQ/dgpKo6ZI4y4lpQA0YxAomWIY+7I7rHWcG02PG+OuPREzMW/5tszQ== @@ -3454,11 +3467,6 @@ copy-to-clipboard@^3.0.6, copy-to-clipboard@^3.0.8: dependencies: toggle-selection "^1.0.6" -copy-to@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/copy-to/-/copy-to-2.0.1.tgz#2680fbb8068a48d08656b6098092bdafc906f4a5" - integrity sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU= - core-js-compat@^3.6.2: version "3.6.5" resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.6.5.tgz#2a51d9a4e25dfd6e690251aa81f99e3c05481f1c" @@ -3995,6 +4003,13 @@ double-ended-queue@^2.1.0-0: resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" integrity sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw= +duck@~0.1.11: + version "0.1.11" + resolved "https://registry.yarnpkg.com/duck/-/duck-0.1.11.tgz#3adc1a3d2fbdd5879ffd3bda05ce0f69355e9093" + integrity sha1-OtwaPS+91Yef/TvaBc4PaTVekJM= + dependencies: + underscore "~1.4.4" + duplexer2@~0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" @@ -4941,6 +4956,11 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +formidable@^1.1.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.2.tgz#bf69aea2972982675f00865342b982986f6b8dd9" + integrity sha512-V8gLm+41I/8kguQ4/o1D3RIHRmhYFG4pnNyonvua+40rqcEmT4+V71yaZ3B457xbbgCsCfjSPi65u/W6vK1U5Q== + fragment-cache@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" @@ -6723,7 +6743,7 @@ jsbn@~0.1.0: resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= -jsdom@^16.2.2: +jsdom@^16.2.0, jsdom@^16.2.2: version "16.4.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.4.0.tgz#36005bde2d136f73eee1a830c6d45e55408edddb" integrity sha512-lYMm3wYdgPhrl7pDcRmvzPhhrGVBeVhPIqeHjzeiHN3DFmD1RBpbExbi8vU7BJdH8VAZYovR8DMt0PNNDM7k8w== @@ -6889,6 +6909,13 @@ jszip@^3.5.0: readable-stream "~2.3.6" set-immediate-shim "~1.0.1" +jszip@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-2.5.0.tgz#7444fd8551ddf3e5da7198fea0c91bc8308cc274" + integrity sha1-dET9hVHd8+XacZj+oMkbyDCMwnQ= + dependencies: + pako "~0.2.5" + jwa@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a" @@ -6966,13 +6993,14 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -koa-bodyparser@4.2.0: +koa-body@^4.2.0: version "4.2.0" - resolved "https://registry.yarnpkg.com/koa-bodyparser/-/koa-bodyparser-4.2.0.tgz#bce6e08bc65f8709b6d1faa9411c7f0d8938aa54" - integrity sha1-vObgi8Zfhwm20fqpQRx/DYk4qlQ= + resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.2.0.tgz#37229208b820761aca5822d14c5fc55cee31b26f" + integrity sha512-wdGu7b9amk4Fnk/ytH8GuWwfs4fsB5iNkY8kZPpgQVb04QZSv85T0M8reb+cJmvLE8cjPYvBzRikD3s6qz8OoA== dependencies: - co-body "^5.1.0" - copy-to "^2.0.1" + "@types/formidable" "^1.0.31" + co-body "^5.1.1" + formidable "^1.1.1" koa-compose@^3.0.0, koa-compose@^3.2.1: version "3.2.1" @@ -7508,6 +7536,15 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3 dependencies: js-tokens "^3.0.0 || ^4.0.0" +lop@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/lop/-/lop-0.4.0.tgz#4f0e4384d5c4f455d0b86d254fd52a9d05593c2c" + integrity sha1-Tw5DhNXE9FXQuG0lT9UqnQVZPCw= + dependencies: + duck "~0.1.11" + option "~0.2.1" + underscore "~1.4.4" + lower-case@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac" @@ -7591,6 +7628,20 @@ makeerror@1.0.x: dependencies: tmpl "1.0.x" +mammoth@^1.4.11: + version "1.4.11" + resolved "https://registry.yarnpkg.com/mammoth/-/mammoth-1.4.11.tgz#cf2d00b09fa61112f1758f3482d9b4507a1d106c" + integrity sha512-OB5/LJfI2QptpFKMfcHxom5nXnobySI6o8UkoeRjzYgbV7ZyC1WtDMATJ/khAfzhBfWHvYVdFGtKffyDX+6kMQ== + dependencies: + argparse "~1.0.3" + bluebird "~3.4.0" + jszip "~2.5.0" + lop "~0.4.0" + path-is-absolute "^1.0.0" + sax "~1.1.1" + underscore "~1.8.3" + xmlbuilder "^10.0.0" + map-age-cleaner@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a" @@ -8310,6 +8361,11 @@ only@~0.0.2: resolved "https://registry.yarnpkg.com/only/-/only-0.0.2.tgz#2afde84d03e50b9a8edc444e30610a70295edfb4" integrity sha1-Kv3oTQPlC5qO3EROMGEKcCle37Q= +option@~0.2.1: + version "0.2.4" + resolved "https://registry.yarnpkg.com/option/-/option-0.2.4.tgz#fd475cdf98dcabb3cb397a3ba5284feb45edbfe4" + integrity sha1-/Udc35jcq7PLOXo7pShP60Xtv+Q= + optionator@^0.8.1: version "0.8.3" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" @@ -8525,6 +8581,11 @@ packet-reader@1.0.0: resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== +pako@~0.2.5: + version "0.2.9" + resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" + integrity sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU= + pako@~1.0.2, pako@~1.0.5: version "1.0.11" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" @@ -9977,6 +10038,11 @@ sax@>=0.6.0: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== +sax@~1.1.1: + version "1.1.6" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.6.tgz#5d616be8a5e607d54e114afae55b7eaf2fcc3240" + integrity sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA= + saxes@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" @@ -11216,6 +11282,13 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turndown@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/turndown/-/turndown-6.0.0.tgz#c083d6109a9366be1b84b86b20af09140ea4b413" + integrity sha512-UVJBhSyRHCpNKtQ00mNWlYUM/i+tcipkb++F0PrOpt0L7EhNd0AX9mWEpL2dRFBu7LWXMp4HgAMA4OeKKnN7og== + dependencies: + jsdom "^16.2.0" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -11342,6 +11415,16 @@ underscore@^1.7.0: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.10.2.tgz#73d6aa3668f3188e4adb0f1943bd12cfd7efaaaf" integrity sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg== +underscore@~1.4.4: + version "1.4.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604" + integrity sha1-YaajIBBiKvoHljvzJSA88SI51gQ= + +underscore@~1.8.3: + version "1.8.3" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" + integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -11994,6 +12077,11 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" +xmlbuilder@^10.0.0: + version "10.1.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0" + integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg== + xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"