From 2c1a111dee9dc55134bd5853f8e10e4017bb694e Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Wed, 16 Sep 2020 21:54:33 -0700 Subject: [PATCH] feat: Support importing .docx or .html files as new documents (#1551) * Support importing .docx as new documents * Add html file support, build types and interface for easily adding file types to importer * fix: Upload embedded images in docx to storage * refactor: Bulk of logic to command * refactor: Do all importing on server, so we're not splitting logic for import into two places * test: Add documentImporter tests Co-authored-by: Lance Whatley --- app/components/DropToImport.js | 9 +- app/menus/CollectionMenu.js | 24 ++- app/stores/DocumentsStore.js | 46 +++++ app/utils/ApiClient.js | 30 +++- app/utils/importFile.js | 58 ------- package.json | 6 +- server/api/documents.js | 205 +++++++++++++---------- server/api/index.js | 9 +- server/auth/index.js | 2 +- server/commands/documentImporter.js | 113 +++++++++++++ server/commands/documentImporter.test.js | 77 +++++++++ server/commands/documentMover.js | 5 +- server/commands/documentMover.test.js | 2 +- server/commands/userInviter.test.js | 2 +- server/test/fixtures/images.docx | Bin 0 -> 9258 bytes server/test/fixtures/markdown.md | 8 + server/test/fixtures/webpage.html | 8 + server/utils/__mocks__/s3.js | 5 + server/utils/dataURItoBuffer.js | 20 +++ server/utils/dataURItoBuffer.test.js | 20 +++ server/utils/parseDocumentIds.test.js | 2 +- server/utils/parseImages.js | 26 +++ server/utils/parseImages.test.js | 24 +++ server/utils/s3.js | 22 +++ yarn.lock | 116 +++++++++++-- 25 files changed, 645 insertions(+), 194 deletions(-) delete mode 100644 app/utils/importFile.js create mode 100644 server/commands/documentImporter.js create mode 100644 server/commands/documentImporter.test.js create mode 100644 server/test/fixtures/images.docx create mode 100644 server/test/fixtures/markdown.md create mode 100644 server/test/fixtures/webpage.html create mode 100644 server/utils/__mocks__/s3.js create mode 100644 server/utils/dataURItoBuffer.js create mode 100644 server/utils/dataURItoBuffer.test.js create mode 100644 server/utils/parseImages.js create mode 100644 server/utils/parseImages.test.js diff --git a/app/components/DropToImport.js b/app/components/DropToImport.js index ecb8c78c..54b936f6 100644 --- a/app/components/DropToImport.js +++ b/app/components/DropToImport.js @@ -8,7 +8,6 @@ import { withRouter, type RouterHistory, type Match } from "react-router-dom"; import { createGlobalStyle } from "styled-components"; import DocumentsStore from "stores/DocumentsStore"; import LoadingIndicator from "components/LoadingIndicator"; -import importFile from "utils/importFile"; const EMPTY_OBJECT = {}; let importingLock = false; @@ -61,12 +60,12 @@ class DropToImport extends React.Component { } for (const file of files) { - const doc = await importFile({ - documents: this.props.documents, + const doc = await this.props.documents.import( file, documentId, collectionId, - }); + { publish: true } + ); if (redirect) { this.props.history.push(doc.url); @@ -95,7 +94,7 @@ class DropToImport extends React.Component { return ( { const files = getDataTransferFiles(ev); try { - const document = await importFile({ - file: files[0], - documents: this.props.documents, - collectionId: this.props.collection.id, - }); + const file = files[0]; + const document = await this.props.documents.import( + file, + null, + this.props.collection.id, + { publish: true } + ); this.props.history.push(document.url); } catch (err) { this.props.ui.showToast(err.message); @@ -103,7 +104,14 @@ class CollectionMenu extends React.Component { }; render() { - const { policies, collection, position, onOpen, onClose } = this.props; + const { + policies, + documents, + collection, + position, + onOpen, + onClose, + } = this.props; const can = policies.abilities(collection.id); return ( @@ -114,7 +122,7 @@ class CollectionMenu extends React.Component { ref={(ref) => (this.file = ref)} onChange={this.onFilePicked} onClick={(ev) => ev.stopPropagation()} - accept="text/markdown, text/plain" + accept={documents.importFileTypes.join(", ")} /> diff --git a/app/stores/DocumentsStore.js b/app/stores/DocumentsStore.js index d1a76959..ff1aa120 100644 --- a/app/stores/DocumentsStore.js +++ b/app/stores/DocumentsStore.js @@ -18,12 +18,23 @@ import Document from "models/Document"; import type { FetchOptions, PaginationParams, SearchResult } from "types"; import { client } from "utils/ApiClient"; +type ImportOptions = { + publish?: boolean, +}; + export default class DocumentsStore extends BaseStore { @observable recentlyViewedIds: string[] = []; @observable searchCache: Map = new Map(); @observable starredIds: Map = new Map(); @observable backlinks: Map = new Map(); + importFileTypes: string[] = [ + "text/markdown", + "text/plain", + "text/html", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]; + constructor(rootStore: RootStore) { super(rootStore, Document); } @@ -455,6 +466,41 @@ export default class DocumentsStore extends BaseStore { return this.add(res.data); }; + @action + import = async ( + file: File, + parentDocumentId: string, + collectionId: string, + options: ImportOptions + ) => { + const title = file.name.replace(/\.[^/.]+$/, ""); + const formData = new FormData(); + + [ + { key: "parentDocumentId", value: parentDocumentId }, + { key: "collectionId", value: collectionId }, + { key: "title", value: title }, + { key: "publish", value: options.publish }, + { key: "file", value: file }, + ].map((info) => { + if (typeof info.value === "string" && info.value) { + formData.append(info.key, info.value); + } + if (typeof info.value === "boolean") { + formData.append(info.key, info.value.toString()); + } + if (info.value instanceof File) { + formData.append(info.key, info.value); + } + }); + + const res = await client.post("/documents.import", formData); + invariant(res && res.data, "Data should be available"); + + this.addPolicies(res.policies); + return this.add(res.data); + }; + _add = this.add; @action diff --git a/app/utils/ApiClient.js b/app/utils/ApiClient.js index 545f0578..1787d9e4 100644 --- a/app/utils/ApiClient.js +++ b/app/utils/ApiClient.js @@ -28,12 +28,13 @@ class ApiClient { fetch = async ( path: string, method: string, - data: ?Object, + data: ?Object | FormData | void, options: Object = {} ) => { let body; let modifiedPath; let urlToFetch; + let isJson; if (method === "GET") { if (data) { @@ -42,7 +43,18 @@ class ApiClient { modifiedPath = path; } } else if (method === "POST" || method === "PUT") { - body = data ? JSON.stringify(data) : undefined; + body = data || undefined; + + // Only stringify data if its a normal object and + // not if it's [object FormData], in addition to + // toggling Content-Type to application/json + if ( + typeof data === "object" && + (data || "").toString() === "[object Object]" + ) { + isJson = true; + body = JSON.stringify(data); + } } if (path.match(/^http/)) { @@ -51,14 +63,20 @@ class ApiClient { urlToFetch = this.baseUrl + (modifiedPath || path); } - // Construct headers - const headers = new Headers({ + let headerOptions: any = { Accept: "application/json", - "Content-Type": "application/json", "cache-control": "no-cache", "x-editor-version": EDITOR_VERSION, pragma: "no-cache", - }); + }; + // for multipart forms or other non JSON requests fetch + // populates the Content-Type without needing to explicitly + // set it. + if (isJson) { + headerOptions["Content-Type"] = "application/json"; + } + const headers = new Headers(headerOptions); + if (stores.auth.authenticated) { invariant(stores.auth.token, "JWT token not set properly"); headers.set("Authorization", `Bearer ${stores.auth.token}`); diff --git a/app/utils/importFile.js b/app/utils/importFile.js deleted file mode 100644 index c7ef1297..00000000 --- a/app/utils/importFile.js +++ /dev/null @@ -1,58 +0,0 @@ -// @flow -import parseTitle from "shared/utils/parseTitle"; -import DocumentsStore from "stores/DocumentsStore"; -import Document from "models/Document"; - -type Options = { - file: File, - documents: DocumentsStore, - collectionId: string, - documentId?: string, -}; - -const importFile = async ({ - documents, - file, - documentId, - collectionId, -}: Options): Promise => { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - - reader.onload = async (ev) => { - let text = ev.target.result; - let title; - - // If the first line of the imported file looks like a markdown heading - // then we can use this as the document title - if (text.trim().startsWith("# ")) { - const result = parseTitle(text); - title = result.title; - text = text.replace(`# ${title}\n`, ""); - - // otherwise, just use the filename without the extension as our best guess - } else { - title = file.name.replace(/\.[^/.]+$/, ""); - } - - let document = new Document( - { - parentDocumentId: documentId, - collectionId, - text, - title, - }, - documents - ); - try { - document = await document.save({ publish: true }); - resolve(document); - } catch (err) { - reject(err); - } - }; - reader.readAsText(file); - }); -}; - -export default importFile; diff --git a/package.json b/package.json index 76f8dc8e..6ef02a6d 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "jsonwebtoken": "^8.5.0", "jszip": "^3.5.0", "koa": "^2.10.0", - "koa-bodyparser": "4.2.0", + "koa-body": "^4.2.0", "koa-compress": "2.0.0", "koa-convert": "1.2.0", "koa-helmet": "5.2.0", @@ -115,6 +115,7 @@ "koa-sslify": "2.1.2", "koa-static": "^4.0.1", "lodash": "^4.17.19", + "mammoth": "^1.4.11", "mobx": "4.6.0", "mobx-react": "^6.2.5", "natural-sort": "^1.0.0", @@ -156,6 +157,7 @@ "styled-normalize": "^8.0.4", "tiny-cookie": "^2.3.1", "tmp": "0.0.33", + "turndown": "^6.0.0", "uuid": "2.0.2", "validator": "5.2.0" }, @@ -193,4 +195,4 @@ "js-yaml": "^3.13.1" }, "version": "0.47.1" -} \ No newline at end of file +} diff --git a/server/api/documents.js b/server/api/documents.js index d570d037..6795d916 100644 --- a/server/api/documents.js +++ b/server/api/documents.js @@ -1,6 +1,7 @@ // @flow import Router from "koa-router"; import Sequelize from "sequelize"; +import documentImporter from "../commands/documentImporter"; import documentMover from "../commands/documentMover"; import { NotFoundError, InvalidRequestError } from "../errors"; import auth from "../middlewares/authentication"; @@ -707,106 +708,23 @@ router.post("documents.unstar", auth(), async (ctx) => { }; }); -router.post("documents.create", auth(), async (ctx) => { - const { - title = "", - text = "", - publish, - collectionId, - parentDocumentId, - templateId, - template, - index, - } = ctx.body; - const editorVersion = ctx.headers["x-editor-version"]; - - ctx.assertUuid(collectionId, "collectionId must be an uuid"); - if (parentDocumentId) { - ctx.assertUuid(parentDocumentId, "parentDocumentId must be an uuid"); - } - - if (index) ctx.assertPositiveInteger(index, "index must be an integer (>=0)"); +router.post("documents.create", auth(), createDocumentFromContext); +router.post("documents.import", auth(), async (ctx) => { + const file: any = Object.values(ctx.request.files)[0]; const user = ctx.state.user; authorize(user, "create", Document); - const collection = await Collection.scope({ - method: ["withMembership", user.id], - }).findOne({ - where: { - id: collectionId, - teamId: user.teamId, - }, - }); - authorize(user, "publish", collection); - - let parentDocument; - if (parentDocumentId) { - parentDocument = await Document.findOne({ - where: { - id: parentDocumentId, - collectionId: collection.id, - }, - }); - authorize(user, "read", parentDocument, { collection }); - } - - let templateDocument; - if (templateId) { - templateDocument = await Document.findByPk(templateId, { userId: user.id }); - authorize(user, "read", templateDocument); - } - - let document = await Document.create({ - parentDocumentId, - editorVersion, - collectionId: collection.id, - teamId: user.teamId, - userId: user.id, - lastModifiedById: user.id, - createdById: user.id, - template, - templateId: templateDocument ? templateDocument.id : undefined, - title: templateDocument ? templateDocument.title : title, - text: templateDocument ? templateDocument.text : text, - }); - - await Event.create({ - name: "documents.create", - documentId: document.id, - collectionId: document.collectionId, - teamId: document.teamId, - actorId: user.id, - data: { title: document.title, templateId }, + const { text, title } = await documentImporter({ + user, + file, ip: ctx.request.ip, }); - if (publish) { - await document.publish(); + ctx.body.text = text; + ctx.body.title = title; - await Event.create({ - name: "documents.publish", - documentId: document.id, - collectionId: document.collectionId, - teamId: document.teamId, - actorId: user.id, - data: { title: document.title }, - ip: ctx.request.ip, - }); - } - - // reload to get all of the data needed to present (user, collection etc) - // we need to specify publishedAt to bypass default scope that only returns - // published documents - document = await Document.findOne({ - where: { id: document.id, publishedAt: document.publishedAt }, - }); - document.collection = collection; - - ctx.body = { - data: await presentDocument(document), - policies: presentPolicies(user, [document]), - }; + await createDocumentFromContext(ctx); }); router.post("documents.templatize", auth(), async (ctx) => { @@ -1073,4 +991,107 @@ router.post("documents.unpublish", auth(), async (ctx) => { }; }); +// TODO: update to actual `ctx` type +export async function createDocumentFromContext(ctx: any) { + const { + title = "", + text = "", + publish, + collectionId, + parentDocumentId, + templateId, + template, + index, + } = ctx.body; + const editorVersion = ctx.headers["x-editor-version"]; + + ctx.assertUuid(collectionId, "collectionId must be an uuid"); + if (parentDocumentId) { + ctx.assertUuid(parentDocumentId, "parentDocumentId must be an uuid"); + } + + if (index) ctx.assertPositiveInteger(index, "index must be an integer (>=0)"); + + const user = ctx.state.user; + authorize(user, "create", Document); + + const collection = await Collection.scope({ + method: ["withMembership", user.id], + }).findOne({ + where: { + id: collectionId, + teamId: user.teamId, + }, + }); + authorize(user, "publish", collection); + + let parentDocument; + if (parentDocumentId) { + parentDocument = await Document.findOne({ + where: { + id: parentDocumentId, + collectionId: collection.id, + }, + }); + authorize(user, "read", parentDocument, { collection }); + } + + let templateDocument; + if (templateId) { + templateDocument = await Document.findByPk(templateId, { userId: user.id }); + authorize(user, "read", templateDocument); + } + + let document = await Document.create({ + parentDocumentId, + editorVersion, + collectionId: collection.id, + teamId: user.teamId, + userId: user.id, + lastModifiedById: user.id, + createdById: user.id, + template, + templateId: templateDocument ? templateDocument.id : undefined, + title: templateDocument ? templateDocument.title : title, + text: templateDocument ? templateDocument.text : text, + }); + + await Event.create({ + name: "documents.create", + documentId: document.id, + collectionId: document.collectionId, + teamId: document.teamId, + actorId: user.id, + data: { title: document.title, templateId }, + ip: ctx.request.ip, + }); + + if (publish) { + await document.publish(); + + await Event.create({ + name: "documents.publish", + documentId: document.id, + collectionId: document.collectionId, + teamId: document.teamId, + actorId: user.id, + data: { title: document.title }, + ip: ctx.request.ip, + }); + } + + // reload to get all of the data needed to present (user, collection etc) + // we need to specify publishedAt to bypass default scope that only returns + // published documents + document = await Document.findOne({ + where: { id: document.id, publishedAt: document.publishedAt }, + }); + document.collection = collection; + + return (ctx.body = { + data: await presentDocument(document), + policies: presentPolicies(user, [document]), + }); +} + export default router; diff --git a/server/api/index.js b/server/api/index.js index c7023163..8fd0f930 100644 --- a/server/api/index.js +++ b/server/api/index.js @@ -1,6 +1,6 @@ // @flow import Koa from "koa"; -import bodyParser from "koa-bodyparser"; +import bodyParser from "koa-body"; import Router from "koa-router"; import { NotFoundError } from "../errors"; @@ -31,8 +31,13 @@ const api = new Koa(); const router = new Router(); // middlewares +api.use( + bodyParser({ + multipart: true, + formidable: { maxFieldsSize: 10 * 1024 * 1024 }, + }) +); api.use(errorHandling()); -api.use(bodyParser()); api.use(methodOverride()); api.use(validation()); api.use(apiWrapper()); diff --git a/server/auth/index.js b/server/auth/index.js index 7aeb7fb0..11095f49 100644 --- a/server/auth/index.js +++ b/server/auth/index.js @@ -1,7 +1,7 @@ // @flow import addMonths from "date-fns/add_months"; import Koa from "koa"; -import bodyParser from "koa-bodyparser"; +import bodyParser from "koa-body"; import Router from "koa-router"; import auth from "../middlewares/authentication"; import validation from "../middlewares/validation"; diff --git a/server/commands/documentImporter.js b/server/commands/documentImporter.js new file mode 100644 index 00000000..efa615c4 --- /dev/null +++ b/server/commands/documentImporter.js @@ -0,0 +1,113 @@ +// @flow +import fs from "fs"; +import File from "formidable/lib/file"; +import mammoth from "mammoth"; +import TurndownService from "turndown"; +import uuid from "uuid"; +import parseTitle from "../../shared/utils/parseTitle"; +import { Attachment, Event, User } from "../models"; +import dataURItoBuffer from "../utils/dataURItoBuffer"; +import parseImages from "../utils/parseImages"; +import { uploadToS3FromBuffer } from "../utils/s3"; + +// https://github.com/domchristie/turndown#options +const turndownService = new TurndownService({ + hr: "---", + bulletListMarker: "-", + headingStyle: "atx", +}); + +interface ImportableFile { + type: string; + getMarkdown: (file: any) => Promise; +} + +const importMapping: ImportableFile[] = [ + { + type: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + getMarkdown: docxToMarkdown, + }, + { + type: "text/html", + getMarkdown: htmlToMarkdown, + }, + { + type: "text/plain", + getMarkdown: fileToMarkdown, + }, + { + type: "text/markdown", + getMarkdown: fileToMarkdown, + }, +]; + +async function fileToMarkdown(file): Promise { + return fs.promises.readFile(file.path, "utf8"); +} + +async function docxToMarkdown(file): Promise { + const { value } = await mammoth.convertToHtml(file); + return turndownService.turndown(value); +} + +async function htmlToMarkdown(file): Promise { + const value = await fs.promises.readFile(file.path, "utf8"); + return turndownService.turndown(value); +} + +export default async function documentImporter({ + file, + user, + ip, +}: { + user: User, + file: File, + ip: string, +}): Promise<{ text: string, title: string }> { + const fileInfo = importMapping.filter((item) => item.type === file.type)[0]; + let title = file.name.replace(/\.[^/.]+$/, ""); + let text = await fileInfo.getMarkdown(file); + + // If the first line of the imported text looks like a markdown heading + // then we can use this as the document title + if (text.trim().startsWith("# ")) { + const result = parseTitle(text); + title = result.title; + text = text.replace(`# ${title}\n`, ""); + } + + // find data urls, convert to blobs, upload and write attachments + const images = parseImages(text); + const dataURIs = images.filter((href) => href.startsWith("data:")); + + for (const uri of dataURIs) { + const name = "imported"; + const key = `uploads/${user.id}/${uuid.v4()}/${name}`; + const acl = process.env.AWS_S3_ACL || "private"; + const { buffer, type } = dataURItoBuffer(uri); + const url = await uploadToS3FromBuffer(buffer, type, key, acl); + + const attachment = await Attachment.create({ + key, + acl, + url, + size: buffer.length, + contentType: type, + teamId: user.teamId, + userId: user.id, + }); + + await Event.create({ + name: "attachments.create", + data: { name }, + teamId: user.teamId, + userId: user.id, + ip, + }); + + text = text.replace(uri, attachment.redirectUrl); + } + + return { text, title }; +} diff --git a/server/commands/documentImporter.test.js b/server/commands/documentImporter.test.js new file mode 100644 index 00000000..2cac5272 --- /dev/null +++ b/server/commands/documentImporter.test.js @@ -0,0 +1,77 @@ +// @flow +import path from "path"; +import File from "formidable/lib/file"; +import { Attachment } from "../models"; +import { buildUser } from "../test/factories"; +import { flushdb } from "../test/support"; +import documentImporter from "./documentImporter"; + +jest.mock("../utils/s3"); + +beforeEach(() => flushdb()); + +describe("documentImporter", () => { + const ip = "127.0.0.1"; + + it("should convert Word Document to markdown", async () => { + const user = await buildUser(); + const name = "images.docx"; + const file = new File({ + name, + type: + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + const response = await documentImporter({ + user, + file, + ip, + }); + + const attachments = await Attachment.count(); + expect(attachments).toEqual(1); + + expect(response.text).toContain("This is a test document for images"); + expect(response.text).toContain("![](/api/attachments.redirect?id="); + expect(response.title).toEqual("images"); + }); + + it("should convert HTML Document to markdown", async () => { + const user = await buildUser(); + const name = "webpage.html"; + const file = new File({ + name, + type: "text/html", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + const response = await documentImporter({ + user, + file, + ip, + }); + + expect(response.text).toContain("Text paragraph"); + expect(response.title).toEqual("Heading 1"); + }); + + it("should load markdown", async () => { + const user = await buildUser(); + const name = "markdown.md"; + const file = new File({ + name, + type: "text/plain", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + const response = await documentImporter({ + user, + file, + ip, + }); + + expect(response.text).toContain("This is a test paragraph"); + expect(response.title).toEqual("Heading 1"); + }); +}); diff --git a/server/commands/documentMover.js b/server/commands/documentMover.js index cdae6d91..353bd455 100644 --- a/server/commands/documentMover.js +++ b/server/commands/documentMover.js @@ -1,6 +1,5 @@ // @flow -import { type Context } from "koa"; -import { Document, Collection, Event } from "../models"; +import { Document, Collection, User, Event } from "../models"; import { sequelize } from "../sequelize"; export default async function documentMover({ @@ -11,7 +10,7 @@ export default async function documentMover({ index, ip, }: { - user: Context, + user: User, document: Document, collectionId: string, parentDocumentId?: string, diff --git a/server/commands/documentMover.test.js b/server/commands/documentMover.test.js index 04dd1020..bee447d6 100644 --- a/server/commands/documentMover.test.js +++ b/server/commands/documentMover.test.js @@ -1,7 +1,7 @@ /* eslint-disable flowtype/require-valid-file-annotation */ -import documentMover from "../commands/documentMover"; import { buildDocument, buildCollection } from "../test/factories"; import { flushdb, seed } from "../test/support"; +import documentMover from "./documentMover"; beforeEach(() => flushdb()); diff --git a/server/commands/userInviter.test.js b/server/commands/userInviter.test.js index 8517a666..327ad367 100644 --- a/server/commands/userInviter.test.js +++ b/server/commands/userInviter.test.js @@ -1,7 +1,7 @@ /* eslint-disable flowtype/require-valid-file-annotation */ -import userInviter from "../commands/userInviter"; import { buildUser } from "../test/factories"; import { flushdb } from "../test/support"; +import userInviter from "./userInviter"; beforeEach(() => flushdb()); diff --git a/server/test/fixtures/images.docx b/server/test/fixtures/images.docx new file mode 100644 index 0000000000000000000000000000000000000000..25a9156a8dfb80c627b6a1d6bc2afdcb5684087d GIT binary patch literal 9258 zcma)i1#p}@vbC8hW~P|gj+rrLW{R0T=9rn8nHghdW@d<)ImVdTeopr7-t7JV-FiJ$ zQ}az#OP^XgYU!MolLQ4r1%iTt0wT93kq7$2kl%khS=$-XTYUxS8`+sznb0`{EDPeK zWqKKq0%W`Jh|gC@&@4sLEG zhqL!h!4Kp6MuWk$J`MTPDO^5W)HoCSd{r`0^4+K8u#;>^YyBE`C~?a1=;0%3nnsZr zJHrv@pJPO-I!D)Sa~j5xojXBa4TaV(WP}bEpCfY;fO>{mk$V{9fGH*(!1)6SmK4)B zU^RmTdbe4(?dcjOeS1a1v5Ts|o{KARsNfv9jy9m0M|!;AL4Qf;s1re+dgHm>1o2M@RK?~V1KS}8fN=dhNjXYI(3=Pq zx9uP{@j;Y`_J3cV2thFv_elJG-D(WUuJcfDq#8DT9fyuWDp?2eKa|JIa2?-^R zBIIQX3~T4ppU~1NFD;QJA0?(gNfhs`L~RV#JpnpX5ggxcwf-mZF2{hNG0RT73#8)se25y zbNMRy5;vxLIHSRuV(WxW%%^A1Mh#Tej?(XHU}d8lkeY-e>|4Z;!*Hm&D&h-gaHx&I z*u7Z35_jQhw)aVx+MIN;+aHj~&eCEtmI{~0^arcK1V+@9+IfiOl!oeYzWPIbB0opE z{ANR4?2r^5lC!#yoexNE@F6=Za4|q3EQ-S4CUO3Ge4TFD$A5rBdmmk_7-*WuP(NQb z>1Ru9{39IVHkeXh26}-*h!i9_o#Dl1)w^(&I?K&+861s(&<3zVBeM1%4 z>G}%LW?Wl%AGn#PWl`wU2I+&tKnHZs16J`fCia|AmI3}#iAls2dT+y z;(GXK2e9jlP**p!rK@Sha({Y?+059YasP2I(VGdf|YI~2~dHffLe&}aZuW`lr z>wUe^{#nn(O4U&N@A8+_%Nybbd_Ah>cZk2%P-JayJ~#*v&=JIcTSNcCy0NvDgOZ-U zrP1$26#pmI^P!MJQ-!?{6Lp*{dCVHZlxB~9nNAngHSyci^56sA`q*uEe_qY`%Ft_;lYex^%+rXCu{X=sm{VOr4_gU&V*oDo_v^+ znjl>|tPVkBzfY3{8Qf2AOjJdU`yWeAxdN%c_F@Cjra!+xoLiad6x)8>p-E4j8LGaf z*?=~EYlTf_oIH1>CNH@6M^fs@A&Ry~YJ<4s;U6cW*wvI@^;_yj+RsfzJwQ<)$|0UI z5L;6xZl*B`Le%4tm4S$IA$`zZfmR`AY zra#69{A=RMg8)D-@8H+?ui!`g7x*1qERB9c|9j%8)iMKWV5RPz_hqNF7>JO(nR5LR zYspbzeXF;hL9p^*gC*ohh35ntNsa_4GTt)~!Bg9y=QHQT_i;@rQ41QhElaBidOQn% ztZgE_vgVtEy)Av?hFn!7eHNw%DR9PA&6s)ODnI}+u3yV8 z$fw!nT_y-Xs-4jwp$>|zk24K_9C(hfHDJNEP=%=XK!sRcPE4y0OuXV=p@MHcj6*_y%t2*3F*%jPrjVFesRSKS`BR2GWsRrxhvVeL9oAggdbyqf%g<6* zf;exU5DTf~G>XeZg_&|ywRC$xAw-$+?LVT(u$m%}}6_94&krf6wc?B;|K;z2tl5Hk}EbIeg+BrleMaz}X z6h4!N8BdRR;P)b{ z%l6He9t8t?#Yk=Pj0=#oE9=B9u8Ai%&84Q>%=5bTq(Mh_RQqHw6I`{*B-&x7 zPeCzF1ivj;v#HvmV&V;%oA9n+j#|mPvKZG;ECX7oe3Q zIzHuoc>MXgFWr-V@U=UkL@yJ!o75V^YxVr9CpaT;`IBI=c11sTNn4{!DyO{9{@J`* zvA8<++-h9#ijw_vLoHj6>1g|^80iaRh74I{9Kyg@9NxSOq-|aS|BloTKS`?*1(?Is zbx`^c9PLEy5kMcRfILE>V?tep=Gze>Q_^XeT{jJca{cV~?w;Ha zo465ld3O9`WEiLn{BZb5`8N2qlfw4uhzvfsQQm=zji{~E8n7%fhTj4dlSq)2vvwgk z@Ux7&D~k_GIukIe?>~UJJm4Ytw?POh;d&ruR|v>`cK6$+|H9@@XkXnPJP{QQGu$|I zBc^aCM4cv$KfNc}PaAH!k|p%yZEPaiGFc(OwdAu@MgW?83k1sv!dA#0IA7x-LvC7~ zNgj6==O830Ckv@68#FIT@gVbH!c5JLGiKbpH^hhDYdTM6po)pDrPtCVEBa6%oGE_K ztrfJ$`XxrZ7gP#a$RYI>W+-FNcLFGE69`8P9WlWnyd&MkCKLV+Ts}&Cym1pGFiMPZ zZzDT-R!3uOqZHNY&4qGi%5f0!D}c^|)IvAF;T4ZZv}Q@0A8Na|`TmEN7NR~ZC?r8L z8N!tzXRF&MSmSMW7_&MmNu<|v(^hEe?&TZZ#n(qGz));cEAfniS&jd9EtE-32$ROd zvoAGU9C#@WvD^8ryP36kBUw%I=y!9ljS44o;}W5AuY|#q(n+(s59zLXn?0tQiQdPz z!?Ru3>CXr}cbWXw@pb0~3VKXAlMImUz%T&>VGFh$2>q$0cYax|0WmQapx^ApQJdmLy$NE2htM9~5-LI^ZUB z?rVu&-d3HLoqe6>Z>O0NjP?_RVvVn;Tn&52tvc+dOA$dNx|sMI7DJO`$(>8#OF>Gc z7a{r^$S-hyNZV!k@(#`bxc>%D+<&G$9XlgSd-^{iO84uZ@;GVp-?x-0uHdDG&{@`a zWR5h2p@t?%xsz&I{TD)M36CcP=maH6jwdqDErg&)R+#GS8i4RwL~x=f$$ zOqx&e9m&|jhn)wrOUwxm^Hm4TAP%;g)`7*<*-4Ghqq}{fIXlc{*~7FPXAA|2!EHmj zEe@`p@yhdg^$*QzK=Q`oEGDD zcBwzSj6%@~Ar32M{M7VRT6$KUJ5sUSY-UL|k+tbg(vBQaJ+q8oF4$QZ9`{VikmM7LvFqm&-pR?vAs( z;|<)~q8Eg=2nf>C^g&-Ncw%1Sw8WdLtm?#1q)eO0lWs&PM`RU$c|C^|f4EHO4bJf( zCfT!Hv7?)(5!R1-Vn$e=KNWZr92Hd9S{w`?%m+vZ*HwJ;y6-k>D-=Zj0Crui)eE3j zqEfE~{OD#%#BdSRcV^ut?tv*JuJZ6f^tSFhaV>Aq7fl{O548oew2t^Z_t7-Erf$Ay z%nhzXpL7;HDny$;u2|7f&1+KJYggDYh5sqJo;G^t$`6=B30+@}%6Y26d}**|fqsSb zgQBDf(h`-Nn`p% zd=bP*P*__lOXK{vZI;&v@MrU)SzA@=`vJHOt*%CM>hTCy@Qw7XHIWxCoV7Cv@bihE z)Ufxj1>X~l(XxC5Kku}IJ77;TF_9YfEwOxT1OGYC3~WrHIa-m}fK!?}OH}Z^+ zr{!&Q+7D9n&Wlw}yVo{bvHkuvtdqba^wsrpEi6bURC z;`-vsn)mH0CR9LSJddW1fbeyO5mpHCm7j2YZvCJgmq z8p>D|@B)!C;`)av5UH-R9XGT9Cm}|aIiNQD&uE+I2j6-FrVEctXx3nN*co6k4DcHL zE;_^=o#;B+Z9$r)q-k1-N{XAu33u5E8RaQF2JWC3qt^=RwijrRpy?jOa#~TpGO)B5 z@&}DbIYfL$N(vDd2zKgc({z#x`h@Z%U85r+sH|(OFRs=fp?aaVHB`(PnH<^Xqim{R z=S18V+FRf&2I+4*cNG#(ma3_*Ab-bMhhV?hxJXh;9FXfqwhKSj9J#vdwuU4cB5#5n z#CiL%8Im$|npO2PsdK&H3`ZaEpk%K{5c1cCY^f?IsQbh^J)?LR%+ydu7QPy-{+6hlxe?TF8T%n2=$XV{kEp^I z7Hncb%;pAKdCMvo1!>0+8AQDbfkdLozbv3W-pP@#K-a!uUxq%xd%!&G>A_B@(=ySr zMxF4hKtz1ESpW-NdnN-pWTryuPHI8(phPW;wcr|IdWVFK%2Bha80uJX)+3Jz1N~1sYKzWd7 z!(}rD*zJbBlGUy_3)_ipWqTD47N?YMv=1TC<|v~KMI-pqIt6{7UXHCk-Sj?S>Y9=j z&E`(qy$aEuJWRz-2RoY&k63DO?-+>mSsfB&6XAs6!MU%;N8b;Kx=^=Zm@CX~?rSX1q#=RiI zgsj*lalEpG#REXC=D~v|QwwadePkSVd^VF|;#nrWndmb^Z@Q}FZ0X32_Y*Iem+H<1 zEW|lSIbNrk9V|mL)t0i%>lk%T1Nr=lo6CH{ipKW#V*AV5QbqT|*+oSIhUDD*8PSsE zRulzq8)OaJmg^_uhfpcqM8pOW&SMZQktA$PD4=gZ#LNT98u_R*YRl& zGIWIX?If1zQfiwfX=379d?`)&$?F#q+b8HI`d!Ac_U`z5H6n?b`WMGdC#WN7+Qwt+ zvn?m6KCa+NCA~Y^D$XOUr1%_4AWlVI+5)f3wei|-kiQZukw&gR%{!&Cg7<$)Hvl6; zGd+4UfS!pFBb|+v$!X#PU;GnBR~DAR@XQ>NLNc)?5gW-$G7U=zdM=H@4;8Hig->uj za2hmA;tTag;zW`Zj0R*q?$l%n1vCZ(shYE6HscHt=Tfh}=jR&)yptT~R~!c%lUI-F z_FuB*YAeW54Zj-#1Ih7cIItrE5%RYg<_q^64D6HeN99!2=c##lGTDdND z#-DH3tB3I}yQj}Eon83!to3MaG+XGHa_pVblOdJll_R56lx}0r+ z31xaXL_!3z^h67`w3k$8t3Xx{#7SMH-G)z6Jr~xczY6NqcbuBt?$zNB8~aDY74OZG zGk!|Ft57AzP5@GD8fi2~kD4!AAQZXBR*&ykHe}<&t^|jp=TiXi@c3~6HQ|7fYxQX} zH|MawU@(*yF=bpBhi*a6`pe-)46hkVqi$X>HW>Qm|!*KjqpRJt@x91MASe`9Yw_UcBY}JSXn2C&=&5uW#BKP!( z7N1U)3jQn6=b^z7853qx{IQGb~?k8AI*8T?V(ND@1uL_rtjrvW*@Q{^ZctM-4+8~UAmK8-wFI9qx%;@ zNe74cdTO6G911ZzMjWQfY>rVoH!ddU?7?;);A1=+Wph1FI+Uc^zZOdGj8`lfspr}W zBe-e-I+ntg=y>?KmZ!&MExSF{{Y~HxZZ*LJL|97O{4@yQ@G(SGi(sp}@2f@RPJ1?MFlg!OBv-bR-YZ8W^?(5h@mkX z?+vLpS1shfAjKTA3+p&*RJv7Z?1&l@<~~uej)%HF zgi2#dZmw2h)K|`fl0AmTG}jB%%(`o9q@8Yrh$1MiF@5|VvO2JqRu%HS>jWEMY{}d^ zIdYkhSD`fK;iZF?$nWRl@3Pw-IBRz-v@da}p#Z$F$8p1hC!^@D5Rl7MIMWk~vNj{< zEorLM&SGIE2w*fp?`N3b9Y@4ep(NAHJ5C8Bz5d8O3@St6Qf-1o=fg^NfEBHs$B5j_ zG_5#IGo*zc5e8j*8al1Z zAxnP|b@W;ka~J!WZomiE)X+mou+ljHe#{33g69oak%Jdyi3U$mHr{U#268!w(`t6p z#8FoKtZHd)+Vf~Q?qHYx9zX#e%30h+{(P=3ftF;f_jF?mNGfi8+tC;}&`L~pym%u#8I8yQB8M(a*K zF*Z{R;_L~2M2^yEAZu9n3`mS`LN%VbN39?g2e5)sCPO@3Fp5^XLA$AsP4w5}a4&o` z@y+>|{l1^81h2@QNMxO$PUVFn2T(t-v-4gvxpg{rz@1Apy&>9kaJ@r%wSe`V%!7lV+C=}3S3 z5gxLq?>w=bYWvN`wkd-j8!qUX6Ak-inw;3jAK@uLgtwwEC1AwwdbvRbgi;UI z>$@L?%KgDSUBs6as?aqeDtOK3BT7Aw$0dA}p#ChrS;-%b!lKKDrV{+1B4xL4R@}TM zFszvxdBLSz+HI2XW7Z?W7IQrI22C{s=l8I7CSt?vVfY0P9 z4bFraw^Ydy_d3A-aB_f*sLyfHurxdr+G&&BZk}?0Ku;e&%AP{_>~X5@tAK(v=%o=L z@x=68`KS@nq|ON6-@SrAp9Uixj!S4~&G~~0tpTQBmgRA3;XCrjtWc?T*C2D{SUka0 z>`URDuRm@B64ZC4taucegHrnA>$H>sm33k|F?oJgPs3d&gN2$qTvR2q%J?csht%W9 z{?{4Kru?A|H=Wzm-N+O~O7&hAGlzZ9W%VQ6>Wv-v3>+cKx~Z#%Q-Tr~CFiDy5l4YDi z|5W7(!%)YjuLu^1?R1u4&lOps^XHG|So&W)RrPLXBhOP#U~>v{c=xE}mYCQg`%`gj z??c)4kufaYsTkbEH`FtfUt_gB(x`36)%y^~nWBhF*Y#1!yPx~6#bWmEtnIPO^44Re z$cEx5sR*7pru?iWnk>!9yQ2Ztjfh6X84^+ie$zw}cL{!RPh z6GR0Ea%$uPP^t>^ORZP*7^zMgj1vk?ls8a?Ig4l^^{ZDkd{-wq*Kgc$?%ga`j90Ip z!tRX8SzvkjQc#!TTIx8=cQZ@~R8qoF$`2-LNY)}Uxkb(p&5#U{`7|)v?)y5}G;>p` zxmO`48T0z-MP=iMKXe{TY8M%;sh6+u^J;0+jtbeZ%H7fu{~;7w%kLzD3l^JUJk$7yaCFV30Vif_viq#2Yq<1gU-U&D1VfW_7 z!bLr)=0}XzUH>ninYCq?52Gu*K31pN^-!NP9bvzr zZl!&u{?@*%fltZj@lEI&Msa&VS2`s+Shi@6E%+KF5nyx{Z&J@)ZiEZQiHQqxSSP=O zbF0z4ylLe71s|;kaEkpgi(csZvKXmdU!ZlA&|{R(4#{g7bo4xb$KEkAe(g=~;&zog2)tl(YN{73iy5-$Jl{3jLs@9+Kw^}o)462E_U z|C9Ut#TWl&>F?a~@4o*7d;Is7{^S;aaixFR5&XX~r+;_+^OpFFdi=}g5dQZS{!T>x zy@@|@`R`2#y+8gQ#osTNe{bT?_4NzDe;F#_{};}3lHl)GTOc5q_fNok?LZXKuc!Y3 DOD}@! literal 0 HcmV?d00001 diff --git a/server/test/fixtures/markdown.md b/server/test/fixtures/markdown.md new file mode 100644 index 00000000..59499fce --- /dev/null +++ b/server/test/fixtures/markdown.md @@ -0,0 +1,8 @@ +# Heading 1 + +## Heading 2 + +This is a test paragraph + +- list item 1 +- list item 2 diff --git a/server/test/fixtures/webpage.html b/server/test/fixtures/webpage.html new file mode 100644 index 00000000..a85c8026 --- /dev/null +++ b/server/test/fixtures/webpage.html @@ -0,0 +1,8 @@ + + + +

Heading 1

+

Text paragraph

+ + + \ No newline at end of file diff --git a/server/utils/__mocks__/s3.js b/server/utils/__mocks__/s3.js new file mode 100644 index 00000000..e0c4056f --- /dev/null +++ b/server/utils/__mocks__/s3.js @@ -0,0 +1,5 @@ +/* eslint-disable flowtype/require-valid-file-annotation */ + +export const uploadToS3FromBuffer = jest.fn().mockReturnValue("/endpoint/key"); + +export const publicS3Endpoint = jest.fn().mockReturnValue("http://mock"); diff --git a/server/utils/dataURItoBuffer.js b/server/utils/dataURItoBuffer.js new file mode 100644 index 00000000..098506ef --- /dev/null +++ b/server/utils/dataURItoBuffer.js @@ -0,0 +1,20 @@ +// @flow + +export default function dataURItoBuffer(dataURI: string) { + const split = dataURI.split(","); + + if (!dataURI.startsWith("data") || split.length <= 1) { + throw new Error("Not a dataURI"); + } + + // separate out the mime component + const type = split[0].split(":")[1].split(";")[0]; + + // convert base64 to buffer + const buffer = Buffer.from(split[1], "base64"); + + return { + buffer, + type, + }; +} diff --git a/server/utils/dataURItoBuffer.test.js b/server/utils/dataURItoBuffer.test.js new file mode 100644 index 00000000..eb23704d --- /dev/null +++ b/server/utils/dataURItoBuffer.test.js @@ -0,0 +1,20 @@ +// @flow +import dataURItoBuffer from "./dataURItoBuffer"; + +it("should parse value data URI", () => { + const response = dataURItoBuffer( + `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAMAAADXqc3KAAAB+FBMVEUAAAA/mUPidDHiLi5Cn0XkNTPmeUrkdUg/m0Q0pEfcpSbwaVdKskg+lUP4zA/iLi3msSHkOjVAmETdJSjtYFE/lkPnRj3sWUs8kkLeqCVIq0fxvhXqUkbVmSjwa1n1yBLepyX1xxP0xRXqUkboST9KukpHpUbuvRrzrhF/ljbwaljuZFM4jELaoSdLtElJrUj1xxP6zwzfqSU4i0HYnydMtUlIqUfywxb60AxZqEXaoifgMCXptR9MtklHpEY2iUHWnSjvvRr70QujkC+pUC/90glMuEnlOjVMt0j70QriLS1LtEnnRj3qUUXfIidOjsxAhcZFo0bjNDH0xxNLr0dIrUdmntVTkMoyfL8jcLBRuErhJyrgKyb4zA/5zg3tYFBBmUTmQTnhMinruBzvvhnxwxZ/st+Ktt5zp9hqota2vtK6y9FemNBblc9HiMiTtMbFtsM6gcPV2r6dwroseLrMrbQrdLGdyKoobKbo3Zh+ynrgVllZulTsXE3rV0pIqUf42UVUo0JyjEHoS0HmsiHRGR/lmRz/1hjqnxjvpRWfwtOhusaz0LRGf7FEfbDVmqHXlJeW0pbXq5bec3fX0nTnzmuJuWvhoFFhm0FtrziBsjaAaDCYWC+uSi6jQS3FsSfLJiTirCOkuCG1KiG+wSC+GBvgyhTszQ64Z77KAAAARXRSTlMAIQRDLyUgCwsE6ebm5ubg2dLR0byXl4FDQzU1NDEuLSUgC+vr6urq6ubb29vb2tra2tG8vLu7u7uXl5eXgYGBgYGBLiUALabIAAABsElEQVQoz12S9VPjQBxHt8VaOA6HE+AOzv1wd7pJk5I2adpCC7RUcHd3d3fXf5PvLkxheD++z+yb7GSRlwD/+Hj/APQCZWxM5M+goF+RMbHK594v+tPoiN1uHxkt+xzt9+R9wnRTZZQpXQ0T5uP1IQxToyOAZiQu5HEpjeA4SWIoksRxNiGC1tRZJ4LNxgHgnU5nJZBDvuDdl8lzQRBsQ+s9PZt7s7Pz8wsL39/DkIfZ4xlB2Gqsq62ta9oxVlVrNZpihFRpGO9fzQw1ms0NDWZz07iGkJmIFH8xxkc3a/WWlubmFkv9AB2SEpDvKxbjidN2faseaNV3zoHXvv7wMODJdkOHAegweAfFPx4G67KluxzottCU9n8CUqXzcIQdXOytAHqXxomvykhEKN9EFutG22p//0rbNvHVxiJywa8yS2KDfV1dfbu31H8jF1RHiTKtWYeHxUvq3bn0pyjCRaiRU6aDO+gb3aEfEeVNsDgm8zzLy9egPa7Qt8TSJdwhjplk06HH43ZNJ3s91KKCHQ5x4sw1fRGYDZ0n1L4FKb9/BP5JLYxToheoFCVxz57PPS8UhhEpLBVeAAAAAElFTkSuQmCC` + ); + expect(response.buffer).toBeTruthy(); + expect(response.type).toBe("image/png"); +}); + +it("should throw an error with junk input", () => { + let err; + try { + dataURItoBuffer("what"); + } catch (error) { + err = error; + } + expect(err).toBeTruthy(); +}); diff --git a/server/utils/parseDocumentIds.test.js b/server/utils/parseDocumentIds.test.js index eb938748..75c210e9 100644 --- a/server/utils/parseDocumentIds.test.js +++ b/server/utils/parseDocumentIds.test.js @@ -1,4 +1,4 @@ -/* eslint-disable flowtype/require-valid-file-annotation */ +// @flow import parseDocumentIds from "./parseDocumentIds"; it("should not return non links", () => { diff --git a/server/utils/parseImages.js b/server/utils/parseImages.js new file mode 100644 index 00000000..d62188d8 --- /dev/null +++ b/server/utils/parseImages.js @@ -0,0 +1,26 @@ +// @flow +import { parser } from "rich-markdown-editor"; + +export default function parseImages(text: string): string[] { + const value = parser.parse(text); + const images = []; + + function findImages(node) { + if (node.type.name === "image") { + if (!images.includes(node.attrs.src)) { + images.push(node.attrs.src); + } + + return; + } + + if (!node.content.size) { + return; + } + + node.content.descendants(findImages); + } + + findImages(value); + return images; +} diff --git a/server/utils/parseImages.test.js b/server/utils/parseImages.test.js new file mode 100644 index 00000000..24cd1a3a --- /dev/null +++ b/server/utils/parseImages.test.js @@ -0,0 +1,24 @@ +// @flow +import parseImages from "./parseImages"; + +it("should not return non images", () => { + expect(parseImages(`# Header`).length).toBe(0); +}); + +it("should return an array of images", () => { + const result = parseImages(`# Header + + ![internal](/attachments/image.png) + `); + + expect(result.length).toBe(1); + expect(result[0]).toBe("/attachments/image.png"); +}); + +it("should not return non document links", () => { + expect(parseImages(`[google](http://www.google.com)`).length).toBe(0); +}); + +it("should not return non document relative links", () => { + expect(parseImages(`[relative](/developers)`).length).toBe(0); +}); diff --git a/server/utils/s3.js b/server/utils/s3.js index 05ed6d9e..4cf11ee2 100644 --- a/server/utils/s3.js +++ b/server/utils/s3.js @@ -89,6 +89,28 @@ export const publicS3Endpoint = (isServerUpload?: boolean) => { }${AWS_S3_UPLOAD_BUCKET_NAME}`; }; +export const uploadToS3FromBuffer = async ( + buffer: Buffer, + contentType: string, + key: string, + acl: string +) => { + await s3 + .putObject({ + ACL: acl, + Bucket: AWS_S3_UPLOAD_BUCKET_NAME, + Key: key, + ContentType: contentType, + ContentLength: buffer.length, + ServerSideEncryption: "AES256", + Body: buffer, + }) + .promise(); + + const endpoint = publicS3Endpoint(true); + return `${endpoint}/${key}`; +}; + export const uploadToS3FromUrl = async ( url: string, key: string, diff --git a/yarn.lock b/yarn.lock index 0d9839a6..601b87b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1510,6 +1510,19 @@ resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ== +"@types/events@*": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== + +"@types/formidable@^1.0.31": + version "1.0.31" + resolved "https://registry.yarnpkg.com/@types/formidable/-/formidable-1.0.31.tgz#274f9dc2d0a1a9ce1feef48c24ca0859e7ec947b" + integrity sha512-dIhM5t8lRP0oWe2HF8MuPvdd1TpPTjhDMAqemcq6oIZQCBQTovhBAdTQ5L5veJB4pdQChadmHuxtB0YzqvfU3Q== + dependencies: + "@types/events" "*" + "@types/node" "*" + "@types/graceful-fs@^4.1.2": version "4.1.3" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.3.tgz#039af35fe26bec35003e8d86d2ee9c586354348f" @@ -1945,7 +1958,7 @@ aproba@^1.1.1: resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== -argparse@^1.0.7: +argparse@^1.0.7, argparse@~1.0.3: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== @@ -2490,7 +2503,7 @@ bluebird@^3.5.5, bluebird@^3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -bluebird@~3.4.1: +bluebird@~3.4.0, bluebird@~3.4.1: version "3.4.7" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM= @@ -3184,7 +3197,7 @@ cluster-key-slot@^1.1.0: resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== -co-body@^5.1.0: +co-body@^5.1.1: version "5.2.0" resolved "https://registry.yarnpkg.com/co-body/-/co-body-5.2.0.tgz#5a0a658c46029131e0e3a306f67647302f71c124" integrity sha512-sX/LQ7LqUhgyaxzbe7IqwPeTr2yfpfUIQ/dgpKo6ZI4y4lpQA0YxAomWIY+7I7rHWcG02PG+OuPREzMW/5tszQ== @@ -3454,11 +3467,6 @@ copy-to-clipboard@^3.0.6, copy-to-clipboard@^3.0.8: dependencies: toggle-selection "^1.0.6" -copy-to@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/copy-to/-/copy-to-2.0.1.tgz#2680fbb8068a48d08656b6098092bdafc906f4a5" - integrity sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU= - core-js-compat@^3.6.2: version "3.6.5" resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.6.5.tgz#2a51d9a4e25dfd6e690251aa81f99e3c05481f1c" @@ -3995,6 +4003,13 @@ double-ended-queue@^2.1.0-0: resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" integrity sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw= +duck@~0.1.11: + version "0.1.11" + resolved "https://registry.yarnpkg.com/duck/-/duck-0.1.11.tgz#3adc1a3d2fbdd5879ffd3bda05ce0f69355e9093" + integrity sha1-OtwaPS+91Yef/TvaBc4PaTVekJM= + dependencies: + underscore "~1.4.4" + duplexer2@~0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" @@ -4941,6 +4956,11 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +formidable@^1.1.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.2.tgz#bf69aea2972982675f00865342b982986f6b8dd9" + integrity sha512-V8gLm+41I/8kguQ4/o1D3RIHRmhYFG4pnNyonvua+40rqcEmT4+V71yaZ3B457xbbgCsCfjSPi65u/W6vK1U5Q== + fragment-cache@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" @@ -6723,7 +6743,7 @@ jsbn@~0.1.0: resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= -jsdom@^16.2.2: +jsdom@^16.2.0, jsdom@^16.2.2: version "16.4.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.4.0.tgz#36005bde2d136f73eee1a830c6d45e55408edddb" integrity sha512-lYMm3wYdgPhrl7pDcRmvzPhhrGVBeVhPIqeHjzeiHN3DFmD1RBpbExbi8vU7BJdH8VAZYovR8DMt0PNNDM7k8w== @@ -6889,6 +6909,13 @@ jszip@^3.5.0: readable-stream "~2.3.6" set-immediate-shim "~1.0.1" +jszip@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-2.5.0.tgz#7444fd8551ddf3e5da7198fea0c91bc8308cc274" + integrity sha1-dET9hVHd8+XacZj+oMkbyDCMwnQ= + dependencies: + pako "~0.2.5" + jwa@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a" @@ -6966,13 +6993,14 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -koa-bodyparser@4.2.0: +koa-body@^4.2.0: version "4.2.0" - resolved "https://registry.yarnpkg.com/koa-bodyparser/-/koa-bodyparser-4.2.0.tgz#bce6e08bc65f8709b6d1faa9411c7f0d8938aa54" - integrity sha1-vObgi8Zfhwm20fqpQRx/DYk4qlQ= + resolved "https://registry.yarnpkg.com/koa-body/-/koa-body-4.2.0.tgz#37229208b820761aca5822d14c5fc55cee31b26f" + integrity sha512-wdGu7b9amk4Fnk/ytH8GuWwfs4fsB5iNkY8kZPpgQVb04QZSv85T0M8reb+cJmvLE8cjPYvBzRikD3s6qz8OoA== dependencies: - co-body "^5.1.0" - copy-to "^2.0.1" + "@types/formidable" "^1.0.31" + co-body "^5.1.1" + formidable "^1.1.1" koa-compose@^3.0.0, koa-compose@^3.2.1: version "3.2.1" @@ -7508,6 +7536,15 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3 dependencies: js-tokens "^3.0.0 || ^4.0.0" +lop@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/lop/-/lop-0.4.0.tgz#4f0e4384d5c4f455d0b86d254fd52a9d05593c2c" + integrity sha1-Tw5DhNXE9FXQuG0lT9UqnQVZPCw= + dependencies: + duck "~0.1.11" + option "~0.2.1" + underscore "~1.4.4" + lower-case@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.4.tgz#9a2cabd1b9e8e0ae993a4bf7d5875c39c42e8eac" @@ -7591,6 +7628,20 @@ makeerror@1.0.x: dependencies: tmpl "1.0.x" +mammoth@^1.4.11: + version "1.4.11" + resolved "https://registry.yarnpkg.com/mammoth/-/mammoth-1.4.11.tgz#cf2d00b09fa61112f1758f3482d9b4507a1d106c" + integrity sha512-OB5/LJfI2QptpFKMfcHxom5nXnobySI6o8UkoeRjzYgbV7ZyC1WtDMATJ/khAfzhBfWHvYVdFGtKffyDX+6kMQ== + dependencies: + argparse "~1.0.3" + bluebird "~3.4.0" + jszip "~2.5.0" + lop "~0.4.0" + path-is-absolute "^1.0.0" + sax "~1.1.1" + underscore "~1.8.3" + xmlbuilder "^10.0.0" + map-age-cleaner@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a" @@ -8310,6 +8361,11 @@ only@~0.0.2: resolved "https://registry.yarnpkg.com/only/-/only-0.0.2.tgz#2afde84d03e50b9a8edc444e30610a70295edfb4" integrity sha1-Kv3oTQPlC5qO3EROMGEKcCle37Q= +option@~0.2.1: + version "0.2.4" + resolved "https://registry.yarnpkg.com/option/-/option-0.2.4.tgz#fd475cdf98dcabb3cb397a3ba5284feb45edbfe4" + integrity sha1-/Udc35jcq7PLOXo7pShP60Xtv+Q= + optionator@^0.8.1: version "0.8.3" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" @@ -8525,6 +8581,11 @@ packet-reader@1.0.0: resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74" integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ== +pako@~0.2.5: + version "0.2.9" + resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" + integrity sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU= + pako@~1.0.2, pako@~1.0.5: version "1.0.11" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" @@ -9977,6 +10038,11 @@ sax@>=0.6.0: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== +sax@~1.1.1: + version "1.1.6" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.1.6.tgz#5d616be8a5e607d54e114afae55b7eaf2fcc3240" + integrity sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA= + saxes@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" @@ -11216,6 +11282,13 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +turndown@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/turndown/-/turndown-6.0.0.tgz#c083d6109a9366be1b84b86b20af09140ea4b413" + integrity sha512-UVJBhSyRHCpNKtQ00mNWlYUM/i+tcipkb++F0PrOpt0L7EhNd0AX9mWEpL2dRFBu7LWXMp4HgAMA4OeKKnN7og== + dependencies: + jsdom "^16.2.0" + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -11342,6 +11415,16 @@ underscore@^1.7.0: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.10.2.tgz#73d6aa3668f3188e4adb0f1943bd12cfd7efaaaf" integrity sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg== +underscore@~1.4.4: + version "1.4.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604" + integrity sha1-YaajIBBiKvoHljvzJSA88SI51gQ= + +underscore@~1.8.3: + version "1.8.3" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" + integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" @@ -11994,6 +12077,11 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" +xmlbuilder@^10.0.0: + version "10.1.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0" + integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg== + xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"