fix: Allow selection of embeds (#1562)

* feat: Support importing .docx or .html files as new documents (#1551)

* Support importing .docx as new documents

* Add html file support, build types and interface for easily adding file types to importer

* fix: Upload embedded images in docx to storage

* refactor: Bulk of logic to command

* refactor: Do all importing on server, so we're not splitting logic for import into two places

* test: Add documentImporter tests


Co-authored-by: Lance Whatley <whatl3y@gmail.com>

* fix: Accessibility audit

* fix: Quick fix, non editable title
closes #1560

* fix: Embed selection

Co-authored-by: Lance Whatley <whatl3y@gmail.com>

server/commands/documentImporter.js

@@ -0,0 +1,113 @@
+// @flow
+import fs from "fs";
+import File from "formidable/lib/file";
+import mammoth from "mammoth";
+import TurndownService from "turndown";
+import uuid from "uuid";
+import parseTitle from "../../shared/utils/parseTitle";
+import { Attachment, Event, User } from "../models";
+import dataURItoBuffer from "../utils/dataURItoBuffer";
+import parseImages from "../utils/parseImages";
+import { uploadToS3FromBuffer } from "../utils/s3";
+
+// https://github.com/domchristie/turndown#options
+const turndownService = new TurndownService({
+  hr: "---",
+  bulletListMarker: "-",
+  headingStyle: "atx",
+});
+
+interface ImportableFile {
+  type: string;
+  getMarkdown: (file: any) => Promise<string>;
+}
+
+const importMapping: ImportableFile[] = [
+  {
+    type:
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    getMarkdown: docxToMarkdown,
+  },
+  {
+    type: "text/html",
+    getMarkdown: htmlToMarkdown,
+  },
+  {
+    type: "text/plain",
+    getMarkdown: fileToMarkdown,
+  },
+  {
+    type: "text/markdown",
+    getMarkdown: fileToMarkdown,
+  },
+];
+
+async function fileToMarkdown(file): Promise<string> {
+  return fs.promises.readFile(file.path, "utf8");
+}
+
+async function docxToMarkdown(file): Promise<string> {
+  const { value } = await mammoth.convertToHtml(file);
+  return turndownService.turndown(value);
+}
+
+async function htmlToMarkdown(file): Promise<string> {
+  const value = await fs.promises.readFile(file.path, "utf8");
+  return turndownService.turndown(value);
+}
+
+export default async function documentImporter({
+  file,
+  user,
+  ip,
+}: {
+  user: User,
+  file: File,
+  ip: string,
+}): Promise<{ text: string, title: string }> {
+  const fileInfo = importMapping.filter((item) => item.type === file.type)[0];
+  let title = file.name.replace(/\.[^/.]+$/, "");
+  let text = await fileInfo.getMarkdown(file);
+
+  // If the first line of the imported text looks like a markdown heading
+  // then we can use this as the document title
+  if (text.trim().startsWith("# ")) {
+    const result = parseTitle(text);
+    title = result.title;
+    text = text.replace(`# ${title}\n`, "");
+  }
+
+  // find data urls, convert to blobs, upload and write attachments
+  const images = parseImages(text);
+  const dataURIs = images.filter((href) => href.startsWith("data:"));
+
+  for (const uri of dataURIs) {
+    const name = "imported";
+    const key = `uploads/${user.id}/${uuid.v4()}/${name}`;
+    const acl = process.env.AWS_S3_ACL || "private";
+    const { buffer, type } = dataURItoBuffer(uri);
+    const url = await uploadToS3FromBuffer(buffer, type, key, acl);
+
+    const attachment = await Attachment.create({
+      key,
+      acl,
+      url,
+      size: buffer.length,
+      contentType: type,
+      teamId: user.teamId,
+      userId: user.id,
+    });
+
+    await Event.create({
+      name: "attachments.create",
+      data: { name },
+      teamId: user.teamId,
+      userId: user.id,
+      ip,
+    });
+
+    text = text.replace(uri, attachment.redirectUrl);
+  }
+
+  return { text, title };
+}

server/commands/documentImporter.test.js

@@ -0,0 +1,77 @@
+// @flow
+import path from "path";
+import File from "formidable/lib/file";
+import { Attachment } from "../models";
+import { buildUser } from "../test/factories";
+import { flushdb } from "../test/support";
+import documentImporter from "./documentImporter";
+
+jest.mock("../utils/s3");
+
+beforeEach(() => flushdb());
+
+describe("documentImporter", () => {
+  const ip = "127.0.0.1";
+
+  it("should convert Word Document to markdown", async () => {
+    const user = await buildUser();
+    const name = "images.docx";
+    const file = new File({
+      name,
+      type:
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      path: path.resolve(__dirname, "..", "test", "fixtures", name),
+    });
+
+    const response = await documentImporter({
+      user,
+      file,
+      ip,
+    });
+
+    const attachments = await Attachment.count();
+    expect(attachments).toEqual(1);
+
+    expect(response.text).toContain("This is a test document for images");
+    expect(response.text).toContain("![](/api/attachments.redirect?id=");
+    expect(response.title).toEqual("images");
+  });
+
+  it("should convert HTML Document to markdown", async () => {
+    const user = await buildUser();
+    const name = "webpage.html";
+    const file = new File({
+      name,
+      type: "text/html",
+      path: path.resolve(__dirname, "..", "test", "fixtures", name),
+    });
+
+    const response = await documentImporter({
+      user,
+      file,
+      ip,
+    });
+
+    expect(response.text).toContain("Text paragraph");
+    expect(response.title).toEqual("Heading 1");
+  });
+
+  it("should load markdown", async () => {
+    const user = await buildUser();
+    const name = "markdown.md";
+    const file = new File({
+      name,
+      type: "text/plain",
+      path: path.resolve(__dirname, "..", "test", "fixtures", name),
+    });
+
+    const response = await documentImporter({
+      user,
+      file,
+      ip,
+    });
+
+    expect(response.text).toContain("This is a test paragraph");
+    expect(response.title).toEqual("Heading 1");
+  });
+});

server/commands/documentMover.js

@@ -1,6 +1,5 @@
 // @flow
-import { type Context } from "koa";
-import { Document, Collection, Event } from "../models";
+import { Document, Collection, User, Event } from "../models";
 import { sequelize } from "../sequelize";
 
 export default async function documentMover({
@@ -11,7 +10,7 @@ export default async function documentMover({
   index,
   ip,
 }: {
-  user: Context,
+  user: User,
   document: Document,
   collectionId: string,
   parentDocumentId?: string,

server/commands/documentMover.test.js

@@ -1,7 +1,7 @@
 /* eslint-disable flowtype/require-valid-file-annotation */
-import documentMover from "../commands/documentMover";
 import { buildDocument, buildCollection } from "../test/factories";
 import { flushdb, seed } from "../test/support";
+import documentMover from "./documentMover";
 
 beforeEach(() => flushdb());
 

server/commands/userInviter.test.js

@@ -1,7 +1,7 @@
 /* eslint-disable flowtype/require-valid-file-annotation */
-import userInviter from "../commands/userInviter";
 import { buildUser } from "../test/factories";
 import { flushdb } from "../test/support";
+import userInviter from "./userInviter";
 
 beforeEach(() => flushdb());