diff --git a/server/api/__snapshots__/collections.test.js.snap b/server/api/__snapshots__/collections.test.js.snap index 95fec2a7..da385b86 100644 --- a/server/api/__snapshots__/collections.test.js.snap +++ b/server/api/__snapshots__/collections.test.js.snap @@ -79,6 +79,15 @@ Object { } `; +exports[`#collections.list should return collections 1`] = ` +Object { + "error": "authentication_required", + "message": "Authentication required", + "ok": false, + "status": 401, +} +`; + exports[`#collections.memberships should require authentication 1`] = ` Object { "error": "authentication_required", diff --git a/server/commands/attachmentCreator.js b/server/commands/attachmentCreator.js new file mode 100644 index 00000000..46f0215c --- /dev/null +++ b/server/commands/attachmentCreator.js @@ -0,0 +1,43 @@ +// @flow +import uuid from "uuid"; +import { Attachment, Event, User } from "../models"; +import { uploadToS3FromBuffer } from "../utils/s3"; + +export default async function attachmentCreator({ + name, + type, + buffer, + user, + ip, +}: { + name: string, + type: string, + buffer: Buffer, + user: User, + ip: string, +}) { + const key = `uploads/${user.id}/${uuid.v4()}/${name}`; + const acl = process.env.AWS_S3_ACL || "private"; + const url = await uploadToS3FromBuffer(buffer, type, key, acl); + + const attachment = await Attachment.create({ + key, + acl, + url, + size: buffer.length, + contentType: type, + teamId: user.teamId, + userId: user.id, + }); + + await Event.create({ + name: "attachments.create", + data: { name }, + modelId: attachment.id, + teamId: user.teamId, + actorId: user.id, + ip, + }); + + return attachment; +} diff --git a/server/commands/documentBatchImporter.js b/server/commands/documentBatchImporter.js index d972f5b3..d0ba6aec 100644 --- a/server/commands/documentBatchImporter.js +++ b/server/commands/documentBatchImporter.js @@ -2,8 +2,10 @@ import fs from "fs"; import path from "path"; import File from "formidable/lib/file"; +import invariant from "invariant"; import JSZip from "jszip"; import { Collection, User } from "../models"; +import attachmentCreator from "./attachmentCreator"; import documentCreator from "./documentCreator"; import documentImporter from "./documentImporter"; @@ -18,27 +20,69 @@ export default async function documentBatchImporter({ type: "outline", ip: string, }) { + // load the zip structure into memory const zipData = await fs.promises.readFile(file.path); const zip = await JSZip.loadAsync(zipData); - async function ingestDocuments( - zip: JSZip, - collectionId: string, - parentDocumentId?: string - ) { - const documents = []; + // store progress and pointers + let attachments = {}; + let collections = {}; + let documents = {}; - let items = []; - zip.forEach(async function (path, item) { - items.push([path, item]); - }); + // this is so we can use async / await a little easier + let folders = []; + zip.forEach(async function (path, item) { + folders.push([path, item]); + }); - // TODO: attachments + for (const [rawPath, item] of folders) { + const itemPath = rawPath.replace(/\/$/, ""); + const itemDir = path.dirname(itemPath); + const name = path.basename(item.name); + const depth = itemPath.split("/").length - 1; - // 2 passes, one for documents and then second for their nested documents - for (const [_, item] of items) { - if (item.dir) return; + // known skippable items + if (itemPath.startsWith("__MACOSX") || itemPath.endsWith(".DS_Store")) { + continue; + } + // all top level items must be directories representing collections + console.log("iterating over", itemPath, depth); + + if (depth === 0 && item.dir && name) { + // check if collection with name exists + let [collection, isCreated] = await Collection.findOrCreate({ + where: { + teamId: user.teamId, + name, + }, + defaults: { + creatorId: user.id, + private: false, + }, + }); + + // create new collection if name already exists, yes it's possible that + // there is also a "Name (Imported)" but this is a case not worth dealing + // with right now + if (!isCreated) { + collection = await Collection.create({ + teamId: user.teamId, + creatorId: user.id, + name: `${name} (Imported)`, + private: false, + }); + } + + collections[itemPath] = collection; + continue; + } + + if (depth > 0 && !item.dir && item.name.endsWith(".md")) { + const collection = collections[itemDir]; + invariant(collection, "Collection must exist for document", itemDir); + + // we have a document const content = await item.async("string"); const name = path.basename(item.name); await fs.promises.writeFile(`/tmp/${name}`, content); @@ -54,70 +98,50 @@ export default async function documentBatchImporter({ ip, }); + // must be a nested document, find the parent + if (depth > 1) { + console.log("nested doc", itemDir); + } + const document = await documentCreator({ title, text, publish: true, - collectionId, - parentDocumentId, + collectionId: collection.id, + parentDocumentId: undefined, user, ip, }); - // Keep track of which documents have been created - documents.push(document); - } - - for (const [filePath, item] of folders) { - const name = path.basename(item.name); - - // treat items in here as nested documents - if (!item.dir) return; - if (name === "uploads") return; - - const document = documents.find((doc) => doc.title === name); - if (!document) { - console.log( - `Couldn't find a matching parent document for folder ${name}` - ); - return; - } - - // ensure document is created first, get parentDocumentId - await ingestDocuments(zip.folder(filePath), collectionId, document.id); - } - } - - let folders = []; - zip.forEach(async function (path, item) { - folders.push([path, item]); - }); - - for (const [folderPath, item] of folders) { - const name = path.basename(item.name); - - if (folderPath.startsWith("__MACOSX") || folderPath.endsWith(".DS_Store")) { + documents[itemPath] = document; continue; } - // all top level items must be directories representing collections - console.log("iterating over", folderPath); - - // treat this as a collection - if (item.dir) { - // create collection if a collection with this name doesn't exist - const [collection, isCreated] = await Collection.findOrCreate({ - where: { - teamId: user.teamId, - name, - }, - defaults: { - private: false, - }, - }); - - console.log(`Collection ${name} ${isCreated ? "created" : "found"}`); - await ingestDocuments(zip.folder(folderPath), collection.id); + if (depth > 0 && item.dir && name !== "uploads") { + // we have a nested document, create if it doesn't exist based on title + continue; } + + if (depth > 0 && !item.dir && itemPath.includes("uploads")) { + // we have an attachment + const buffer = await item.async("nodebuffer"); + const attachment = await attachmentCreator({ + name, + type, + buffer, + user, + ip, + }); + attachments[itemPath] = attachment; + continue; + } + + console.log(`Skipped ${itemPath}`); } + + return { + documents, + collections, + attachments, + }; } diff --git a/server/commands/documentBatchImporter.test.js b/server/commands/documentBatchImporter.test.js index 4f462dc5..cf97c71a 100644 --- a/server/commands/documentBatchImporter.test.js +++ b/server/commands/documentBatchImporter.test.js @@ -20,13 +20,16 @@ describe("documentBatchImporter", () => { type: "application/zip", path: path.resolve(__dirname, "..", "test", "fixtures", name), }); - console.log(file); - await documentBatchImporter({ + const response = await documentBatchImporter({ type: "outline", user, file, ip, }); + + expect(Object.keys(response.collections).length).toEqual(1); + expect(Object.keys(response.documents).length).toEqual(15); + expect(Object.keys(response.attachments).length).toEqual(6); }); }); diff --git a/server/commands/documentImporter.js b/server/commands/documentImporter.js index a38dee6c..cb8ff43d 100644 --- a/server/commands/documentImporter.js +++ b/server/commands/documentImporter.js @@ -7,13 +7,12 @@ import mammoth from "mammoth"; import quotedPrintable from "quoted-printable"; import TurndownService from "turndown"; import utf8 from "utf8"; -import uuid from "uuid"; import parseTitle from "../../shared/utils/parseTitle"; import { FileImportError, InvalidRequestError } from "../errors"; -import { Attachment, Event, User } from "../models"; +import { User } from "../models"; import dataURItoBuffer from "../utils/dataURItoBuffer"; import parseImages from "../utils/parseImages"; -import { uploadToS3FromBuffer } from "../utils/s3"; +import attachmentCreator from "./attachmentCreator"; // https://github.com/domchristie/turndown#options const turndownService = new TurndownService({ @@ -170,26 +169,13 @@ export default async function documentImporter({ for (const uri of dataURIs) { const name = "imported"; - const key = `uploads/${user.id}/${uuid.v4()}/${name}`; - const acl = process.env.AWS_S3_ACL || "private"; const { buffer, type } = dataURItoBuffer(uri); - const url = await uploadToS3FromBuffer(buffer, type, key, acl); - const attachment = await Attachment.create({ - key, - acl, - url, - size: buffer.length, - contentType: type, - teamId: user.teamId, - userId: user.id, - }); - - await Event.create({ - name: "attachments.create", - data: { name }, - teamId: user.teamId, - userId: user.id, + const attachment = await attachmentCreator({ + name, + type, + buffer, + user, ip, }); diff --git a/server/test/fixtures/outline.zip b/server/test/fixtures/outline.zip index b1074a43..bc361a7a 100644 Binary files a/server/test/fixtures/outline.zip and b/server/test/fixtures/outline.zip differ