diff --git a/server/commands/documentBatchImporter.js b/server/commands/documentBatchImporter.js index 0fb58d85..d972f5b3 100644 --- a/server/commands/documentBatchImporter.js +++ b/server/commands/documentBatchImporter.js @@ -1,5 +1,6 @@ // @flow import fs from "fs"; +import path from "path"; import File from "formidable/lib/file"; import JSZip from "jszip"; import { Collection, User } from "../models"; @@ -17,7 +18,7 @@ export default async function documentBatchImporter({ type: "outline", ip: string, }) { - const zipData = await fs.promises.readFile(file.path, "utf8"); + const zipData = await fs.promises.readFile(file.path); const zip = await JSZip.loadAsync(zipData); async function ingestDocuments( @@ -27,14 +28,25 @@ export default async function documentBatchImporter({ ) { const documents = []; + let items = []; + zip.forEach(async function (path, item) { + items.push([path, item]); + }); + // TODO: attachments // 2 passes, one for documents and then second for their nested documents - zip.forEach(async function (filePath, item) { + for (const [_, item] of items) { if (item.dir) return; - const fileData = await item.async("blob"); - const file = new File([fileData], item.name); + const content = await item.async("string"); + const name = path.basename(item.name); + await fs.promises.writeFile(`/tmp/${name}`, content); + const file = new File({ + name, + type: "text/markdown", + path: `/tmp/${name}`, + }); const { text, title } = await documentImporter({ file, @@ -54,28 +66,41 @@ export default async function documentBatchImporter({ // Keep track of which documents have been created documents.push(document); - }); + } + + for (const [filePath, item] of folders) { + const name = path.basename(item.name); - zip.forEach(async function (filePath, item) { // treat items in here as nested documents if (!item.dir) return; - if (item.name === "uploads") return; + if (name === "uploads") return; - const document = documents.find((doc) => doc.title === item.name); + const document = documents.find((doc) => doc.title === name); if (!document) { console.log( - `Couldn't find a matching parent document for folder ${item.name}` + `Couldn't find a matching parent document for folder ${name}` ); return; } // ensure document is created first, get parentDocumentId await ingestDocuments(zip.folder(filePath), collectionId, document.id); - }); + } } - zip.forEach(async function (folderPath, item) { - // all top level items must be directories representing collections + let folders = []; + zip.forEach(async function (path, item) { + folders.push([path, item]); + }); + + for (const [folderPath, item] of folders) { + const name = path.basename(item.name); + + if (folderPath.startsWith("__MACOSX") || folderPath.endsWith(".DS_Store")) { + continue; + } + + // all top level items must be directories representing collections console.log("iterating over", folderPath); // treat this as a collection @@ -84,16 +109,15 @@ export default async function documentBatchImporter({ const [collection, isCreated] = await Collection.findOrCreate({ where: { teamId: user.teamId, - name: item.name, + name, }, defaults: { private: false, }, }); - console.log(`Collection ${item.name} ${isCreated ? "created" : "found"}`); - + console.log(`Collection ${name} ${isCreated ? "created" : "found"}`); await ingestDocuments(zip.folder(folderPath), collection.id); } - }); + } } diff --git a/server/commands/documentBatchImporter.test.js b/server/commands/documentBatchImporter.test.js index 78d114e2..4f462dc5 100644 --- a/server/commands/documentBatchImporter.test.js +++ b/server/commands/documentBatchImporter.test.js @@ -20,6 +20,7 @@ describe("documentBatchImporter", () => { type: "application/zip", path: path.resolve(__dirname, "..", "test", "fixtures", name), }); + console.log(file); await documentBatchImporter({ type: "outline", diff --git a/server/test/fixtures/outline.zip b/server/test/fixtures/outline.zip new file mode 100644 index 00000000..b1074a43 Binary files /dev/null and b/server/test/fixtures/outline.zip differ