wip

server/commands/documentBatchImporter.js

@@ -1,5 +1,6 @@
 // @flow
 import fs from "fs";
+import path from "path";
 import File from "formidable/lib/file";
 import JSZip from "jszip";
 import { Collection, User } from "../models";
@@ -17,7 +18,7 @@ export default async function documentBatchImporter({
   type: "outline",
   ip: string,
 }) {
-  const zipData = await fs.promises.readFile(file.path, "utf8");
+  const zipData = await fs.promises.readFile(file.path);
   const zip = await JSZip.loadAsync(zipData);
 
   async function ingestDocuments(
@@ -27,14 +28,25 @@ export default async function documentBatchImporter({
   ) {
     const documents = [];
 
+    let items = [];
+    zip.forEach(async function (path, item) {
+      items.push([path, item]);
+    });
+
     // TODO: attachments
 
     // 2 passes, one for documents and then second for their nested documents
-    zip.forEach(async function (filePath, item) {
+    for (const [_, item] of items) {
       if (item.dir) return;
 
-      const fileData = await item.async("blob");
-      const file = new File([fileData], item.name);
+      const content = await item.async("string");
+      const name = path.basename(item.name);
+      await fs.promises.writeFile(`/tmp/${name}`, content);
+      const file = new File({
+        name,
+        type: "text/markdown",
+        path: `/tmp/${name}`,
+      });
 
       const { text, title } = await documentImporter({
         file,
@@ -54,28 +66,41 @@ export default async function documentBatchImporter({
 
       // Keep track of which documents have been created
       documents.push(document);
-    });
+    }
+
+    for (const [filePath, item] of folders) {
+      const name = path.basename(item.name);
 
-    zip.forEach(async function (filePath, item) {
       // treat items in here as nested documents
       if (!item.dir) return;
-      if (item.name === "uploads") return;
+      if (name === "uploads") return;
 
-      const document = documents.find((doc) => doc.title === item.name);
+      const document = documents.find((doc) => doc.title === name);
       if (!document) {
         console.log(
-          `Couldn't find a matching parent document for folder ${item.name}`
+          `Couldn't find a matching parent document for folder ${name}`
         );
         return;
       }
 
       // ensure document is created first, get parentDocumentId
       await ingestDocuments(zip.folder(filePath), collectionId, document.id);
-    });
+    }
   }
 
-  zip.forEach(async function (folderPath, item) {
-    // all top level items must be directories representing collections
+  let folders = [];
+  zip.forEach(async function (path, item) {
+    folders.push([path, item]);
+  });
+
+  for (const [folderPath, item] of folders) {
+    const name = path.basename(item.name);
+
+    if (folderPath.startsWith("__MACOSX") || folderPath.endsWith(".DS_Store")) {
+      continue;
+    }
+
+    // all top level  items must be directories representing collections
     console.log("iterating over", folderPath);
 
     // treat this as a collection
@@ -84,16 +109,15 @@ export default async function documentBatchImporter({
       const [collection, isCreated] = await Collection.findOrCreate({
         where: {
           teamId: user.teamId,
-          name: item.name,
+          name,
         },
         defaults: {
           private: false,
         },
       });
 
-      console.log(`Collection ${item.name} ${isCreated ? "created" : "found"}`);
-
+      console.log(`Collection ${name} ${isCreated ? "created" : "found"}`);
       await ingestDocuments(zip.folder(folderPath), collection.id);
     }
-  });
+  }
 }

server/commands/documentBatchImporter.test.js

@@ -20,6 +20,7 @@ describe("documentBatchImporter", () => {
       type: "application/zip",
       path: path.resolve(__dirname, "..", "test", "fixtures", name),
     });
+    console.log(file);
 
     await documentBatchImporter({
       type: "outline",