From b98e4bb1ff5c973895aefecd593fde86ec953810 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Thu, 17 Dec 2020 21:19:31 -0800 Subject: [PATCH] stash --- server/api/documents.js | 38 +++++-- server/commands/documentBatchImporter.js | 99 +++++++++++++++++++ server/commands/documentBatchImporter.test.js | 31 ++++++ server/commands/documentCreator.js | 6 +- server/policies/document.js | 6 ++ 5 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 server/commands/documentBatchImporter.js create mode 100644 server/commands/documentBatchImporter.test.js diff --git a/server/api/documents.js b/server/api/documents.js index b6715fae..faebcdee 100644 --- a/server/api/documents.js +++ b/server/api/documents.js @@ -2,6 +2,7 @@ import Router from "koa-router"; import Sequelize from "sequelize"; import { subtractDate } from "../../shared/utils/date"; +import documentBatchImporter from "../commands/documentBatchImporter"; import documentCreator from "../commands/documentCreator"; import documentImporter from "../commands/documentImporter"; import documentMover from "../commands/documentMover"; @@ -1105,13 +1106,38 @@ router.post("documents.unpublish", auth(), async (ctx) => { }; }); +router.post("documents.batchImport", auth(), async (ctx) => { + const { type } = ctx.body; + ctx.assertIn(type, ["outline"], "type must be one of 'outline'"); + + if (!ctx.is("multipart/form-data")) { + throw new InvalidRequestError("Request type must be multipart/form-data"); + } + + const file: any = Object.values(ctx.request.files)[0]; + ctx.assertPresent(file, "file is required"); + + if (file.type !== "application/zip") { + throw new InvalidRequestError("File type must be a zip"); + } + + const user = ctx.state.user; + authorize(user, "batchImport", Document); + + await documentBatchImporter({ + file, + user, + type, + ip: ctx.request.ip, + }); + + ctx.body = { + success: true, + }; +}); + router.post("documents.import", auth(), async (ctx) => { - const { - publish, - collectionId, - parentDocumentId, - index, - } = ctx.body; + const { publish, collectionId, parentDocumentId, index } = ctx.body; if (!ctx.is("multipart/form-data")) { throw new InvalidRequestError("Request type must be multipart/form-data"); diff --git a/server/commands/documentBatchImporter.js b/server/commands/documentBatchImporter.js new file mode 100644 index 00000000..0fb58d85 --- /dev/null +++ b/server/commands/documentBatchImporter.js @@ -0,0 +1,99 @@ +// @flow +import fs from "fs"; +import File from "formidable/lib/file"; +import JSZip from "jszip"; +import { Collection, User } from "../models"; +import documentCreator from "./documentCreator"; +import documentImporter from "./documentImporter"; + +export default async function documentBatchImporter({ + file, + type, + user, + ip, +}: { + file: File, + user: User, + type: "outline", + ip: string, +}) { + const zipData = await fs.promises.readFile(file.path, "utf8"); + const zip = await JSZip.loadAsync(zipData); + + async function ingestDocuments( + zip: JSZip, + collectionId: string, + parentDocumentId?: string + ) { + const documents = []; + + // TODO: attachments + + // 2 passes, one for documents and then second for their nested documents + zip.forEach(async function (filePath, item) { + if (item.dir) return; + + const fileData = await item.async("blob"); + const file = new File([fileData], item.name); + + const { text, title } = await documentImporter({ + file, + user, + ip, + }); + + const document = await documentCreator({ + title, + text, + publish: true, + collectionId, + parentDocumentId, + user, + ip, + }); + + // Keep track of which documents have been created + documents.push(document); + }); + + zip.forEach(async function (filePath, item) { + // treat items in here as nested documents + if (!item.dir) return; + if (item.name === "uploads") return; + + const document = documents.find((doc) => doc.title === item.name); + if (!document) { + console.log( + `Couldn't find a matching parent document for folder ${item.name}` + ); + return; + } + + // ensure document is created first, get parentDocumentId + await ingestDocuments(zip.folder(filePath), collectionId, document.id); + }); + } + + zip.forEach(async function (folderPath, item) { + // all top level items must be directories representing collections + console.log("iterating over", folderPath); + + // treat this as a collection + if (item.dir) { + // create collection if a collection with this name doesn't exist + const [collection, isCreated] = await Collection.findOrCreate({ + where: { + teamId: user.teamId, + name: item.name, + }, + defaults: { + private: false, + }, + }); + + console.log(`Collection ${item.name} ${isCreated ? "created" : "found"}`); + + await ingestDocuments(zip.folder(folderPath), collection.id); + } + }); +} diff --git a/server/commands/documentBatchImporter.test.js b/server/commands/documentBatchImporter.test.js new file mode 100644 index 00000000..78d114e2 --- /dev/null +++ b/server/commands/documentBatchImporter.test.js @@ -0,0 +1,31 @@ +// @flow +import path from "path"; +import File from "formidable/lib/file"; +import { buildUser } from "../test/factories"; +import { flushdb } from "../test/support"; +import documentBatchImporter from "./documentBatchImporter"; + +jest.mock("../utils/s3"); + +beforeEach(() => flushdb()); + +describe("documentBatchImporter", () => { + const ip = "127.0.0.1"; + + it("should import documents in outline format", async () => { + const user = await buildUser(); + const name = "outline.zip"; + const file = new File({ + name, + type: "application/zip", + path: path.resolve(__dirname, "..", "test", "fixtures", name), + }); + + await documentBatchImporter({ + type: "outline", + user, + file, + ip, + }); + }); +}); diff --git a/server/commands/documentCreator.js b/server/commands/documentCreator.js index 471721f9..0652fc90 100644 --- a/server/commands/documentCreator.js +++ b/server/commands/documentCreator.js @@ -12,7 +12,7 @@ export default async function documentCreator({ index, user, editorVersion, - ip + ip, }: { title: string, text: string, @@ -24,7 +24,7 @@ export default async function documentCreator({ index?: number, user: User, editorVersion?: string, - ip: string + ip: string, }): Document { const templateId = templateDocument ? templateDocument.id : undefined; let document = await Document.create({ @@ -71,4 +71,4 @@ export default async function documentCreator({ return Document.findOne({ where: { id: document.id, publishedAt: document.publishedAt }, }); -} \ No newline at end of file +} diff --git a/server/policies/document.js b/server/policies/document.js index 310734a8..06356c8b 100644 --- a/server/policies/document.js +++ b/server/policies/document.js @@ -1,5 +1,6 @@ // @flow import invariant from "invariant"; +import { AdminRequiredError } from "../errors"; import { Document, Revision, User } from "../models"; import policy from "./policy"; @@ -7,6 +8,11 @@ const { allow, cannot } = policy; allow(User, "create", Document); +allow(User, "batchImport", Document, actor => { + if (actor.isAdmin) return true; + throw new AdminRequiredError(); +}); + allow(User, ["read", "download"], Document, (user, document) => { // existence of collection option is not required here to account for share tokens if (document.collection && cannot(user, "read", document.collection)) {