From a51af98d43bb666f2d18d390a3dc22ea562dba68 Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Thu, 24 Dec 2020 10:18:53 -0800 Subject: [PATCH] refactor --- .../__snapshots__/collections.test.js.snap | 9 + server/commands/attachmentCreator.js | 43 +++++ server/commands/documentBatchImporter.js | 158 ++++++++++-------- server/commands/documentBatchImporter.test.js | 7 +- server/commands/documentImporter.js | 28 +--- server/test/fixtures/outline.zip | Bin 619744 -> 612222 bytes 6 files changed, 155 insertions(+), 90 deletions(-) create mode 100644 server/commands/attachmentCreator.js diff --git a/server/api/__snapshots__/collections.test.js.snap b/server/api/__snapshots__/collections.test.js.snap index 95fec2a7..da385b86 100644 --- a/server/api/__snapshots__/collections.test.js.snap +++ b/server/api/__snapshots__/collections.test.js.snap @@ -79,6 +79,15 @@ Object { } `; +exports[`#collections.list should return collections 1`] = ` +Object { + "error": "authentication_required", + "message": "Authentication required", + "ok": false, + "status": 401, +} +`; + exports[`#collections.memberships should require authentication 1`] = ` Object { "error": "authentication_required", diff --git a/server/commands/attachmentCreator.js b/server/commands/attachmentCreator.js new file mode 100644 index 00000000..46f0215c --- /dev/null +++ b/server/commands/attachmentCreator.js @@ -0,0 +1,43 @@ +// @flow +import uuid from "uuid"; +import { Attachment, Event, User } from "../models"; +import { uploadToS3FromBuffer } from "../utils/s3"; + +export default async function attachmentCreator({ + name, + type, + buffer, + user, + ip, +}: { + name: string, + type: string, + buffer: Buffer, + user: User, + ip: string, +}) { + const key = `uploads/${user.id}/${uuid.v4()}/${name}`; + const acl = process.env.AWS_S3_ACL || "private"; + const url = await uploadToS3FromBuffer(buffer, type, key, acl); + + const attachment = await Attachment.create({ + key, + acl, + url, + size: buffer.length, + contentType: type, + teamId: user.teamId, + userId: user.id, + }); + + await Event.create({ + name: "attachments.create", + data: { name }, + modelId: attachment.id, + teamId: user.teamId, + actorId: user.id, + ip, + }); + + return attachment; +} diff --git a/server/commands/documentBatchImporter.js b/server/commands/documentBatchImporter.js index d972f5b3..d0ba6aec 100644 --- a/server/commands/documentBatchImporter.js +++ b/server/commands/documentBatchImporter.js @@ -2,8 +2,10 @@ import fs from "fs"; import path from "path"; import File from "formidable/lib/file"; +import invariant from "invariant"; import JSZip from "jszip"; import { Collection, User } from "../models"; +import attachmentCreator from "./attachmentCreator"; import documentCreator from "./documentCreator"; import documentImporter from "./documentImporter"; @@ -18,27 +20,69 @@ export default async function documentBatchImporter({ type: "outline", ip: string, }) { + // load the zip structure into memory const zipData = await fs.promises.readFile(file.path); const zip = await JSZip.loadAsync(zipData); - async function ingestDocuments( - zip: JSZip, - collectionId: string, - parentDocumentId?: string - ) { - const documents = []; + // store progress and pointers + let attachments = {}; + let collections = {}; + let documents = {}; - let items = []; - zip.forEach(async function (path, item) { - items.push([path, item]); - }); + // this is so we can use async / await a little easier + let folders = []; + zip.forEach(async function (path, item) { + folders.push([path, item]); + }); - // TODO: attachments + for (const [rawPath, item] of folders) { + const itemPath = rawPath.replace(/\/$/, ""); + const itemDir = path.dirname(itemPath); + const name = path.basename(item.name); + const depth = itemPath.split("/").length - 1; - // 2 passes, one for documents and then second for their nested documents - for (const [_, item] of items) { - if (item.dir) return; + // known skippable items + if (itemPath.startsWith("__MACOSX") || itemPath.endsWith(".DS_Store")) { + continue; + } + // all top level items must be directories representing collections + console.log("iterating over", itemPath, depth); + + if (depth === 0 && item.dir && name) { + // check if collection with name exists + let [collection, isCreated] = await Collection.findOrCreate({ + where: { + teamId: user.teamId, + name, + }, + defaults: { + creatorId: user.id, + private: false, + }, + }); + + // create new collection if name already exists, yes it's possible that + // there is also a "Name (Imported)" but this is a case not worth dealing + // with right now + if (!isCreated) { + collection = await Collection.create({ + teamId: user.teamId, + creatorId: user.id, + name: `${name} (Imported)`, + private: false, + }); + } + + collections[itemPath] = collection; + continue; + } + + if (depth > 0 && !item.dir && item.name.endsWith(".md")) { + const collection = collections[itemDir]; + invariant(collection, "Collection must exist for document", itemDir); + + // we have a document const content = await item.async("string"); const name = path.basename(item.name); await fs.promises.writeFile(`/tmp/${name}`, content); @@ -54,70 +98,50 @@ export default async function documentBatchImporter({ ip, }); + // must be a nested document, find the parent + if (depth > 1) { + console.log("nested doc", itemDir); + } + const document = await documentCreator({ title, text, publish: true, - collectionId, - parentDocumentId, + collectionId: collection.id, + parentDocumentId: undefined, user, ip, }); - // Keep track of which documents have been created - documents.push(document); - } - - for (const [filePath, item] of folders) { - const name = path.basename(item.name); - - // treat items in here as nested documents - if (!item.dir) return; - if (name === "uploads") return; - - const document = documents.find((doc) => doc.title === name); - if (!document) { - console.log( - `Couldn't find a matching parent document for folder ${name}` - ); - return; - } - - // ensure document is created first, get parentDocumentId - await ingestDocuments(zip.folder(filePath), collectionId, document.id); - } - } - - let folders = []; - zip.forEach(async function (path, item) { - folders.push([path, item]); - }); - - for (const [folderPath, item] of folders) { - const name = path.basename(item.name); - - if (folderPath.startsWith("__MACOSX") || folderPath.endsWith(".DS_Store")) { + documents[itemPath] = document; continue; } - // all top level items must be directories representing collections - console.log("iterating over", folderPath); - - // treat this as a collection - if (item.dir) { - // create collection if a collection with this name doesn't exist - const [collection, isCreated] = await Collection.findOrCreate({ - where: { - teamId: user.teamId, - name, - }, - defaults: { - private: false, - }, - }); - - console.log(`Collection ${name} ${isCreated ? "created" : "found"}`); - await ingestDocuments(zip.folder(folderPath), collection.id); + if (depth > 0 && item.dir && name !== "uploads") { + // we have a nested document, create if it doesn't exist based on title + continue; } + + if (depth > 0 && !item.dir && itemPath.includes("uploads")) { + // we have an attachment + const buffer = await item.async("nodebuffer"); + const attachment = await attachmentCreator({ + name, + type, + buffer, + user, + ip, + }); + attachments[itemPath] = attachment; + continue; + } + + console.log(`Skipped ${itemPath}`); } + + return { + documents, + collections, + attachments, + }; } diff --git a/server/commands/documentBatchImporter.test.js b/server/commands/documentBatchImporter.test.js index 4f462dc5..cf97c71a 100644 --- a/server/commands/documentBatchImporter.test.js +++ b/server/commands/documentBatchImporter.test.js @@ -20,13 +20,16 @@ describe("documentBatchImporter", () => { type: "application/zip", path: path.resolve(__dirname, "..", "test", "fixtures", name), }); - console.log(file); - await documentBatchImporter({ + const response = await documentBatchImporter({ type: "outline", user, file, ip, }); + + expect(Object.keys(response.collections).length).toEqual(1); + expect(Object.keys(response.documents).length).toEqual(15); + expect(Object.keys(response.attachments).length).toEqual(6); }); }); diff --git a/server/commands/documentImporter.js b/server/commands/documentImporter.js index a38dee6c..cb8ff43d 100644 --- a/server/commands/documentImporter.js +++ b/server/commands/documentImporter.js @@ -7,13 +7,12 @@ import mammoth from "mammoth"; import quotedPrintable from "quoted-printable"; import TurndownService from "turndown"; import utf8 from "utf8"; -import uuid from "uuid"; import parseTitle from "../../shared/utils/parseTitle"; import { FileImportError, InvalidRequestError } from "../errors"; -import { Attachment, Event, User } from "../models"; +import { User } from "../models"; import dataURItoBuffer from "../utils/dataURItoBuffer"; import parseImages from "../utils/parseImages"; -import { uploadToS3FromBuffer } from "../utils/s3"; +import attachmentCreator from "./attachmentCreator"; // https://github.com/domchristie/turndown#options const turndownService = new TurndownService({ @@ -170,26 +169,13 @@ export default async function documentImporter({ for (const uri of dataURIs) { const name = "imported"; - const key = `uploads/${user.id}/${uuid.v4()}/${name}`; - const acl = process.env.AWS_S3_ACL || "private"; const { buffer, type } = dataURItoBuffer(uri); - const url = await uploadToS3FromBuffer(buffer, type, key, acl); - const attachment = await Attachment.create({ - key, - acl, - url, - size: buffer.length, - contentType: type, - teamId: user.teamId, - userId: user.id, - }); - - await Event.create({ - name: "attachments.create", - data: { name }, - teamId: user.teamId, - userId: user.id, + const attachment = await attachmentCreator({ + name, + type, + buffer, + user, ip, }); diff --git a/server/test/fixtures/outline.zip b/server/test/fixtures/outline.zip index b1074a4365eff92e44480ab550e7db74cff6d969..bc361a7adf204f5c73512db4ae0dfd3f13095827 100644 GIT binary patch delta 4618 zcmai%4Ny~87RO&8A-NDp2mv*Qlmuf7LIghutRPT@cCc1H6cMl~5fLj?sN%3?+w7~% zqE&Z=S9q?Y(yeq_x6%d84BIc=E^X<;u4vb(Uj@;^R#&^*acqlAqkHdrxq0`6q#owY zFnZ4af6l%4z58SAruqK6X6CFWN@1laDwK+$*lZh!l1E<|x9P8pGgUG5s+sU#ZMK!! z%T^VY%(vKtgj$$Dt3OgyODIZ(N<$05Ok8@hLpnpE(s|W-GWgnFCR3?2wnJ+G6Q9_; z{mBNqrM7a->QGAF@!}O5ZevD7t5Qoyv7IF#l5@g6tO=kfHR_?W0L=3AFeRy$C}$Ie zj3ZD_)N%d_2KsZgSJw%(j4A3J60VgDoHa_N;jn_aVKf9nrIWz4SfbPu8cb=dF_5o_XUc<aht1= zxs@nnk5qtkcO@D_D=p-!5}88_&Fo8d0OS#0T*v)0PwUjmaf$-TaSi1?C?FoPeWq}C6WFhp(v>qW`Dc(gXjB`lwE~KlP-U{ z;R)4x*<|Fv=-wFDEBCWphXN`edw=SmO{zdzB@(lQW9|7m1X-m%xEj;|;GR~xr;z^?qR zxTjQiZMb4(YFz0RJgm^*#G$ZImg0x#K{&k?svow-YoCUsoUxJ zFl$tEVGVs(TDtoU5v&C~xFgb;zS{9S^@ASKQG*HU5J$E4=Tgf-gUA zI=CmZ^|+x|cJ7(n_$8^YR+|o&-RxtZUUKn6$L*nsyBj}0kn~Q6>2JMn4|G31{PJYu zrIhsa^mo<2a_nlKJ#ne`++fr9zHcaIHa!1Yf(Y&<$u=d!AlJ^RZMBiFD4}zse~g_e_+G;0n(Nmi0 zdpTL(+&MbA`{jRqHn{K3?Te=sjg95Oo%t)|ij>8w>Ta}waG=j2n(z#sAiG{T+D z_MDl3q7oJ!nx!H543NjRDw#Nd(HUl@Jf1v%@zc*9+p`gnhmU_3pG#JI4l--20v6&B z`x4P@`WX?nwGI?A2FpV_VY)02X~gvcFf$M4KSYQ9&*q*VEyeR5@$XOH+TvY<~uTe_CZ;&q?mp1=n3)yM1@aBN(#~dQOS4k~p8)s-!bb`p8GM4^ zBkyZ=Dh|~sex+gB;C;KkYCSa4^0o9wUaK^m8XTP%+j z=48>vvCqB$G;$j+)AyMfl<9rwz|1r*3G18+CUm-&0jqqiTjJI-&=c%K0w$T+&h2o? zP$sW)j6Jgo1hFADpw%Gaz~Q9PZvtMxFFo734#cpJlmcZmq7WQTx()v7k4f0xa#(b= z04OzxN^m%7hXl%d2n5N)%!FvfC^($-d+=5zl|A{ed9EER^~bQHwXgtVi%|P^Amtxo zgjsZm7;rf0VyWOL0vmpi$$U5n!V?@$I#UJ*Awh>5M2Bbvhm#%(5@bp2L472634Q?I zB>`^43ollERUxQ)6A{<1;nqt5bM3GVj7CC7F9J#gO8oL8Qo`Y+m%<}PEI-LjoV;~n zOF`QSn2!NUC|wXjQGRiXUm@|rFHs6!BwcYvXh#Bb3=ChC3P`OK=&YmHXL2_QjOq<5zcEE z4Wc+4uIv$5ftik*L5FA#hl_qPLQwRu9U*zI0;n|z@o>1ZS3~l0UkGC0Bi=rQd^lY6 zIIO@6h3;|@03tKJ3ElrB86X10(M3l_K}$CedN=^`09PB2YRNT2?0}<-&V?&Znu|cf zGpW{V$lzJ-A-$l2KjY%@4#Hg0AxXp-el3P{yoHE2eCe0X_SPry!>|5cJ^bK!qlhnk zJ-oU^{gY(kSAM0z-M%-a__7#*$s!98wD|Q1gBv*wV0`Hxfr(;$l4tmXd3+1}$RnD; z;iQL+?p-F>hB(CtaBw*3@8iTC-nfJMieckwr@17}bGs1u!w(@g7=7K_z#E1*3q$vg z-z;F9Bj^gJKo5+&9O-QKXP1bLlW|EccHX$L) mm%c6CTi-)miEiSudHr3Qzd{DdfTEJ%&l&i2IG6#iW$OP5vBQ!8 delta 9987 zcmb7J2|yEP7M^fsxQq}?kN^P$A(9YO1VK~|6^kO5ViDz7Sfn5n1hrKgM@zj{6b2cs zR$CWsw~9weJ;D1Z3m#S6T92xz;8F3a*9yA-pD@WJbHF|w1DG*V0nJ1OCR#rTy?b_QC-I z%7>D&XP4v_Ba7 z{h2S=17Z&5;Yw?XjXGI+f{?&dx6phhPcn!pmhR8v&nzUCFk9Lv z_TUOz8}UpYg19kZKO2dPb=dVh5lXW+5@0(qU?*i{=4a1V%*xW_w(4^Y ze;5OvYyGEW0GFE#8%L@e)vWq8x(YE9GrLnnC8i~o6c%S|+>He3BA?QE)(Fq+6`-jp zxX0_ltd5Tsl-)eJh_^v5e{Jy+dpn+7{;t;L&ayCB%lVh3FEcIO zzZ68&FFCNHF()mO|Ex^?F0#hq#{O03hlHqpL`yVZf9CYfC3Bbi(E|?_&Av3)&h7Nt zRSql6LoPn3oz$j!Q4&rn5Nj3qy6h@#~(s3@2tZGZ6*K0?jjJn+9R{yKl4deRh3XU?ZG30ARbL_BdV*sw?6c0Z+}P(tR9eah4|goV!ihm3-Q}~!Q0l) zEG{|)1GeOsdtH14iN)dWwodqNC$k_=jyeOu(%9-dr$OZpO2(Zb#O_X$W))}ry+l{x z)%prIj&*J9_}G(5L4CX95`n$sYJtDgnd|eA?{8a3uFeT{@ZE6QF1mhG^HYwk)`DQ25vZx@aWTMb&WyJRBhwE3Ac5E*A?L%pqhwqr|X{eibjWgFa<}4{*_e1-&qy8}2Ge*jrE`VhF^D5RzM*DFmnmuRwjKf(a5IQqtoWLSc zp>9r1w_VPYcyst8zKBFc)CmHIplWD$?UTcw9I#0#m=YAl{RIqo!67fYC5YvDywX1t!%C@DQl^kAgHxO;%q?5e!?!U z9t>y2<~AF&=>37)#43(gnmrj zU;1f&VMf+${ZS0hRPcavTXPhLw1DV6xMS|d&>hJT0{8|;a)THL%NY;JbCg38e`~p& z`;6Spr=5|icpielX<=L*6LSl)W@YEjF3B#I!jd8LNjcGs`~Bxhhpdh(g*RR}nu}a6 z#$DSsSr9Z`C<>}b*l=Y=^u-wavrjg}#y%Xd@wQF-@R;)SFRO3WtJIAjhfG@L@R}d- zOZHRO8(;4cMErR3pxQa`l04+A)f*P?TwY(aBx%LHm`%x-W&Yy5K61wkO9~xbtOc*rk_lFOU24i2u9vum3#*cI~7^DWL?& zA`Dl5MobxjkF>Ue6Ei{qazMK%k-efw<4T9VLK zT36iuF&;4)uz34e5q}NePjhk()+s(-TP{N+n(k3YKdOd0M8A%^}$n0Uu$J`uXfwr{k3h?OW}Sze395O062C- za1UEC+<_=1_YTk@sdS8jO0fgjB8H%PS;}A+#>*AATV=81O+7S8x5(f{K~Ot#h7IeX zpN%~kF~rxPWpnno77sAP_R$35YdOsH-U$Rdt?y19=jz(~x)_g}R#pP%zQpeJT)dw--2s(W&B? zvjX>Vu!XSX0F}Z4I*x-gUf?jI8@v`z5egkLS%5bS6`IK4ja)OGg#t-F2Qa}Khfb&F z!jpQjgjde7jGp{ey}y|^6f98+%U1iECWua#+f1+cwr2dH?|c?6K0F#7kXU0gd9G{2 znKu`bQ#y#n&yMuDhZ;B69ul;MtnI}(=4tII?@xj-xc89t45myeA zwB`-3xl(ks@=|jC3pAyA@vqf;Cf&LwJD7RAQuN8XFV3f(TGw*6v&QCx@*jEOp&u28 zJ{fFtV*lvQckX!Jfj)B{uD<&wD^`|MEPs70_KwH!YL&zAwL=b6ze{gWZ?toZO?K_$ zuK@q>{*v7J^pV@%J}=*WXnNAFI-%St78TA2%3I*V_kPnb>~9O52aY~8e486D{!QIe zyPI9>Q=1m&#QfTJ>B->k^b!S98{Q+NQ*}8tjU)peOZoSt(^IT zyac+JI@lcA5|VM24|Ih;b&^vms1_GWS?iPNlYn~Rq0T6!NPF}^txs(-~azMf0stWF>Pzu%?npKTD2pS$cd64?HH|F$Iqqc+A0L0UpbJ$G7k| z=1Z!3h*ndkL_;5RD2>Qa9&5Y=EI*=8U(}vtL-_PXpZ6FwTsNHNDAk4C5v)Q&FU?JVpVJmh|D+5s79`0)lKzCd9>;$awBEpglcJW`fJQJa*P8uW*MWRlw zkHf*Sr@J9&h7Uy*B=$t1Vq#4+YDVmjLX9MQvnIam`^3ww3jRdZt;71w&Fk^5{mH#NNp$15r8pF*wTvHT54Zyg2dGVD$fyJaOsK=?a zpfg|<2`Oyg-HM7fH{=={3(gk2@m028|=p@LX=T_#P9mDq9; zWd|l1YufKa1n2=MCXuGbO2K=JaT*PzC4KH>P+uZVjmZVBu-n({4FFChOJ&FqBGzUZpbavG!8s{;-Csd zni@L<-tIUO#05>+Oi2u!B0A9qz#Ep^VJA9kx@tpmWh=L+{+wLtfFwg0e4<{C(P}~Z z9E3U3*>OuTBvV~h6QHs~ni^{Xh#Xnv2B)$fqtj#nI23zGQ)9nN=&a!YrAkILo>A2H zP-7Fo{gQ*HF&ved;?Y7A?@T}^2Rw#jA<~u3j4Jt1X0hSK69Ju$jgEl~q;0cpVW2^w z5JZ|9a|YiS&kl+#q;F3-REJ1YV-4UNHqT(zQnVrnAysV6{HF_v2)tlWSNRY?3oGFqUsv8H`kLGBA*ISXrq2nOPW1Jgh9-l41$yHR${b6 zt)4ybplVRsXbvZ#Hby+;Ud%jbdK4uehPz`K(E|^vLZwA