From ba2552f69fa831f0d3864711f15f79669171d10d Mon Sep 17 00:00:00 2001 From: Tom Moor Date: Fri, 25 Dec 2020 18:04:38 -0800 Subject: [PATCH] fix --- server/commands/documentBatchImporter.js | 46 +++++++++++------- server/commands/documentBatchImporter.test.js | 2 +- server/policies/document.js | 2 +- server/test/fixtures/outline.zip | Bin 612222 -> 615122 bytes 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/server/commands/documentBatchImporter.js b/server/commands/documentBatchImporter.js index d0ba6aec..aa1f1209 100644 --- a/server/commands/documentBatchImporter.js +++ b/server/commands/documentBatchImporter.js @@ -4,7 +4,8 @@ import path from "path"; import File from "formidable/lib/file"; import invariant from "invariant"; import JSZip from "jszip"; -import { Collection, User } from "../models"; +import { values, keys } from "lodash"; +import { Attachment, Document, Collection, User } from "../models"; import attachmentCreator from "./attachmentCreator"; import documentCreator from "./documentCreator"; import documentImporter from "./documentImporter"; @@ -25,9 +26,9 @@ export default async function documentBatchImporter({ const zip = await JSZip.loadAsync(zipData); // store progress and pointers - let attachments = {}; - let collections = {}; - let documents = {}; + let collections: { string: Collection } = {}; + let documents: { string: Document } = {}; + let attachments: { string: Attachment } = {}; // this is so we can use async / await a little easier let folders = []; @@ -46,9 +47,6 @@ export default async function documentBatchImporter({ continue; } - // all top level items must be directories representing collections - console.log("iterating over", itemPath, depth); - if (depth === 0 && item.dir && name) { // check if collection with name exists let [collection, isCreated] = await Collection.findOrCreate({ @@ -79,8 +77,9 @@ export default async function documentBatchImporter({ } if (depth > 0 && !item.dir && item.name.endsWith(".md")) { - const collection = collections[itemDir]; - invariant(collection, "Collection must exist for document", itemDir); + const collectionDir = itemDir.split("/")[0]; + const collection = collections[collectionDir]; + invariant(collection, `Collection must exist for document ${itemDir}`); // we have a document const content = await item.async("string"); @@ -98,9 +97,12 @@ export default async function documentBatchImporter({ ip, }); - // must be a nested document, find the parent + // must be a nested document, find and reference the parent document + let parentDocumentId; if (depth > 1) { - console.log("nested doc", itemDir); + const parentDocument = documents[`${itemDir}.md`] || documents[itemDir]; + invariant(parentDocument, `Document must exist for parent ${itemDir}`); + parentDocumentId = parentDocument.id; } const document = await documentCreator({ @@ -108,7 +110,7 @@ export default async function documentBatchImporter({ text, publish: true, collectionId: collection.id, - parentDocumentId: undefined, + parentDocumentId, user, ip, }); @@ -117,11 +119,6 @@ export default async function documentBatchImporter({ continue; } - if (depth > 0 && item.dir && name !== "uploads") { - // we have a nested document, create if it doesn't exist based on title - continue; - } - if (depth > 0 && !item.dir && itemPath.includes("uploads")) { // we have an attachment const buffer = await item.async("nodebuffer"); @@ -139,6 +136,21 @@ export default async function documentBatchImporter({ console.log(`Skipped ${itemPath}`); } + // All collections, documents, and attachments have been created – time to + // update the documents to point to newly uploaded attachments where possible + for (const attachmentPath of keys(attachments)) { + const attachment = attachments[attachmentPath]; + + for (const document of values(documents)) { + document.text = document.text + .replace(attachmentPath, attachment.redirectUrl) + .replace(`/${attachmentPath}`, attachment.redirectUrl); + + // does nothing if the document text is unchanged + await document.save(); + } + } + return { documents, collections, diff --git a/server/commands/documentBatchImporter.test.js b/server/commands/documentBatchImporter.test.js index cf97c71a..53b94d96 100644 --- a/server/commands/documentBatchImporter.test.js +++ b/server/commands/documentBatchImporter.test.js @@ -29,7 +29,7 @@ describe("documentBatchImporter", () => { }); expect(Object.keys(response.collections).length).toEqual(1); - expect(Object.keys(response.documents).length).toEqual(15); + expect(Object.keys(response.documents).length).toEqual(8); expect(Object.keys(response.attachments).length).toEqual(6); }); }); diff --git a/server/policies/document.js b/server/policies/document.js index 06356c8b..203d8055 100644 --- a/server/policies/document.js +++ b/server/policies/document.js @@ -8,7 +8,7 @@ const { allow, cannot } = policy; allow(User, "create", Document); -allow(User, "batchImport", Document, actor => { +allow(User, "batchImport", Document, (actor) => { if (actor.isAdmin) return true; throw new AdminRequiredError(); }); diff --git a/server/test/fixtures/outline.zip b/server/test/fixtures/outline.zip index bc361a7adf204f5c73512db4ae0dfd3f13095827..030598dcbc55d5aa5151ee3c62b3eb4adf04b148 100644 GIT binary patch delta 6628 zcmai33p`Y58=o1sIT_7h48w#lh>S}vD;XvV8*-;qLs+#X6>a)d)70vwln$M0`_^Wq zHmqd6^eJ7G%C`AhX}7yB8@nsTc2No6duE)ObIh6c>G$@lywCstJkR@{=lQ?ChJWO@ zxA48AW@%}&DX1Oax<6X}htc2Y6l+RcMo#YHWtpz=aRwB^zX z+HXM&@>=xHNe<`5n+=Z?H1D6}vSoS!fByZw9R0(moD&>fIGp1*HhQi+SCa8n<~ZZb z%#(ig`YtUU9mzX>$qSQ&pJ|EfynD%Er?-&nddu9{wx9DxBwp)j9uXe?`scPe^G-(H zdl!|~J!^CJ7sDfEIq%Qqw0)~ZNpYN{M|GlX*l;n|zj|DA?X^db&lJzU`h=S6c7dj4 z#7Z7Y7R)=f{Bd(ZVGkpBYjxd=)5{+}(cSDH@Xohkg|OFc<>@Us>k}A~XV1IuUw$vU zcy(yn!B@V z;BsJWQ^4&GS<*`}UDv{*XDC8?{z-Y@Pph?)^DEOFA_WKMo_XneBzk`sZF|n#!H;um zO0OT~q`qnS4^OuLQDDc`mU`jI#xrBi!Tr?SiTgKi@d$lwALGGPUurr6jiWYwMD{aeAvJRZe~ow?NcX`p{*+ zwTym3B``fuqyP?5#A=6VO8J1DW zS7;R?_|}2M`o{^P@46l}h4Uki-p=zgJUXo+%y7oj+5-b861RlJL}veu)T}l-q}>xT z=dXp;{t|mv-wqMI>9Y*vf%E5vl;!8+h1^~L$&__F4`0}yCJFdX@)~oc(ft^21+|ao|)VTIieq{puV{MP1Ztz3s zK$VSOHvgN?uX!iNzr4RL`R4H3XC+qz}9iC3{~;QHAb;g{F0uDSoBsy~_;+xRM}=N`X(T4mbAq1)EFYjY>S)oEGm zfvuipi|^KX7rF@Np6}>MN^a4z(aG=d{o+ER?TKxD2TJoeZ5MR%tOV{l>&JIqVD=e* zZ0m4Hao;003%AX7j}Tt;T*DE!MyJd!qW!P-@cDhe+-d!d7jxj);1|!^ z{d$)S{*>5jQP$Mh>gTolhF4G-wOel)_5Sq_H%~fQXV$Wu8CBH`=jV5J9@{?R9?%=- zc_m~xq%7XN-?g|qHmaa$f&WNpb2LwW=h1`E^!oCU}DH;iVm15`2C192x7}8&s;E5 z<{NCIvMovG0J1&^GL>&r1ag6oZ~P&Pb3<1x6lA5#8~@&a-4l-vDY#Rs*o{Ym=PD4B zJ1O96;3g*9j)lu>)$*m;sp&b2q@2gy6QfI!lsWu@#qj6=Q`MZDY1CI99OjCfjqq+r zn}G(2odLWLnt+W)_zZNA?LdNyjEEp>+-PiK`G5#aa4^M|91Nw{Z-uEf2?!_ywhI@K zr;tja-~+-PYFz5Vr5QO`sY`IBUabGziY!o9l{cKqFu{jx0fUfH34vv<+Ukn5;lu{~ zUa!hZK$d{Y13peC%8nN-7K4ieM|KB8mRhKEu{HFfvh7IVkbz)mqL!i*^1BXq8?p4g zR3Na_LI67&8=x@YKFg2Z3_bb)5;S<3@w8E}8h9?7tv`1RP#N1p1;l2t(@8)OY6z4% zic$w#aaj670*Wh~$}y2=wT9D8WrvYKfS}zFaVW&WWi~@Cn0#kG{4Xw+nGu2VVgD9Jb0i$P{h~aK(F5l8z zP6dkwmKIprVCjIR3zl9(b2hbOZjF)P6t6;%U%&^GRi(FA0@cw8X}5?tdTzhAci^mV8cu%or!ZIJl_-^ zDUlGv25rEY$)tC|A$;mZr*K>%1XSpI4l|jw9ZYttk0bck9gYfp;$bF}vVj^=qryG- zkh)-q7lgj}@Y)3({Fqfp^>tLNH*AT(gWkwc|WD8R3$`C1wkwiR9ay+ zBr0U8V$meDPz38xF*~Gf3vk5LmJ&mH2VjsKss`;PFQ|RbI-31L0Fen6F~6%URYZ6> zhC2vU>tH66mKdtj6C|3Vl$HJv%N!MHn8~EOjZ{>a7$cOPk199JWKt(%bta;YlLz_v zvnA1cP5?Jd%&8EpKyM+rrW&Q4DrpyD(tai+p{OoCUhXNQf1OI{*MX3}~r^z;`QpBu6 zx}K}fsnm_AgQ8W)?>QHhj+hOJ_~uFuypWWKCcs~9R83+wB;5zx#$IhqTjB_PW;Ce2}cDd>cm6PdfK3p6SE;{Fi)LHDM003lV-&1i}vt}+)D*_g?s$Rq;QSjnyMlYauJ?#4_e zZE+`MR8!&dkoo=*3-{=(qEIj?uEL>DA@bvdIWAiw_t~QwxW{UB1$oU^MK%kU(kYl= zSCLbX5y%Ph9mkGpbj)N@Pg98O775f9I|?y*Y`x7O{P##A+~pcot~iNVQ_MB+;a%eWV+?PQ1(8=x*W>af@|t|6ffujz72XyJYP?ClL|(k=*GMS|)8X( Y%OYx)jxKssp5WI9(50JaK;Uox2VK&=dH?_b delta 3981 zcma)8Yitx%6rQD9c5ge|9fs{=U$9jQYbi@9f;2#(LCUiQ`cR-HOA8eURAPXHzz-cw z1eG5x*Lann5>pd-)Yx{Cs$i5K#vc#`CB}rnuNp&Q2r=;wx~=EVox5}IK8h!qr0JgX zecw6v&V1+R@2e(0ufqL$$n8^N3Q2B8WuEBW@nq!gIQ~U{#a{vcO(vTg*KBFqzA%w= z68a#aQu}7!ewU*7A!*Iwy}hya-Xlj3yu2^bx3{C+r+D5yaW`p$jg}XbyAmN8ZK@iE zKa3yxv?zkIbq9{FKi0#M-*b@h(n5HNi6p(M5P>^>!d}hB(4GS>j zS`&&`zdBOXI96C~b8j#0G}dzXXz!6+s!o?(UMRxB@TRO`B-h=v6Y|<76y?IOKI6Q9 zwjhiiLps6ksj|VPD0EEi-93jKLBavx1A#v?$CNvnkEsyGR7}-5I!ALwL367U8p3nO{lzpPZu)0?g~YjK`H-j zL9=M>=$?utE_|@ahlheOJGB}=4z6`CplOxc(E@xt1#!$1wA0~LG%xo+L&3W$0`K9OK@V2 z#&`!hQ7e8N@D#!`9Ujlv$R+RjL+YEdz68bDVL@=3XI(;* zgq{@f2a)dSW;7%oM1M(Y={2Q!^;Qap>57Ul^uWl~Fu)`k%-EeU>WoM@#ZCnhfbp76pv^p8~D!gk``jvESD`mR3y$;>~@IBVs&P~vpJe(5y8vPBQ@i&>WY>8R%uNz4tPQOHzJMPUyIb~sm3wCk~Ub16d@1c*ATT=9w{RQ zN&IG~&DP3(YR|R)ZJ+jdKl}%>x$E-6L1SD4vD||heEk-Hj|lLCMuYCa<14{R>UWM( zojNp+RO3BJmi$^!P2HMD%2+?sfhF9#8Sd4sNYkl%^GG$e2RN?njgZGX;7-tm zfrn}QRB1!w$*{D4unOzi(Ite3Y5e_+L?N#vv?@jpcEaaE=Bb?zJmQ+d)XUKKFF4Ry zOACYktqY(}N`iu)@ab5aq`1T|61t@DFpcN^NE(r}vREXuJZFA2zk~|DyEw%kYsfx{ z>`lhH>T*Mhp1~#VHH>Rw-HB)=Tt>d&I9$tTa;~g(hxH|M71laj^6q1uNwPcZOja-A i1X=4)E`sC=hahV=B3F7)MM