fix: Refactor collection exports to not send email attachment (#2460)

Co-authored-by: Tom Moor <tom.moor@gmail.com>

app/components/Button.js

@@ -128,11 +128,11 @@ export type Props = {|
   fullwidth?: boolean,
   autoFocus?: boolean,
   style?: Object,
-  as?: React.ComponentType<any>,
+  as?: React.ComponentType<any> | string,
   to?: string,
   onClick?: (event: SyntheticEvent<>) => mixed,
   borderOnHover?: boolean,
-
+  href?: string,
   "data-on"?: string,
   "data-event-category"?: string,
   "data-event-action"?: string,

app/components/SocketProvider.js

@@ -8,6 +8,7 @@ import AuthStore from "stores/AuthStore";
 import CollectionsStore from "stores/CollectionsStore";
 import DocumentPresenceStore from "stores/DocumentPresenceStore";
 import DocumentsStore from "stores/DocumentsStore";
+import FileOperationsStore from "stores/FileOperationsStore";
 import GroupsStore from "stores/GroupsStore";
 import MembershipsStore from "stores/MembershipsStore";
 import PoliciesStore from "stores/PoliciesStore";
@@ -28,6 +29,7 @@ type Props = {
   views: ViewsStore,
   auth: AuthStore,
   toasts: ToastsStore,
+  fileOperations: FileOperationsStore,
 };
 
 @observer
@@ -80,6 +82,7 @@ class SocketProvider extends React.Component<Props> {
       policies,
       presence,
       views,
+      fileOperations,
     } = this.props;
     if (!auth.token) return;
 
@@ -287,6 +290,21 @@ class SocketProvider extends React.Component<Props> {
       }
     });
 
+    this.socket.on("fileOperations.update", async (event) => {
+      const user = auth.user;
+      let collection = null;
+
+      if (event.collectionId)
+        collection = await collections.fetch(event.collectionId);
+      if (user) {
+        fileOperations.add({
+          ...event,
+          user,
+          collection,
+        });
+      }
+    });
+
     // received a message from the API server that we should request
     // to join a specific room. Forward that to the ws server.
     this.socket.on("join", (event) => {
@@ -345,5 +363,6 @@ export default inject(
   "memberships",
   "presence",
   "policies",
-  "views"
+  "views",
+  "fileOperations"
 )(SocketProvider);

app/menus/CollectionMenu.js

@@ -22,6 +22,7 @@ import ContextMenu from "components/ContextMenu";
 import OverflowMenuButton from "components/ContextMenu/OverflowMenuButton";
 import Template from "components/ContextMenu/Template";
 import Modal from "components/Modal";
+import useCurrentTeam from "hooks/useCurrentTeam";
 import useStores from "hooks/useStores";
 import useToasts from "hooks/useToasts";
 import getDataTransferFiles from "utils/getDataTransferFiles";
@@ -46,6 +47,7 @@ function CollectionMenu({
 }: Props) {
   const menu = useMenuState({ modal, placement });
   const [renderModals, setRenderModals] = React.useState(false);
+  const team = useCurrentTeam();
   const { documents, policies } = useStores();
   const { showToast } = useToasts();
   const { t } = useTranslation();
@@ -120,6 +122,8 @@ function CollectionMenu({
   );
 
   const can = policies.abilities(collection.id);
+  const canUserInTeam = policies.abilities(team.id);
+
   const items = React.useMemo(
     () => [
       {
@@ -151,7 +155,7 @@ function CollectionMenu({
       },
       {
         title: `${t("Export")}…`,
-        visible: !!(collection && can.export),
+        visible: !!(collection && canUserInTeam.export),
         onClick: () => setShowCollectionExport(true),
         icon: <ExportIcon />,
       },
@@ -165,7 +169,15 @@ function CollectionMenu({
         icon: <TrashIcon />,
       },
     ],
-    [can, collection, handleNewDocument, handleImportDocument, t]
+    [
+      t,
+      can.update,
+      can.delete,
+      handleNewDocument,
+      handleImportDocument,
+      collection,
+      canUserInTeam.export,
+    ]
   );
 
   if (!items.length) {

app/models/Collection.js

@@ -124,10 +124,6 @@ export default class Collection extends BaseModel {
   };
 
   export = () => {
-    return client.get(
+    return client.get("/collections.export", { id: this.id });
-      "/collections.export",
-      { id: this.id },
-      { download: true }
-    );
   };
 }

app/models/FileOperation.js

@@ -0,0 +1,27 @@
+// @flow
+import { computed } from "mobx";
+import BaseModal from "./BaseModel";
+import Collection from "./Collection";
+import User from "./User";
+
+class FileOperation extends BaseModal {
+  id: string;
+  state: string;
+  collection: ?Collection;
+  size: number;
+  type: string;
+  user: User;
+  createdAt: string;
+
+  @computed
+  get sizeInMB(): string {
+    const inKB = this.size / 1024;
+    if (inKB < 1024) {
+      return inKB.toFixed(2) + "KB";
+    }
+
+    return (inKB / 1024).toFixed(2) + "MB";
+  }
+}
+
+export default FileOperation;

app/scenes/CollectionExport.js

@@ -6,7 +6,7 @@ import Collection from "models/Collection";
 import Button from "components/Button";
 import Flex from "components/Flex";
 import HelpText from "components/HelpText";
-
+import useToasts from "hooks/useToasts";
 type Props = {
   collection: Collection,
   onSubmit: () => void,
@@ -15,6 +15,7 @@ type Props = {
 function CollectionExport({ collection, onSubmit }: Props) {
   const [isLoading, setIsLoading] = React.useState();
   const { t } = useTranslation();
+  const { showToast } = useToasts();
 
   const handleSubmit = React.useCallback(
     async (ev: SyntheticEvent<>) => {
@@ -23,9 +24,12 @@ function CollectionExport({ collection, onSubmit }: Props) {
       setIsLoading(true);
       await collection.export();
       setIsLoading(false);
+      showToast(
+        t("Export started, you will receive an email when it’s complete.")
+      );
       onSubmit();
     },
-    [collection, onSubmit]
+    [collection, onSubmit, showToast, t]
   );
 
   return (
@@ -33,7 +37,7 @@ function CollectionExport({ collection, onSubmit }: Props) {
       <form onSubmit={handleSubmit}>
         <HelpText>
           <Trans
-            defaults="Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be downloaded as a zip of folders with files in Markdown format."
+            defaults="Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be a zip of folders with files in Markdown format. Please visit the Export section on settings to get the zip."
             values={{ collectionName: collection.name }}
             components={{ em: <strong /> }}
           />

app/scenes/Settings/ImportExport.js

@@ -11,7 +11,10 @@ import Button from "components/Button";
 import Heading from "components/Heading";
 import HelpText from "components/HelpText";
 import Notice from "components/Notice";
+import PaginatedList from "components/PaginatedList";
 import Scene from "components/Scene";
+import Subheading from "components/Subheading";
+import FileOperationListItem from "./components/FileOperationListItem";
 import useCurrentUser from "hooks/useCurrentUser";
 import useStores from "hooks/useStores";
 import useToasts from "hooks/useToasts";
@@ -22,7 +25,7 @@ function ImportExport() {
   const { t } = useTranslation();
   const user = useCurrentUser();
   const fileRef = React.useRef();
-  const { collections } = useStores();
+  const { fileOperations, collections } = useStores();
   const { showToast } = useToasts();
   const [isLoading, setLoading] = React.useState(false);
   const [isImporting, setImporting] = React.useState(false);
@@ -178,11 +181,10 @@ function ImportExport() {
           {t("Choose File")}…
         </Button>
       )}
-
       <Heading>{t("Export")}</Heading>
       <HelpText>
         <Trans
-          defaults="A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <em>{{ userEmail }}</em>."
+          defaults="A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it's complete."
           values={{ userEmail: user.email }}
           components={{ em: <strong /> }}
         />
@@ -199,6 +201,24 @@ function ImportExport() {
           ? `${t("Requesting Export")}…`
           : t("Export Data")}
       </Button>
+      <br />
+      <br />
+      <PaginatedList
+        items={fileOperations.orderedDataExports}
+        fetch={fileOperations.fetchPage}
+        options={{ type: "export" }}
+        heading={
+          <Subheading>
+            <Trans>Recent exports</Trans>
+          </Subheading>
+        }
+        renderItem={(item) => (
+          <FileOperationListItem
+            key={item.id + item.state}
+            fileOperation={item}
+          />
+        )}
+      />
     </Scene>
   );
 }

app/scenes/Settings/components/FileOperationListItem.js

@@ -0,0 +1,61 @@
+// @flow
+import * as React from "react";
+import { useTranslation } from "react-i18next";
+import FileOperation from "models/FileOperation";
+import Button from "components/Button";
+import ListItem from "components/List/Item";
+import Time from "components/Time";
+
+type Props = {|
+  fileOperation: FileOperation,
+|};
+
+const FileOperationListItem = ({ fileOperation }: Props) => {
+  const { t } = useTranslation();
+
+  const stateMapping = {
+    creating: t("Processing"),
+    expired: t("Expired"),
+    uploading: t("Processing"),
+    error: t("Error"),
+  };
+
+  return (
+    <ListItem
+      title={
+        fileOperation.collection
+          ? fileOperation.collection.name
+          : t("All collections")
+      }
+      subtitle={
+        <>
+          {fileOperation.state !== "complete" && (
+            <>{stateMapping[fileOperation.state]}&nbsp;•&nbsp;</>
+          )}
+          {t(`{{userName}} requested`, {
+            userName:
+              fileOperation.id === fileOperation.user.id
+                ? t("You")
+                : fileOperation.user.name,
+          })}
+          &nbsp;
+          <Time dateTime={fileOperation.createdAt} addSuffix shorten />
+          &nbsp;•&nbsp;{fileOperation.sizeInMB}
+        </>
+      }
+      actions={
+        fileOperation.state === "complete" ? (
+          <Button
+            as="a"
+            href={`/api/fileOperations.redirect?id=${fileOperation.id}`}
+            neutral
+          >
+            {t("Download")}
+          </Button>
+        ) : undefined
+      }
+    />
+  );
+};
+
+export default FileOperationListItem;

app/stores/FileOperationsStore.js

@@ -0,0 +1,27 @@
+// @flow
+import { orderBy } from "lodash";
+import { computed } from "mobx";
+import FileOperation from "models/FileOperation";
+import BaseStore from "./BaseStore";
+import RootStore from "./RootStore";
+
+export default class FileOperationsStore extends BaseStore<FileOperation> {
+  actions = ["list", "info"];
+
+  constructor(rootStore: RootStore) {
+    super(rootStore, FileOperation);
+  }
+
+  @computed
+  get exports(): FileOperation[] {
+    return Array.from(this.data.values()).reduce(
+      (acc, fileOp) => (fileOp.type === "export" ? [...acc, fileOp] : acc),
+      []
+    );
+  }
+
+  @computed
+  get orderedDataExports(): FileOperation[] {
+    return orderBy(this.exports, "createdAt", "desc");
+  }
+}

app/stores/RootStore.js

@@ -6,6 +6,7 @@ import CollectionsStore from "./CollectionsStore";
 import DocumentPresenceStore from "./DocumentPresenceStore";
 import DocumentsStore from "./DocumentsStore";
 import EventsStore from "./EventsStore";
+import FileOperationsStore from "./FileOperationsStore";
 import GroupMembershipsStore from "./GroupMembershipsStore";
 import GroupsStore from "./GroupsStore";
 import IntegrationsStore from "./IntegrationsStore";
@@ -39,6 +40,7 @@ export default class RootStore {
   users: UsersStore;
   views: ViewsStore;
   toasts: ToastsStore;
+  fileOperations: FileOperationsStore;
 
   constructor() {
     // PoliciesStore must be initialized before AuthStore
@@ -60,6 +62,7 @@ export default class RootStore {
     this.ui = new UiStore();
     this.users = new UsersStore(this);
     this.views = new ViewsStore(this);
+    this.fileOperations = new FileOperationsStore(this);
     this.toasts = new ToastsStore();
   }
 
@@ -79,6 +82,7 @@ export default class RootStore {
     this.policies.clear();
     this.revisions.clear();
     this.shares.clear();
+    this.fileOperations.clear();
     // this.ui omitted to keep ui settings between sessions
     this.users.clear();
     this.views.clear();

server/api/attachments.js

@@ -11,7 +11,7 @@ import {
   getSignature,
   publicS3Endpoint,
   makeCredential,
-  getSignedImageUrl,
+  getSignedUrl,
 } from "../utils/s3";
 
 const { authorize } = policy;
@@ -146,7 +146,7 @@ router.post("attachments.redirect", auth(), async (ctx) => {
       authorize(user, "read", document);
     }
 
-    const accessUrl = await getSignedImageUrl(attachment.key);
+    const accessUrl = await getSignedUrl(attachment.key);
     ctx.redirect(accessUrl);
   } else {
     ctx.redirect(attachment.url);

server/api/collections.js

@@ -1,5 +1,4 @@
 // @flow
-import fs from "fs";
 import fractionalIndex from "fractional-index";
 import Router from "koa-router";
 import { ValidationError } from "../errors";
@@ -14,6 +13,7 @@ import {
   User,
   Group,
   Attachment,
+  FileOperation,
 } from "../models";
 import policy from "../policies";
 import {
@@ -23,12 +23,13 @@ import {
   presentMembership,
   presentGroup,
   presentCollectionGroupMembership,
+  presentFileOperation,
 } from "../presenters";
 import { Op, sequelize } from "../sequelize";
 
 import collectionIndexing from "../utils/collectionIndexing";
 import removeIndexCollision from "../utils/removeIndexCollision";
-import { archiveCollection, archiveCollections } from "../utils/zip";
+import { getAWSKeyForFileOp } from "../utils/s3";
 import pagination from "./middlewares/pagination";
 
 const { authorize } = policy;
@@ -454,59 +455,70 @@ router.post("collections.export", auth(), async (ctx) => {
   ctx.assertUuid(id, "id is required");
 
   const user = ctx.state.user;
+  const team = await Team.findByPk(user.teamId);
+  authorize(user, "export", team);
+
   const collection = await Collection.scope({
     method: ["withMembership", user.id],
   }).findByPk(id);
-  authorize(user, "export", collection);
 
-  const filePath = await archiveCollection(collection);
+  ctx.assertPresent(collection, "Collection should be present");
+  authorize(user, "read", collection);
 
-  await Event.create({
+  const key = getAWSKeyForFileOp(team.id, collection.name);
-    name: "collections.export",
+
-    collectionId: collection.id,
+  let exportData;
-    teamId: user.teamId,
+  exportData = await FileOperation.create({
-    actorId: user.id,
+    type: "export",
-    data: { title: collection.title },
+    state: "creating",
-    ip: ctx.request.ip,
+    key,
+    url: null,
+    size: 0,
+    collectionId: id,
+    userId: user.id,
+    teamId: team.id,
   });
 
-  ctx.attachment(`${collection.name}.zip`);
+  exportCollections(user.teamId, user.id, user.email, exportData.id, id);
-  ctx.set("Content-Type", "application/force-download");
+
-  ctx.body = fs.createReadStream(filePath);
+  exportData.user = user;
+  exportData.collection = collection;
+
+  ctx.body = {
+    success: true,
+    data: { fileOperation: presentFileOperation(exportData) },
+  };
 });
 
 router.post("collections.export_all", auth(), async (ctx) => {
-  const { download = false } = ctx.body;
-
   const user = ctx.state.user;
   const team = await Team.findByPk(user.teamId);
   authorize(user, "export", team);
 
-  await Event.create({
+  const key = getAWSKeyForFileOp(team.id, team.name);
-    name: "collections.export",
+
-    teamId: user.teamId,
+  let exportData;
-    actorId: user.id,
+  exportData = await FileOperation.create({
-    ip: ctx.request.ip,
+    type: "export",
+    state: "creating",
+    key,
+    url: null,
+    size: 0,
+    collectionId: null,
+    userId: user.id,
+    teamId: team.id,
   });
 
-  if (download) {
+  // async operation to upload zip archive to cloud and email user with link
-    const collections = await Collection.findAll({
+  exportCollections(user.teamId, user.id, user.email, exportData.id);
-      where: { teamId: team.id },
-      order: [["name", "ASC"]],
-    });
-    const filePath = await archiveCollections(collections);
 
-    ctx.attachment(`${team.name}.zip`);
+  exportData.user = user;
-    ctx.set("Content-Type", "application/force-download");
+  exportData.collection = null;
-    ctx.body = fs.createReadStream(filePath);
-  } else {
-    // async operation to create zip archive and email user
-    exportCollections(user.teamId, user.email);
 
-    ctx.body = {
+  ctx.body = {
-      success: true,
+    success: true,
-    };
+    data: { fileOperation: presentFileOperation(exportData) },
-  }
+  };
 });
 
 router.post("collections.update", auth(), async (ctx) => {

server/api/collections.test.js

@@ -264,53 +264,53 @@ describe("#collections.move", () => {
 
 describe("#collections.export", () => {
   it("should not allow export of private collection not a member", async () => {
-    const { user } = await seed();
+    const { admin } = await seed();
     const collection = await buildCollection({
       permission: null,
-      teamId: user.teamId,
+      teamId: admin.teamId,
     });
     const res = await server.post("/api/collections.export", {
-      body: { token: user.getJwtToken(), id: collection.id },
+      body: { token: admin.getJwtToken(), id: collection.id },
     });
 
     expect(res.status).toEqual(403);
   });
 
   it("should allow export of private collection when the actor is a member", async () => {
-    const { user, collection } = await seed();
+    const { admin, collection } = await seed();
     collection.permission = null;
     await collection.save();
 
     await CollectionUser.create({
-      createdById: user.id,
+      createdById: admin.id,
       collectionId: collection.id,
-      userId: user.id,
+      userId: admin.id,
       permission: "read_write",
     });
 
     const res = await server.post("/api/collections.export", {
-      body: { token: user.getJwtToken(), id: collection.id },
+      body: { token: admin.getJwtToken(), id: collection.id },
     });
 
     expect(res.status).toEqual(200);
   });
 
   it("should allow export of private collection when the actor is a group member", async () => {
-    const user = await buildUser();
+    const admin = await buildAdmin();
     const collection = await buildCollection({
       permission: null,
-      teamId: user.teamId,
+      teamId: admin.teamId,
     });
 
-    const group = await buildGroup({ teamId: user.teamId });
+    const group = await buildGroup({ teamId: admin.teamId });
-    await group.addUser(user, { through: { createdById: user.id } });
+    await group.addUser(admin, { through: { createdById: admin.id } });
 
     await collection.addGroup(group, {
-      through: { permission: "read_write", createdById: user.id },
+      through: { permission: "read_write", createdById: admin.id },
     });
 
     const res = await server.post("/api/collections.export", {
-      body: { token: user.getJwtToken(), id: collection.id },
+      body: { token: admin.getJwtToken(), id: collection.id },
     });
 
     expect(res.status).toEqual(200);
@@ -324,13 +324,29 @@ describe("#collections.export", () => {
     expect(body).toMatchSnapshot();
   });
 
-  it("should return success", async () => {
+  it("should return unauthorized if user is not admin", async () => {
     const { user, collection } = await seed();
     const res = await server.post("/api/collections.export", {
       body: { token: user.getJwtToken(), id: collection.id },
     });
 
-    expect(res.status).toEqual(200);
+    expect(res.status).toEqual(403);
+  });
+
+  it("should return file operation associated with export", async () => {
+    const admin = await buildAdmin();
+    const collection = await buildCollection({
+      teamId: admin.teamId,
+    });
+
+    const res = await server.post("/api/collections.export", {
+      body: { token: admin.getJwtToken(), id: collection.id },
+    });
+
+    const body = await res.json();
+    expect(res.status).toBe(200);
+    expect(body.data.fileOperation.id).toBeTruthy();
+    expect(body.data.fileOperation.state).toBe("creating");
   });
 });
 
@@ -359,18 +375,6 @@ describe("#collections.export_all", () => {
 
     expect(res.status).toEqual(200);
   });
-
-  it("should allow downloading directly", async () => {
-    const { admin } = await seed();
-    const res = await server.post("/api/collections.export_all", {
-      body: { token: admin.getJwtToken(), download: true },
-    });
-
-    expect(res.status).toEqual(200);
-    expect(res.headers.get("content-type")).toEqual(
-      "application/force-download"
-    );
-  });
 });
 
 describe("#collections.add_user", () => {
@@ -1026,7 +1030,6 @@ describe("#collections.create", () => {
     expect(body.data.sort.direction).toBe("asc");
     expect(body.policies.length).toBe(1);
     expect(body.policies[0].abilities.read).toBeTruthy();
-    expect(body.policies[0].abilities.export).toBeTruthy();
   });
 
   it("should error when index is invalid", async () => {
@@ -1060,7 +1063,6 @@ describe("#collections.create", () => {
     expect(body.data.permission).toEqual(null);
     expect(body.policies.length).toBe(1);
     expect(body.policies[0].abilities.read).toBeTruthy();
-    expect(body.policies[0].abilities.export).toBeTruthy();
   });
 
   it("if index collision, should updated index of other collection", async () => {

server/api/fileOperations.js

@@ -0,0 +1,99 @@
+// @flow
+import Router from "koa-router";
+import { NotFoundError, ValidationError } from "../errors";
+import auth from "../middlewares/authentication";
+import { FileOperation, Team } from "../models";
+import policy from "../policies";
+import { presentFileOperation } from "../presenters";
+import { getSignedUrl } from "../utils/s3";
+import pagination from "./middlewares/pagination";
+
+const { authorize } = policy;
+const router = new Router();
+
+router.post("fileOperations.info", auth(), async (ctx) => {
+  const { id } = ctx.body;
+  ctx.assertUuid(id, "id is required");
+  const user = ctx.state.user;
+  const team = await Team.findByPk(user.teamId);
+
+  const fileOperation = await FileOperation.findByPk(id);
+
+  authorize(user, fileOperation.type, team);
+
+  if (!fileOperation) {
+    throw new NotFoundError();
+  }
+
+  ctx.body = {
+    data: presentFileOperation(fileOperation),
+  };
+});
+
+router.post("fileOperations.list", auth(), pagination(), async (ctx) => {
+  let { sort = "createdAt", direction, type } = ctx.body;
+
+  ctx.assertPresent(type, "type is required");
+  ctx.assertIn(
+    type,
+    ["import", "export"],
+    "type must be one of 'import' or 'export'"
+  );
+
+  if (direction !== "ASC") direction = "DESC";
+
+  const user = ctx.state.user;
+
+  const where = {
+    teamId: user.teamId,
+    type,
+  };
+
+  const team = await Team.findByPk(user.teamId);
+  authorize(user, type, team);
+
+  const [exports, total] = await Promise.all([
+    await FileOperation.findAll({
+      where,
+      order: [[sort, direction]],
+      offset: ctx.state.pagination.offset,
+      limit: ctx.state.pagination.limit,
+    }),
+    await FileOperation.count({
+      where,
+    }),
+  ]);
+
+  ctx.body = {
+    pagination: {
+      ...ctx.state.pagination,
+      total,
+    },
+    data: exports.map(presentFileOperation),
+  };
+});
+
+router.post("fileOperations.redirect", auth(), async (ctx) => {
+  const { id } = ctx.body;
+  ctx.assertUuid(id, "id is required");
+
+  const user = ctx.state.user;
+  const team = await Team.findByPk(user.teamId);
+  const fileOp = await FileOperation.unscoped().findByPk(id);
+
+  if (!fileOp) {
+    throw new NotFoundError();
+  }
+
+  authorize(user, fileOp.type, team);
+
+  if (fileOp.state !== "complete") {
+    throw new ValidationError("file operation is not complete yet");
+  }
+
+  const accessUrl = await getSignedUrl(fileOp.key);
+
+  ctx.redirect(accessUrl);
+});
+
+export default router;

server/api/fileOperations.test.js

@@ -0,0 +1,283 @@
+/* eslint-disable flowtype/require-valid-file-annotation */
+import TestServer from "fetch-test-server";
+
+import { Collection, User } from "../models";
+import webService from "../services/web";
+import {
+  buildAdmin,
+  buildCollection,
+  buildFileOperation,
+  buildTeam,
+  buildUser,
+} from "../test/factories";
+import { flushdb } from "../test/support";
+
+const app = webService();
+const server = new TestServer(app.callback());
+
+beforeEach(() => flushdb());
+afterAll(() => server.close());
+
+describe("#fileOperations.info", () => {
+  it("should return fileOperation", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+    });
+
+    const res = await server.post("/api/fileOperations.info", {
+      body: {
+        id: exportData.id,
+        token: admin.getJwtToken(),
+        type: "export",
+      },
+    });
+
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.id).toBe(exportData.id);
+    expect(body.data.state).toBe(exportData.state);
+  });
+
+  it("should require user to be an admin", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const user = await buildUser({ teamId: team.id });
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+    });
+
+    const res = await server.post("/api/fileOperations.info", {
+      body: {
+        id: exportData.id,
+        token: user.getJwtToken(),
+        type: "export",
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});
+
+describe("#fileOperations.list", () => {
+  it("should return fileOperations list", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+    });
+
+    const res = await server.post("/api/fileOperations.list", {
+      body: {
+        token: admin.getJwtToken(),
+        type: "export",
+      },
+    });
+
+    const body = await res.json();
+    const data = body.data[0];
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toBe(1);
+    expect(data.id).toBe(exportData.id);
+    expect(data.key).toBe(undefined);
+    expect(data.state).toBe(exportData.state);
+  });
+
+  it("should return exports with collection data", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const collection = await buildCollection({
+      userId: admin.id,
+      teamId: team.id,
+    });
+
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+      collectionId: collection.id,
+    });
+
+    const res = await server.post("/api/fileOperations.list", {
+      body: {
+        token: admin.getJwtToken(),
+        type: "export",
+      },
+    });
+
+    const body = await res.json();
+    const data = body.data[0];
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toBe(1);
+    expect(data.id).toBe(exportData.id);
+    expect(data.key).toBe(undefined);
+    expect(data.state).toBe(exportData.state);
+    expect(data.collection.id).toBe(collection.id);
+  });
+
+  it("should return exports with collection data even if collection is deleted", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const collection = await buildCollection({
+      userId: admin.id,
+      teamId: team.id,
+    });
+
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+      collectionId: collection.id,
+    });
+
+    await collection.destroy();
+
+    const isCollectionPresent = await Collection.findByPk(collection.id);
+
+    expect(isCollectionPresent).toBe(null);
+
+    const res = await server.post("/api/fileOperations.list", {
+      body: {
+        token: admin.getJwtToken(),
+        type: "export",
+      },
+    });
+
+    const body = await res.json();
+    const data = body.data[0];
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toBe(1);
+    expect(data.id).toBe(exportData.id);
+    expect(data.key).toBe(undefined);
+    expect(data.state).toBe(exportData.state);
+    expect(data.collection.id).toBe(collection.id);
+  });
+
+  it("should return exports with user data even if user is deleted", async () => {
+    const team = await buildTeam();
+    const admin2 = await buildAdmin({ teamId: team.id });
+    const admin = await buildAdmin({ teamId: team.id });
+    const collection = await buildCollection({
+      userId: admin.id,
+      teamId: team.id,
+    });
+
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+      collectionId: collection.id,
+    });
+
+    await admin.destroy();
+
+    const isAdminPresent = await User.findByPk(admin.id);
+
+    expect(isAdminPresent).toBe(null);
+
+    const res = await server.post("/api/fileOperations.list", {
+      body: {
+        token: admin2.getJwtToken(),
+        type: "export",
+      },
+    });
+
+    const body = await res.json();
+    const data = body.data[0];
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toBe(1);
+    expect(data.id).toBe(exportData.id);
+    expect(data.key).toBe(undefined);
+    expect(data.state).toBe(exportData.state);
+    expect(data.user.id).toBe(admin.id);
+  });
+
+  it("should require authorization", async () => {
+    const user = await buildUser();
+
+    const res = await server.post("/api/fileOperations.list", {
+      body: { token: user.getJwtToken(), type: "export" },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});
+
+describe("#fileOperations.redirect", () => {
+  it("should not redirect when file operation is not complete", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+    });
+
+    const res = await server.post("/api/fileOperations.redirect", {
+      body: {
+        token: admin.getJwtToken(),
+        id: exportData.id,
+      },
+    });
+
+    const body = await res.json();
+    expect(res.status).toEqual(400);
+    expect(body.message).toEqual("file operation is not complete yet");
+  });
+});
+
+describe("#fileOperations.info", () => {
+  it("should return file operation", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+    });
+
+    const res = await server.post("/api/fileOperations.info", {
+      body: {
+        token: admin.getJwtToken(),
+        id: exportData.id,
+      },
+    });
+
+    const body = await res.json();
+    expect(res.status).toBe(200);
+    expect(body.data.id).toBe(exportData.id);
+    expect(body.data.user.id).toBe(admin.id);
+  });
+
+  it("should require authorization", async () => {
+    const team = await buildTeam();
+    const admin = await buildAdmin({ teamId: team.id });
+    const user = await buildUser({ teamId: team.id });
+    const exportData = await buildFileOperation({
+      type: "export",
+      teamId: team.id,
+      userId: admin.id,
+    });
+
+    const res = await server.post("/api/fileOperations.info", {
+      body: {
+        token: user.getJwtToken(),
+        id: exportData.id,
+      },
+    });
+
+    expect(res.status).toBe(403);
+  });
+});

server/api/index.js

@@ -14,6 +14,7 @@ import authenticationProviders from "./authenticationProviders";
 import collections from "./collections";
 import documents from "./documents";
 import events from "./events";
+import fileOperationsRoute from "./fileOperations";
 import groups from "./groups";
 import hooks from "./hooks";
 import integrations from "./integrations";
@@ -62,6 +63,7 @@ router.use("/", notificationSettings.routes());
 router.use("/", attachments.routes());
 router.use("/", utils.routes());
 router.use("/", groups.routes());
+router.use("/", fileOperationsRoute.routes());
 
 router.post("*", (ctx) => {
   ctx.throw(new NotFoundError("Endpoint not found"));

server/api/utils.js

@@ -4,7 +4,7 @@ import debug from "debug";
 import Router from "koa-router";
 import { documentPermanentDeleter } from "../commands/documentPermanentDeleter";
 import { AuthenticationError } from "../errors";
-import { Document } from "../models";
+import { Document, FileOperation } from "../models";
 import { Op } from "../sequelize";
 
 const router = new Router();
@@ -34,6 +34,26 @@ router.post("utils.gc", async (ctx) => {
 
   log(`Destroyed ${countDeletedDocument} documents`);
 
+  log(`Expiring all the collection export older than 30 days…`);
+
+  const exports = await FileOperation.unscoped().findAll({
+    where: {
+      type: "export",
+      createdAt: {
+        [Op.lt]: subDays(new Date(), 30),
+      },
+      state: {
+        [Op.ne]: "expired",
+      },
+    },
+  });
+
+  await Promise.all(
+    exports.map(async (e) => {
+      await e.expire();
+    })
+  );
+
   ctx.body = {
     success: true,
   };

server/api/utils.test.js

@@ -1,9 +1,10 @@
 /* eslint-disable flowtype/require-valid-file-annotation */
 import { subDays } from "date-fns";
 import TestServer from "fetch-test-server";
-import { Document } from "../models";
+import { Document, FileOperation } from "../models";
+import { Op } from "../sequelize";
 import webService from "../services/web";
-import { buildDocument } from "../test/factories";
+import { buildDocument, buildFileOperation } from "../test/factories";
 import { flushdb } from "../test/support";
 
 const app = webService();
@@ -83,6 +84,68 @@ describe("#utils.gc", () => {
     expect(await Document.unscoped().count({ paranoid: false })).toEqual(0);
   });
 
+  it("should expire exports older than 30 days ago", async () => {
+    await buildFileOperation({
+      type: "export",
+      state: "complete",
+      createdAt: subDays(new Date(), 30),
+    });
+
+    await buildFileOperation({
+      type: "export",
+      state: "complete",
+    });
+
+    const res = await server.post("/api/utils.gc", {
+      body: {
+        token: process.env.UTILS_SECRET,
+      },
+    });
+
+    const data = await FileOperation.count({
+      where: {
+        type: "export",
+        state: {
+          [Op.eq]: "expired",
+        },
+      },
+    });
+
+    expect(res.status).toEqual(200);
+    expect(data).toEqual(1);
+  });
+
+  it("should not expire exports made less than 30 days ago", async () => {
+    await buildFileOperation({
+      type: "export",
+      state: "complete",
+      createdAt: subDays(new Date(), 29),
+    });
+
+    await buildFileOperation({
+      type: "export",
+      state: "complete",
+    });
+
+    const res = await server.post("/api/utils.gc", {
+      body: {
+        token: process.env.UTILS_SECRET,
+      },
+    });
+
+    const data = await FileOperation.count({
+      where: {
+        type: "export",
+        state: {
+          [Op.eq]: "expired",
+        },
+      },
+    });
+
+    expect(res.status).toEqual(200);
+    expect(data).toEqual(0);
+  });
+
   it("should require authentication", async () => {
     const res = await server.post("/api/utils.gc");
     expect(res.status).toEqual(401);

server/emails/ExportEmail.js

@@ -1,36 +0,0 @@
-// @flow
-import * as React from "react";
-import Body from "./components/Body";
-import Button from "./components/Button";
-import EmailTemplate from "./components/EmailLayout";
-import EmptySpace from "./components/EmptySpace";
-import Footer from "./components/Footer";
-import Header from "./components/Header";
-import Heading from "./components/Heading";
-
-export const exportEmailText = `
-Your Data Export
-
-Your requested data export is attached as a zip file to this email.
-`;
-
-export const ExportEmail = () => {
-  return (
-    <EmailTemplate>
-      <Header />
-
-      <Body>
-        <Heading>Your Data Export</Heading>
-        <p>
-          Your requested data export is attached as a zip file to this email.
-        </p>
-        <EmptySpace height={10} />
-        <p>
-          <Button href={`${process.env.URL}/home`}>Go to dashboard</Button>
-        </p>
-      </Body>
-
-      <Footer />
-    </EmailTemplate>
-  );
-};

server/emails/ExportFailureEmail.js

@@ -0,0 +1,47 @@
+// @flow
+import * as React from "react";
+import Body from "./components/Body";
+import Button from "./components/Button";
+import EmailTemplate from "./components/EmailLayout";
+import EmptySpace from "./components/EmptySpace";
+import Footer from "./components/Footer";
+import Header from "./components/Header";
+import Heading from "./components/Heading";
+
+export const exportEmailFailureText = `
+Your Data Export
+
+Sorry, your requested data export has failed, please visit the admin
+section to try again – if the problem persists please contact support.
+`;
+
+export const ExportFailureEmail = ({ teamUrl }: { teamUrl: string }) => {
+  return (
+    <EmailTemplate>
+      <Header />
+
+      <Body>
+        <Heading>Your Data Export</Heading>
+        <p>
+          Sorry, your requested data export has failed, please visit the{" "}
+          <a
+            href={`${teamUrl}/settings/import-export`}
+            rel="noreferrer"
+            target="_blank"
+          >
+            admin section
+          </a>
+          . to try again – if the problem persists please contact support.
+        </p>
+        <EmptySpace height={10} />
+        <p>
+          <Button href={`${teamUrl}/settings/import-export`}>
+            Go to export
+          </Button>
+        </p>
+      </Body>
+
+      <Footer />
+    </EmailTemplate>
+  );
+};

server/emails/ExportSuccessEmail.js

@@ -0,0 +1,53 @@
+// @flow
+import * as React from "react";
+import Body from "./components/Body";
+import Button from "./components/Button";
+import EmailTemplate from "./components/EmailLayout";
+import EmptySpace from "./components/EmptySpace";
+import Footer from "./components/Footer";
+import Header from "./components/Header";
+import Heading from "./components/Heading";
+
+export const exportEmailSuccessText = `
+Your Data Export
+
+Your requested data export is complete, the exported files are also available in the admin section.
+`;
+
+export const ExportSuccessEmail = ({
+  id,
+  teamUrl,
+}: {
+  id: string,
+  teamUrl: string,
+}) => {
+  return (
+    <EmailTemplate>
+      <Header />
+
+      <Body>
+        <Heading>Your Data Export</Heading>
+        <p>
+          Your requested data export is complete, the exported files are also
+          available in the{" "}
+          <a
+            href={`${teamUrl}/settings/import-export`}
+            rel="noreferrer"
+            target="_blank"
+          >
+            admin section
+          </a>
+          .
+        </p>
+        <EmptySpace height={10} />
+        <p>
+          <Button href={`${teamUrl}/api/fileOperations.redirect?id=${id}`}>
+            Download
+          </Button>
+        </p>
+      </Body>
+
+      <Footer />
+    </EmailTemplate>
+  );
+};

server/exporter.js

@@ -1,8 +1,10 @@
 // @flow
+import fs from "fs";
 import debug from "debug";
 import mailer from "./mailer";
-import { Collection, Team } from "./models";
+import { FileOperation, Collection, Team, Event, User } from "./models";
 import { createQueue } from "./utils/queue";
+import { uploadToS3FromBuffer } from "./utils/s3";
 
 const log = debug("exporter");
 const exporterQueue = createQueue("exporter");
@@ -15,27 +17,137 @@ const queueOptions = {
   },
 };
 
-async function exportAndEmailCollections(teamId: string, email: string) {
+async function fileOperationsUpdate(teamId, userId, exportData) {
+  await Event.add({
+    name: "fileOperations.update",
+    teamId: teamId,
+    actorId: userId,
+    data: {
+      type: exportData.type,
+      id: exportData.id,
+      state: exportData.state,
+      size: exportData.size,
+      collectionId: exportData.collectionId,
+      createdAt: exportData.createdAt,
+    },
+  });
+}
+
+type exportAndEmailCollectionsType = {|
+  teamId: string,
+  userId: string,
+  email: string,
+  fileOperationId: string,
+  collectionId?: string,
+|};
+
+// TODO: Refactor to use command pattern
+async function exportAndEmailCollections({
+  teamId,
+  userId,
+  email,
+  collectionId,
+  fileOperationId,
+}: exportAndEmailCollectionsType) {
   log("Archiving team", teamId);
   const { archiveCollections } = require("./utils/zip");
   const team = await Team.findByPk(teamId);
-  const collections = await Collection.findAll({
+  const user = await User.findByPk(userId);
-    where: { teamId },
+
-    order: [["name", "ASC"]],
+  let collections;
-  });
+  if (!collectionId) {
+    const collectionIds = await user.collectionIds();
+
+    collections = await Promise.all(
+      collectionIds.map(
+        async (collectionId) => await Collection.findByPk(collectionId)
+      )
+    );
+  } else {
+    collections = [await Collection.findByPk(collectionId)];
+  }
+
+  let exportData;
+  let state;
+  let key;
+
+  exportData = await FileOperation.findByPk(fileOperationId);
+  state = exportData.state;
+  key = exportData.key;
+  await fileOperationsUpdate(teamId, userId, exportData);
+
   const filePath = await archiveCollections(collections);
 
   log("Archive path", filePath);
 
-  mailer.export({
+  let url;
-    to: email,
+  try {
-    attachments: [
+    const readBuffer = await fs.promises.readFile(filePath);
-      {
+    state = "uploading";
-        filename: `${team.name} Export.zip`,
+    exportData.state = state;
-        path: filePath,
+    const stat = await fs.promises.stat(filePath);
-      },
+    exportData.size = stat.size;
-    ],
+
-  });
+    await exportData.save();
+    await fileOperationsUpdate(teamId, userId, exportData);
+
+    url = await uploadToS3FromBuffer(
+      readBuffer,
+      "application/zip",
+      key,
+      "private"
+    );
+
+    state = "complete";
+  } catch (e) {
+    log("Failed to export data", e);
+    state = "error";
+    url = null;
+  } finally {
+    exportData.state = state;
+    exportData.url = url;
+    await exportData.save();
+
+    await fileOperationsUpdate(teamId, userId, exportData);
+
+    if (collectionId) {
+      await Event.create({
+        name: "collections.export",
+        collectionId,
+        teamId: teamId,
+        actorId: userId,
+        data: { name: collections[0].name, exportId: exportData.id },
+      });
+    } else {
+      const collectionsExported = collections.map((c) => ({
+        name: c.name,
+        id: c.id,
+      }));
+
+      await Event.create({
+        name: "collections.export_all",
+        teamId: teamId,
+        actorId: userId,
+        data: {
+          exportId: exportData.id,
+          collections: collectionsExported,
+        },
+      });
+    }
+
+    if (state === "error") {
+      mailer.exportFailure({
+        to: email,
+        teamUrl: team.url,
+      });
+    } else {
+      mailer.exportSuccess({
+        to: email,
+        id: exportData.id,
+        teamUrl: team.url,
+      });
+    }
+  }
 }
 
 exporterQueue.process(async function exportProcessor(job) {
@@ -43,17 +155,33 @@ exporterQueue.process(async function exportProcessor(job) {
 
   switch (job.data.type) {
     case "export-collections":
-      return await exportAndEmailCollections(job.data.teamId, job.data.email);
+      const { teamId, userId, email, collectionId, fileOperationId } = job.data;
+      return await exportAndEmailCollections({
+        teamId,
+        userId,
+        email,
+        fileOperationId,
+        collectionId,
+      });
     default:
   }
 });
 
-export const exportCollections = (teamId: string, email: string) => {
+export const exportCollections = (
+  teamId: string,
+  userId: string,
+  email: string,
+  fileOperationId: string,
+  collectionId?: string
+) => {
   exporterQueue.add(
     {
       type: "export-collections",
       teamId,
+      userId,
       email,
+      fileOperationId,
+      collectionId,
     },
     queueOptions
   );

server/mailer.js

@@ -14,7 +14,15 @@ import {
   DocumentNotificationEmail,
   documentNotificationEmailText,
 } from "./emails/DocumentNotificationEmail";
-import { ExportEmail, exportEmailText } from "./emails/ExportEmail";
+import {
+  ExportFailureEmail,
+  exportEmailFailureText,
+} from "./emails/ExportFailureEmail";
+
+import {
+  ExportSuccessEmail,
+  exportEmailSuccessText,
+} from "./emails/ExportSuccessEmail";
 import {
   type Props as InviteEmailT,
   InviteEmail,
@@ -155,14 +163,34 @@ export class Mailer {
     });
   };
 
-  export = async (opts: { to: string, attachments: Object[] }) => {
+  exportSuccess = async (opts: {
+    to: string,
+    attachments?: Object[],
+    id: string,
+    teamUrl: string,
+  }) => {
     this.sendMail({
       to: opts.to,
       attachments: opts.attachments,
       title: "Your requested export",
       previewText: "Here's your request data export from Outline",
-      html: <ExportEmail />,
+      html: <ExportSuccessEmail id={opts.id} teamUrl={opts.teamUrl} />,
-      text: exportEmailText,
+      text: exportEmailSuccessText,
+    });
+  };
+
+  exportFailure = async (opts: {
+    to: string,
+    attachments?: Object[],
+    teamUrl: string,
+  }) => {
+    this.sendMail({
+      to: opts.to,
+      attachments: opts.attachments,
+      title: "Your requested export",
+      previewText: "Sorry, your requested data export has failed",
+      html: <ExportFailureEmail teamUrl={opts.teamUrl} />,
+      text: exportEmailFailureText,
     });
   };
 

server/migrations/20210730210120-add-fileOperations.js

@@ -0,0 +1,65 @@
+'use strict';
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.createTable("file_operations",{
+      id: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        primaryKey: true,
+      },
+      state: {
+        type: Sequelize.ENUM("creating", "uploading", "complete", "error","expired"),
+        allowNull: false,
+      },
+      type: {
+        type: Sequelize.ENUM("import", "export"),
+        allowNull: false,
+      },
+      key: {
+        type: Sequelize.STRING,
+      },
+      url: {
+        type: Sequelize.STRING,
+      },
+      size: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+      },
+      userId: {
+        type: Sequelize.UUID,       
+        allowNull: false,
+        references: {
+          model: "users"
+        }
+      },
+      collectionId: {
+        type: Sequelize.UUID,       
+        references: {
+          model: "collections"
+        }
+      },
+      teamId: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        references: {
+          model: "teams"
+        }
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+    })
+    await queryInterface.addIndex('file_operations', ["type", "state"])
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    await queryInterface.removeIndex('file_operations', ["type", "state"]);
+    await queryInterface.dropTable('file_operations');
+  }
+};

server/models/Event.js

@@ -83,6 +83,7 @@ Event.AUDIT_EVENTS = [
   "collections.add_group",
   "collections.remove_group",
   "collections.delete",
+  "collections.export_all",
   "documents.create",
   "documents.publish",
   "documents.update",

server/models/FileOperation.js

@@ -0,0 +1,72 @@
+// @flow
+import { DataTypes, sequelize } from "../sequelize";
+import { deleteFromS3 } from "../utils/s3";
+
+const FileOperation = sequelize.define("file_operations", {
+  id: {
+    type: DataTypes.UUID,
+    defaultValue: DataTypes.UUIDV4,
+    primaryKey: true,
+  },
+  type: {
+    type: DataTypes.ENUM("import", "export"),
+    allowNull: false,
+  },
+  state: {
+    type: DataTypes.ENUM(
+      "creating",
+      "uploading",
+      "complete",
+      "error",
+      "expired"
+    ),
+    allowNull: false,
+  },
+  key: {
+    type: DataTypes.STRING,
+  },
+  url: {
+    type: DataTypes.STRING,
+  },
+  size: {
+    type: DataTypes.BIGINT,
+    allowNull: false,
+  },
+});
+
+FileOperation.prototype.expire = async function () {
+  this.state = "expired";
+  await deleteFromS3(this.key);
+  this.save();
+};
+
+FileOperation.associate = (models) => {
+  FileOperation.belongsTo(models.User, {
+    as: "user",
+    foreignKey: "userId",
+  });
+  FileOperation.belongsTo(models.Collection, {
+    as: "collection",
+    foreignKey: "collectionId",
+  });
+  FileOperation.belongsTo(models.Team, {
+    as: "team",
+    foreignKey: "teamId",
+  });
+  FileOperation.addScope("defaultScope", {
+    include: [
+      {
+        model: models.User,
+        as: "user",
+        paranoid: false,
+      },
+      {
+        model: models.Collection,
+        as: "collection",
+        paranoid: false,
+      },
+    ],
+  });
+};
+
+export default FileOperation;

server/models/index.js

@@ -8,6 +8,7 @@ import CollectionGroup from "./CollectionGroup";
 import CollectionUser from "./CollectionUser";
 import Document from "./Document";
 import Event from "./Event";
+import FileOperation from "./FileOperation";
 import Group from "./Group";
 import GroupUser from "./GroupUser";
 import Integration from "./Integration";
@@ -47,6 +48,7 @@ const models = {
   User,
   UserAuthentication,
   View,
+  FileOperation,
 };
 
 // based on https://github.com/sequelize/express-example/blob/master/models/index.js
@@ -80,4 +82,5 @@ export {
   User,
   UserAuthentication,
   View,
+  FileOperation,
 };

server/policies/collection.js

@@ -47,7 +47,7 @@ allow(User, "read", Collection, (user, collection) => {
   return true;
 });
 
-allow(User, ["share", "export"], Collection, (user, collection) => {
+allow(User, "share", Collection, (user, collection) => {
   if (user.isViewer) return false;
   if (!collection || user.teamId !== collection.teamId) return false;
   if (!collection.sharing) return false;

server/policies/collection.test.js

@@ -16,7 +16,6 @@ describe("read_write permission", () => {
     });
     const abilities = serialize(user, collection);
     expect(abilities.read).toEqual(true);
-    expect(abilities.export).toEqual(true);
     expect(abilities.update).toEqual(true);
     expect(abilities.share).toEqual(true);
   });
@@ -43,7 +42,6 @@ describe("read_write permission", () => {
 
     const abilities = serialize(user, collection);
     expect(abilities.read).toEqual(true);
-    expect(abilities.export).toEqual(true);
     expect(abilities.update).toEqual(true);
     expect(abilities.share).toEqual(true);
   });
@@ -59,7 +57,6 @@ describe("read permission", () => {
     });
     const abilities = serialize(user, collection);
     expect(abilities.read).toEqual(true);
-    expect(abilities.export).toEqual(false);
     expect(abilities.update).toEqual(false);
     expect(abilities.share).toEqual(false);
   });
@@ -86,7 +83,6 @@ describe("read permission", () => {
 
     const abilities = serialize(user, collection);
     expect(abilities.read).toEqual(true);
-    expect(abilities.export).toEqual(true);
     expect(abilities.update).toEqual(true);
     expect(abilities.share).toEqual(true);
   });
@@ -102,7 +98,6 @@ describe("no permission", () => {
     });
     const abilities = serialize(user, collection);
     expect(abilities.read).toEqual(false);
-    expect(abilities.export).toEqual(false);
     expect(abilities.update).toEqual(false);
     expect(abilities.share).toEqual(false);
   });
@@ -129,7 +124,6 @@ describe("no permission", () => {
 
     const abilities = serialize(user, collection);
     expect(abilities.read).toEqual(true);
-    expect(abilities.export).toEqual(true);
     expect(abilities.update).toEqual(true);
     expect(abilities.share).toEqual(true);
   });

server/presenters/document.js

@@ -1,7 +1,7 @@
 // @flow
 import { Attachment, Document } from "../models";
 import parseAttachmentIds from "../utils/parseAttachmentIds";
-import { getSignedImageUrl } from "../utils/s3";
+import { getSignedUrl } from "../utils/s3";
 import presentUser from "./user";
 
 type Options = {
@@ -16,7 +16,7 @@ async function replaceImageAttachments(text: string) {
     attachmentIds.map(async (id) => {
       const attachment = await Attachment.findByPk(id);
       if (attachment) {
-        const accessUrl = await getSignedImageUrl(attachment.key);
+        const accessUrl = await getSignedUrl(attachment.key);
         text = text.replace(attachment.redirectUrl, accessUrl);
       }
     })

server/presenters/fileOperation.js

@@ -0,0 +1,15 @@
+// @flow
+import { FileOperation } from "../models";
+import { presentCollection, presentUser } from ".";
+
+export default function present(data: FileOperation) {
+  return {
+    id: data.id,
+    type: data.type,
+    state: data.state,
+    collection: data.collection ? presentCollection(data.collection) : null,
+    size: data.size,
+    user: presentUser(data.user),
+    createdAt: data.createdAt,
+  };
+}

server/presenters/index.js

@@ -5,6 +5,7 @@ import presentCollection from "./collection";
 import presentCollectionGroupMembership from "./collectionGroupMembership";
 import presentDocument from "./document";
 import presentEvent from "./event";
+import presentFileOperation from "./fileOperation";
 import presentGroup from "./group";
 import presentGroupMembership from "./groupMembership";
 import presentIntegration from "./integration";
@@ -20,6 +21,7 @@ import presentView from "./view";
 
 export {
   presentApiKey,
+  presentFileOperation,
   presentAuthenticationProvider,
   presentUser,
   presentView,

server/queues/processors/websockets.js

@@ -322,6 +322,13 @@ export default class WebsocketsProcessor {
         }
         return;
       }
+
+      case "fileOperations.update": {
+        return socketio
+          .to(`user-${event.actorId}`)
+          .emit("fileOperations.update", event.data);
+      }
+
       case "groups.create":
       case "groups.update": {
         const group = await Group.findByPk(event.modelId, {

server/test/factories.js

@@ -13,6 +13,7 @@ import {
   IntegrationAuthentication,
   Integration,
   AuthenticationProvider,
+  FileOperation,
 } from "../models";
 
 let count = 1;
@@ -255,6 +256,31 @@ export async function buildDocument(overrides: Object = {}) {
   });
 }
 
+export async function buildFileOperation(overrides: Object = {}) {
+  if (!overrides.teamId) {
+    const team = await buildTeam();
+    overrides.teamId = team.id;
+  }
+
+  if (!overrides.userId) {
+    const user = await buildAdmin({ teamId: overrides.teamId });
+    overrides.userId = user.id;
+  }
+
+  if (!overrides.collectionId) {
+    const collection = await buildCollection(overrides);
+    overrides.collectionId = collection.id;
+  }
+
+  return FileOperation.create({
+    state: "creating",
+    size: 0,
+    key: "key/to/aws/file.zip",
+    url: "https://www.urltos3file.com/file.zip",
+    ...overrides,
+  });
+}
+
 export async function buildAttachment(overrides: Object = {}) {
   if (!overrides.teamId) {
     const team = await buildTeam();

server/types.js

@@ -116,6 +116,30 @@ export type CollectionImportEvent = {
   ip: string,
 };
 
+export type CollectionExportAll = {
+  name: "collections.export_all",
+  teamId: string,
+  actorId: string,
+  data: {
+    exportId: string,
+    collections: [{ name: string, id: string }],
+  },
+};
+
+export type FileOperationEvent = {
+  name: "fileOperations.update",
+  teamId: string,
+  actorId: string,
+  data: {
+    type: string,
+    state: string,
+    id: string,
+    size: number,
+    createdAt: string,
+    collectionId: string,
+  },
+};
+
 export type CollectionEvent =
   | {
   name: | "collections.create" // eslint-disable-line
@@ -192,6 +216,8 @@ export type Event =
   | DocumentEvent
   | CollectionEvent
   | CollectionImportEvent
+  | CollectionExportAll
+  | FileOperationEvent
   | IntegrationEvent
   | GroupEvent
   | RevisionEvent

server/utils/s3.js

@@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";
 import AWS from "aws-sdk";
 import { addHours, format } from "date-fns";
 import fetch from "fetch-with-proxy";
+import { v4 as uuidv4 } from "uuid";
 
 const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
 const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
@@ -167,7 +168,7 @@ export const deleteFromS3 = (key: string) => {
     .promise();
 };
 
-export const getSignedImageUrl = async (key: string) => {
+export const getSignedUrl = async (key: string) => {
   const isDocker = process.env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
 
   const params = {
@@ -181,6 +182,12 @@ export const getSignedImageUrl = async (key: string) => {
     : s3.getSignedUrl("getObject", params);
 };
 
+// function assumes that acl is private
+export const getAWSKeyForFileOp = (teamId: string, name: string) => {
+  const bucket = "uploads";
+  return `${bucket}/${teamId}/${uuidv4()}/${name}-export.zip`;
+};
+
 export const getFileByKey = async (key: string) => {
   const params = {
     Bucket: AWS_S3_UPLOAD_BUCKET_NAME,

server/utils/zip.js

@@ -69,17 +69,6 @@ async function archiveToPath(zip) {
   });
 }
 
-export async function archiveCollection(collection: Collection) {
-  const zip = new JSZip();
-
-  if (collection.documentStructure) {
-    const folder = zip.folder(collection.name);
-    await addToArchive(folder, collection.documentStructure);
-  }
-
-  return archiveToPath(zip);
-}
-
 export async function archiveCollections(collections: Collection[]) {
   const zip = new JSZip();
 

shared/i18n/locales/en_US/translation.json

@@ -279,7 +279,8 @@
   "Public sharing is currently disabled in the team security settings.": "Public sharing is currently disabled in the team security settings.",
   "Saving": "Saving",
   "Save": "Save",
-  "Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be downloaded as a zip of folders with files in Markdown format.": "Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be downloaded as a zip of folders with files in Markdown format.",
+  "Export started, you will receive an email when it’s complete.": "Export started, you will receive an email when it’s complete.",
+  "Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be a zip of folders with files in Markdown format. Please visit the Export section on settings to get the zip.": "Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be a zip of folders with files in Markdown format. Please visit the Export section on settings to get the zip.",
   "Exporting": "Exporting",
   "Export Collection": "Export Collection",
   "Collections are for grouping your documents. They work best when organized around a topic or internal team — Product or Engineering for example.": "Collections are for grouping your documents. They work best when organized around a topic or internal team — Product or Engineering for example.",
@@ -469,6 +470,11 @@
   "No documents found for your search filters. <1></1>": "No documents found for your search filters. <1></1>",
   "Create a new document?": "Create a new document?",
   "Clear filters": "Clear filters",
+  "Processing": "Processing",
+  "Expired": "Expired",
+  "Error": "Error",
+  "All collections": "All collections",
+  "{{userName}} requested": "{{userName}} requested",
   "Last active": "Last active",
   "Role": "Role",
   "Viewer": "Viewer",
@@ -501,10 +507,11 @@
   "Uploading": "Uploading",
   "Confirm & Import": "Confirm & Import",
   "Choose File": "Choose File",
-  "A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <em>{{ userEmail }}</em>.": "A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <em>{{ userEmail }}</em>.",
+  "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it's complete.": "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it's complete.",
   "Export Requested": "Export Requested",
   "Requesting Export": "Requesting Export",
   "Export Data": "Export Data",
+  "Recent exports": "Recent exports",
   "Document published": "Document published",
   "Receive a notification whenever a new document is published": "Receive a notification whenever a new document is published",
   "Document updated": "Document updated",