outline/server/exporter.js

// @flow
import fs from "fs";
import debug from "debug";
import mailer from "./mailer";
import { FileOperation, Collection, Team, Event, User } from "./models";
import { createQueue } from "./utils/queue";
import { uploadToS3FromBuffer } from "./utils/s3";

const log = debug("exporter");
const exporterQueue = createQueue("exporter");
const queueOptions = {
  attempts: 2,
  removeOnComplete: true,
  backoff: {
    type: "exponential",
    delay: 60 * 1000,
  },
};

async function fileOperationsUpdate(teamId, userId, exportData) {
  await Event.add({
    name: "fileOperations.update",
    teamId: teamId,
    actorId: userId,
    data: {
      type: exportData.type,
      id: exportData.id,
      state: exportData.state,
      size: exportData.size,
      collectionId: exportData.collectionId,
      createdAt: exportData.createdAt,
    },
  });
}

type exportAndEmailCollectionsType = {|
  teamId: string,
  userId: string,
  email: string,
  fileOperationId: string,
  collectionId?: string,
|};

// TODO: Refactor to use command pattern
async function exportAndEmailCollections({
  teamId,
  userId,
  email,
  collectionId,
  fileOperationId,
}: exportAndEmailCollectionsType) {
  log("Archiving team", teamId);
  const { archiveCollections } = require("./utils/zip");
  const team = await Team.findByPk(teamId);
  const user = await User.findByPk(userId);

  let collections;
  if (!collectionId) {
    const collectionIds = await user.collectionIds();

    collections = await Promise.all(
      collectionIds.map(
        async (collectionId) => await Collection.findByPk(collectionId)
      )
    );
  } else {
    collections = [await Collection.findByPk(collectionId)];
  }

  let exportData;
  let state;
  let key;

  exportData = await FileOperation.findByPk(fileOperationId);
  state = exportData.state;
  key = exportData.key;
  await fileOperationsUpdate(teamId, userId, exportData);

  const filePath = await archiveCollections(collections);

  log("Archive path", filePath);

  let url;
  try {
    const readBuffer = await fs.promises.readFile(filePath);
    state = "uploading";
    exportData.state = state;
    const stat = await fs.promises.stat(filePath);
    exportData.size = stat.size;

    await exportData.save();
    await fileOperationsUpdate(teamId, userId, exportData);

    url = await uploadToS3FromBuffer(
      readBuffer,
      "application/zip",
      key,
      "private"
    );

    state = "complete";
  } catch (e) {
    log("Failed to export data", e);
    state = "error";
    url = null;
  } finally {
    exportData.state = state;
    exportData.url = url;
    await exportData.save();

    await fileOperationsUpdate(teamId, userId, exportData);

    if (collectionId) {
      await Event.create({
        name: "collections.export",
        collectionId,
        teamId: teamId,
        actorId: userId,
        data: { name: collections[0].name, exportId: exportData.id },
      });
    } else {
      const collectionsExported = collections.map((c) => ({
        name: c.name,
        id: c.id,
      }));

      await Event.create({
        name: "collections.export_all",
        teamId: teamId,
        actorId: userId,
        data: {
          exportId: exportData.id,
          collections: collectionsExported,
        },
      });
    }

    if (state === "error") {
      mailer.exportFailure({
        to: email,
        teamUrl: team.url,
      });
    } else {
      mailer.exportSuccess({
        to: email,
        id: exportData.id,
        teamUrl: team.url,
      });
    }
  }
}

exporterQueue.process(async function exportProcessor(job) {
  log("Process", job.data);

  switch (job.data.type) {
    case "export-collections":
      const { teamId, userId, email, collectionId, fileOperationId } = job.data;
      return await exportAndEmailCollections({
        teamId,
        userId,
        email,
        fileOperationId,
        collectionId,
      });
    default:
  }
});

export const exportCollections = (
  teamId: string,
  userId: string,
  email: string,
  fileOperationId: string,
  collectionId?: string
) => {
  exporterQueue.add(
    {
      type: "export-collections",
      teamId,
      userId,
      email,
      fileOperationId,
      collectionId,
    },
    queueOptions
  );
};