chore: Permanent team deletion (#2493)

server/commands/documentPermanentDeleter.js

@@ -4,7 +4,7 @@ import { Document, Attachment } from "../models";
 import { sequelize } from "../sequelize";
 import parseAttachmentIds from "../utils/parseAttachmentIds";
 
-export async function documentPermanentDeleter(documents: Document[]) {
+export default async function documentPermanentDeleter(documents: Document[]) {
   const activeDocument = documents.find((doc) => !doc.deletedAt);
 
   if (activeDocument) {

server/commands/documentPermanentDeleter.test.js

@@ -3,7 +3,7 @@ import { subDays } from "date-fns";
 import { Attachment, Document } from "../models";
 import { buildAttachment, buildDocument } from "../test/factories";
 import { flushdb } from "../test/support";
-import { documentPermanentDeleter } from "./documentPermanentDeleter";
+import documentPermanentDeleter from "./documentPermanentDeleter";
 
 jest.mock("aws-sdk", () => {
   const mS3 = { deleteObject: jest.fn().mockReturnThis(), promise: jest.fn() };

server/commands/teamPermanentDeleter.js

@@ -0,0 +1,177 @@
+// @flow
+import Logger from "../logging/logger";
+import {
+  ApiKey,
+  Attachment,
+  AuthenticationProvider,
+  Collection,
+  Document,
+  Event,
+  FileOperation,
+  Group,
+  Team,
+  NotificationSetting,
+  User,
+  UserAuthentication,
+  Integration,
+  SearchQuery,
+  Share,
+} from "../models";
+import { sequelize } from "../sequelize";
+
+export default async function teamPermanentDeleter(team: Team) {
+  if (!team.deletedAt) {
+    throw new Error(
+      `Cannot permanently delete ${team.id} team. Please delete it and try again.`
+    );
+  }
+
+  Logger.info(
+    "commands",
+    `Permanently deleting team ${team.name} (${team.id})`
+  );
+
+  const teamId = team.id;
+  let transaction;
+
+  try {
+    transaction = await sequelize.transaction();
+
+    await Attachment.findAllInBatches(
+      {
+        where: {
+          teamId,
+        },
+        limit: 100,
+        offset: 0,
+      },
+      async (attachments, options) => {
+        Logger.info(
+          "commands",
+          `Deleting attachments ${options.offset} – ${
+            options.offset + options.limit
+          }…`
+        );
+
+        await Promise.all(
+          attachments.map((attachment) => attachment.destroy({ transaction }))
+        );
+      }
+    );
+
+    // Destroy user-relation models
+    await User.findAllInBatches(
+      {
+        attributes: ["id"],
+        where: {
+          teamId,
+        },
+        limit: 100,
+        offset: 0,
+      },
+      async (users) => {
+        const userIds = users.map((user) => user.id);
+
+        await UserAuthentication.destroy({
+          where: { userId: userIds },
+          force: true,
+          transaction,
+        });
+
+        await ApiKey.destroy({
+          where: { userId: userIds },
+          force: true,
+          transaction,
+        });
+      }
+    );
+
+    // Destory team-relation models
+    await AuthenticationProvider.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    // events must be first due to db constraints
+    await Event.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await Collection.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await Document.unscoped().destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await FileOperation.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await Group.unscoped().destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await Integration.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await NotificationSetting.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await SearchQuery.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await Share.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await User.destroy({
+      where: { teamId },
+      force: true,
+      transaction,
+    });
+
+    await team.destroy({
+      force: true,
+      transaction,
+    });
+
+    await Event.create(
+      {
+        name: "teams.destroy",
+        modelId: teamId,
+      },
+      { transaction }
+    );
+
+    await transaction.commit();
+  } catch (err) {
+    if (transaction) {
+      await transaction.rollback();
+    }
+    throw err;
+  }
+}

server/commands/teamPermanentDeleter.test.js

@@ -0,0 +1,100 @@
+// @flow
+import { subDays } from "date-fns";
+import { Attachment, User, Document, Collection, Team } from "../models";
+import {
+  buildAttachment,
+  buildUser,
+  buildTeam,
+  buildDocument,
+} from "../test/factories";
+import { flushdb } from "../test/support";
+import teamPermanentDeleter from "./teamPermanentDeleter";
+
+jest.mock("aws-sdk", () => {
+  const mS3 = { deleteObject: jest.fn().mockReturnThis(), promise: jest.fn() };
+  return {
+    S3: jest.fn(() => mS3),
+    Endpoint: jest.fn(),
+  };
+});
+
+beforeEach(() => flushdb());
+
+describe("teamPermanentDeleter", () => {
+  it("should destroy related data", async () => {
+    const team = await buildTeam({
+      deletedAt: subDays(new Date(), 90),
+    });
+    const user = await buildUser({
+      teamId: team.id,
+    });
+    await buildDocument({
+      teamId: team.id,
+      userId: user.id,
+    });
+
+    await teamPermanentDeleter(team);
+
+    expect(await Team.count()).toEqual(0);
+    expect(await User.count()).toEqual(0);
+    expect(await Document.unscoped().count({ paranoid: false })).toEqual(0);
+    expect(await Collection.unscoped().count({ paranoid: false })).toEqual(0);
+  });
+
+  it("should not destroy unrelated data", async () => {
+    const team = await buildTeam({
+      deletedAt: subDays(new Date(), 90),
+    });
+
+    await buildUser();
+    await buildTeam();
+    await buildDocument();
+
+    await teamPermanentDeleter(team);
+
+    expect(await Team.count()).toEqual(4); // each build command creates a team
+    expect(await User.count()).toEqual(2);
+    expect(await Document.unscoped().count({ paranoid: false })).toEqual(1);
+    expect(await Collection.unscoped().count({ paranoid: false })).toEqual(1);
+  });
+
+  it("should destroy attachments", async () => {
+    const team = await buildTeam({
+      deletedAt: subDays(new Date(), 90),
+    });
+    const user = await buildUser({
+      teamId: team.id,
+    });
+    const document = await buildDocument({
+      teamId: team.id,
+      userId: user.id,
+    });
+    await buildAttachment({
+      teamId: document.teamId,
+      documentId: document.id,
+    });
+
+    await teamPermanentDeleter(team);
+
+    expect(await Team.count()).toEqual(0);
+    expect(await User.count()).toEqual(0);
+    expect(await Attachment.count()).toEqual(0);
+    expect(await Document.unscoped().count({ paranoid: false })).toEqual(0);
+    expect(await Collection.unscoped().count({ paranoid: false })).toEqual(0);
+  });
+
+  it("should error when trying to destroy undeleted team", async () => {
+    const team = await buildTeam();
+
+    let error;
+    try {
+      await teamPermanentDeleter(team);
+    } catch (err) {
+      error = err.message;
+    }
+
+    expect(error).toEqual(
+      `Cannot permanently delete ${team.id} team. Please delete it and try again.`
+    );
+  });
+});

server/migrations/20210921031555-missing-cascades.js

@@ -0,0 +1,78 @@
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    let tableName, constraintName;
+
+    tableName = 'collection_users';
+    constraintName = 'collection_users_collectionId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"
+        add constraint "${constraintName}" foreign key("collectionId") references "collections" ("id")
+        on delete cascade`
+    );
+
+    constraintName = 'collection_users_userId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"\
+        add constraint "${constraintName}" foreign key("userId") references "users" ("id")
+        on delete cascade`
+    );
+
+    tableName = 'group_users';
+    constraintName = 'group_users_groupId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"
+        add constraint "${constraintName}" foreign key("groupId") references "groups" ("id")
+        on delete cascade`
+    );
+
+    constraintName = 'group_users_userId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"
+        add constraint "${constraintName}" foreign key("userId") references "users" ("id")
+        on delete cascade`
+    );
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    let tableName, constraintName;
+
+    tableName = 'collection_users';
+    constraintName = 'collection_users_collectionId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"\
+        add constraint "${constraintName}" foreign key("collectionId") references "collections" ("id")
+        on delete no action`
+    );
+
+    constraintName = 'collection_users_userId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"\
+        add constraint "${constraintName}" foreign key("userId") references "users" ("id")
+        on delete no action`
+    );
+
+    tableName = 'group_users';
+    constraintName = 'group_users_groupId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"
+        add constraint "${constraintName}" foreign key("groupId") references "groups" ("id")
+        on delete no action`
+    );
+
+    constraintName = 'group_users_userId_fkey';
+    await queryInterface.sequelize.query(`alter table "${tableName}" drop constraint "${constraintName}"`)
+    await queryInterface.sequelize.query(
+      `alter table "${tableName}"
+        add constraint "${constraintName}" foreign key("userId") references "users" ("id")
+        on delete no action`
+    );
+  },
+};

server/models/Attachment.js

@@ -54,6 +54,22 @@ const Attachment = sequelize.define(
   }
 );
 
+Attachment.findAllInBatches = async (
+  query,
+  callback: (attachments: Array<Attachment>, query: Object) => Promise<void>
+) => {
+  if (!query.offset) query.offset = 0;
+  if (!query.limit) query.limit = 10;
+  let results;
+
+  do {
+    results = await Attachment.findAll(query);
+
+    await callback(results, query);
+    query.offset += query.limit;
+  } while (results.length >= query.limit);
+};
+
 Attachment.beforeDestroy(async (model) => {
   await deleteFromS3(model.key);
 });

server/models/User.js

@@ -304,6 +304,22 @@ User.getCounts = async function (teamId: string) {
   };
 };
 
+User.findAllInBatches = async (
+  query,
+  callback: (users: Array<User>, query: Object) => Promise<void>
+) => {
+  if (!query.offset) query.offset = 0;
+  if (!query.limit) query.limit = 10;
+  let results;
+
+  do {
+    results = await User.findAll(query);
+
+    await callback(results, query);
+    query.offset += query.limit;
+  } while (results.length >= query.limit);
+};
+
 User.prototype.demote = async function (
   teamId: string,
   to: "member" | "viewer"

server/models/View.js

@@ -4,26 +4,20 @@ import { USER_PRESENCE_INTERVAL } from "../../shared/constants";
 import { User } from "../models";
 import { DataTypes, Op, sequelize } from "../sequelize";
 
-const View = sequelize.define(
-  "view",
-  {
-    id: {
-      type: DataTypes.UUID,
-      defaultValue: DataTypes.UUIDV4,
-      primaryKey: true,
-    },
-    lastEditingAt: {
-      type: DataTypes.DATE,
-    },
-    count: {
-      type: DataTypes.INTEGER,
-      defaultValue: 1,
-    },
+const View = sequelize.define("view", {
+  id: {
+    type: DataTypes.UUID,
+    defaultValue: DataTypes.UUIDV4,
+    primaryKey: true,
   },
-  {
-    classMethods: {},
-  }
-);
+  lastEditingAt: {
+    type: DataTypes.DATE,
+  },
+  count: {
+    type: DataTypes.INTEGER,
+    defaultValue: 1,
+  },
+});
 
 View.associate = (models) => {
   View.belongsTo(models.Document);

server/routes/api/documents.js

@@ -5,7 +5,7 @@ import { subtractDate } from "../../../shared/utils/date";
 import documentCreator from "../../commands/documentCreator";
 import documentImporter from "../../commands/documentImporter";
 import documentMover from "../../commands/documentMover";
-import { documentPermanentDeleter } from "../../commands/documentPermanentDeleter";
+import documentPermanentDeleter from "../../commands/documentPermanentDeleter";
 import env from "../../env";
 import {
   NotFoundError,

server/routes/api/utils.js

@@ -1,10 +1,11 @@
 // @flow
 import { subDays } from "date-fns";
 import Router from "koa-router";
-import { documentPermanentDeleter } from "../../commands/documentPermanentDeleter";
+import documentPermanentDeleter from "../../commands/documentPermanentDeleter";
+import teamPermanentDeleter from "../../commands/teamPermanentDeleter";
 import { AuthenticationError } from "../../errors";
 import Logger from "../../logging/logger";
-import { Document, FileOperation } from "../../models";
+import { Document, Team, FileOperation } from "../../models";
 import { Op } from "../../sequelize";
 
 const router = new Router();
@@ -59,6 +60,25 @@ router.post("utils.gc", async (ctx) => {
     })
   );
 
+  Logger.info(
+    "utils",
+    `Permanently destroying upto ${limit} teams older than 30 days…`
+  );
+
+  const teams = await Team.findAll({
+    where: {
+      deletedAt: {
+        [Op.lt]: subDays(new Date(), 30),
+      },
+    },
+    paranoid: false,
+    limit,
+  });
+
+  for (const team of teams) {
+    await teamPermanentDeleter(team);
+  }
+
   ctx.body = {
     success: true,
   };