chore: Refactoring event processors and service architecture (#2495)

This commit is contained in:
Tom Moor
2021-08-27 21:42:13 -07:00
committed by GitHub
parent 86f008293a
commit 28aef82af9
61 changed files with 1610 additions and 1498 deletions

View File

@ -1,503 +1,242 @@
// @flow
import { subHours } from "date-fns";
import type { Event } from "../events";
import { socketio } from "../main";
import {
Document,
Collection,
Group,
CollectionGroup,
GroupUser,
} from "../models";
import { Op } from "../sequelize";
import http from "http";
import Koa from "koa";
import IO from "socket.io";
import socketRedisAdapter from "socket.io-redis";
import SocketAuth from "socketio-auth";
import env from "../env";
import { Document, Collection, View } from "../models";
import policy from "../policies";
import { websocketsQueue } from "../queues";
import WebsocketsProcessor from "../queues/processors/websockets";
import { client, subscriber } from "../redis";
import Sentry from "../sentry";
import { getUserForJWT } from "../utils/jwt";
import * as metrics from "../utils/metrics";
export default class Websockets {
async on(event: Event) {
if (!socketio) {
return;
const { can } = policy;
const websockets = new WebsocketsProcessor();
export default function init(app: Koa, server: http.Server) {
const io = IO(server, {
path: "/realtime",
serveClient: false,
cookie: false,
});
io.adapter(
socketRedisAdapter({
pubClient: client,
subClient: subscriber,
})
);
io.origins((_, callback) => {
callback(null, true);
});
io.of("/").adapter.on("error", (err) => {
if (err.name === "MaxRetriesPerRequestError") {
console.error(`Redis error: ${err.message}. Shutting down now.`);
throw err;
} else {
console.error(`Redis error: ${err.message}`);
}
});
switch (event.name) {
case "documents.publish":
case "documents.restore":
case "documents.archive":
case "documents.unarchive": {
const document = await Document.findByPk(event.documentId, {
paranoid: false,
});
io.on("connection", (socket) => {
metrics.increment("websockets.connected");
metrics.gaugePerInstance(
"websockets.count",
socket.client.conn.server.clientsCount
);
const channel = document.publishedAt
? `collection-${document.collectionId}`
: `user-${event.actorId}`;
socket.on("disconnect", () => {
metrics.increment("websockets.disconnected");
metrics.gaugePerInstance(
"websockets.count",
socket.client.conn.server.clientsCount
);
});
});
return socketio.to(channel).emit("entities", {
event: event.name,
documentIds: [
{
id: document.id,
updatedAt: document.updatedAt,
},
],
collectionIds: [
{
id: document.collectionId,
},
],
});
SocketAuth(io, {
authenticate: async (socket, data, callback) => {
const { token } = data;
try {
const user = await getUserForJWT(token);
socket.client.user = user;
// store the mapping between socket id and user id in redis
// so that it is accessible across multiple server nodes
await client.hset(socket.id, "userId", user.id);
return callback(null, true);
} catch (err) {
return callback(err);
}
case "documents.delete": {
const document = await Document.findByPk(event.documentId, {
paranoid: false,
});
},
postAuthenticate: async (socket, data) => {
const { user } = socket.client;
if (!document.publishedAt) {
return socketio.to(`user-${document.createdById}`).emit("entities", {
event: event.name,
documentIds: [
{
id: document.id,
updatedAt: document.updatedAt,
},
],
});
}
// the rooms associated with the current team
// and user so we can send authenticated events
let rooms = [`team-${user.teamId}`, `user-${user.id}`];
return socketio
.to(`collection-${document.collectionId}`)
.emit("entities", {
event: event.name,
documentIds: [
{
id: document.id,
updatedAt: document.updatedAt,
},
],
collectionIds: [
{
id: document.collectionId,
},
],
});
}
case "documents.permanent_delete": {
return socketio
.to(`collection-${event.collectionId}`)
.emit(event.name, {
documentId: event.documentId,
});
}
case "documents.pin":
case "documents.unpin":
case "documents.update": {
const document = await Document.findByPk(event.documentId, {
paranoid: false,
});
// the rooms associated with collections this user
// has access to on connection. New collection subscriptions
// are managed from the client as needed through the 'join' event
const collectionIds = await user.collectionIds();
collectionIds.forEach((collectionId) =>
rooms.push(`collection-${collectionId}`)
);
const channel = document.publishedAt
? `collection-${document.collectionId}`
: `user-${event.actorId}`;
// join all of the rooms at once
socket.join(rooms);
return socketio.to(channel).emit("entities", {
event: event.name,
documentIds: [
{
id: document.id,
updatedAt: document.updatedAt,
},
],
});
}
case "documents.create": {
const document = await Document.findByPk(event.documentId);
return socketio.to(`user-${event.actorId}`).emit("entities", {
event: event.name,
documentIds: [
{
id: document.id,
updatedAt: document.updatedAt,
},
],
collectionIds: [
{
id: document.collectionId,
},
],
});
}
case "documents.star":
case "documents.unstar": {
return socketio.to(`user-${event.actorId}`).emit(event.name, {
documentId: event.documentId,
});
}
case "documents.move": {
const documents = await Document.findAll({
where: {
id: event.data.documentIds,
},
paranoid: false,
});
documents.forEach((document) => {
socketio.to(`collection-${document.collectionId}`).emit("entities", {
event: event.name,
documentIds: [
{
id: document.id,
updatedAt: document.updatedAt,
},
],
});
});
event.data.collectionIds.forEach((collectionId) => {
socketio.to(`collection-${collectionId}`).emit("entities", {
event: event.name,
collectionIds: [{ id: collectionId }],
});
});
return;
}
case "collections.create": {
const collection = await Collection.findByPk(event.collectionId, {
paranoid: false,
});
socketio
.to(
collection.permission
? `team-${collection.teamId}`
: `collection-${collection.id}`
)
.emit("entities", {
event: event.name,
collectionIds: [
{
id: collection.id,
updatedAt: collection.updatedAt,
},
],
});
return socketio
.to(
collection.permission
? `team-${collection.teamId}`
: `collection-${collection.id}`
)
.emit("join", {
event: event.name,
collectionId: collection.id,
});
}
case "collections.update":
case "collections.delete": {
const collection = await Collection.findByPk(event.collectionId, {
paranoid: false,
});
return socketio.to(`team-${collection.teamId}`).emit("entities", {
event: event.name,
collectionIds: [
{
id: collection.id,
updatedAt: collection.updatedAt,
},
],
});
}
case "collections.move": {
return socketio
.to(`collection-${event.collectionId}`)
.emit("collections.update_index", {
collectionId: event.collectionId,
index: event.data.index,
});
}
case "collections.add_user": {
// the user being added isn't yet in the websocket channel for the collection
// so they need to be notified separately
socketio.to(`user-${event.userId}`).emit(event.name, {
event: event.name,
userId: event.userId,
collectionId: event.collectionId,
});
// let everyone with access to the collection know a user was added
socketio.to(`collection-${event.collectionId}`).emit(event.name, {
event: event.name,
userId: event.userId,
collectionId: event.collectionId,
});
// tell any user clients to connect to the websocket channel for the collection
return socketio.to(`user-${event.userId}`).emit("join", {
event: event.name,
collectionId: event.collectionId,
});
}
case "collections.remove_user": {
const membershipUserIds = await Collection.membershipUserIds(
event.collectionId
);
if (membershipUserIds.includes(event.userId)) {
// Even though we just removed a user from the collection
// the user still has access through some means
// treat this like an add, so that the client re-syncs policies
socketio.to(`user-${event.userId}`).emit("collections.add_user", {
event: "collections.add_user",
userId: event.userId,
collectionId: event.collectionId,
});
} else {
// let everyone with access to the collection know a user was removed
socketio
.to(`collection-${event.collectionId}`)
.emit("collections.remove_user", {
event: event.name,
userId: event.userId,
collectionId: event.collectionId,
});
// tell any user clients to disconnect from the websocket channel for the collection
socketio.to(`user-${event.userId}`).emit("leave", {
event: event.name,
collectionId: event.collectionId,
});
}
return;
}
case "collections.add_group": {
const group = await Group.findByPk(event.data.groupId);
// the users being added are not yet in the websocket channel for the collection
// so they need to be notified separately
for (const groupMembership of group.groupMemberships) {
socketio
.to(`user-${groupMembership.userId}`)
.emit("collections.add_user", {
event: event.name,
userId: groupMembership.userId,
collectionId: event.collectionId,
});
// tell any user clients to connect to the websocket channel for the collection
socketio.to(`user-${groupMembership.userId}`).emit("join", {
event: event.name,
collectionId: event.collectionId,
});
}
return;
}
case "collections.remove_group": {
const group = await Group.findByPk(event.data.groupId);
const membershipUserIds = await Collection.membershipUserIds(
event.collectionId
);
for (const groupMembership of group.groupMemberships) {
if (membershipUserIds.includes(groupMembership.userId)) {
// the user still has access through some means...
// treat this like an add, so that the client re-syncs policies
socketio
.to(`user-${groupMembership.userId}`)
.emit("collections.add_user", {
event: event.name,
userId: groupMembership.userId,
collectionId: event.collectionId,
});
} else {
// let users in the channel know they were removed
socketio
.to(`user-${groupMembership.userId}`)
.emit("collections.remove_user", {
event: event.name,
userId: groupMembership.userId,
collectionId: event.collectionId,
});
// tell any user clients to disconnect to the websocket channel for the collection
socketio.to(`user-${groupMembership.userId}`).emit("leave", {
event: event.name,
collectionId: event.collectionId,
});
}
}
return;
}
case "groups.create":
case "groups.update": {
const group = await Group.findByPk(event.modelId, {
paranoid: false,
});
return socketio.to(`team-${group.teamId}`).emit("entities", {
event: event.name,
groupIds: [
{
id: group.id,
updatedAt: group.updatedAt,
},
],
});
}
case "groups.add_user": {
// do an add user for every collection that the group is a part of
const collectionGroupMemberships = await CollectionGroup.findAll({
where: { groupId: event.modelId },
});
for (const collectionGroup of collectionGroupMemberships) {
// the user being added isn't yet in the websocket channel for the collection
// so they need to be notified separately
socketio.to(`user-${event.userId}`).emit("collections.add_user", {
event: event.name,
userId: event.userId,
collectionId: collectionGroup.collectionId,
});
// let everyone with access to the collection know a user was added
socketio
.to(`collection-${collectionGroup.collectionId}`)
.emit("collections.add_user", {
event: event.name,
userId: event.userId,
collectionId: collectionGroup.collectionId,
});
// tell any user clients to connect to the websocket channel for the collection
return socketio.to(`user-${event.userId}`).emit("join", {
event: event.name,
collectionId: collectionGroup.collectionId,
});
}
return;
}
case "groups.remove_user": {
const collectionGroupMemberships = await CollectionGroup.findAll({
where: { groupId: event.modelId },
});
for (const collectionGroup of collectionGroupMemberships) {
// if the user has any memberships remaining on the collection
// we need to emit add instead of remove
// allow the client to request to join rooms
socket.on("join", async (event) => {
// user is joining a collection channel, because their permissions have
// changed, granting them access.
if (event.collectionId) {
const collection = await Collection.scope({
method: ["withMembership", event.userId],
}).findByPk(collectionGroup.collectionId);
method: ["withMembership", user.id],
}).findByPk(event.collectionId);
if (!collection) {
continue;
}
const hasMemberships =
collection.memberships.length > 0 ||
collection.collectionGroupMemberships.length > 0;
if (hasMemberships) {
// the user still has access through some means...
// treat this like an add, so that the client re-syncs policies
socketio.to(`user-${event.userId}`).emit("collections.add_user", {
event: event.name,
userId: event.userId,
collectionId: collectionGroup.collectionId,
});
} else {
// let everyone with access to the collection know a user was removed
socketio
.to(`collection-${collectionGroup.collectionId}`)
.emit("collections.remove_user", {
event: event.name,
userId: event.userId,
collectionId: collectionGroup.collectionId,
});
// tell any user clients to disconnect from the websocket channel for the collection
socketio.to(`user-${event.userId}`).emit("leave", {
event: event.name,
collectionId: collectionGroup.collectionId,
if (can(user, "read", collection)) {
socket.join(`collection-${event.collectionId}`, () => {
metrics.increment("websockets.collections.join");
});
}
}
return;
}
case "groups.delete": {
const group = await Group.findByPk(event.modelId, {
paranoid: false,
});
socketio.to(`team-${group.teamId}`).emit("entities", {
event: event.name,
groupIds: [
{
id: group.id,
updatedAt: group.updatedAt,
},
],
});
// user is joining a document channel, because they have navigated to
// view a document.
if (event.documentId) {
const document = await Document.findByPk(event.documentId, {
userId: user.id,
});
// we the users and collection relations that were just severed as a result of the group deletion
// since there are cascading deletes, we approximate this by looking for the recently deleted
// items in the GroupUser and CollectionGroup tables
const groupUsers = await GroupUser.findAll({
paranoid: false,
where: {
groupId: event.modelId,
deletedAt: {
[Op.gt]: subHours(new Date(), 1),
},
},
});
if (can(user, "read", document)) {
const room = `document-${event.documentId}`;
const collectionGroupMemberships = await CollectionGroup.findAll({
paranoid: false,
where: {
groupId: event.modelId,
deletedAt: {
[Op.gt]: subHours(new Date(), 1),
},
},
});
await View.touch(event.documentId, user.id, event.isEditing);
const editing = await View.findRecentlyEditingByDocument(
event.documentId
);
for (const collectionGroup of collectionGroupMemberships) {
const membershipUserIds = await Collection.membershipUserIds(
collectionGroup.collectionId
socket.join(room, () => {
metrics.increment("websockets.documents.join");
// let everyone else in the room know that a new user joined
io.to(room).emit("user.join", {
userId: user.id,
documentId: event.documentId,
isEditing: event.isEditing,
});
// let this user know who else is already present in the room
io.in(room).clients(async (err, sockets) => {
if (err) {
if (process.env.SENTRY_DSN) {
Sentry.withScope(function (scope) {
scope.setExtra("clients", sockets);
Sentry.captureException(err);
});
} else {
console.error(err);
}
return;
}
// because a single user can have multiple socket connections we
// need to make sure that only unique userIds are returned. A Map
// makes this easy.
let userIds = new Map();
for (const socketId of sockets) {
const userId = await client.hget(socketId, "userId");
userIds.set(userId, userId);
}
socket.emit("document.presence", {
documentId: event.documentId,
userIds: Array.from(userIds.keys()),
editingIds: editing.map((view) => view.userId),
});
});
});
}
}
});
// allow the client to request to leave rooms
socket.on("leave", (event) => {
if (event.collectionId) {
socket.leave(`collection-${event.collectionId}`, () => {
metrics.increment("websockets.collections.leave");
});
}
if (event.documentId) {
const room = `document-${event.documentId}`;
socket.leave(room, () => {
metrics.increment("websockets.documents.leave");
io.to(room).emit("user.leave", {
userId: user.id,
documentId: event.documentId,
});
});
}
});
socket.on("disconnecting", () => {
const rooms = Object.keys(socket.rooms);
rooms.forEach((room) => {
if (room.startsWith("document-")) {
const documentId = room.replace("document-", "");
io.to(room).emit("user.leave", {
userId: user.id,
documentId,
});
}
});
});
socket.on("presence", async (event) => {
metrics.increment("websockets.presence");
const room = `document-${event.documentId}`;
if (event.documentId && socket.rooms[room]) {
const view = await View.touch(
event.documentId,
user.id,
event.isEditing
);
view.user = user;
for (const groupUser of groupUsers) {
if (membershipUserIds.includes(groupUser.userId)) {
// the user still has access through some means...
// treat this like an add, so that the client re-syncs policies
socketio
.to(`user-${groupUser.userId}`)
.emit("collections.add_user", {
event: event.name,
userId: groupUser.userId,
collectionId: collectionGroup.collectionId,
});
} else {
// let everyone with access to the collection know a user was removed
socketio
.to(`collection-${collectionGroup.collectionId}`)
.emit("collections.remove_user", {
event: event.name,
userId: groupUser.userId,
collectionId: collectionGroup.collectionId,
});
// tell any user clients to disconnect from the websocket channel for the collection
socketio.to(`user-${groupUser.userId}`).emit("leave", {
event: event.name,
collectionId: collectionGroup.collectionId,
});
}
}
io.to(room).emit("user.presence", {
userId: user.id,
documentId: event.documentId,
isEditing: event.isEditing,
});
}
return;
}
});
},
});
default:
}
}
websocketsQueue.process(async function websocketEventsProcessor(job) {
const event = job.data;
websockets.on(event, io).catch((error) => {
if (env.SENTRY_DSN) {
Sentry.withScope(function (scope) {
scope.setExtra("event", event);
Sentry.captureException(error);
});
} else {
throw error;
}
});
});
}