coop-cloud-chaos-patchs/outline
coop-cloud-chaos-patchs
/
outline
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix: Refactor collection exports to not send email attachment (#2460) 

Co-authored-by: Tom Moor <tom.moor@gmail.com>
This commit is contained in:
Saumya Pandey 2021-08-29 02:57:07 +05:30 committed by GitHub
parent 28aef82af9
commit 00ba65f3ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 1252 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/components/Button.js
View File

@ -128,11 +128,11 @@ export type Props = {|
fullwidth?: boolean,
autoFocus?: boolean,
style?: Object,
as?: React.ComponentType<any>,
as?: React.ComponentType<any> | string,
to?: string,
onClick?: (event: SyntheticEvent<>) => mixed,
borderOnHover?: boolean,
href?: string,
"data-on"?: string,
"data-event-category"?: string,
"data-event-action"?: string,

21
app/components/SocketProvider.js
View File

@ -8,6 +8,7 @@ import AuthStore from "stores/AuthStore";
import CollectionsStore from "stores/CollectionsStore";
import DocumentPresenceStore from "stores/DocumentPresenceStore";
import DocumentsStore from "stores/DocumentsStore";
import FileOperationsStore from "stores/FileOperationsStore";
import GroupsStore from "stores/GroupsStore";
import MembershipsStore from "stores/MembershipsStore";
import PoliciesStore from "stores/PoliciesStore";
@ -28,6 +29,7 @@ type Props = {
views: ViewsStore,
auth: AuthStore,
toasts: ToastsStore,
fileOperations: FileOperationsStore,
};
@observer
@ -80,6 +82,7 @@ class SocketProvider extends React.Component<Props> {
policies,
presence,
views,
fileOperations,
} = this.props;
if (!auth.token) return;
@ -287,6 +290,21 @@ class SocketProvider extends React.Component<Props> {
}
});
this.socket.on("fileOperations.update", async (event) => {
const user = auth.user;
let collection = null;
if (event.collectionId)
collection = await collections.fetch(event.collectionId);
if (user) {
fileOperations.add({
...event,
user,
collection,
});
}
});
// received a message from the API server that we should request
// to join a specific room. Forward that to the ws server.
this.socket.on("join", (event) => {
@ -345,5 +363,6 @@ export default inject(
"memberships",
"presence",
"policies",
"views"
"views",
"fileOperations"
)(SocketProvider);

16
app/menus/CollectionMenu.js
View File

@ -22,6 +22,7 @@ import ContextMenu from "components/ContextMenu";
import OverflowMenuButton from "components/ContextMenu/OverflowMenuButton";
import Template from "components/ContextMenu/Template";
import Modal from "components/Modal";
import useCurrentTeam from "hooks/useCurrentTeam";
import useStores from "hooks/useStores";
import useToasts from "hooks/useToasts";
import getDataTransferFiles from "utils/getDataTransferFiles";
@ -46,6 +47,7 @@ function CollectionMenu({
}: Props) {
const menu = useMenuState({ modal, placement });
const [renderModals, setRenderModals] = React.useState(false);
const team = useCurrentTeam();
const { documents, policies } = useStores();
const { showToast } = useToasts();
const { t } = useTranslation();
@ -120,6 +122,8 @@ function CollectionMenu({
);
const can = policies.abilities(collection.id);
const canUserInTeam = policies.abilities(team.id);
const items = React.useMemo(
() => [
{
@ -151,7 +155,7 @@ function CollectionMenu({
},
{
title: `${t("Export")}…`,
visible: !!(collection && can.export),
visible: !!(collection && canUserInTeam.export),
onClick: () => setShowCollectionExport(true),
icon: <ExportIcon />,
},
@ -165,7 +169,15 @@ function CollectionMenu({
icon: <TrashIcon />,
},
],
[can, collection, handleNewDocument, handleImportDocument, t]
[
t,
can.update,
can.delete,
handleNewDocument,
handleImportDocument,
collection,
canUserInTeam.export,
]
);
if (!items.length) {

6
app/models/Collection.js
View File

@ -124,10 +124,6 @@ export default class Collection extends BaseModel {
};
export = () => {
return client.get(
"/collections.export",
{ id: this.id },
{ download: true }
);
return client.get("/collections.export", { id: this.id });
};
}

27
app/models/FileOperation.js Normal file
View File

@ -0,0 +1,27 @@
// @flow
import { computed } from "mobx";
import BaseModal from "./BaseModel";
import Collection from "./Collection";
import User from "./User";
class FileOperation extends BaseModal {
id: string;
state: string;
collection: ?Collection;
size: number;
type: string;
user: User;
createdAt: string;
@computed
get sizeInMB(): string {
const inKB = this.size / 1024;
if (inKB < 1024) {
return inKB.toFixed(2) + "KB";
}
return (inKB / 1024).toFixed(2) + "MB";
}
}
export default FileOperation;

10
app/scenes/CollectionExport.js
View File

@ -6,7 +6,7 @@ import Collection from "models/Collection";
import Button from "components/Button";
import Flex from "components/Flex";
import HelpText from "components/HelpText";
import useToasts from "hooks/useToasts";
type Props = {
collection: Collection,
onSubmit: () => void,
@ -15,6 +15,7 @@ type Props = {
function CollectionExport({ collection, onSubmit }: Props) {
const [isLoading, setIsLoading] = React.useState();
const { t } = useTranslation();
const { showToast } = useToasts();
const handleSubmit = React.useCallback(
async (ev: SyntheticEvent<>) => {
@ -23,9 +24,12 @@ function CollectionExport({ collection, onSubmit }: Props) {
setIsLoading(true);
await collection.export();
setIsLoading(false);
showToast(
t("Export started, you will receive an email when it’s complete.")
);
onSubmit();
},
[collection, onSubmit]
[collection, onSubmit, showToast, t]
);
return (
@ -33,7 +37,7 @@ function CollectionExport({ collection, onSubmit }: Props) {
<form onSubmit={handleSubmit}>
<HelpText>
<Trans
defaults="Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be downloaded as a zip of folders with files in Markdown format."
defaults="Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be a zip of folders with files in Markdown format. Please visit the Export section on settings to get the zip."
values={{ collectionName: collection.name }}
components={{ em: <strong /> }}
/>

26
app/scenes/Settings/ImportExport.js
View File

@ -11,7 +11,10 @@ import Button from "components/Button";
import Heading from "components/Heading";
import HelpText from "components/HelpText";
import Notice from "components/Notice";
import PaginatedList from "components/PaginatedList";
import Scene from "components/Scene";
import Subheading from "components/Subheading";
import FileOperationListItem from "./components/FileOperationListItem";
import useCurrentUser from "hooks/useCurrentUser";
import useStores from "hooks/useStores";
import useToasts from "hooks/useToasts";
@ -22,7 +25,7 @@ function ImportExport() {
const { t } = useTranslation();
const user = useCurrentUser();
const fileRef = React.useRef();
const { collections } = useStores();
const { fileOperations, collections } = useStores();
const { showToast } = useToasts();
const [isLoading, setLoading] = React.useState(false);
const [isImporting, setImporting] = React.useState(false);
@ -178,11 +181,10 @@ function ImportExport() {
{t("Choose File")}…
</Button>
)}
<Heading>{t("Export")}</Heading>
<HelpText>
<Trans
defaults="A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <em>{{ userEmail }}</em>."
defaults="A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it's complete."
values={{ userEmail: user.email }}
components={{ em: <strong /> }}
/>
@ -199,6 +201,24 @@ function ImportExport() {
? `${t("Requesting Export")}…`
: t("Export Data")}
</Button>
<br />
<br />
<PaginatedList
items={fileOperations.orderedDataExports}
fetch={fileOperations.fetchPage}
options={{ type: "export" }}
heading={
<Subheading>
<Trans>Recent exports</Trans>
</Subheading>
}
renderItem={(item) => (
<FileOperationListItem
key={item.id + item.state}
fileOperation={item}
/>
)}
/>
</Scene>
);
}

61
app/scenes/Settings/components/FileOperationListItem.js Normal file
View File

@ -0,0 +1,61 @@
// @flow
import * as React from "react";
import { useTranslation } from "react-i18next";
import FileOperation from "models/FileOperation";
import Button from "components/Button";
import ListItem from "components/List/Item";
import Time from "components/Time";
type Props = {|
fileOperation: FileOperation,
|};
const FileOperationListItem = ({ fileOperation }: Props) => {
const { t } = useTranslation();
const stateMapping = {
creating: t("Processing"),
expired: t("Expired"),
uploading: t("Processing"),
error: t("Error"),
};
return (
<ListItem
title={
fileOperation.collection
? fileOperation.collection.name
: t("All collections")
}
subtitle={
<>
{fileOperation.state !== "complete" && (
<>{stateMapping[fileOperation.state]}&nbsp;•&nbsp;</>
)}
{t(`{{userName}} requested`, {
userName:
fileOperation.id === fileOperation.user.id
? t("You")
: fileOperation.user.name,
})}
&nbsp;
<Time dateTime={fileOperation.createdAt} addSuffix shorten />
&nbsp;•&nbsp;{fileOperation.sizeInMB}
</>
}
actions={
fileOperation.state === "complete" ? (
<Button
as="a"
href={`/api/fileOperations.redirect?id=${fileOperation.id}`}
neutral
>
{t("Download")}
</Button>
) : undefined
}
/>
);
};
export default FileOperationListItem;

27
app/stores/FileOperationsStore.js Normal file
View File

@ -0,0 +1,27 @@
// @flow
import { orderBy } from "lodash";
import { computed } from "mobx";
import FileOperation from "models/FileOperation";
import BaseStore from "./BaseStore";
import RootStore from "./RootStore";
export default class FileOperationsStore extends BaseStore<FileOperation> {
actions = ["list", "info"];
constructor(rootStore: RootStore) {
super(rootStore, FileOperation);
}
@computed
get exports(): FileOperation[] {
return Array.from(this.data.values()).reduce(
(acc, fileOp) => (fileOp.type === "export" ? [...acc, fileOp] : acc),
[]
);
}
@computed
get orderedDataExports(): FileOperation[] {
return orderBy(this.exports, "createdAt", "desc");
}
}

4
app/stores/RootStore.js
View File

@ -6,6 +6,7 @@ import CollectionsStore from "./CollectionsStore";
import DocumentPresenceStore from "./DocumentPresenceStore";
import DocumentsStore from "./DocumentsStore";
import EventsStore from "./EventsStore";
import FileOperationsStore from "./FileOperationsStore";
import GroupMembershipsStore from "./GroupMembershipsStore";
import GroupsStore from "./GroupsStore";
import IntegrationsStore from "./IntegrationsStore";
@ -39,6 +40,7 @@ export default class RootStore {
users: UsersStore;
views: ViewsStore;
toasts: ToastsStore;
fileOperations: FileOperationsStore;
constructor() {
// PoliciesStore must be initialized before AuthStore
@ -60,6 +62,7 @@ export default class RootStore {
this.ui = new UiStore();
this.users = new UsersStore(this);
this.views = new ViewsStore(this);
this.fileOperations = new FileOperationsStore(this);
this.toasts = new ToastsStore();
}
@ -79,6 +82,7 @@ export default class RootStore {
this.policies.clear();
this.revisions.clear();
this.shares.clear();
this.fileOperations.clear();
// this.ui omitted to keep ui settings between sessions
this.users.clear();
this.views.clear();

4
server/api/attachments.js
View File

@ -11,7 +11,7 @@ import {
getSignature,
publicS3Endpoint,
makeCredential,
getSignedImageUrl,
getSignedUrl,
} from "../utils/s3";
const { authorize } = policy;
@ -146,7 +146,7 @@ router.post("attachments.redirect", auth(), async (ctx) => {
authorize(user, "read", document);
}
const accessUrl = await getSignedImageUrl(attachment.key);
const accessUrl = await getSignedUrl(attachment.key);
ctx.redirect(accessUrl);
} else {
ctx.redirect(attachment.url);

86
server/api/collections.js
View File

@ -1,5 +1,4 @@
// @flow
import fs from "fs";
import fractionalIndex from "fractional-index";
import Router from "koa-router";
import { ValidationError } from "../errors";
@ -14,6 +13,7 @@ import {
User,
Group,
Attachment,
FileOperation,
} from "../models";
import policy from "../policies";
import {
@ -23,12 +23,13 @@ import {
presentMembership,
presentGroup,
presentCollectionGroupMembership,
presentFileOperation,
} from "../presenters";
import { Op, sequelize } from "../sequelize";
import collectionIndexing from "../utils/collectionIndexing";
import removeIndexCollision from "../utils/removeIndexCollision";
import { archiveCollection, archiveCollections } from "../utils/zip";
import { getAWSKeyForFileOp } from "../utils/s3";
import pagination from "./middlewares/pagination";
const { authorize } = policy;
@ -454,59 +455,70 @@ router.post("collections.export", auth(), async (ctx) => {
ctx.assertUuid(id, "id is required");
const user = ctx.state.user;
const team = await Team.findByPk(user.teamId);
authorize(user, "export", team);
const collection = await Collection.scope({
method: ["withMembership", user.id],
}).findByPk(id);
authorize(user, "export", collection);
const filePath = await archiveCollection(collection);
ctx.assertPresent(collection, "Collection should be present");
authorize(user, "read", collection);
await Event.create({
name: "collections.export",
collectionId: collection.id,
teamId: user.teamId,
actorId: user.id,
data: { title: collection.title },
ip: ctx.request.ip,
const key = getAWSKeyForFileOp(team.id, collection.name);
let exportData;
exportData = await FileOperation.create({
type: "export",
state: "creating",
key,
url: null,
size: 0,
collectionId: id,
userId: user.id,
teamId: team.id,
});
ctx.attachment(`${collection.name}.zip`);
ctx.set("Content-Type", "application/force-download");
ctx.body = fs.createReadStream(filePath);
exportCollections(user.teamId, user.id, user.email, exportData.id, id);
exportData.user = user;
exportData.collection = collection;
ctx.body = {
success: true,
data: { fileOperation: presentFileOperation(exportData) },
};
});
router.post("collections.export_all", auth(), async (ctx) => {
const { download = false } = ctx.body;
const user = ctx.state.user;
const team = await Team.findByPk(user.teamId);
authorize(user, "export", team);
await Event.create({
name: "collections.export",
teamId: user.teamId,
actorId: user.id,
ip: ctx.request.ip,
const key = getAWSKeyForFileOp(team.id, team.name);
let exportData;
exportData = await FileOperation.create({
type: "export",
state: "creating",
key,
url: null,
size: 0,
collectionId: null,
userId: user.id,
teamId: team.id,
});
if (download) {
const collections = await Collection.findAll({
where: { teamId: team.id },
order: [["name", "ASC"]],
});
const filePath = await archiveCollections(collections);
// async operation to upload zip archive to cloud and email user with link
exportCollections(user.teamId, user.id, user.email, exportData.id);
ctx.attachment(`${team.name}.zip`);
ctx.set("Content-Type", "application/force-download");
ctx.body = fs.createReadStream(filePath);
} else {
// async operation to create zip archive and email user
exportCollections(user.teamId, user.email);
exportData.user = user;
exportData.collection = null;
ctx.body = {
success: true,
};
}
ctx.body = {
success: true,
data: { fileOperation: presentFileOperation(exportData) },
};
});
router.post("collections.update", auth(), async (ctx) => {

60
server/api/collections.test.js
View File

@ -264,53 +264,53 @@ describe("#collections.move", () => {
describe("#collections.export", () => {
it("should not allow export of private collection not a member", async () => {
const { user } = await seed();
const { admin } = await seed();
const collection = await buildCollection({
permission: null,
teamId: user.teamId,
teamId: admin.teamId,
});
const res = await server.post("/api/collections.export", {
body: { token: user.getJwtToken(), id: collection.id },
body: { token: admin.getJwtToken(), id: collection.id },
});
expect(res.status).toEqual(403);
});
it("should allow export of private collection when the actor is a member", async () => {
const { user, collection } = await seed();
const { admin, collection } = await seed();
collection.permission = null;
await collection.save();
await CollectionUser.create({
createdById: user.id,
createdById: admin.id,
collectionId: collection.id,
userId: user.id,
userId: admin.id,
permission: "read_write",
});
const res = await server.post("/api/collections.export", {
body: { token: user.getJwtToken(), id: collection.id },
body: { token: admin.getJwtToken(), id: collection.id },
});
expect(res.status).toEqual(200);
});
it("should allow export of private collection when the actor is a group member", async () => {
const user = await buildUser();
const admin = await buildAdmin();
const collection = await buildCollection({
permission: null,
teamId: user.teamId,
teamId: admin.teamId,
});
const group = await buildGroup({ teamId: user.teamId });
await group.addUser(user, { through: { createdById: user.id } });
const group = await buildGroup({ teamId: admin.teamId });
await group.addUser(admin, { through: { createdById: admin.id } });
await collection.addGroup(group, {
through: { permission: "read_write", createdById: user.id },
through: { permission: "read_write", createdById: admin.id },
});
const res = await server.post("/api/collections.export", {
body: { token: user.getJwtToken(), id: collection.id },
body: { token: admin.getJwtToken(), id: collection.id },
});
expect(res.status).toEqual(200);
@ -324,13 +324,29 @@ describe("#collections.export", () => {
expect(body).toMatchSnapshot();
});
it("should return success", async () => {
it("should return unauthorized if user is not admin", async () => {
const { user, collection } = await seed();
const res = await server.post("/api/collections.export", {
body: { token: user.getJwtToken(), id: collection.id },
});
expect(res.status).toEqual(200);
expect(res.status).toEqual(403);
});
it("should return file operation associated with export", async () => {
const admin = await buildAdmin();
const collection = await buildCollection({
teamId: admin.teamId,
});
const res = await server.post("/api/collections.export", {
body: { token: admin.getJwtToken(), id: collection.id },
});
const body = await res.json();
expect(res.status).toBe(200);
expect(body.data.fileOperation.id).toBeTruthy();
expect(body.data.fileOperation.state).toBe("creating");
});
});
@ -359,18 +375,6 @@ describe("#collections.export_all", () => {
expect(res.status).toEqual(200);
});
it("should allow downloading directly", async () => {
const { admin } = await seed();
const res = await server.post("/api/collections.export_all", {
body: { token: admin.getJwtToken(), download: true },
});
expect(res.status).toEqual(200);
expect(res.headers.get("content-type")).toEqual(
"application/force-download"
);
});
});
describe("#collections.add_user", () => {
@ -1026,7 +1030,6 @@ describe("#collections.create", () => {
expect(body.data.sort.direction).toBe("asc");
expect(body.policies.length).toBe(1);
expect(body.policies[0].abilities.read).toBeTruthy();
expect(body.policies[0].abilities.export).toBeTruthy();
});
it("should error when index is invalid", async () => {
@ -1060,7 +1063,6 @@ describe("#collections.create", () => {
expect(body.data.permission).toEqual(null);
expect(body.policies.length).toBe(1);
expect(body.policies[0].abilities.read).toBeTruthy();
expect(body.policies[0].abilities.export).toBeTruthy();
});
it("if index collision, should updated index of other collection", async () => {

99
server/api/fileOperations.js Normal file
View File

@ -0,0 +1,99 @@
// @flow
import Router from "koa-router";
import { NotFoundError, ValidationError } from "../errors";
import auth from "../middlewares/authentication";
import { FileOperation, Team } from "../models";
import policy from "../policies";
import { presentFileOperation } from "../presenters";
import { getSignedUrl } from "../utils/s3";
import pagination from "./middlewares/pagination";
const { authorize } = policy;
const router = new Router();
router.post("fileOperations.info", auth(), async (ctx) => {
const { id } = ctx.body;
ctx.assertUuid(id, "id is required");
const user = ctx.state.user;
const team = await Team.findByPk(user.teamId);
const fileOperation = await FileOperation.findByPk(id);
authorize(user, fileOperation.type, team);
if (!fileOperation) {
throw new NotFoundError();
}
ctx.body = {
data: presentFileOperation(fileOperation),
};
});
router.post("fileOperations.list", auth(), pagination(), async (ctx) => {
let { sort = "createdAt", direction, type } = ctx.body;
ctx.assertPresent(type, "type is required");
ctx.assertIn(
type,
["import", "export"],
"type must be one of 'import' or 'export'"
);
if (direction !== "ASC") direction = "DESC";
const user = ctx.state.user;
const where = {
teamId: user.teamId,
type,
};
const team = await Team.findByPk(user.teamId);
authorize(user, type, team);
const [exports, total] = await Promise.all([
await FileOperation.findAll({
where,
order: [[sort, direction]],
offset: ctx.state.pagination.offset,
limit: ctx.state.pagination.limit,
}),
await FileOperation.count({
where,
}),
]);
ctx.body = {
pagination: {
...ctx.state.pagination,
total,
},
data: exports.map(presentFileOperation),
};
});
router.post("fileOperations.redirect", auth(), async (ctx) => {
const { id } = ctx.body;
ctx.assertUuid(id, "id is required");
const user = ctx.state.user;
const team = await Team.findByPk(user.teamId);
const fileOp = await FileOperation.unscoped().findByPk(id);
if (!fileOp) {
throw new NotFoundError();
}
authorize(user, fileOp.type, team);
if (fileOp.state !== "complete") {
throw new ValidationError("file operation is not complete yet");
}
const accessUrl = await getSignedUrl(fileOp.key);
ctx.redirect(accessUrl);
});
export default router;

283
server/api/fileOperations.test.js Normal file
View File

@ -0,0 +1,283 @@
/* eslint-disable flowtype/require-valid-file-annotation */
import TestServer from "fetch-test-server";
import { Collection, User } from "../models";
import webService from "../services/web";
import {
buildAdmin,
buildCollection,
buildFileOperation,
buildTeam,
buildUser,
} from "../test/factories";
import { flushdb } from "../test/support";
const app = webService();
const server = new TestServer(app.callback());
beforeEach(() => flushdb());
afterAll(() => server.close());
describe("#fileOperations.info", () => {
it("should return fileOperation", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.info", {
body: {
id: exportData.id,
token: admin.getJwtToken(),
type: "export",
},
});
const body = await res.json();
expect(res.status).toEqual(200);
expect(body.data.id).toBe(exportData.id);
expect(body.data.state).toBe(exportData.state);
});
it("should require user to be an admin", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const user = await buildUser({ teamId: team.id });
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.info", {
body: {
id: exportData.id,
token: user.getJwtToken(),
type: "export",
},
});
expect(res.status).toEqual(403);
});
});
describe("#fileOperations.list", () => {
it("should return fileOperations list", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin.getJwtToken(),
type: "export",
},
});
const body = await res.json();
const data = body.data[0];
expect(res.status).toEqual(200);
expect(body.data.length).toBe(1);
expect(data.id).toBe(exportData.id);
expect(data.key).toBe(undefined);
expect(data.state).toBe(exportData.state);
});
it("should return exports with collection data", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const collection = await buildCollection({
userId: admin.id,
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
collectionId: collection.id,
});
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin.getJwtToken(),
type: "export",
},
});
const body = await res.json();
const data = body.data[0];
expect(res.status).toEqual(200);
expect(body.data.length).toBe(1);
expect(data.id).toBe(exportData.id);
expect(data.key).toBe(undefined);
expect(data.state).toBe(exportData.state);
expect(data.collection.id).toBe(collection.id);
});
it("should return exports with collection data even if collection is deleted", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const collection = await buildCollection({
userId: admin.id,
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
collectionId: collection.id,
});
await collection.destroy();
const isCollectionPresent = await Collection.findByPk(collection.id);
expect(isCollectionPresent).toBe(null);
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin.getJwtToken(),
type: "export",
},
});
const body = await res.json();
const data = body.data[0];
expect(res.status).toEqual(200);
expect(body.data.length).toBe(1);
expect(data.id).toBe(exportData.id);
expect(data.key).toBe(undefined);
expect(data.state).toBe(exportData.state);
expect(data.collection.id).toBe(collection.id);
});
it("should return exports with user data even if user is deleted", async () => {
const team = await buildTeam();
const admin2 = await buildAdmin({ teamId: team.id });
const admin = await buildAdmin({ teamId: team.id });
const collection = await buildCollection({
userId: admin.id,
teamId: team.id,
});
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
collectionId: collection.id,
});
await admin.destroy();
const isAdminPresent = await User.findByPk(admin.id);
expect(isAdminPresent).toBe(null);
const res = await server.post("/api/fileOperations.list", {
body: {
token: admin2.getJwtToken(),
type: "export",
},
});
const body = await res.json();
const data = body.data[0];
expect(res.status).toEqual(200);
expect(body.data.length).toBe(1);
expect(data.id).toBe(exportData.id);
expect(data.key).toBe(undefined);
expect(data.state).toBe(exportData.state);
expect(data.user.id).toBe(admin.id);
});
it("should require authorization", async () => {
const user = await buildUser();
const res = await server.post("/api/fileOperations.list", {
body: { token: user.getJwtToken(), type: "export" },
});
expect(res.status).toEqual(403);
});
});
describe("#fileOperations.redirect", () => {
it("should not redirect when file operation is not complete", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.redirect", {
body: {
token: admin.getJwtToken(),
id: exportData.id,
},
});
const body = await res.json();
expect(res.status).toEqual(400);
expect(body.message).toEqual("file operation is not complete yet");
});
});
describe("#fileOperations.info", () => {
it("should return file operation", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.info", {
body: {
token: admin.getJwtToken(),
id: exportData.id,
},
});
const body = await res.json();
expect(res.status).toBe(200);
expect(body.data.id).toBe(exportData.id);
expect(body.data.user.id).toBe(admin.id);
});
it("should require authorization", async () => {
const team = await buildTeam();
const admin = await buildAdmin({ teamId: team.id });
const user = await buildUser({ teamId: team.id });
const exportData = await buildFileOperation({
type: "export",
teamId: team.id,
userId: admin.id,
});
const res = await server.post("/api/fileOperations.info", {
body: {
token: user.getJwtToken(),
id: exportData.id,
},
});
expect(res.status).toBe(403);
});
});

2
server/api/index.js
View File

@ -14,6 +14,7 @@ import authenticationProviders from "./authenticationProviders";
import collections from "./collections";
import documents from "./documents";
import events from "./events";
import fileOperationsRoute from "./fileOperations";
import groups from "./groups";
import hooks from "./hooks";
import integrations from "./integrations";
@ -62,6 +63,7 @@ router.use("/", notificationSettings.routes());
router.use("/", attachments.routes());
router.use("/", utils.routes());
router.use("/", groups.routes());
router.use("/", fileOperationsRoute.routes());
router.post("*", (ctx) => {
ctx.throw(new NotFoundError("Endpoint not found"));

22
server/api/utils.js
View File

@ -4,7 +4,7 @@ import debug from "debug";
import Router from "koa-router";
import { documentPermanentDeleter } from "../commands/documentPermanentDeleter";
import { AuthenticationError } from "../errors";
import { Document } from "../models";
import { Document, FileOperation } from "../models";
import { Op } from "../sequelize";
const router = new Router();
@ -34,6 +34,26 @@ router.post("utils.gc", async (ctx) => {
log(`Destroyed ${countDeletedDocument} documents`);
log(`Expiring all the collection export older than 30 days…`);
const exports = await FileOperation.unscoped().findAll({
where: {
type: "export",
createdAt: {
[Op.lt]: subDays(new Date(), 30),
},
state: {
[Op.ne]: "expired",
},
},
});
await Promise.all(
exports.map(async (e) => {
await e.expire();
})
);
ctx.body = {
success: true,
};

67
server/api/utils.test.js
View File

@ -1,9 +1,10 @@
/* eslint-disable flowtype/require-valid-file-annotation */
import { subDays } from "date-fns";
import TestServer from "fetch-test-server";
import { Document } from "../models";
import { Document, FileOperation } from "../models";
import { Op } from "../sequelize";
import webService from "../services/web";
import { buildDocument } from "../test/factories";
import { buildDocument, buildFileOperation } from "../test/factories";
import { flushdb } from "../test/support";
const app = webService();
@ -83,6 +84,68 @@ describe("#utils.gc", () => {
expect(await Document.unscoped().count({ paranoid: false })).toEqual(0);
});
it("should expire exports older than 30 days ago", async () => {
await buildFileOperation({
type: "export",
state: "complete",
createdAt: subDays(new Date(), 30),
});
await buildFileOperation({
type: "export",
state: "complete",
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
const data = await FileOperation.count({
where: {
type: "export",
state: {
[Op.eq]: "expired",
},
},
});
expect(res.status).toEqual(200);
expect(data).toEqual(1);
});
it("should not expire exports made less than 30 days ago", async () => {
await buildFileOperation({
type: "export",
state: "complete",
createdAt: subDays(new Date(), 29),
});
await buildFileOperation({
type: "export",
state: "complete",
});
const res = await server.post("/api/utils.gc", {
body: {
token: process.env.UTILS_SECRET,
},
});
const data = await FileOperation.count({
where: {
type: "export",
state: {
[Op.eq]: "expired",
},
},
});
expect(res.status).toEqual(200);
expect(data).toEqual(0);
});
it("should require authentication", async () => {
const res = await server.post("/api/utils.gc");
expect(res.status).toEqual(401);

36
server/emails/ExportEmail.js
View File

@ -1,36 +0,0 @@
// @flow
import * as React from "react";
import Body from "./components/Body";
import Button from "./components/Button";
import EmailTemplate from "./components/EmailLayout";
import EmptySpace from "./components/EmptySpace";
import Footer from "./components/Footer";
import Header from "./components/Header";
import Heading from "./components/Heading";
export const exportEmailText = `
Your Data Export
Your requested data export is attached as a zip file to this email.
`;
export const ExportEmail = () => {
return (
<EmailTemplate>
<Header />
<Body>
<Heading>Your Data Export</Heading>
<p>
Your requested data export is attached as a zip file to this email.
</p>
<EmptySpace height={10} />
<p>
<Button href={`${process.env.URL}/home`}>Go to dashboard</Button>
</p>
</Body>
<Footer />
</EmailTemplate>
);
};

47
server/emails/ExportFailureEmail.js Normal file
View File

@ -0,0 +1,47 @@
// @flow
import * as React from "react";
import Body from "./components/Body";
import Button from "./components/Button";
import EmailTemplate from "./components/EmailLayout";
import EmptySpace from "./components/EmptySpace";
import Footer from "./components/Footer";
import Header from "./components/Header";
import Heading from "./components/Heading";
export const exportEmailFailureText = `
Your Data Export
Sorry, your requested data export has failed, please visit the admin
section to try again – if the problem persists please contact support.
`;
export const ExportFailureEmail = ({ teamUrl }: { teamUrl: string }) => {
return (
<EmailTemplate>
<Header />
<Body>
<Heading>Your Data Export</Heading>
<p>
Sorry, your requested data export has failed, please visit the{" "}
<a
href={`${teamUrl}/settings/import-export`}
rel="noreferrer"
target="_blank"
>
admin section
</a>
. to try again – if the problem persists please contact support.
</p>
<EmptySpace height={10} />
<p>
<Button href={`${teamUrl}/settings/import-export`}>
Go to export
</Button>
</p>
</Body>
<Footer />
</EmailTemplate>
);
};

53
server/emails/ExportSuccessEmail.js Normal file
View File

@ -0,0 +1,53 @@
// @flow
import * as React from "react";
import Body from "./components/Body";
import Button from "./components/Button";
import EmailTemplate from "./components/EmailLayout";
import EmptySpace from "./components/EmptySpace";
import Footer from "./components/Footer";
import Header from "./components/Header";
import Heading from "./components/Heading";
export const exportEmailSuccessText = `
Your Data Export
Your requested data export is complete, the exported files are also available in the admin section.
`;
export const ExportSuccessEmail = ({
id,
teamUrl,
}: {
id: string,
teamUrl: string,
}) => {
return (
<EmailTemplate>
<Header />
<Body>
<Heading>Your Data Export</Heading>
<p>
Your requested data export is complete, the exported files are also
available in the{" "}
<a
href={`${teamUrl}/settings/import-export`}
rel="noreferrer"
target="_blank"
>
admin section
</a>
.
</p>
<EmptySpace height={10} />
<p>
<Button href={`${teamUrl}/api/fileOperations.redirect?id=${id}`}>
Download
</Button>
</p>
</Body>
<Footer />
</EmailTemplate>
);
};

162
server/exporter.js
View File

@ -1,8 +1,10 @@
// @flow
import fs from "fs";
import debug from "debug";
import mailer from "./mailer";
import { Collection, Team } from "./models";
import { FileOperation, Collection, Team, Event, User } from "./models";
import { createQueue } from "./utils/queue";
import { uploadToS3FromBuffer } from "./utils/s3";
const log = debug("exporter");
const exporterQueue = createQueue("exporter");
@ -15,27 +17,137 @@ const queueOptions = {
},
};
async function exportAndEmailCollections(teamId: string, email: string) {
async function fileOperationsUpdate(teamId, userId, exportData) {
await Event.add({
name: "fileOperations.update",
teamId: teamId,
actorId: userId,
data: {
type: exportData.type,
id: exportData.id,
state: exportData.state,
size: exportData.size,
collectionId: exportData.collectionId,
createdAt: exportData.createdAt,
},
});
}
type exportAndEmailCollectionsType = {|
teamId: string,
userId: string,
email: string,
fileOperationId: string,
collectionId?: string,
|};
// TODO: Refactor to use command pattern
async function exportAndEmailCollections({
teamId,
userId,
email,
collectionId,
fileOperationId,
}: exportAndEmailCollectionsType) {
log("Archiving team", teamId);
const { archiveCollections } = require("./utils/zip");
const team = await Team.findByPk(teamId);
const collections = await Collection.findAll({
where: { teamId },
order: [["name", "ASC"]],
});
const user = await User.findByPk(userId);
let collections;
if (!collectionId) {
const collectionIds = await user.collectionIds();
collections = await Promise.all(
collectionIds.map(
async (collectionId) => await Collection.findByPk(collectionId)
)
);
} else {
collections = [await Collection.findByPk(collectionId)];
}
let exportData;
let state;
let key;
exportData = await FileOperation.findByPk(fileOperationId);
state = exportData.state;
key = exportData.key;
await fileOperationsUpdate(teamId, userId, exportData);
const filePath = await archiveCollections(collections);
log("Archive path", filePath);
mailer.export({
to: email,
attachments: [
{
filename: `${team.name} Export.zip`,
path: filePath,
},
],
});
let url;
try {
const readBuffer = await fs.promises.readFile(filePath);
state = "uploading";
exportData.state = state;
const stat = await fs.promises.stat(filePath);
exportData.size = stat.size;
await exportData.save();
await fileOperationsUpdate(teamId, userId, exportData);
url = await uploadToS3FromBuffer(
readBuffer,
"application/zip",
key,
"private"
);
state = "complete";
} catch (e) {
log("Failed to export data", e);
state = "error";
url = null;
} finally {
exportData.state = state;
exportData.url = url;
await exportData.save();
await fileOperationsUpdate(teamId, userId, exportData);
if (collectionId) {
await Event.create({
name: "collections.export",
collectionId,
teamId: teamId,
actorId: userId,
data: { name: collections[0].name, exportId: exportData.id },
});
} else {
const collectionsExported = collections.map((c) => ({
name: c.name,
id: c.id,
}));
await Event.create({
name: "collections.export_all",
teamId: teamId,
actorId: userId,
data: {
exportId: exportData.id,
collections: collectionsExported,
},
});
}
if (state === "error") {
mailer.exportFailure({
to: email,
teamUrl: team.url,
});
} else {
mailer.exportSuccess({
to: email,
id: exportData.id,
teamUrl: team.url,
});
}
}
}
exporterQueue.process(async function exportProcessor(job) {
@ -43,17 +155,33 @@ exporterQueue.process(async function exportProcessor(job) {
switch (job.data.type) {
case "export-collections":
return await exportAndEmailCollections(job.data.teamId, job.data.email);
const { teamId, userId, email, collectionId, fileOperationId } = job.data;
return await exportAndEmailCollections({
teamId,
userId,
email,
fileOperationId,
collectionId,
});
default:
}
});
export const exportCollections = (teamId: string, email: string) => {
export const exportCollections = (
teamId: string,
userId: string,
email: string,
fileOperationId: string,
collectionId?: string
) => {
exporterQueue.add(
{
type: "export-collections",
teamId,
userId,
email,
fileOperationId,
collectionId,
},
queueOptions
);

36
server/mailer.js
View File

@ -14,7 +14,15 @@ import {
DocumentNotificationEmail,
documentNotificationEmailText,
} from "./emails/DocumentNotificationEmail";
import { ExportEmail, exportEmailText } from "./emails/ExportEmail";
import {
ExportFailureEmail,
exportEmailFailureText,
} from "./emails/ExportFailureEmail";
import {
ExportSuccessEmail,
exportEmailSuccessText,
} from "./emails/ExportSuccessEmail";
import {
type Props as InviteEmailT,
InviteEmail,
@ -155,14 +163,34 @@ export class Mailer {
});
};
export = async (opts: { to: string, attachments: Object[] }) => {
exportSuccess = async (opts: {
to: string,
attachments?: Object[],
id: string,
teamUrl: string,
}) => {
this.sendMail({
to: opts.to,
attachments: opts.attachments,
title: "Your requested export",
previewText: "Here's your request data export from Outline",
html: <ExportEmail />,
text: exportEmailText,
html: <ExportSuccessEmail id={opts.id} teamUrl={opts.teamUrl} />,
text: exportEmailSuccessText,
});
};
exportFailure = async (opts: {
to: string,
attachments?: Object[],
teamUrl: string,
}) => {
this.sendMail({
to: opts.to,
attachments: opts.attachments,
title: "Your requested export",
previewText: "Sorry, your requested data export has failed",
html: <ExportFailureEmail teamUrl={opts.teamUrl} />,
text: exportEmailFailureText,
});
};

65
server/migrations/20210730210120-add-fileOperations.js Normal file
View File

@ -0,0 +1,65 @@
'use strict';
module.exports = {
up: async (queryInterface, Sequelize) => {
await queryInterface.createTable("file_operations",{
id: {
type: Sequelize.UUID,
allowNull: false,
primaryKey: true,
},
state: {
type: Sequelize.ENUM("creating", "uploading", "complete", "error","expired"),
allowNull: false,
},
type: {
type: Sequelize.ENUM("import", "export"),
allowNull: false,
},
key: {
type: Sequelize.STRING,
},
url: {
type: Sequelize.STRING,
},
size: {
type: Sequelize.BIGINT,
allowNull: false,
},
userId: {
type: Sequelize.UUID,
allowNull: false,
references: {
model: "users"
}
},
collectionId: {
type: Sequelize.UUID,
references: {
model: "collections"
}
},
teamId: {
type: Sequelize.UUID,
allowNull: false,
references: {
model: "teams"
}
},
createdAt: {
type: Sequelize.DATE,
allowNull: false,
},
updatedAt: {
type: Sequelize.DATE,
allowNull: false,
},
})
await queryInterface.addIndex('file_operations', ["type", "state"])
},
down: async (queryInterface, Sequelize) => {
await queryInterface.removeIndex('file_operations', ["type", "state"]);
await queryInterface.dropTable('file_operations');
}
};

1
server/models/Event.js
View File

@ -83,6 +83,7 @@ Event.AUDIT_EVENTS = [
"collections.add_group",
"collections.remove_group",
"collections.delete",
"collections.export_all",
"documents.create",
"documents.publish",
"documents.update",

72
server/models/FileOperation.js Normal file
View File

@ -0,0 +1,72 @@
// @flow
import { DataTypes, sequelize } from "../sequelize";
import { deleteFromS3 } from "../utils/s3";
const FileOperation = sequelize.define("file_operations", {
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
primaryKey: true,
},
type: {
type: DataTypes.ENUM("import", "export"),
allowNull: false,
},
state: {
type: DataTypes.ENUM(
"creating",
"uploading",
"complete",
"error",
"expired"
),
allowNull: false,
},
key: {
type: DataTypes.STRING,
},
url: {
type: DataTypes.STRING,
},
size: {
type: DataTypes.BIGINT,
allowNull: false,
},
});
FileOperation.prototype.expire = async function () {
this.state = "expired";
await deleteFromS3(this.key);
this.save();
};
FileOperation.associate = (models) => {
FileOperation.belongsTo(models.User, {
as: "user",
foreignKey: "userId",
});
FileOperation.belongsTo(models.Collection, {
as: "collection",
foreignKey: "collectionId",
});
FileOperation.belongsTo(models.Team, {
as: "team",
foreignKey: "teamId",
});
FileOperation.addScope("defaultScope", {
include: [
{
model: models.User,
as: "user",
paranoid: false,
},
{
model: models.Collection,
as: "collection",
paranoid: false,
},
],
});
};
export default FileOperation;

3
server/models/index.js
View File

@ -8,6 +8,7 @@ import CollectionGroup from "./CollectionGroup";
import CollectionUser from "./CollectionUser";
import Document from "./Document";
import Event from "./Event";
import FileOperation from "./FileOperation";
import Group from "./Group";
import GroupUser from "./GroupUser";
import Integration from "./Integration";
@ -47,6 +48,7 @@ const models = {
User,
UserAuthentication,
View,
FileOperation,
};
// based on https://github.com/sequelize/express-example/blob/master/models/index.js
@ -80,4 +82,5 @@ export {
User,
UserAuthentication,
View,
FileOperation,
};

2
server/policies/collection.js
View File

@ -47,7 +47,7 @@ allow(User, "read", Collection, (user, collection) => {
return true;
});
allow(User, ["share", "export"], Collection, (user, collection) => {
allow(User, "share", Collection, (user, collection) => {
if (user.isViewer) return false;
if (!collection || user.teamId !== collection.teamId) return false;
if (!collection.sharing) return false;

6
server/policies/collection.test.js
View File

@ -16,7 +16,6 @@ describe("read_write permission", () => {
});
const abilities = serialize(user, collection);
expect(abilities.read).toEqual(true);
expect(abilities.export).toEqual(true);
expect(abilities.update).toEqual(true);
expect(abilities.share).toEqual(true);
});
@ -43,7 +42,6 @@ describe("read_write permission", () => {
const abilities = serialize(user, collection);
expect(abilities.read).toEqual(true);
expect(abilities.export).toEqual(true);
expect(abilities.update).toEqual(true);
expect(abilities.share).toEqual(true);
});
@ -59,7 +57,6 @@ describe("read permission", () => {
});
const abilities = serialize(user, collection);
expect(abilities.read).toEqual(true);
expect(abilities.export).toEqual(false);
expect(abilities.update).toEqual(false);
expect(abilities.share).toEqual(false);
});
@ -86,7 +83,6 @@ describe("read permission", () => {
const abilities = serialize(user, collection);
expect(abilities.read).toEqual(true);
expect(abilities.export).toEqual(true);
expect(abilities.update).toEqual(true);
expect(abilities.share).toEqual(true);
});
@ -102,7 +98,6 @@ describe("no permission", () => {
});
const abilities = serialize(user, collection);
expect(abilities.read).toEqual(false);
expect(abilities.export).toEqual(false);
expect(abilities.update).toEqual(false);
expect(abilities.share).toEqual(false);
});
@ -129,7 +124,6 @@ describe("no permission", () => {
const abilities = serialize(user, collection);
expect(abilities.read).toEqual(true);
expect(abilities.export).toEqual(true);
expect(abilities.update).toEqual(true);
expect(abilities.share).toEqual(true);
});

4
server/presenters/document.js
View File

@ -1,7 +1,7 @@
// @flow
import { Attachment, Document } from "../models";
import parseAttachmentIds from "../utils/parseAttachmentIds";
import { getSignedImageUrl } from "../utils/s3";
import { getSignedUrl } from "../utils/s3";
import presentUser from "./user";
type Options = {
@ -16,7 +16,7 @@ async function replaceImageAttachments(text: string) {
attachmentIds.map(async (id) => {
const attachment = await Attachment.findByPk(id);
if (attachment) {
const accessUrl = await getSignedImageUrl(attachment.key);
const accessUrl = await getSignedUrl(attachment.key);
text = text.replace(attachment.redirectUrl, accessUrl);
}
})

15
server/presenters/fileOperation.js Normal file
View File

@ -0,0 +1,15 @@
// @flow
import { FileOperation } from "../models";
import { presentCollection, presentUser } from ".";
export default function present(data: FileOperation) {
return {
id: data.id,
type: data.type,
state: data.state,
collection: data.collection ? presentCollection(data.collection) : null,
size: data.size,
user: presentUser(data.user),
createdAt: data.createdAt,
};
}

2
server/presenters/index.js
View File

@ -5,6 +5,7 @@ import presentCollection from "./collection";
import presentCollectionGroupMembership from "./collectionGroupMembership";
import presentDocument from "./document";
import presentEvent from "./event";
import presentFileOperation from "./fileOperation";
import presentGroup from "./group";
import presentGroupMembership from "./groupMembership";
import presentIntegration from "./integration";
@ -20,6 +21,7 @@ import presentView from "./view";
export {
presentApiKey,
presentFileOperation,
presentAuthenticationProvider,
presentUser,
presentView,

7
server/queues/processors/websockets.js
View File

@ -322,6 +322,13 @@ export default class WebsocketsProcessor {
}
return;
}
case "fileOperations.update": {
return socketio
.to(`user-${event.actorId}`)
.emit("fileOperations.update", event.data);
}
case "groups.create":
case "groups.update": {
const group = await Group.findByPk(event.modelId, {

26
server/test/factories.js
View File

@ -13,6 +13,7 @@ import {
IntegrationAuthentication,
Integration,
AuthenticationProvider,
FileOperation,
} from "../models";
let count = 1;
@ -255,6 +256,31 @@ export async function buildDocument(overrides: Object = {}) {
});
}
export async function buildFileOperation(overrides: Object = {}) {
if (!overrides.teamId) {
const team = await buildTeam();
overrides.teamId = team.id;
}
if (!overrides.userId) {
const user = await buildAdmin({ teamId: overrides.teamId });
overrides.userId = user.id;
}
if (!overrides.collectionId) {
const collection = await buildCollection(overrides);
overrides.collectionId = collection.id;
}
return FileOperation.create({
state: "creating",
size: 0,
key: "key/to/aws/file.zip",
url: "https://www.urltos3file.com/file.zip",
...overrides,
});
}
export async function buildAttachment(overrides: Object = {}) {
if (!overrides.teamId) {
const team = await buildTeam();

26
server/types.js
View File

@ -116,6 +116,30 @@ export type CollectionImportEvent = {
ip: string,
};
export type CollectionExportAll = {
name: "collections.export_all",
teamId: string,
actorId: string,
data: {
exportId: string,
collections: [{ name: string, id: string }],
},
};
export type FileOperationEvent = {
name: "fileOperations.update",
teamId: string,
actorId: string,
data: {
type: string,
state: string,
id: string,
size: number,
createdAt: string,
collectionId: string,
},
};
export type CollectionEvent =
| {
name: | "collections.create" // eslint-disable-line
@ -192,6 +216,8 @@ export type Event =
| DocumentEvent
| CollectionEvent
| CollectionImportEvent
| CollectionExportAll
| FileOperationEvent
| IntegrationEvent
| GroupEvent
| RevisionEvent

9
server/utils/s3.js
View File

@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";
import AWS from "aws-sdk";
import { addHours, format } from "date-fns";
import fetch from "fetch-with-proxy";
import { v4 as uuidv4 } from "uuid";
const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
@ -167,7 +168,7 @@ export const deleteFromS3 = (key: string) => {
.promise();
};
export const getSignedImageUrl = async (key: string) => {
export const getSignedUrl = async (key: string) => {
const isDocker = process.env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
const params = {
@ -181,6 +182,12 @@ export const getSignedImageUrl = async (key: string) => {
: s3.getSignedUrl("getObject", params);
};
// function assumes that acl is private
export const getAWSKeyForFileOp = (teamId: string, name: string) => {
const bucket = "uploads";
return `${bucket}/${teamId}/${uuidv4()}/${name}-export.zip`;
};
export const getFileByKey = async (key: string) => {
const params = {
Bucket: AWS_S3_UPLOAD_BUCKET_NAME,

11
server/utils/zip.js
View File

@ -69,17 +69,6 @@ async function archiveToPath(zip) {
});
}
export async function archiveCollection(collection: Collection) {
const zip = new JSZip();
if (collection.documentStructure) {
const folder = zip.folder(collection.name);
await addToArchive(folder, collection.documentStructure);
}
return archiveToPath(zip);
}
export async function archiveCollections(collections: Collection[]) {
const zip = new JSZip();

11
shared/i18n/locales/en_US/translation.json
View File

@ -279,7 +279,8 @@
"Public sharing is currently disabled in the team security settings.": "Public sharing is currently disabled in the team security settings.",
"Saving": "Saving",
"Save": "Save",
"Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be downloaded as a zip of folders with files in Markdown format.": "Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be downloaded as a zip of folders with files in Markdown format.",
"Export started, you will receive an email when it’s complete.": "Export started, you will receive an email when it’s complete.",
"Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be a zip of folders with files in Markdown format. Please visit the Export section on settings to get the zip.": "Exporting the collection <em>{{collectionName}}</em> may take a few seconds. Your documents will be a zip of folders with files in Markdown format. Please visit the Export section on settings to get the zip.",
"Exporting": "Exporting",
"Export Collection": "Export Collection",
"Collections are for grouping your documents. They work best when organized around a topic or internal team — Product or Engineering for example.": "Collections are for grouping your documents. They work best when organized around a topic or internal team — Product or Engineering for example.",
@ -469,6 +470,11 @@
"No documents found for your search filters. <1></1>": "No documents found for your search filters. <1></1>",
"Create a new document?": "Create a new document?",
"Clear filters": "Clear filters",
"Processing": "Processing",
"Expired": "Expired",
"Error": "Error",
"All collections": "All collections",
"{{userName}} requested": "{{userName}} requested",
"Last active": "Last active",
"Role": "Role",
"Viewer": "Viewer",
@ -501,10 +507,11 @@
"Uploading": "Uploading",
"Confirm & Import": "Confirm & Import",
"Choose File": "Choose File",
"A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <em>{{ userEmail }}</em>.": "A full export might take some time, consider exporting a single document or collection if possible. We’ll put together a zip of all your documents in Markdown format and email it to <em>{{ userEmail }}</em>.",
"A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it's complete.": "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it's complete.",
"Export Requested": "Export Requested",
"Requesting Export": "Requesting Export",
"Export Data": "Export Data",
"Recent exports": "Recent exports",
"Document published": "Document published",
"Receive a notification whenever a new document is published": "Receive a notification whenever a new document is published",
"Document updated": "Document updated",