feat: private content (#1137)
* save images as private and serve via signed url from images.info api * download private images to directory on export * fix lint errors * private s3 default, AWS.s3 module level scope, default s3 url expiry * combine regex to one, and only replace when there are matches * fix lint * code not needed anymore, remove * updates after pulling master * revert the uploadToS3FromUrl url return * use model gettr to compact code, rename to attachments api * basic checking of document read permission to allow attachment viewing * fix: Continue to upload avatars as public fix: Allow redirect for non-private attachments * add support for publicly shared documents * catch errors which crash the app during zip export and user creation * add tests * enable AWS signature v4 for s3 * switch to use factories to build models for testing * add isDocker flag for local serving of attachment redirect url * fix redirect tests Co-authored-by: Tom Moor <tom.moor@gmail.com>
This commit is contained in:
parent
064d8cea44
commit
8e2b19dc7a
|
@ -49,6 +49,9 @@ AWS_REGION=xx-xxxx-x
|
||||||
AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569
|
AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569
|
||||||
AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here
|
AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here
|
||||||
AWS_S3_UPLOAD_MAX_SIZE=26214400
|
AWS_S3_UPLOAD_MAX_SIZE=26214400
|
||||||
|
# uploaded s3 objects permission level, default is private
|
||||||
|
# set to "public-read" to allow public access
|
||||||
|
AWS_S3_ACL=private
|
||||||
|
|
||||||
# Emails configuration (optional)
|
# Emails configuration (optional)
|
||||||
SMTP_HOST=
|
SMTP_HOST=
|
||||||
|
|
|
@ -49,10 +49,11 @@ class DropToImport extends React.Component<Props> {
|
||||||
const canvas = this.avatarEditorRef.getImage();
|
const canvas = this.avatarEditorRef.getImage();
|
||||||
const imageBlob = dataUrlToBlob(canvas.toDataURL());
|
const imageBlob = dataUrlToBlob(canvas.toDataURL());
|
||||||
try {
|
try {
|
||||||
const asset = await uploadFile(imageBlob, {
|
const attachment = await uploadFile(imageBlob, {
|
||||||
name: this.file.name,
|
name: this.file.name,
|
||||||
|
public: true,
|
||||||
});
|
});
|
||||||
this.props.onSuccess(asset.url);
|
this.props.onSuccess(attachment.url);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.props.onError(err.message);
|
this.props.onError(err.message);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
@ -5,6 +5,7 @@ import invariant from 'invariant';
|
||||||
type Options = {
|
type Options = {
|
||||||
name?: string,
|
name?: string,
|
||||||
documentId?: string,
|
documentId?: string,
|
||||||
|
public?: boolean,
|
||||||
};
|
};
|
||||||
|
|
||||||
export const uploadFile = async (
|
export const uploadFile = async (
|
||||||
|
@ -13,6 +14,7 @@ export const uploadFile = async (
|
||||||
) => {
|
) => {
|
||||||
const name = file instanceof File ? file.name : options.name;
|
const name = file instanceof File ? file.name : options.name;
|
||||||
const response = await client.post('/users.s3Upload', {
|
const response = await client.post('/users.s3Upload', {
|
||||||
|
public: options.public,
|
||||||
documentId: options.documentId,
|
documentId: options.documentId,
|
||||||
contentType: file.type,
|
contentType: file.type,
|
||||||
size: file.size,
|
size: file.size,
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
// @flow
|
||||||
|
import Router from 'koa-router';
|
||||||
|
import auth from '../middlewares/authentication';
|
||||||
|
import { Attachment, Document } from '../models';
|
||||||
|
import { getSignedImageUrl } from '../utils/s3';
|
||||||
|
|
||||||
|
import policy from '../policies';
|
||||||
|
|
||||||
|
const { authorize } = policy;
|
||||||
|
const router = new Router();
|
||||||
|
|
||||||
|
router.post('attachments.redirect', auth(), async ctx => {
|
||||||
|
const { id } = ctx.body;
|
||||||
|
ctx.assertPresent(id, 'id is required');
|
||||||
|
|
||||||
|
const user = ctx.state.user;
|
||||||
|
const attachment = await Attachment.findByPk(id);
|
||||||
|
|
||||||
|
if (attachment.isPrivate) {
|
||||||
|
const document = await Document.findByPk(attachment.documentId, {
|
||||||
|
userId: user.id,
|
||||||
|
});
|
||||||
|
authorize(user, 'read', document);
|
||||||
|
|
||||||
|
const accessUrl = await getSignedImageUrl(attachment.key);
|
||||||
|
ctx.redirect(accessUrl);
|
||||||
|
} else {
|
||||||
|
ctx.redirect(attachment.url);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
export default router;
|
|
@ -0,0 +1,86 @@
|
||||||
|
/* eslint-disable flowtype/require-valid-file-annotation */
|
||||||
|
import TestServer from 'fetch-test-server';
|
||||||
|
import app from '../app';
|
||||||
|
import { flushdb } from '../test/support';
|
||||||
|
import {
|
||||||
|
buildUser,
|
||||||
|
buildCollection,
|
||||||
|
buildAttachment,
|
||||||
|
buildDocument,
|
||||||
|
} from '../test/factories';
|
||||||
|
|
||||||
|
const server = new TestServer(app.callback());
|
||||||
|
|
||||||
|
beforeEach(flushdb);
|
||||||
|
afterAll(server.close);
|
||||||
|
|
||||||
|
describe('#attachments.redirect', async () => {
|
||||||
|
it('should require authentication', async () => {
|
||||||
|
const res = await server.post('/api/attachments.redirect');
|
||||||
|
expect(res.status).toEqual(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should return a redirect for an attachment belonging to a document user has access to', async () => {
|
||||||
|
const user = await buildUser();
|
||||||
|
const attachment = await buildAttachment({
|
||||||
|
teamId: user.teamId,
|
||||||
|
userId: user.id,
|
||||||
|
});
|
||||||
|
const res = await server.post('/api/attachments.redirect', {
|
||||||
|
body: { token: user.getJwtToken(), id: attachment.id },
|
||||||
|
redirect: 'manual',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(res.status).toEqual(302);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should always return a redirect for a public attachment', async () => {
|
||||||
|
const user = await buildUser();
|
||||||
|
const collection = await buildCollection({
|
||||||
|
teamId: user.teamId,
|
||||||
|
userId: user.id,
|
||||||
|
private: true,
|
||||||
|
});
|
||||||
|
const document = await buildDocument({
|
||||||
|
teamId: user.teamId,
|
||||||
|
userId: user.id,
|
||||||
|
collectionId: collection.id,
|
||||||
|
});
|
||||||
|
const attachment = await buildAttachment({
|
||||||
|
teamId: user.teamId,
|
||||||
|
userId: user.id,
|
||||||
|
documentId: document.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await server.post('/api/attachments.redirect', {
|
||||||
|
body: { token: user.getJwtToken(), id: attachment.id },
|
||||||
|
redirect: 'manual',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(res.status).toEqual(302);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not return a redirect for a private attachment belonging to a document user does not have access to', async () => {
|
||||||
|
const user = await buildUser();
|
||||||
|
const collection = await buildCollection({
|
||||||
|
private: true,
|
||||||
|
});
|
||||||
|
const document = await buildDocument({
|
||||||
|
teamId: collection.teamId,
|
||||||
|
userId: collection.userId,
|
||||||
|
collectionId: collection.id,
|
||||||
|
});
|
||||||
|
const attachment = await buildAttachment({
|
||||||
|
teamId: document.teamId,
|
||||||
|
userId: document.userId,
|
||||||
|
documentId: document.id,
|
||||||
|
acl: 'private',
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await server.post('/api/attachments.redirect', {
|
||||||
|
body: { token: user.getJwtToken(), id: attachment.id },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(res.status).toEqual(403);
|
||||||
|
});
|
||||||
|
});
|
|
@ -16,6 +16,7 @@ import team from './team';
|
||||||
import integrations from './integrations';
|
import integrations from './integrations';
|
||||||
import notificationSettings from './notificationSettings';
|
import notificationSettings from './notificationSettings';
|
||||||
import utils from './utils';
|
import utils from './utils';
|
||||||
|
import attachments from './attachments';
|
||||||
|
|
||||||
import { NotFoundError } from '../errors';
|
import { NotFoundError } from '../errors';
|
||||||
import errorHandling from '../middlewares/errorHandling';
|
import errorHandling from '../middlewares/errorHandling';
|
||||||
|
@ -48,6 +49,7 @@ router.use('/', shares.routes());
|
||||||
router.use('/', team.routes());
|
router.use('/', team.routes());
|
||||||
router.use('/', integrations.routes());
|
router.use('/', integrations.routes());
|
||||||
router.use('/', notificationSettings.routes());
|
router.use('/', notificationSettings.routes());
|
||||||
|
router.use('/', attachments.routes());
|
||||||
router.use('/', utils.routes());
|
router.use('/', utils.routes());
|
||||||
router.post('*', ctx => {
|
router.post('*', ctx => {
|
||||||
ctx.throw(new NotFoundError('Endpoint not found'));
|
ctx.throw(new NotFoundError('Endpoint not found'));
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
// @flow
|
// @flow
|
||||||
import Router from 'koa-router';
|
import Router from 'koa-router';
|
||||||
import { Team } from '../models';
|
import { Team } from '../models';
|
||||||
import { publicS3Endpoint } from '../utils/s3';
|
|
||||||
|
|
||||||
import auth from '../middlewares/authentication';
|
import auth from '../middlewares/authentication';
|
||||||
import { presentTeam, presentPolicies } from '../presenters';
|
import { presentTeam, presentPolicies } from '../presenters';
|
||||||
|
@ -19,8 +18,6 @@ router.post('team.update', auth(), async ctx => {
|
||||||
guestSignin,
|
guestSignin,
|
||||||
documentEmbeds,
|
documentEmbeds,
|
||||||
} = ctx.body;
|
} = ctx.body;
|
||||||
const endpoint = publicS3Endpoint();
|
|
||||||
|
|
||||||
const user = ctx.state.user;
|
const user = ctx.state.user;
|
||||||
const team = await Team.findByPk(user.teamId);
|
const team = await Team.findByPk(user.teamId);
|
||||||
authorize(user, 'update', team);
|
authorize(user, 'update', team);
|
||||||
|
@ -33,9 +30,7 @@ router.post('team.update', auth(), async ctx => {
|
||||||
if (sharing !== undefined) team.sharing = sharing;
|
if (sharing !== undefined) team.sharing = sharing;
|
||||||
if (documentEmbeds !== undefined) team.documentEmbeds = documentEmbeds;
|
if (documentEmbeds !== undefined) team.documentEmbeds = documentEmbeds;
|
||||||
if (guestSignin !== undefined) team.guestSignin = guestSignin;
|
if (guestSignin !== undefined) team.guestSignin = guestSignin;
|
||||||
if (avatarUrl && avatarUrl.startsWith(`${endpoint}/uploads/${user.id}`)) {
|
if (avatarUrl !== undefined) team.avatarUrl = avatarUrl;
|
||||||
team.avatarUrl = avatarUrl;
|
|
||||||
}
|
|
||||||
await team.save();
|
await team.save();
|
||||||
|
|
||||||
ctx.body = {
|
ctx.body = {
|
||||||
|
|
|
@ -17,6 +17,7 @@ import userInviter from '../commands/userInviter';
|
||||||
import { presentUser } from '../presenters';
|
import { presentUser } from '../presenters';
|
||||||
import policy from '../policies';
|
import policy from '../policies';
|
||||||
|
|
||||||
|
const AWS_S3_ACL = process.env.AWS_S3_ACL || 'private';
|
||||||
const { authorize } = policy;
|
const { authorize } = policy;
|
||||||
const router = new Router();
|
const router = new Router();
|
||||||
|
|
||||||
|
@ -61,12 +62,9 @@ router.post('users.info', auth(), async ctx => {
|
||||||
router.post('users.update', auth(), async ctx => {
|
router.post('users.update', auth(), async ctx => {
|
||||||
const { user } = ctx.state;
|
const { user } = ctx.state;
|
||||||
const { name, avatarUrl } = ctx.body;
|
const { name, avatarUrl } = ctx.body;
|
||||||
const endpoint = publicS3Endpoint();
|
|
||||||
|
|
||||||
if (name) user.name = name;
|
if (name) user.name = name;
|
||||||
if (avatarUrl && avatarUrl.startsWith(`${endpoint}/uploads/${user.id}`)) {
|
if (avatarUrl) user.avatarUrl = avatarUrl;
|
||||||
user.avatarUrl = avatarUrl;
|
|
||||||
}
|
|
||||||
|
|
||||||
await user.save();
|
await user.save();
|
||||||
|
|
||||||
|
@ -89,14 +87,19 @@ router.post('users.s3Upload', auth(), async ctx => {
|
||||||
const { user } = ctx.state;
|
const { user } = ctx.state;
|
||||||
const s3Key = uuid.v4();
|
const s3Key = uuid.v4();
|
||||||
const key = `uploads/${user.id}/${s3Key}/${name}`;
|
const key = `uploads/${user.id}/${s3Key}/${name}`;
|
||||||
|
const acl =
|
||||||
|
ctx.body.public === undefined
|
||||||
|
? AWS_S3_ACL
|
||||||
|
: ctx.body.public ? 'public-read' : 'private';
|
||||||
const credential = makeCredential();
|
const credential = makeCredential();
|
||||||
const longDate = format(new Date(), 'YYYYMMDDTHHmmss\\Z');
|
const longDate = format(new Date(), 'YYYYMMDDTHHmmss\\Z');
|
||||||
const policy = makePolicy(credential, longDate);
|
const policy = makePolicy(credential, longDate, acl);
|
||||||
const endpoint = publicS3Endpoint();
|
const endpoint = publicS3Endpoint();
|
||||||
const url = `${endpoint}/${key}`;
|
const url = `${endpoint}/${key}`;
|
||||||
|
|
||||||
await Attachment.create({
|
const attachment = await Attachment.create({
|
||||||
key,
|
key,
|
||||||
|
acl,
|
||||||
size,
|
size,
|
||||||
url,
|
url,
|
||||||
contentType,
|
contentType,
|
||||||
|
@ -120,7 +123,7 @@ router.post('users.s3Upload', auth(), async ctx => {
|
||||||
form: {
|
form: {
|
||||||
'Cache-Control': 'max-age=31557600',
|
'Cache-Control': 'max-age=31557600',
|
||||||
'Content-Type': contentType,
|
'Content-Type': contentType,
|
||||||
acl: 'public-read',
|
acl,
|
||||||
key,
|
key,
|
||||||
policy,
|
policy,
|
||||||
'x-amz-algorithm': 'AWS4-HMAC-SHA256',
|
'x-amz-algorithm': 'AWS4-HMAC-SHA256',
|
||||||
|
@ -131,7 +134,7 @@ router.post('users.s3Upload', auth(), async ctx => {
|
||||||
asset: {
|
asset: {
|
||||||
contentType,
|
contentType,
|
||||||
name,
|
name,
|
||||||
url,
|
url: attachment.redirectUrl,
|
||||||
size,
|
size,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
@ -40,6 +40,12 @@ const Attachment = sequelize.define(
|
||||||
name: function() {
|
name: function() {
|
||||||
return path.parse(this.key).base;
|
return path.parse(this.key).base;
|
||||||
},
|
},
|
||||||
|
redirectUrl: function() {
|
||||||
|
return `/api/attachments.redirect?id=${this.id}`;
|
||||||
|
},
|
||||||
|
isPrivate: function() {
|
||||||
|
return this.acl === 'private';
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
|
@ -91,12 +91,18 @@ Team.associate = models => {
|
||||||
|
|
||||||
const uploadAvatar = async model => {
|
const uploadAvatar = async model => {
|
||||||
const endpoint = publicS3Endpoint();
|
const endpoint = publicS3Endpoint();
|
||||||
|
const { avatarUrl } = model;
|
||||||
|
|
||||||
if (model.avatarUrl && !model.avatarUrl.startsWith(endpoint)) {
|
if (
|
||||||
|
avatarUrl &&
|
||||||
|
!avatarUrl.startsWith('/api') &&
|
||||||
|
!avatarUrl.startsWith(endpoint)
|
||||||
|
) {
|
||||||
try {
|
try {
|
||||||
const newUrl = await uploadToS3FromUrl(
|
const newUrl = await uploadToS3FromUrl(
|
||||||
model.avatarUrl,
|
avatarUrl,
|
||||||
`avatars/${model.id}/${uuid.v4()}`
|
`avatars/${model.id}/${uuid.v4()}`,
|
||||||
|
'public-read'
|
||||||
);
|
);
|
||||||
if (newUrl) model.avatarUrl = newUrl;
|
if (newUrl) model.avatarUrl = newUrl;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
|
@ -128,14 +128,21 @@ const uploadAvatar = async model => {
|
||||||
|
|
||||||
if (
|
if (
|
||||||
avatarUrl &&
|
avatarUrl &&
|
||||||
|
!avatarUrl.startsWith('/api') &&
|
||||||
!avatarUrl.startsWith(endpoint) &&
|
!avatarUrl.startsWith(endpoint) &&
|
||||||
!avatarUrl.startsWith(DEFAULT_AVATAR_HOST)
|
!avatarUrl.startsWith(DEFAULT_AVATAR_HOST)
|
||||||
) {
|
) {
|
||||||
const newUrl = await uploadToS3FromUrl(
|
try {
|
||||||
avatarUrl,
|
const newUrl = await uploadToS3FromUrl(
|
||||||
`avatars/${model.id}/${uuid.v4()}`
|
avatarUrl,
|
||||||
);
|
`avatars/${model.id}/${uuid.v4()}`,
|
||||||
if (newUrl) model.avatarUrl = newUrl;
|
'public-read'
|
||||||
|
);
|
||||||
|
if (newUrl) model.avatarUrl = newUrl;
|
||||||
|
} catch (err) {
|
||||||
|
// we can try again next time
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,24 +1,46 @@
|
||||||
// @flow
|
// @flow
|
||||||
import { takeRight } from 'lodash';
|
import { takeRight } from 'lodash';
|
||||||
import { User, Document } from '../models';
|
import { User, Document, Attachment } from '../models';
|
||||||
|
import { getSignedImageUrl } from '../utils/s3';
|
||||||
import presentUser from './user';
|
import presentUser from './user';
|
||||||
|
|
||||||
type Options = {
|
type Options = {
|
||||||
isPublic?: boolean,
|
isPublic?: boolean,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const attachmentRegex = /!\[.*\]\(\/api\/attachments\.redirect\?id=(?<id>.*)\)/gi;
|
||||||
|
|
||||||
|
// replaces attachments.redirect urls with signed/authenticated url equivalents
|
||||||
|
async function replaceImageAttachments(text) {
|
||||||
|
const attachmentIds = [...text.matchAll(attachmentRegex)].map(
|
||||||
|
match => match.groups && match.groups.id
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const id of attachmentIds) {
|
||||||
|
const attachment = await Attachment.findByPk(id);
|
||||||
|
const accessUrl = await getSignedImageUrl(attachment.key);
|
||||||
|
text = text.replace(attachment.redirectUrl, accessUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
|
||||||
export default async function present(document: Document, options: ?Options) {
|
export default async function present(document: Document, options: ?Options) {
|
||||||
options = {
|
options = {
|
||||||
isPublic: false,
|
isPublic: false,
|
||||||
...options,
|
...options,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const text = options.isPublic
|
||||||
|
? await replaceImageAttachments(document.text)
|
||||||
|
: document.text;
|
||||||
|
|
||||||
const data = {
|
const data = {
|
||||||
id: document.id,
|
id: document.id,
|
||||||
url: document.url,
|
url: document.url,
|
||||||
urlId: document.urlId,
|
urlId: document.urlId,
|
||||||
title: document.title,
|
title: document.title,
|
||||||
text: document.text,
|
text,
|
||||||
emoji: document.emoji,
|
emoji: document.emoji,
|
||||||
createdAt: document.createdAt,
|
createdAt: document.createdAt,
|
||||||
createdBy: undefined,
|
createdBy: undefined,
|
||||||
|
|
|
@ -1,5 +1,13 @@
|
||||||
// @flow
|
// @flow
|
||||||
import { Share, Team, User, Event, Document, Collection } from '../models';
|
import {
|
||||||
|
Share,
|
||||||
|
Team,
|
||||||
|
User,
|
||||||
|
Event,
|
||||||
|
Document,
|
||||||
|
Collection,
|
||||||
|
Attachment,
|
||||||
|
} from '../models';
|
||||||
import uuid from 'uuid';
|
import uuid from 'uuid';
|
||||||
|
|
||||||
let count = 0;
|
let count = 0;
|
||||||
|
@ -104,3 +112,38 @@ export async function buildDocument(overrides: Object = {}) {
|
||||||
...overrides,
|
...overrides,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function buildAttachment(overrides: Object = {}) {
|
||||||
|
count++;
|
||||||
|
|
||||||
|
if (!overrides.teamId) {
|
||||||
|
const team = await buildTeam();
|
||||||
|
overrides.teamId = team.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!overrides.userId) {
|
||||||
|
const user = await buildUser();
|
||||||
|
overrides.userId = user.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!overrides.collectionId) {
|
||||||
|
const collection = await buildCollection(overrides);
|
||||||
|
overrides.collectionId = collection.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!overrides.documentId) {
|
||||||
|
const document = await buildDocument(overrides);
|
||||||
|
overrides.documentId = document.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Attachment.create({
|
||||||
|
key: `uploads/key/to/file ${count}.png`,
|
||||||
|
url: `https://redirect.url.com/uploads/key/to/file ${count}.png`,
|
||||||
|
contentType: 'image/png',
|
||||||
|
size: 100,
|
||||||
|
acl: 'public-read',
|
||||||
|
createdAt: new Date('2018-01-02T00:00:00.000Z'),
|
||||||
|
updatedAt: new Date('2018-01-02T00:00:00.000Z'),
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
|
@ -12,6 +12,14 @@ const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
|
||||||
const AWS_REGION = process.env.AWS_REGION;
|
const AWS_REGION = process.env.AWS_REGION;
|
||||||
const AWS_S3_UPLOAD_BUCKET_NAME = process.env.AWS_S3_UPLOAD_BUCKET_NAME;
|
const AWS_S3_UPLOAD_BUCKET_NAME = process.env.AWS_S3_UPLOAD_BUCKET_NAME;
|
||||||
|
|
||||||
|
const s3 = new AWS.S3({
|
||||||
|
s3ForcePathStyle: true,
|
||||||
|
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||||
|
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
||||||
|
endpoint: new AWS.Endpoint(process.env.AWS_S3_UPLOAD_BUCKET_URL),
|
||||||
|
signatureVersion: 'v4',
|
||||||
|
});
|
||||||
|
|
||||||
const hmac = (key: string, message: string, encoding: any) => {
|
const hmac = (key: string, message: string, encoding: any) => {
|
||||||
return crypto
|
return crypto
|
||||||
.createHmac('sha256', key)
|
.createHmac('sha256', key)
|
||||||
|
@ -30,13 +38,17 @@ export const makeCredential = () => {
|
||||||
return credential;
|
return credential;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const makePolicy = (credential: string, longDate: string) => {
|
export const makePolicy = (
|
||||||
|
credential: string,
|
||||||
|
longDate: string,
|
||||||
|
acl: string
|
||||||
|
) => {
|
||||||
const tomorrow = addHours(new Date(), 24);
|
const tomorrow = addHours(new Date(), 24);
|
||||||
const policy = {
|
const policy = {
|
||||||
conditions: [
|
conditions: [
|
||||||
{ bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME },
|
{ bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME },
|
||||||
['starts-with', '$key', ''],
|
['starts-with', '$key', ''],
|
||||||
{ acl: 'public-read' },
|
{ acl },
|
||||||
['content-length-range', 0, +process.env.AWS_S3_UPLOAD_MAX_SIZE],
|
['content-length-range', 0, +process.env.AWS_S3_UPLOAD_MAX_SIZE],
|
||||||
['starts-with', '$Content-Type', 'image'],
|
['starts-with', '$Content-Type', 'image'],
|
||||||
['starts-with', '$Cache-Control', ''],
|
['starts-with', '$Cache-Control', ''],
|
||||||
|
@ -77,13 +89,11 @@ export const publicS3Endpoint = (isServerUpload?: boolean) => {
|
||||||
}`;
|
}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const uploadToS3FromUrl = async (url: string, key: string) => {
|
export const uploadToS3FromUrl = async (
|
||||||
const s3 = new AWS.S3({
|
url: string,
|
||||||
s3ForcePathStyle: true,
|
key: string,
|
||||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
acl: string
|
||||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
) => {
|
||||||
endpoint: new AWS.Endpoint(process.env.AWS_S3_UPLOAD_BUCKET_URL),
|
|
||||||
});
|
|
||||||
invariant(AWS_S3_UPLOAD_BUCKET_NAME, 'AWS_S3_UPLOAD_BUCKET_NAME not set');
|
invariant(AWS_S3_UPLOAD_BUCKET_NAME, 'AWS_S3_UPLOAD_BUCKET_NAME not set');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -92,7 +102,7 @@ export const uploadToS3FromUrl = async (url: string, key: string) => {
|
||||||
const buffer = await res.buffer();
|
const buffer = await res.buffer();
|
||||||
await s3
|
await s3
|
||||||
.putObject({
|
.putObject({
|
||||||
ACL: 'public-read',
|
ACL: acl,
|
||||||
Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME,
|
Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME,
|
||||||
Key: key,
|
Key: key,
|
||||||
ContentType: res.headers['content-type'],
|
ContentType: res.headers['content-type'],
|
||||||
|
@ -112,3 +122,36 @@ export const uploadToS3FromUrl = async (url: string, key: string) => {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const getSignedImageUrl = async (key: string) => {
|
||||||
|
invariant(AWS_S3_UPLOAD_BUCKET_NAME, 'AWS_S3_UPLOAD_BUCKET_NAME not set');
|
||||||
|
const isDocker = process.env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
|
||||||
|
|
||||||
|
const params = {
|
||||||
|
Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME,
|
||||||
|
Key: key,
|
||||||
|
Expires: 60,
|
||||||
|
};
|
||||||
|
|
||||||
|
return isDocker
|
||||||
|
? `${publicS3Endpoint()}/${key}`
|
||||||
|
: s3.getSignedUrl('getObject', params);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getImageByKey = async (key: string) => {
|
||||||
|
const params = {
|
||||||
|
Bucket: process.env.AWS_S3_UPLOAD_BUCKET_NAME,
|
||||||
|
Key: key,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const data = await s3.getObject(params).promise();
|
||||||
|
return data.Body;
|
||||||
|
} catch (err) {
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
bugsnag.notify(err);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
|
@ -3,13 +3,25 @@ import fs from 'fs';
|
||||||
import JSZip from 'jszip';
|
import JSZip from 'jszip';
|
||||||
import tmp from 'tmp';
|
import tmp from 'tmp';
|
||||||
import unescape from '../../shared/utils/unescape';
|
import unescape from '../../shared/utils/unescape';
|
||||||
import { Collection, Document } from '../models';
|
import { Attachment, Collection, Document } from '../models';
|
||||||
|
import { getImageByKey } from './s3';
|
||||||
|
import bugsnag from 'bugsnag';
|
||||||
|
|
||||||
async function addToArchive(zip, documents) {
|
async function addToArchive(zip, documents) {
|
||||||
for (const doc of documents) {
|
for (const doc of documents) {
|
||||||
const document = await Document.findByPk(doc.id);
|
const document = await Document.findByPk(doc.id);
|
||||||
|
let text = unescape(document.text);
|
||||||
|
|
||||||
zip.file(`${document.title}.md`, unescape(document.text));
|
const attachments = await Attachment.findAll({
|
||||||
|
where: { documentId: document.id },
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const attachment of attachments) {
|
||||||
|
await addImageToArchive(zip, attachment.key);
|
||||||
|
text = text.replace(attachment.redirectUrl, encodeURI(attachment.key));
|
||||||
|
}
|
||||||
|
|
||||||
|
zip.file(`${document.title}.md`, text);
|
||||||
|
|
||||||
if (doc.children && doc.children.length) {
|
if (doc.children && doc.children.length) {
|
||||||
const folder = zip.folder(document.title);
|
const folder = zip.folder(document.title);
|
||||||
|
@ -18,6 +30,20 @@ async function addToArchive(zip, documents) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function addImageToArchive(zip, key) {
|
||||||
|
try {
|
||||||
|
const img = await getImageByKey(key);
|
||||||
|
zip.file(key, img, { createFolders: true });
|
||||||
|
} catch (err) {
|
||||||
|
if (process.env.NODE_ENV === 'production') {
|
||||||
|
bugsnag.notify(err);
|
||||||
|
} else {
|
||||||
|
// error during file retrieval
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function archiveToPath(zip) {
|
async function archiveToPath(zip) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
tmp.file({ prefix: 'export-', postfix: '.zip' }, (err, path) => {
|
tmp.file({ prefix: 'export-', postfix: '.zip' }, (err, path) => {
|
||||||
|
|
Reference in New Issue