Added feature to send emails in a background-task

This commit is contained in:
OzzieIsaacs 2018-07-30 20:12:41 +02:00
parent 4d6299f0d5
commit b0cbd0a37a
6 changed files with 367 additions and 76 deletions

236
cps/asyncmail.py Normal file
View File

@ -0,0 +1,236 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import smtplib
import threading
from datetime import datetime
import logging
import time
import socket
import sys
from email.generator import Generator
import web
from flask_babel import gettext as _
# from babel.dates import format_datetime
import re
try:
from StringIO import StringIO
except ImportError as e:
from io import StringIO
chunksize = 8192
STAT_WAITING = 0
STAT_FAIL = 1
STAT_STARTED = 2
STAT_FINISH_SUCCESS = 3
class email(smtplib.SMTP):
transferSize = 0
progress = 0
def __init__(self, *args, **kwargs):
smtplib.SMTP.__init__(self, *args, **kwargs)
def data(self, msg):
self.transferSize = len(msg)
(code, resp) = smtplib.SMTP.data(self, msg)
self.progress = 0
return (code, resp)
def send(self, str):
"""Send `str' to the server."""
if self.debuglevel > 0:
print>> sys.stderr, 'send:', repr(str)
if hasattr(self, 'sock') and self.sock:
try:
if self.transferSize:
lock=threading.Lock()
lock.acquire()
self.transferSize = len(str)
lock.release()
for i in range(0, self.transferSize, chunksize):
self.sock.send(str[i:i+chunksize])
lock.acquire()
self.progress = i
lock.release()
else:
self.sock.sendall(str)
except socket.error:
self.close()
raise smtplib.SMTPServerDisconnected('Server not connected')
else:
raise smtplib.SMTPServerDisconnected('please run connect() first')
def getTransferStatus(self):
if self.transferSize:
lock2 = threading.Lock()
lock2.acquire()
value = round(float(self.progress) / float(self.transferSize),2)*100
lock2.release()
return str(value) + ' %'
else:
return "100 %"
class email_SSL(email):
def __init__(self, *args, **kwargs):
smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
class EMailThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.status = 0
self.current = 0
self.last = 0
self.queue=list()
self.UIqueue = list()
self.asyncSMTP=None
def run(self):
while 1:
doLock = threading.Lock()
doLock.acquire()
if self.current != self.last:
doLock.release()
self.send_raw_email()
self.current += 1
time.sleep(1)
def get_send_status(self):
if self.asyncSMTP:
return self.asyncSMTP.getTransferStatus()
else:
return "0 %"
def delete_completed_tasks(self):
# muss gelockt werden
for index, task in reversed(list(enumerate(self.UIqueue))):
if task['progress'] == "100 %":
# delete tasks
self.queue.pop(index)
self.UIqueue.pop(index)
# if we are deleting entries before the current index, adjust the index
# if self.current >= index:
self.current -= 1
self.last = len(self.queue)
def get_taskstatus(self):
if self.current < len(self.queue):
if self.queue[self.current]['status'] == STAT_STARTED:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def add_email(self, data, settings, recipient, user_name):
# if more than 50 entries in the list, clean the list
addLock = threading.Lock()
addLock.acquire()
if self.last >= 3:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.queue.append({'data':data, 'settings':settings, 'recipent':recipient, 'starttime': 0,
'status': STAT_WAITING})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': 'E-Mail',
'runtime': '0 s', 'status': _('Waiting') })
# access issue
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
obj=self.queue[self.current]
# settings = ub.get_mail_settings()
obj['data']['From'] = obj['settings']["mail_from"]
obj['data']['To'] = obj['recipent']
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(obj['data'])
obj['data'] = fp.getvalue()
# send email
try:
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
#org_stderr2 = smtplib.stderr
sys.stderr = StderrLogger()
#smtplib.stderr = StderrLogger()
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
else:
self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
# link to logginglevel
if web.ub.config.config_log_level != logging.DEBUG:
self.asyncSMTP.set_debuglevel(0)
else:
self.asyncSMTP.set_debuglevel(1)
if use_ssl == 1:
self.asyncSMTP.starttls()
if obj['settings']["mail_password"]:
self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], obj['data'])
self.asyncSMTP.quit()
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
sys.stderr = org_stderr
#smtplib.stderr = org_stderr2
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
web.app.logger.error(e)
return None
def _formatRuntime(self, runtime):
val = re.split('\:|\.', str(runtime))[0:3]
erg = list()
for v in val:
if int(v) > 0:
erg.append(v)
retVal = (':'.join(erg)).lstrip('0') + ' s'
if retVal == ' s':
retVal = '0 s'
return retVal
class StderrLogger(object):
buffer = ''
def __init__(self):
self.logger = web.app.logger
def write(self, message):
if message == '\n':
self.logger.debug(self.buffer)
print self.buffer
self.buffer = ''
else:
self.buffer += message

View File

@ -5,9 +5,7 @@ import db
import ub
from flask import current_app as app
import logging
import smtplib
from tempfile import gettempdir
import socket
import sys
import os
import traceback
@ -15,6 +13,7 @@ import re
import unicodedata
from io import BytesIO
import converter
import asyncmail
try:
from StringIO import StringIO
@ -28,11 +27,9 @@ except ImportError as e:
from email.mime.text import MIMEText
from email import encoders
from email.generator import Generator
from email.utils import formatdate
from email.utils import make_msgid
from flask_babel import gettext as _
import subprocess
import threading
import shutil
import requests
@ -52,11 +49,22 @@ except ImportError:
# Global variables
updater_thread = None
global_eMailThread = asyncmail.EMailThread()
global_eMailThread.start()
RET_SUCCESS = 1
RET_FAIL = 0
def update_download(book_id, user_id):
check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id ==
book_id).first()
if not check:
new_download = ub.Downloads(user_id=user_id, book_id=book_id)
ub.session.add(new_download)
ub.session.commit()
def make_mobi(book_id, calibrepath):
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
@ -73,74 +81,16 @@ def make_mobi(book_id, calibrepath):
return error_message, RET_FAIL
class StderrLogger(object):
buffer = ''
def __init__(self):
self.logger = logging.getLogger('cps.web')
def write(self, message):
if message == '\n':
self.logger.debug(self.buffer)
self.buffer = ''
else:
self.buffer += message
def send_raw_email(kindle_mail, msg):
settings = ub.get_mail_settings()
msg['From'] = settings["mail_from"]
msg['To'] = kindle_mail
use_ssl = int(settings.get('mail_use_ssl', 0))
# convert MIME message to string
fp = StringIO()
gen = Generator(fp, mangle_from_=False)
gen.flatten(msg)
msg = fp.getvalue()
# send email
try:
timeout = 600 # set timeout to 5mins
org_stderr = sys.stderr
sys.stderr = StderrLogger()
if use_ssl == 2:
mailserver = smtplib.SMTP_SSL(settings["mail_server"], settings["mail_port"], timeout)
else:
mailserver = smtplib.SMTP(settings["mail_server"], settings["mail_port"], timeout)
mailserver.set_debuglevel(1)
if use_ssl == 1:
mailserver.starttls()
if settings["mail_password"]:
mailserver.login(str(settings["mail_login"]), str(settings["mail_password"]))
mailserver.sendmail(settings["mail_from"], kindle_mail, msg)
mailserver.quit()
smtplib.stderr = org_stderr
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as ex:
app.logger.error(traceback.print_exc())
return _("Failed to send mail: %s" % str(ex))
return None
def send_test_mail(kindle_mail):
def send_test_mail(kindle_mail, user_name):
msg = MIMEMultipart()
msg['Subject'] = _(u'Calibre-web test email')
text = _(u'This email has been sent via calibre web.')
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
return send_raw_email(kindle_mail, msg)
global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name)
return # send_raw_email(kindle_mail, msg)
def send_mail(book_id, kindle_mail, calibrepath):
def send_mail(book_id, kindle_mail, calibrepath, user_id):
"""Send email with attachments"""
# create MIME message
msg = MIMEMultipart()
@ -179,8 +129,8 @@ def send_mail(book_id, kindle_mail, calibrepath):
msg.attach(get_attachment(formats['pdf']))
else:
return _("Could not find any formats suitable for sending by email")
return send_raw_email(kindle_mail, msg)
global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id)
return None # send_raw_email(kindle_mail, msg)
def get_attachment(file_path):

View File

@ -67,6 +67,9 @@
</li>
{% endif %}
{% endif %}
{% if g.user.role_admin() %}
<li><a id="top_tasks" href="{{url_for('get_tasks_status')}}"><span class="glyphicon glyphicon-tasks"></span><span class="hidden-sm"> {{_('Tasks')}}</span></a></li>
{% endif %}
{% if g.user.role_admin() %}
<li><a id="top_admin" href="{{url_for('admin')}}"><span class="glyphicon glyphicon-dashboard"></span><span class="hidden-sm"> {{_('Admin')}}</span></a></li>
{% endif %}
@ -222,6 +225,7 @@
</div>
{% block modal %}{% endblock %}
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<!--script src="https://code.jquery.com/jquery.js"></script-->
<script src="{{ url_for('static', filename='js/libs/jquery.min.js') }}"></script>

48
cps/templates/tasks.html Normal file
View File

@ -0,0 +1,48 @@
{% extends "layout.html" %}
{% block header %}
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table.css">
{% endblock %}
{% block body %}
<div class="discover">
<h2>{{_('Tasks list')}}</h2>
<table class="table table-no-bordered" id="table" data-url="{{'/ajax/emailstat'}}">
<thead>
<tr>
{% if g.user.role_admin() %}
<th data-field="user">{{_('User')}}</th>
{% endif %}
<th data-field="type">{{_('Task')}}</th>
<th data-field="status">{{_('Status')}}</th>
<th data-field="progress">{{_('Progress')}}</th>
<th data-field="runtime">{{_('Runtime')}}</th>
<th data-field="starttime">{{_('Starttime')}}</th>
</tr>
</thead>
</table>
<!--div class="btn btn-default" id="tasks_delete">{{_('Delete finished tasks')}}</div>
<div class="btn btn-default" id="tasks_hide">{{_('Hide all tasks')}}</div-->
</div>
{% endblock %}
{% block js %}
<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table.js"></script>
<script>
$('#table').bootstrapTable({
formatNoMatches: function () {
return '';
},
striped: true
});
setInterval(function() {
$.ajax({
method:"get",
url: "{{'/ajax/emailstat'}}",
async: true,
timeout: 900,
success:function(data){
$('#table').bootstrapTable("load", data);
}
});
}, 1000);
</script>
{% endblock %}

View File

@ -10,7 +10,6 @@ import sys
import os
import logging
from werkzeug.security import generate_password_hash
from flask_babel import gettext as _
import json
import datetime
from binascii import hexlify

View File

@ -912,13 +912,41 @@ def get_metadata_calibre_companion(uuid):
else:
return ""
@app.route("/ajax/emailstat")
@login_required
def get_email_status_json():
answer=list()
tasks=helper.global_eMailThread.get_taskstatus()
if not current_user.role_admin():
for task in tasks:
if task['user'] == current_user.nickname:
if task['formStarttime']:
task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
task['formStarttime'] = ""
else:
if 'starttime' not in task:
task['starttime'] = ""
answer.append(task)
else:
for task in tasks:
if task['formStarttime']:
task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
task['formStarttime'] = ""
else:
if 'starttime' not in task:
task['starttime'] = ""
answer = tasks
js=json.dumps(answer)
response = make_response(js)
response.headers["Content-Type"] = "application/json; charset=utf-8"
return response
@app.route("/get_authors_json", methods=['GET', 'POST'])
@login_required_if_no_ano
def get_authors_json():
if request.method == "GET":
query = request.args.get('q')
# entries = db.session.execute("select name from authors where name like '%" + query + "%'")
entries = db.session.query(db.Authors).filter(db.Authors.name.ilike("%" + query + "%")).all()
json_dumps = json.dumps([dict(name=r.name) for r in entries])
return json_dumps
@ -929,10 +957,7 @@ def get_authors_json():
def get_tags_json():
if request.method == "GET":
query = request.args.get('q')
# entries = db.session.execute("select name from tags where name like '%" + query + "%'")
entries = db.session.query(db.Tags).filter(db.Tags.name.ilike("%" + query + "%")).all()
# for x in entries:
# alfa = dict(name=x.name)
json_dumps = json.dumps([dict(name=r.name) for r in entries])
return json_dumps
@ -1421,6 +1446,35 @@ def bookmark(book_id, book_format):
ub.session.commit()
return "", 201
@app.route("/tasks")
@login_required
def get_tasks_status():
# if current user admin, show all email, otherwise only own emails
answer=list()
tasks=helper.global_eMailThread.get_taskstatus()
if not current_user.role_admin():
for task in tasks:
if task['user'] == current_user.nickname:
if task['formStarttime']:
task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
task['formStarttime'] = ""
else:
if 'starttime' not in task:
task['starttime'] = ""
answer.append(task)
else:
for task in tasks:
if task['formStarttime']:
task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
task['formStarttime'] = ""
else:
if 'starttime' not in task:
task['starttime'] = ""
answer = tasks
# foreach row format row
return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"))
@app.route("/admin")
@login_required
@ -2147,9 +2201,9 @@ def send_to_kindle(book_id):
if settings.get("mail_server", "mail.example.com") == "mail.example.com":
flash(_(u"Please configure the SMTP mail settings first..."), category="error")
elif current_user.kindle_mail:
result = helper.send_mail(book_id, current_user.kindle_mail, config.config_calibre_dir)
result = helper.send_mail(book_id, current_user.kindle_mail, config.config_calibre_dir, current_user.nickname)
if result is None:
flash(_(u"Book successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail),
flash(_(u"Book successfully queued for sending to %(kindlemail)s", kindlemail=current_user.kindle_mail),
category="success")
ub.update_download(book_id, int(current_user.id))
else:
@ -2851,7 +2905,7 @@ def edit_mailsettings():
flash(e, category="error")
if "test" in to_save and to_save["test"]:
if current_user.kindle_mail:
result = helper.send_test_mail(current_user.kindle_mail)
result = helper.send_test_mail(current_user.kindle_mail, current_user.nickname)
if result is None:
flash(_(u"Test E-Mail successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail),
category="success")