diff --git a/constraints.txt b/constraints.txt
index b510a27c9fda073bb05db6da5ac65a024f9d54cf..18e05ba613732671a30c101753f10d1c621c7fc0 100644
--- a/constraints.txt
+++ b/constraints.txt
@@ -39,6 +39,7 @@ six==1.16.0
SQLAlchemy==1.4.40
sqlalchemy-stubs==0.4
sqlalchemy2-stubs==0.0.2a25
+token-bucket==0.3.0
tomli==2.0.1
types-bleach==5.0.3
types-Flask-SQLAlchemy==2.5.9
diff --git a/etc/config.py.example b/etc/config.py.example
index dff58ab5b8d8dbfc9fc26045c110e58e59d00592..6883a276f39cbe9d612d19e2d56d82a680986da7 100644
--- a/etc/config.py.example
+++ b/etc/config.py.example
@@ -25,6 +25,9 @@ MAIL_CONTACT = "osmo@mo.mff.cuni.cz"
# Odesilatel generovaných mailů (není-li definován, neposílají se)
# MAIL_FROM = "osmo-auto@mo.mff.cuni.cz"
+# Kam posíláme maily o interních chybách (není-li definováno, neposílají se)
+# MAIL_ERRORS_TO = "osmo@mo.mff.cuni.cz"
+
# Pro testování je možné všechny odesílané maily přesměrovat na jinou adresu
# MAIL_INSTEAD = "mares@kam.mff.cuni.cz"
diff --git a/mo/email.py b/mo/email.py
index 7b9a99e999698caa4ec700ccb71fa79c60039f68..d2fa1572952b2192241d1d2cce6484506cda35ff 100644
--- a/mo/email.py
+++ b/mo/email.py
@@ -5,16 +5,17 @@ import email.message
import email.headerregistry
import subprocess
import textwrap
+import token_bucket
+import traceback
+from typing import Mapping, Optional
import urllib.parse
import mo.db as db
import mo.config as config
-from mo.util import logger
+from mo.util import logger, ExceptionInfo
-def send_user_email(user: db.User, subject: str, body: str, add_footer: bool = False) -> bool:
- logger.info(f'Mail: "{subject}" -> {user.email} (#{user.user_id})')
-
+def send_email(send_to: str, full_name: str, subject: str, body: str) -> bool:
mail_from = getattr(config, 'MAIL_FROM', None)
if mail_from is None:
logger.error('Mail: V configu chybí nastavení MAIL_FROM')
@@ -27,28 +28,21 @@ def send_user_email(user: db.User, subject: str, body: str, add_footer: bool = F
)
msg['To'] = [
email.headerregistry.Address(
- display_name=user.full_name(),
- addr_spec=user.email,
+ display_name=full_name,
+ addr_spec=send_to,
)
]
msg['Reply-To'] = email.headerregistry.Address(
display_name='Správce OSMO',
addr_spec=config.MAIL_CONTACT,
)
- msg['Subject'] = 'OSMO – ' + subject
+ msg['Subject'] = subject
msg['Date'] = datetime.datetime.now()
- if add_footer:
- body += "\n" + ("=" * 76) + "\n"
- body += "Pokud nechcete tyto e-maily dostávat, vypněte si notifikace v nastavení\n"
- body += f"svého účtu na {settings_url()}."
-
msg.set_content(body, cte='quoted-printable')
mail_instead = getattr(config, 'MAIL_INSTEAD', None)
- if mail_instead is None:
- send_to = user.email
- else:
+ if mail_instead is not None:
send_to = mail_instead
sm = subprocess.Popen(
@@ -70,6 +64,17 @@ def send_user_email(user: db.User, subject: str, body: str, add_footer: bool = F
return True
+def send_user_email(user: db.User, subject: str, body: str, add_footer: bool = False) -> bool:
+ logger.info(f'Mail: "{subject}" -> {user.email} (#{user.user_id})')
+
+ if add_footer:
+ body += "\n" + ("=" * 76) + "\n"
+ body += "Pokud nechcete tyto e-maily dostávat, vypněte si notifikace v nastavení\n"
+ body += f"svého účtu na {settings_url()}."
+
+ return send_email(user.email, user.full_name(), 'OSMO – ' + subject, body)
+
+
def activate_url(token: str) -> str:
return config.WEB_ROOT + 'acct/activate?' + urllib.parse.urlencode({'token': token}, safe=':')
@@ -175,3 +180,36 @@ def send_grading_info_email(dest: db.User, round: db.Round) -> bool:
Váš OSMO
'''), add_footer=True)
+
+
+# Omezení rychlosti odesílání mailů o chybách děravým kyblíkem
+# (pozor, každý proces má svůj kyblík)
+error_tbf = token_bucket.Limiter(rate=1/300, capacity=10, storage=token_bucket.MemoryStorage())
+errors_supressed = 0
+
+
+def send_internal_error_email(place: str, attrs: Mapping[str, Optional[str]], exc_info: ExceptionInfo):
+ errors_to = getattr(config, 'MAIL_ERRORS_TO', None)
+ if errors_to is None:
+ return
+
+ global errors_supressed
+ if not error_tbf.consume(key="K"):
+ errors_supressed += 1
+ logger.info('Mail: Příliš mnoho chybových zpráv')
+ return
+
+ exc_type, exc_value, exc_traceback = exc_info
+
+ lines = [f'{key+":":10s} {val if val is not None else "-"}\n' for key, val in sorted(attrs.items())]
+
+ if exc_traceback is not None:
+ lines.append('\n')
+ lines.extend(traceback.format_exception(exc_type, exc_value, exc_traceback))
+
+ if errors_supressed > 0:
+ lines.append('\n')
+ lines.append(f'### Předchozích {errors_supressed} chyb neohlášeno\n')
+
+ logger.info('Mail: Zpráva o interní chybě')
+ send_email(errors_to, "", f'Interní chyba ({place})', "".join(lines))
diff --git a/mo/jobs/__init__.py b/mo/jobs/__init__.py
index 56e68e5f40d390c1781c6060f9520f296195d477..67bff065b55cdb901fbc81829f5a232097848721 100644
--- a/mo/jobs/__init__.py
+++ b/mo/jobs/__init__.py
@@ -4,14 +4,16 @@ from datetime import timedelta
import os
import shutil
from sqlalchemy import or_
+import sys
from typing import Optional, Dict, Callable, List
import mo
import mo.config as config
import mo.db as db
+import mo.email
import mo.rights
import mo.util
-from mo.util import logger
+from mo.util import logger, ExceptionInfo
def send_notify():
@@ -141,6 +143,8 @@ class TheJob:
job.out_file = None
sess.commit()
+ exc_info: ExceptionInfo = (None, None, None)
+
try:
self.gatekeeper = mo.rights.Gatekeeper(job.user)
self.expires_in_minutes = config.JOB_EXPIRATION
@@ -159,7 +163,8 @@ class TheJob:
logger.info(f'Job: Úspěšně dokončen job #{job.job_id} ({job.result})')
job.state = db.JobState.done
except Exception as e:
- logger.error(f'Job: Chyba při zpracování jobu #{job.job_id}: %s', e, exc_info=e)
+ exc_info = sys.exc_info()
+ logger.error(f'Job: Chyba při zpracování jobu #{job.job_id}: %s', e, exc_info=exc_info)
job.state = db.JobState.internal_error
job.result = 'Interní chyba, informujte prosím správce systému.'
@@ -168,6 +173,16 @@ class TheJob:
job.expires_at = job.finished_at + timedelta(minutes=self.expires_in_minutes)
sess.commit()
+ # Maily o interních chybách posíláme mimo transakci
+ if job.state == db.JobState.internal_error:
+ err_attrs = {
+ 'Job': f'#{job.job_id}',
+ 'Type': str(job.type),
+ 'User': f'#{job.user_id}',
+ 'Desc': job.description,
+ }
+ mo.email.send_internal_error_email(f'Job #{job.job_id}', err_attrs, exc_info)
+
def process_jobs():
sess = db.get_session()
diff --git a/mo/util.py b/mo/util.py
index ede00bc0375a1ee9d7da29ec5a6f2956528d8ce3..a5fec2e3d3831da784b87b1e6a653655e5ce0338 100644
--- a/mo/util.py
+++ b/mo/util.py
@@ -10,7 +10,8 @@ import os
import re
import secrets
import sys
-from typing import Any, Optional, NoReturn, Tuple, TypeVar, List
+from types import TracebackType
+from typing import Any, Optional, NoReturn, Tuple, TypeVar, List, Union
import mo
import mo.db as db
@@ -25,6 +26,9 @@ logger = logging.getLogger('mo')
logger.setLevel(logging.DEBUG)
logger.propagate = True
+# Typ vracený sys.exc_info()
+ExceptionInfo = Union[Tuple[type, BaseException, TracebackType], Tuple[None, None, None]]
+
def get_now() -> datetime.datetime:
return datetime.datetime.now(tz=dateutil.tz.UTC)
diff --git a/mo/web/__init__.py b/mo/web/__init__.py
index c15c9abb0777718a37e93c534c57efc11f8a125d..c7eb334ada58c52337d76f35b15eedf52aeb16fb 100644
--- a/mo/web/__init__.py
+++ b/mo/web/__init__.py
@@ -13,6 +13,7 @@ import werkzeug.formparser
import mo
import mo.config as config
+import mo.email
import mo.db as db
import mo.ext.assets
import mo.jobs
@@ -22,6 +23,19 @@ import mo.users
import mo.util
+class MOFlask(Flask):
+
+ def log_exception(self, exc_info: mo.util.ExceptionInfo) -> None:
+ err_attrs = {
+ 'Method': request.method,
+ 'Path': request.path,
+ 'Referrer': request.referrer,
+ 'Client': request.remote_addr,
+ }
+ mo.email.send_internal_error_email(request.path, err_attrs, exc_info)
+ super().log_exception(exc_info)
+
+
# Ohýbáme Flask, aby uploadované soubory ukládal do adresáře podle našeho přání,
# aby se pak daly zařadit mezi datové soubory prostým hardlinkováním. Za tímto účelem
# subclassujeme Request, aby použil subclassovaný FormDataParser, který použije naši
@@ -64,7 +78,7 @@ mo.config.DATA_DIR = os.path.abspath(mo.config.DATA_DIR)
static_dir = os.path.abspath('static')
# Aplikační objekt
-app = Flask(__name__, static_folder=static_dir)
+app = MOFlask(__name__, static_folder=static_dir)
app.config.from_object(config)
app.request_class = Request
db.flask_db = SQLAlchemy(app, metadata=db.metadata)
diff --git a/setup.py b/setup.py
index 41d797367b7b28eac72e96f6a48f0ce357441feb..418460678597f1bd540b50de2722ab61bc436aab 100644
--- a/setup.py
+++ b/setup.py
@@ -48,6 +48,7 @@ setuptools.setup(
'pyzbar',
'requests',
'sqlalchemy[mypy]',
+ 'token-bucket',
'uwsgidecorators',
# Používáme pro vývoj, ale aby je pylsp našel, musí být ve stejném virtualenvu
# jako ostatní knihovny.