Skip to content
Snippets Groups Projects
Commit be678729 authored by Martin Mareš's avatar Martin Mareš
Browse files

Import: Přepsání importovacího mechanismu

Druh importu je určen enumem, podle něj se automaticky vybere
podtřída, která implementuje konkrétní typ dat.

Na formáty souborů se používá FileFormat.

Zmizela spousta opakovaného kódu.
parent 889e0c28
No related branches found
No related tags found
1 merge request!15WIP: Reforma importů
This commit is part of merge request !15. Comments created here will be created in the context of that merge request.
from dataclasses import dataclass from dataclasses import dataclass
from enum import auto
import io import io
import re import re
from typing import List, Optional, Any, Dict, Callable, Type, TypeVar from typing import List, Optional, Any, Dict, Type
import mo.csv import mo.csv
from mo.csv import FileFormat
import mo.db as db import mo.db as db
import mo.rights import mo.rights
import mo.users import mo.users
import mo.util import mo.util
from mo.util import logger from mo.util import logger
RowType = TypeVar('RowType', bound=mo.csv.Row)
class ImportType(db.MOEnum):
participants = auto()
proctors = auto()
judges = auto()
@dataclass def friendly_name(self) -> str:
class ContestImportRow(mo.csv.Row): return import_type_names[self]
email: str = ""
krestni: str = ""
prijmeni: str = ""
kod_skoly: str = ""
rocnik: str = ""
rok_naroz: str = ""
kod_mista: str = ""
kod_oblasti: str = ""
@dataclass
class ProctorImportRow(mo.csv.Row):
email: str = ""
krestni: str = ""
prijmeni: str = ""
kod_mista: str = ""
@dataclass import_type_names = {
class JudgeImportRow(mo.csv.Row): ImportType.participants.name: 'účastníci',
email: str = "" ImportType.proctors.name: 'dozor',
krestni: str = "" ImportType.judges.name: 'opravovatelé',
prijmeni: str = "" }
kod_oblasti: str = ""
class Import: class Import:
...@@ -50,10 +38,15 @@ class Import: ...@@ -50,10 +38,15 @@ class Import:
cnt_new_participations: int = 0 cnt_new_participations: int = 0
cnt_new_roles: int = 0 cnt_new_roles: int = 0
# Interní: Co zrovna importujeme # Interní: Co a jak zrovna importujeme
user: db.User user: db.User
round: Optional[db.Round] round: Optional[db.Round]
contest: Optional[db.Contest] contest: Optional[db.Contest]
fmt: FileFormat
row_class: Type[mo.csv.Row]
row_example: mo.csv.Row
log_msg_prefix: str
log_event_name: str
# Interní: Stav importu # Interní: Stav importu
place_cache: Dict[str, db.Place] place_cache: Dict[str, db.Place]
...@@ -63,15 +56,18 @@ class Import: ...@@ -63,15 +56,18 @@ class Import:
new_user_ids: List[int] new_user_ids: List[int]
line_number: int = 0 line_number: int = 0
def __init__(self, user: db.User): def __init__(self):
self.errors = [] self.errors = []
self.user = user
self.rr = None self.rr = None
self.place_cache = {} self.place_cache = {}
self.school_place_cache = {} self.school_place_cache = {}
self.place_rights_cache = {} self.place_rights_cache = {}
self.new_user_ids = [] self.new_user_ids = []
def setup(self):
# Definováno odvozenými třídami
assert NotImplementedError()
def error(self, msg: str) -> Any: def error(self, msg: str) -> Any:
if self.line_number > 0: if self.line_number > 0:
msg = f"Řádek {self.line_number}: {msg}" msg = f"Řádek {self.line_number}: {msg}"
...@@ -180,9 +176,9 @@ class Import: ...@@ -180,9 +176,9 @@ class Import:
return self.error(f'Osoba již registrována s odlišným jménem {user.full_name()}') return self.error(f'Osoba již registrována s odlišným jménem {user.full_name()}')
if (user.is_admin or user.is_org) != is_org: if (user.is_admin or user.is_org) != is_org:
if is_org: if is_org:
return self.error(f'Nelze předefinovat účastníka na organizátora') return self.error('Nelze předefinovat účastníka na organizátora')
else: else:
return self.error(f'Nelze předefinovat organizátora na účastníka') return self.error('Nelze předefinovat organizátora na účastníka')
else: else:
user = db.User(email=email, first_name=krestni, last_name=prijmeni, is_org=is_org) user = db.User(email=email, first_name=krestni, last_name=prijmeni, is_org=is_org)
sess.add(user) sess.add(user)
...@@ -241,7 +237,7 @@ class Import: ...@@ -241,7 +237,7 @@ class Import:
elif len(pions) == 1: elif len(pions) == 1:
pion = pions[0] pion = pions[0]
if pion.place != place: if pion.place != place:
return self.error('Již se tohoto kola účastní v jiné oblasti') return self.error(f'Již se tohoto kola účastní v jiné oblasti ({pion.place.get_code()})')
else: else:
return self.error('Již se tohoto kola účastní ve vice oblastech, což by nemělo být možné') return self.error('Již se tohoto kola účastní ve vice oblastech, což by nemělo být možné')
...@@ -286,7 +282,121 @@ class Import: ...@@ -286,7 +282,121 @@ class Import:
) )
self.cnt_new_roles += 1 self.cnt_new_roles += 1
def import_contest_row(self, r: ContestImportRow): def import_row(self, r: mo.csv.Row):
# Definováno odvozenými třídami
assert NotImplementedError()
def log_start(self, path):
args = [f'user=#{self.user.user_id}', f'fmt={self.fmt.name}']
if self.round is not None:
args.append(f'round=#{self.round.round_id}')
if self.contest is not None:
args.append(f'contest=#{self.contest.contest_id}')
logger.info('Import: %s ze souboru %s: %s', self.log_msg_prefix, path, " ".join(args))
def log_end(self):
logger.info(f'Import: Hotovo (rows={self.cnt_rows} users={self.cnt_new_users} p-ants={self.cnt_new_participants} p-ions={self.cnt_new_participations} roles={self.cnt_new_roles})')
if self.contest is not None:
mo.util.log(
type=db.LogType.contest,
what=self.contest.contest_id,
details={'action': 'import'}
)
elif self.round is not None:
mo.util.log(
type=db.LogType.round,
what=self.round.round_id,
details={'action': self.log_event_name}
)
else:
assert False
def generic_import(self, path: str) -> bool:
try:
with open(path, encoding=self.fmt.get_charset()) as file:
rows: List[mo.csv.Row] = mo.csv.read(file=file, fmt=self.fmt, row_class=self.row_class)
except UnicodeDecodeError:
return self.error(f'Soubor není v kódování {self.fmt.get_charset()}')
except Exception as e:
return self.error(f'Chybná struktura tabulky: {e}')
self.line_number = 2
for row in rows:
self.cnt_rows += 1
self.import_row(row)
if len(self.errors) >= 100:
self.errors.append('Import přerušen pro příliš mnoho chyb')
break
self.line_number += 1
return len(self.errors) == 0
def notify_users(self):
# Projde všechny uživatele a těm, kteří ještě nemají nastavené heslo,
# ani nepožádali o jeho reset, pošle mail s odkazem na reset. Každého
# uživatele zpracováváme ve zvlášť transakci, aby se po chybě neztratila
# informace o tom, že už jsme nějaké maily rozeslali.
sess = db.get_session()
for uid in self.new_user_ids:
u = sess.query(db.User).get(uid)
if u and not u.password_hash and not u.reset_at:
token = mo.users.ask_reset_password(u)
sess.commit()
mo.util.send_new_account_email(u, token)
else:
sess.rollback()
def get_template(self) -> str:
out = io.StringIO()
mo.csv.write(file=out, fmt=self.fmt, row_class=self.row_class, rows=[self.row_example])
return out.getvalue()
def run(self, path: str) -> bool:
self.log_start(path)
if not self.generic_import(path):
logger.info('Import: Rollback')
db.get_session().rollback()
return False
self.log_end()
db.get_session().commit()
self.notify_users()
return True
@dataclass
class ContestImportRow(mo.csv.Row):
email: str = ""
krestni: str = ""
prijmeni: str = ""
kod_skoly: str = ""
rocnik: str = ""
rok_naroz: str = ""
kod_mista: str = ""
kod_oblasti: str = ""
class ContestImport(Import):
row_class = ContestImportRow
row_example = ContestImportRow(
email="nekdo@example.org",
krestni="Pokusný",
prijmeni="Králík",
kod_skoly="#3333",
rocnik="1/8",
rok_naroz="2000",
)
log_msg_prefix = 'Účastníci'
log_event_name = 'import'
def setup(self):
assert self.round is not None
def import_row(self, r: mo.csv.Row):
assert isinstance(r, ContestImportRow)
num_prev_errs = len(self.errors) num_prev_errs = len(self.errors)
email = self.parse_email(r.email) email = self.parse_email(r.email)
krestni = self.parse_name(r.krestni) krestni = self.parse_name(r.krestni)
...@@ -315,9 +425,36 @@ class Import: ...@@ -315,9 +425,36 @@ class Import:
return return
contest = self.obtain_contest(oblast) contest = self.obtain_contest(oblast)
if contest is None:
return
self.find_or_create_participation(user, contest, misto) self.find_or_create_participation(user, contest, misto)
def import_proctor_row(self, r: ProctorImportRow):
@dataclass
class ProctorImportRow(mo.csv.Row):
email: str = ""
krestni: str = ""
prijmeni: str = ""
kod_mista: str = ""
class ProctorImport(Import):
row_class = ProctorImportRow
row_example = ProctorImportRow(
email='nekdo@example.org',
krestni='Pokusný',
prijmeni='Králík',
kod_mista='#3333',
)
log_msg_prefix = 'Dozor'
log_event_name = 'import-proctors'
def setup(self):
assert self.round is not None
def import_row(self, r: mo.csv.Row):
assert isinstance(r, ProctorImportRow)
num_prev_errs = len(self.errors) num_prev_errs = len(self.errors)
email = self.parse_email(r.email) email = self.parse_email(r.email)
krestni = self.parse_name(r.krestni) krestni = self.parse_name(r.krestni)
...@@ -339,7 +476,31 @@ class Import: ...@@ -339,7 +476,31 @@ class Import:
self.add_role(user, misto, db.RoleType.dozor) self.add_role(user, misto, db.RoleType.dozor)
def import_judge_row(self, r: JudgeImportRow):
@dataclass
class JudgeImportRow(mo.csv.Row):
email: str = ""
krestni: str = ""
prijmeni: str = ""
kod_oblasti: str = ""
class JudgeImport(Import):
row_class = JudgeImportRow
row_example = JudgeImportRow(
email='nekdo@example.org',
krestni='Pokusný',
prijmeni='Králík',
kod_oblasti='B',
)
log_msg_prefix = 'Opravovatelé'
log_event_name = 'import-judges'
def setup(self):
assert self.round is not None
def import_row(self, r: mo.csv.Row):
assert isinstance(r, JudgeImportRow)
num_prev_errs = len(self.errors) num_prev_errs = len(self.errors)
email = self.parse_email(r.email) email = self.parse_email(r.email)
krestni = self.parse_name(r.krestni) krestni = self.parse_name(r.krestni)
...@@ -357,124 +518,28 @@ class Import: ...@@ -357,124 +518,28 @@ class Import:
return return
contest = self.obtain_contest(oblast, allow_none=True) contest = self.obtain_contest(oblast, allow_none=True)
place = contest.place if contest else None place = contest.place if contest else db.get_root_place()
self.add_role(user, place, db.RoleType.opravovatel) if not self.check_rights(place):
return self.error(f'K místu "{place.get_code()}" nemáte práva na správu soutěže')
def generic_import(self, path: str, row_class: Type[RowType], process_row: Callable[['Import', RowType], None]) -> bool:
try:
with open(path) as file:
rows: List[RowType] = mo.csv.read(file=file, dialect='excel', row_class=row_class)
except UnicodeDecodeError:
return self.error('Soubor není v kódování UTF-8')
except Exception as e:
return self.error(f'Chybná struktura tabulky: {e}')
self.line_number = 2
for row in rows:
self.cnt_rows += 1
process_row(self, row)
if len(self.errors) >= 100:
self.errors.append('Import přerušen pro příliš mnoho chyb')
break
self.line_number += 1
if self.errors:
logger.info('Import: Rollback')
return False
else:
logger.info(f'Import: Hotovo (rows={self.cnt_rows} users={self.cnt_new_users} p-ants={self.cnt_new_participants} p-ions={self.cnt_new_participations} roles={self.cnt_new_roles})')
return True
def import_contest(self, round: db.Round, contest: Optional[db.Contest], path: str) -> bool:
self.round = round
self.contest = contest
logger.info(f'Import: Účastníci ze souboru {path}: uid={self.user.user_id} round=#{round.round_id}')
if not self.generic_import(path, ContestImportRow, Import.import_contest_row):
db.get_session().rollback()
return False
mo.util.log(
type=db.LogType.contest,
what=contest.contest_id,
details={'action': 'import'}
)
db.get_session().commit()
mo.util.log(
type=db.LogType.round,
what=round.round_id,
details={'action': 'import'}
)
db.get_session().commit()
self.notify_users()
return True
def import_proctors(self, round: db.Round, path: str) -> bool:
self.round = round
self.contest = None
logger.info(f'Import: Dozor ze souboru {path}: uid={self.user.user_id}')
if not self.generic_import(path, ProctorImportRow, Import.import_proctor_row):
return False
mo.util.log(
type=db.LogType.round,
what=round.round_id,
details={'action': 'import-proctors'}
)
db.get_session().commit()
self.notify_users()
return True
def import_judges(self, round: db.Round, contest: Optional[db.Contest], path: str) -> bool:
self.round = round
self.contest = contest
logger.info(f'Import: Opravovatelé ze souboru {path}: uid={self.user.user_id}')
if not self.generic_import(path, JudgeImportRow, Import.import_judge_row):
return False
mo.util.log(
type=db.LogType.round,
what=round.round_id,
details={'action': 'import-judges'}
)
db.get_session().commit()
self.notify_users() self.add_role(user, place, db.RoleType.opravovatel)
return True
def notify_users(self):
# Projde všechny uživatele a těm, kteří ještě nemají nastavené heslo,
# ani nepožádali o jeho reset, pošle mail s odkazem na reset. Každého
# uživatele zpracováváme ve zvlášť transakci, aby se po chybě neztratila
# informace o tom, že už jsme nějaké maily rozeslali.
sess = db.get_session() def create_import(user: db.User, type: ImportType, fmt: FileFormat, round: Optional[db.Round] = None, contest: Optional[db.Contest] = None):
for uid in self.new_user_ids: imp: Import
u = sess.query(db.User).get(uid) if type == ImportType.participants:
if u and not u.password_hash and not u.reset_at: imp = ContestImport()
token = mo.users.ask_reset_password(u) elif type == ImportType.proctors:
sess.commit() imp = ProctorImport()
mo.util.send_new_account_email(u, token) elif type == ImportType.judges:
imp = JudgeImport()
else: else:
sess.rollback() assert False, "Neznámý typ importu"
def generic_template(row_class: Type[RowType]) -> str:
out = io.StringIO()
mo.csv.write(file=out, dialect='excel', row_class=row_class, rows=[])
return out.getvalue()
def contest_template() -> str:
return generic_template(ContestImportRow)
def proctor_template() -> str:
return generic_template(ProctorImportRow)
imp.user = user
imp.round = round
imp.contest = contest
imp.fmt = fmt
imp.setup()
def judge_template() -> str: return imp
return generic_template(JudgeImportRow)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment