diff --git a/mo/jobs/submit.py b/mo/jobs/submit.py index 0fca647803c37c0292e6d35ce36245ad4017358f..a981f8d9ed2ee508100896f876a590c3858a4ba9 100644 --- a/mo/jobs/submit.py +++ b/mo/jobs/submit.py @@ -1,28 +1,34 @@ # Implementace jobů pracujících se submity +from dataclasses import dataclass import os +import re +import shutil from sqlalchemy.orm import joinedload from tempfile import NamedTemporaryFile -from typing import List +from typing import List, Optional import unicodedata import werkzeug.utils import zipfile -from mo.util import logger, data_dir import mo.db as db -from mo.jobs import TheJob, job_handler +from mo.jobs import TheJob, job_handler, job_file_path +import mo.rights +from mo.submit import Submitter, SubmitException +from mo.util import logger, data_dir +from mo.util_format import inflect_number, inflect_by_number def schedule_download_submits(paper_ids: List[int], description: str, for_user: db.User): - tj = TheJob() - job = tj.create(db.JobType.download_submits, for_user) + the_job = TheJob() + job = the_job.create(db.JobType.download_submits, for_user) job.description = description job.in_json = {'papers': paper_ids} - tj.submit() + the_job.submit() @job_handler(db.JobType.download_submits) -def handle_download_submits(tj: TheJob): +def handle_download_submits(the_job: TheJob): """Zazipuje zadané papíry. Vstupní JSON: @@ -32,10 +38,9 @@ def handle_download_submits(tj: TheJob): null """ - job = tj.job + job = the_job.job assert job.in_json is not None - # FIXME: Typování JSONu... - ids = job.in_json['papers'] + ids = job.in_json['papers'] # type: ignore sess = db.get_session() papers = (sess.query(db.Paper) @@ -43,10 +48,10 @@ def handle_download_submits(tj: TheJob): .options(joinedload(db.Paper.for_user_obj), joinedload(db.Paper.task)) .all()) + papers.sort(key=lambda p: (p.for_user_obj.sort_key(), p.task.code)) temp_file = NamedTemporaryFile(suffix='.zip', dir=data_dir('tmp'), mode='w+b') logger.debug('Job: Vytvářím archiv %s', temp_file.name) - # FIXME: Setřídit soubory cnt = 0 with zipfile.ZipFile(temp_file, mode='w') as zip: @@ -62,10 +67,175 @@ def handle_download_submits(tj: TheJob): zip.write(filename=os.path.join(data_dir('submits'), p.file_name), arcname=fn) - job.out_file = tj.attach_file(temp_file.name, '.zip') + job.out_file = the_job.attach_file(temp_file.name, '.zip') + job.result = 'Celkem ' + inflect_number(cnt, 'soubor', 'soubory', 'souborů') temp_file.close() +def schedule_upload_feedback(round: db.Round, tmp_file: str, description: str, for_user: db.User): + the_job = TheJob() + job = the_job.create(db.JobType.upload_feedback, for_user) + job.description = description + job.in_json = {'round_id': round.round_id} + job.in_file = the_job.attach_file(tmp_file, '.zip') + the_job.submit() + + +@dataclass +class UploadFeedback: + file_name: str + task_code: str + user_id: int + task: Optional[db.Task] = None + user: Optional[db.User] = None + tmp_name: Optional[str] = None + + +def parse_feedback_name(name: str) -> Optional[UploadFeedback]: + name = os.path.basename(name) + m = re.match(r'(?P<task>[^_]+)_(?P<order>\d+)_(?P<user_id>\d+)_', name) + if m: + return UploadFeedback( + file_name=name, + task_code=m['task'], + user_id=int(m['user_id']), + ) + else: + return None + + @job_handler(db.JobType.upload_feedback) -def handle_upload_feedback(tj: TheJob): - raise NotImplementedError() +def handle_upload_feedback(the_job: TheJob): + """Uploaduje opravená řešení. + + Vstupní JSON: + { 'round_id': <id> } + + Výstupní JSON: + null + """ + + job = the_job.job + assert job.in_json is not None + assert job.in_file is not None + round_id = job.in_json['round_id'] # type: ignore + + sess = db.get_session() + round = sess.query(db.Round).get(round_id) + assert round is not None + + files: List[UploadFeedback] = [] + + def parse_zip(in_path: str): + try: + with zipfile.ZipFile(in_path, mode='r') as zip: + contents = zip.infolist() + for item in contents: + if not item.is_dir(): + fb = parse_feedback_name(item.filename) + if fb: + tmp_file = NamedTemporaryFile(dir=data_dir('tmp'), mode='w+b', delete=False) + logger.debug(f'Job: Extrahuji {item.filename} do {tmp_file.name}') + with zip.open(item) as item_file: + shutil.copyfileobj(item_file, tmp_file) + tmp_file.close() + fb.tmp_name = tmp_file.name + files.append(fb) + else: + the_job.error(f'Nerozpoznáno jméno souboru {item.filename}') + except zipfile.BadZipFile: + the_job.error('Chybný formát souboru. Je to opravdu ZIP?') + + def resolve_tasks(files): + task_dict = {f.task_code: None for f in files} + tasks = sess.query(db.Task).filter_by(round=round).filter(db.Task.code.in_(task_dict.keys())).all() + for task in tasks: + task_dict[task.code] = task + for code, task in task_dict.items(): + if task is None: + the_job.error(f'Neznámá úloha {code}') + for f in files: + f.task = task_dict[f.task_code] + if f.task is None: + the_job.error(f'{f.file_name}: Neznámá úloha {code}') + + def resolve_users(files): + user_dict = {f.user_id: None for f in files} + rows = (sess.query(db.User, db.Participation, db.Contest) + .select_from(db.Participation) + .join(db.User, db.User.user_id == db.Participation.user_id) + .join(db.Contest, db.Contest.contest_id == db.Participation.contest_id) + .filter(db.Contest.round == round) + .filter(db.Participation.user_id.in_(user_dict.keys())) + .all()) + + rr = mo.rights.Rights(job.user) + rights_cache = {} + user_rights = {} + for user, pion, contest in rows: + user_dict[user.user_id] = user + if contest.contest_id not in rights_cache: + rr.get_for_contest(contest) + rights_cache[contest.contest_id] = ( + rr.have_right(mo.rights.Right.upload_submits) + or (rr.have_right(mo.rights.Right.upload_feedback) and round.state == db.RoundState.grading)) + user_rights[user.user_id] = rights_cache[contest.contest_id] + + for f in files: + f.user = user_dict[f.user_id] + if not f.user: + the_job.error(f'{f.file_name}: Neznámý účastník #{f.user_id}') + elif not user_rights[f.user_id]: + the_job.error(f'{f.file_name}: K tomuto účastníkovi nemáte dostatečná oprávnění') + + def process_file(fb: UploadFeedback) -> bool: + assert fb.user and fb.task + paper = db.Paper( + for_user_obj=fb.user, + task=fb.task, + type=db.PaperType.feedback, + uploaded_by_obj=job.user, + ) + try: + assert fb.tmp_name + smtr = Submitter() + smtr.submit_paper(paper, fb.tmp_name) + fb.tmp_name = None # Soubor byl přesunut, není ho třeba mazat + sess.add(paper) + + sol = (sess.query(db.Solution) + .filter_by(task=fb.task, user=fb.user) + .with_for_update() + .one()) + sol.final_feedback_obj = paper + + sess.commit() + return True + except SubmitException as e: + the_job.error(f'{fb.file_name}: {e}') + return False + + cnt_good = 0 + parse_zip(job_file_path(job.in_file)) + + if not the_job.errors: + resolve_tasks(files) + resolve_users(files) + + if not the_job.errors: + for f in files: + if process_file(f): + cnt_good += 1 + + for f in files: + if f.tmp_name is not None: + os.unlink(f.tmp_name) + + job.result = (inflect_by_number(cnt_good, 'Nahrán', 'Nahrány', 'Nahráno') + + f' {cnt_good} z ' + + inflect_number(len(files), 'souboru', 'souborů', 'souborů')) + if the_job.errors: + job.result += (', ' + + inflect_by_number(len(the_job.errors), 'nastala', 'nastaly', 'nastalo') + + ' ' + + inflect_number(len(the_job.errors), 'chyba', 'chyby', 'chyb'))