Skip to content
Snippets Groups Projects
Select Git revision
  • master
1 result

LICENCE

Blame
  • This project is licensed under the GNU Affero General Public License v3.0. Learn more
    submit.py 11.61 KiB
    # Implementace jobů pracujících se submity
    
    from dataclasses import dataclass
    import os
    import re
    import shutil
    from sqlalchemy import and_
    from sqlalchemy.orm import joinedload
    from tempfile import NamedTemporaryFile
    from typing import List, Optional
    import unicodedata
    import werkzeug.utils
    import zipfile
    
    import mo.db as db
    from mo.jobs import TheJob, job_handler, job_file_path
    from mo.submit import Submitter, SubmitException
    from mo.util import logger, data_dir
    from mo.util_format import inflect_number, inflect_by_number, data_size
    
    
    def schedule_download_submits(paper_ids: List[int], description: str, for_user: db.User, want_subdirs: bool):
        the_job = TheJob()
        job = the_job.create(db.JobType.download_submits, for_user)
        job.description = description
        job.in_json = {'papers': paper_ids, 'want_subdirs': want_subdirs}
        the_job.submit()
    
    
    @job_handler(db.JobType.download_submits)
    def handle_download_submits(the_job: TheJob):
        """Zazipuje zadané papíry.
    
        Vstupní JSON:
            { 'papers': [ seznam paper_id ke stažení ],
              'want_feedback': true/false,
            }
    
        Výstupní JSON:
            null
        """
    
        job = the_job.job
        assert job.in_json is not None
        ids: List[int] = job.in_json['papers']  # type: ignore
        want_subdirs: bool = job.in_json['want_subdirs']  # type: ignore
    
        sess = db.get_session()
        papers = (sess.query(db.Paper, db.User, db.Task.code, db.Place)
                  .select_from(db.Paper)
                  .filter(db.Paper.paper_id.in_(ids))
                  .join(db.User, db.User.user_id == db.Paper.for_user)
                  .join(db.Task, db.Task.task_id == db.Paper.for_task)
                  .join(db.Participation, db.Participation.user_id == db.Paper.for_user)
                  .join(db.Contest, and_(db.Contest.master_contest_id == db.Participation.contest_id, db.Contest.round_id == db.Task.round_id))
                  .join(db.Place, db.Place.place_id == db.Contest.place_id)
                  .all())
        papers.sort(key=lambda p: (p[1].sort_key(), p[2]))
    
        temp_file = NamedTemporaryFile(suffix='.zip', dir=data_dir('tmp'), mode='w+b')
        logger.debug('Job: Vytvářím archiv %s', temp_file.name)
    
        cnt = 0
        with zipfile.ZipFile(temp_file, mode='w') as zip:
            for p, u, task_code, place in papers:
                cnt += 1
                full_name = u.full_name()
                ascii_name = (unicodedata.normalize('NFD', full_name)
                              .encode('ascii', 'ignore')
                              .decode('utf-8'))
                fn = f'{task_code}_{cnt:04d}_{u.user_id}_{p.paper_id}_{ascii_name}.pdf'
                fn = werkzeug.utils.secure_filename(fn)
                if want_subdirs:
                    fn = f'{place.get_code()}/{fn}'
                logger.debug('Job: Přidávám %s', fn)
                zip.write(filename=os.path.join(data_dir('submits'), p.file_name or p.orig_file_name),
                          arcname=fn)
    
        job.out_file = the_job.attach_file(temp_file.name, '.zip')
        out_size = temp_file.tell()
        job.result = 'Celkem ' + inflect_number(cnt, 'soubor', 'soubory', 'souborů') + ', ' + data_size(out_size)
        temp_file.close()
    
    
    def schedule_upload_feedback(round: db.Round, tmp_file: str, description: str, for_user: db.User,
                                 only_contest: Optional[db.Contest],
                                 only_site: Optional[db.Place],
                                 only_region: Optional[db.Place],
                                 only_task: Optional[db.Task]):
        the_job = TheJob()
        job = the_job.create(db.JobType.upload_feedback, for_user)
        job.description = description
        job.in_json = {
            'round_id': round.round_id,
            'only_contest_id': only_contest.contest_id if only_contest is not None else None,
            'only_site_id': only_site.place_id if only_site is not None else None,
            'only_region_id': only_region.place_id if only_region is not None else None,
            'only_task_id': only_task.task_id if only_task is not None else None,
        }
        job.in_file = the_job.attach_file(tmp_file, '.zip')
        the_job.submit()
    
    
    @dataclass
    class UploadFeedback:
        file_name: str
        task_code: str
        user_id: int
        task: Optional[db.Task] = None
        user: Optional[db.User] = None
        tmp_name: Optional[str] = None
    
    
    def parse_feedback_name(name: str) -> Optional[UploadFeedback]:
        name = os.path.basename(name)
    
        # Formát jmen generovaný dávkovým stahováním
        m = re.match(r'(?P<task>[^_]+)_(?P<order>\d+)_(?P<user_id>\d+)_', name)
        if m:
            return UploadFeedback(
                file_name=name,
                task_code=m['task'],
                user_id=int(m['user_id']),
            )
    
        # Formát jmen, pod kterými se ukládají jednotlivě stahovaná řešení
        m = re.match(r'(?P<task>[^_]+)_(reseni|opravene)_(?P<user_id>\d+)_', name)
        if m:
            return UploadFeedback(
                file_name=name,
                task_code=m['task'],
                user_id=int(m['user_id']),
            )
    
        # Formát jmen, pod kterými se dříve ukládala jednotlivě stahovaná řešení
        m = re.match(r'(?P<task>.+)-(reseni|opravene)-(?P<paper_id>\d+)\.', name)
        if m:
            paper = db.get_session().query(db.Paper).get(int(m['paper_id']))
            if paper:
                return UploadFeedback(
                    file_name=name,
                    task_code=m['task'],
                    user_id=paper.for_user,
                )
    
        return None
    
    
    @job_handler(db.JobType.upload_feedback)
    def handle_upload_feedback(the_job: TheJob):
        """Uploaduje opravená řešení.
    
        Vstupní JSON:
            { 'round_id': <id>,
              'only_task_id': <id_or_null>,
              'only_contest_id': <id_or_null>,
              'only_site_id': <id_or_null>,
            }
    
        Výstupní JSON:
            null
        """
    
        job = the_job.job
        assert job.in_file is not None
        in_json = job.in_json
        assert in_json is not None
        round_id: int = in_json['round_id']  # type: ignore
        only_contest_id: Optional[int] = in_json['only_contest_id']  # type: ignore
        only_site_id: Optional[int] = in_json['only_site_id']  # type: ignore
        only_region_id: Optional[int] = in_json['only_region_id']  # type: ignore
        only_task_id: Optional[int] = in_json['only_task_id']  # type: ignore
    
        sess = db.get_session()
        round = sess.query(db.Round).get(round_id)
        assert round is not None
    
        if only_region_id is not None:
            only_region = sess.query(db.Place).get(only_region_id)
            assert only_region is not None
        else:
            only_region = None
    
        files: List[UploadFeedback] = []
    
        def parse_zip(in_path: str):
            try:
                with zipfile.ZipFile(in_path, mode='r') as zip:
                    contents = zip.infolist()
                    for item in contents:
                        if not item.is_dir():
                            fb = parse_feedback_name(item.filename)
                            if fb:
                                tmp_file = NamedTemporaryFile(dir=data_dir('tmp'), mode='w+b', delete=False)
                                logger.debug(f'Job: Extrahuji {item.filename} do {tmp_file.name}')
                                with zip.open(item) as item_file:
                                    shutil.copyfileobj(item_file, tmp_file)
                                tmp_file.close()
                                fb.tmp_name = tmp_file.name
                                files.append(fb)
                            else:
                                the_job.error(f'Nerozpoznáno jméno souboru {item.filename}')
            except zipfile.BadZipFile:
                the_job.error('Chybný formát souboru. Je to opravdu ZIP?')
    
        def resolve_tasks(files):
            task_dict = {f.task_code: None for f in files}
            tasks = sess.query(db.Task).filter_by(round=round).filter(db.Task.code.in_(task_dict.keys())).all()
            for task in tasks:
                task_dict[task.code] = task
            for code, task in task_dict.items():
                if task is None:
                    the_job.error(f'Neznámá úloha {code}')
            for f in files:
                f.task = task_dict[f.task_code]
                if f.task is None:
                    the_job.error(f'{f.file_name}: Neznámá úloha {code}')
                elif only_task_id is not None and f.task.task_id != only_task_id:
                    the_job.error(f'{f.file_name}: Tato úloha nebyla vybraná k nahrávání')
    
        def resolve_users(files):
            user_dict = {f.user_id: None for f in files}
            contest_dict = {f.user_id: None for f in files}
            site_id_dict = {f.user_id: None for f in files}
    
            rows = (sess.query(db.User, db.Participation, db.Contest)
                    .select_from(db.Participation)
                    .join(db.User, db.User.user_id == db.Participation.user_id)
                    .join(db.Contest, db.Contest.master_contest_id == db.Participation.contest_id)
                    .filter(db.Contest.round == round)
                    .filter(db.Participation.user_id.in_(user_dict.keys()))
                    .options(joinedload(db.Contest.place))
                    .all())
    
            user_rights = {}
            for user, pion, contest in rows:
                user_dict[user.user_id] = user
                contest_dict[user.user_id] = contest
                site_id_dict[user.user_id] = pion.place_id
                rr = the_job.gatekeeper.rights_for_contest(contest)
                user_rights[user.user_id] = rr.can_upload_feedback()
    
            for f in files:
                f.user = user_dict[f.user_id]
                if not f.user:
                    the_job.error(f'{f.file_name}: Neznámý účastník #{f.user_id}')
                elif only_contest_id is not None and contest_dict[f.user_id].contest_id != only_contest_id:
                    the_job.error(f'{f.file_name}: Účastník leží mimo vybranou soutěž')
                elif only_site_id is not None and site_id_dict[f.user_id] != only_site_id:
                    the_job.error(f'{f.file_name}: Účastník leží mimo vybrané soutěžní místo')
                elif only_region is not None and not the_job.gatekeeper.is_ancestor_of(only_region, contest_dict[f.user_id].place):
                    the_job.error(f'{f.file_name}: Účastník leží mimo vybraný region')
                elif not user_rights[f.user_id]:
                    the_job.error(f'{f.file_name}: K tomuto účastníkovi nemáte dostatečná oprávnění')
    
        def process_file(fb: UploadFeedback) -> bool:
            assert fb.user and fb.task
            paper = db.Paper(
                for_user_obj=fb.user,
                task=fb.task,
                type=db.PaperType.feedback,
                uploaded_by_obj=job.user,
            )
            try:
                assert fb.tmp_name
                smtr = Submitter()
                smtr.submit_paper(paper, fb.tmp_name)
                sess.add(paper)
    
                sol = (sess.query(db.Solution)
                       .filter_by(task=fb.task, user=fb.user)
                       .with_for_update()
                       .one())
                sol.final_feedback_obj = paper
    
                sess.commit()
                return True
            except SubmitException as e:
                the_job.error(f'{fb.file_name}: {e}')
                return False
    
        cnt_good = 0
        parse_zip(job_file_path(job.in_file))
    
        if not the_job.errors:
            resolve_tasks(files)
            resolve_users(files)
    
        if not the_job.errors:
            for f in files:
                if process_file(f):
                    cnt_good += 1
    
        for f in files:
            if f.tmp_name is not None:
                os.unlink(f.tmp_name)
    
        job.result = (inflect_by_number(cnt_good, 'Nahrán', 'Nahrány', 'Nahráno')
                      + f' {cnt_good} z '
                      + inflect_number(len(files), 'souboru', 'souborů', 'souborů'))
        if the_job.errors:
            job.result += (', '
                           + inflect_by_number(len(the_job.errors), 'nastala', 'nastaly', 'nastalo')
                           + ' '
                           + inflect_number(len(the_job.errors), 'chyba', 'chyby', 'chyb'))