diff --git a/constraints.txt b/constraints.txt index 6499845736ec89bccc04f150c16a7a4af437fb62..0fba56cb92a81b77389414ce6b606e64914e0228 100644 --- a/constraints.txt +++ b/constraints.txt @@ -22,6 +22,7 @@ pkg-resources==0.0.0 psycopg2==2.8.6 pycparser==2.20 pyparsing==2.4.7 +PyPDF2==1.26.0 python-dateutil==2.8.1 python-poppler==0.2.2 pytz==2020.5 diff --git a/db/upgrade-20210701.sql b/db/upgrade-20211026.sql similarity index 93% rename from db/upgrade-20210701.sql rename to db/upgrade-20211026.sql index b226fba273bc7edc886805b4ae6497b37685d9f5..ff4d76cb0dd7c8bf3bfe967ebd218b6a5bf84006 100644 --- a/db/upgrade-20210701.sql +++ b/db/upgrade-20211026.sql @@ -2,6 +2,7 @@ SET ROLE 'mo_osmo'; ALTER TYPE job_type ADD VALUE 'create_protocols'; ALTER TYPE job_type ADD VALUE 'process_scans'; +ALTER TYPE job_type ADD VALUE 'sort_scans'; ALTER TYPE job_state ADD VALUE 'preparing'; diff --git a/mo/db.py b/mo/db.py index a756eb8bf1e94d11dec586e4a8b6c65f04383056..a822d3902068c4b3bd1adb80f2ef13c4bea47a46 100644 --- a/mo/db.py +++ b/mo/db.py @@ -689,6 +689,7 @@ class JobType(MOEnum): upload_feedback = auto() create_protocols = auto() process_scans = auto() + sort_scans = auto() class JobState(MOEnum): @@ -835,6 +836,12 @@ class ScanPage(Base): user = relationship('User') task = relationship('Task') + def is_empty(self) -> bool: + return self.seq_id == SCAN_PAGE_EMPTY and self.user_id is None and self.task_id is None + + def is_ok(self) -> bool: + return self.user_id is not None and self.user_id > 0 and self.task_id is not None and self.task_id > 0 and self.seq_id >= 0 + # Speciální seq_id ve ScanPage SCAN_PAGE_FIX = -1 diff --git a/mo/jobs/protocols.py b/mo/jobs/protocols.py index 0bf16b94d719476d9ff6f34f92b0897c9ccdb7c3..9fb4774cca5d8033add718da070ac3cbd7d360a7 100644 --- a/mo/jobs/protocols.py +++ b/mo/jobs/protocols.py @@ -11,7 +11,8 @@ from sqlalchemy import delete from sqlalchemy.orm import joinedload from sqlalchemy.orm.query import Query import subprocess -from typing import List, Optional +from typing import Dict, List, Optional, Tuple +import PyPDF2 import mo import mo.config as config @@ -174,7 +175,7 @@ def handle_create_protocols(the_job: TheJob): # -def schedule_process_scans(contest: db.Contest, site: Optional[db.Place], for_user: db.User, tasks: List[db.Task], in_file_names: List[str]): +def schedule_process_scans(contest: db.Contest, site: Optional[db.Place], for_user: db.User, tasks: List[db.Task], in_file_names: List[str]) -> int: place = site or contest.place the_job = TheJob() @@ -197,6 +198,7 @@ def schedule_process_scans(contest: db.Contest, site: Optional[db.Place], for_us 'in_files': in_files, } the_job.submit() + return the_job.job_id @dataclass @@ -270,7 +272,7 @@ def handle_process_scans(the_job: TheJob): return 'Neznámý účastník' sp.user_id = user_id sp.task_id = tasks_by_code[fields[2]].task_id - sp.seq_id = 0 + sp.seq_id = 1 return None return 'Neznamý formát kódu' @@ -352,3 +354,167 @@ def _process_scan_file(args: ScanJobArgs) -> List[ScanJobPage]: logger.debug(f'Scan: Strana #{page_nr}: {qr}') return output + + +# +# Job sort_scans: Roztřídí nascanované protokoly a založí jednotlivá řešení +# +# Je to recyklovaný process_scans job. +# +# Vstupní JSON (beze změny z process_scans): +# { 'contest_id': ID contestu, +# 'site_id': ID soutěžního místa nebo none, +# 'task_ids': [task_id, ...], +# 'in_files': [názvy vstupních souborů] +# } +# +# Výstupní JSON: +# null + + +def schedule_sort_scans(job_id: int, for_user: db.User) -> int: + # Znovupoužijeme starý job, jen mu změníme typ + the_job = TheJob(job_id) + job = the_job.load() + assert job is not None + + sess = db.get_session() + contest = sess.query(db.Contest).options(joinedload(db.Contest.round)).get(job.in_json['contest_id']) + assert contest is not None + + job.type = db.JobType.sort_scans + job.created_at = mo.now + job.expires_at = None + job.user = for_user + job.description = f'Rozdělení již roztříděných scanů {contest.round.round_code_short()}' + + the_job.submit() + return the_job.job_id + + +class SortScansPaper: + paper: db.Paper + pages: List[db.ScanPage] + + def __init__(self, paper: db.Paper) -> None: + self.paper = paper + self.pages = [] + + def filename(self) -> str: + return f"out_{self.paper.task.task_id}_{self.paper.for_user_obj.user_id}.pdf" + + +@job_handler(db.JobType.sort_scans) +def handle_sort_scans(the_job: TheJob): + job = the_job.job + assert job.in_json is not None + contest_id = job.in_json['contest_id'] # type: ignore + site_id = job.in_json['site_id'] # type: ignore + task_ids = job.in_json['task_ids'] # type: ignore + in_files: List[str] = job.in_json['in_files'] # type: ignore + + sess = db.get_session() + contest = sess.query(db.Contest).options(joinedload(db.Contest.round)).get(contest_id) + assert contest is not None + + user_ids = set(u[0] for u in _get_user_id_query(contest, site_id).all()) + users = sess.query(db.User).filter(db.User.user_id.in_(user_ids)).all() + users_by_id = {u.user_id: u for u in users} + + tasks = sess.query(db.Task).filter(db.Task.task_id.in_(task_ids)).all() + tasks_by_id = {t.task_id: t for t in tasks} + + pages = sess.query(db.ScanPage).filter_by(job_id=the_job.job_id).all() + + sols = sess.query(db.Solution).filter( + db.Solution.task_id.in_(task_ids), + db.Solution.user_id.in_(user_ids), + ).all() + + # Jelikož se plánujeme zamyslet na dlouhou dobu, uzavřeme databázovou session. + sess.commit() + + # Nejdříve si vše naplánujeme + sols_map = {(sol.task_id, sol.user_id): sol for sol in sols} + sols_to_create: Dict[Tuple[int, int], db.Solution] = {} + papers: Dict[Tuple[int, int], SortScansPaper] = {} + + for p in pages: + if p.is_empty(): + continue + task = tasks_by_id[p.task_id] + user = users_by_id[p.user_id] + + index = (p.task_id, p.user_id) + if index in sols_map: + sol = sols_map[index] + elif index in sols_to_create: + sol = sols_to_create[index] + else: + sol = db.Solution(task=task, user=user) + sols_to_create[index] = sol + + if index not in papers: + papers[index] = SortScansPaper(db.Paper( + task=task, + for_user_obj=user, + uploaded_by_obj=job.user, + type=db.PaperType.solution, + note='Z hromadného skenování', + )) + + papers[index].pages.append(p) + + for index in papers: + papers[index].pages.sort(key=lambda p: p.seq_id) + + # Poté poskládáme výsledné PDF soubory + readers: Dict[int, PyPDF2.PdfFileReader] = {} + for index in papers: + paper = papers[index] + writer = PyPDF2.PdfFileWriter() + for p in paper.pages: + if p.file_nr not in readers: + readers[p.file_nr] = PyPDF2.PdfFileReader(job.file_path(in_files[p.file_nr]), strict=False) + # Přihodíme správnou stránku na výstup + writer.addPage( + readers[p.file_nr].getPage(p.page_nr) + ) + # Zapíšeme vše do správného souboru + with open(job.file_path(paper.filename()), 'wb') as f: + writer.write(f) + + # ... a uložíme je na správné místo + submitter = mo.submit.Submitter() + + for index in papers: + paper = papers[index] + try: + submitter.submit_paper(paper.paper, job.file_path(paper.filename())) + except mo.submit.SubmitException as e: + logger.error(f"Paper task:{paper.paper.for_task}, user:{paper.paper.for_user}: {e}") + + # Nakonec vše uložíme do databáze + for index in sols_to_create: + sol = sols_to_create[index] + sess.add(sol) + mo.util.log( + type=db.LogType.participant, + what=sol.user.user_id, + details={ + 'action': 'solution-created', + 'task': sol.task.task_id, + }, + ) + + for index in papers: + paper = papers[index] + sess.add(paper.paper) + if index in sols_map: + sols_map[index].final_submit_obj = paper.paper + elif index in sols_to_create: + sols_to_create[index].final_submit_obj = paper.paper + + sess.commit() + job.result = 'Celkem ' + mo.util_format.inflect_number(len(papers), 'roztříděné řešení', 'roztříděná řešení', 'roztříděných řešení') + the_job.expires_in_minutes = config.JOB_EXPIRATION_LONG diff --git a/mo/web/__init__.py b/mo/web/__init__.py index 0079dbfe289d2726f93abd0bd5122d89bc889bdd..8d5271effe5fccd8aceb78edd41a283ce7f62f43 100644 --- a/mo/web/__init__.py +++ b/mo/web/__init__.py @@ -116,6 +116,7 @@ app.assets.add_assets([ 'mo.css', 'js/news-reloader.js', 'js/osmo.js', + 'js/autocomplete.js' ]) diff --git a/mo/web/org_contest.py b/mo/web/org_contest.py index 4a70b4cafa082be95202151ca0c2e36e1444957f..71837e02f15ebff4b8c488d76348e1a6995d19a7 100644 --- a/mo/web/org_contest.py +++ b/mo/web/org_contest.py @@ -1,6 +1,9 @@ +import os from flask import render_template, g, redirect, url_for, flash, request +from flask.helpers import send_file from flask_wtf import FlaskForm import flask_wtf.file +import json import locale from markupsafe import Markup from sqlalchemy import func, and_, select @@ -1649,11 +1652,6 @@ class GenProtoForm(FlaskForm): gen_protos = wtforms.SubmitField('Vytvořit protokoly') -class ProcessScansForm(FlaskForm): - files = wtforms.MultipleFileField('Soubory PDF se scany', validators=[validators.required()]) - process_scans = wtforms.SubmitField('Zpracovat scany') - - @app.route('/org/contest/c/<int:ct_id>/protocols', methods=('GET', 'POST')) @app.route('/org/contest/c/<int:ct_id>/site/<int:site_id>/protocols', methods=('GET', 'POST')) def org_contest_protocols(ct_id: int, site_id: Optional[int] = None): @@ -1664,20 +1662,13 @@ def org_contest_protocols(ct_id: int, site_id: Optional[int] = None): class GPF(GenProtoForm): pass - class PSF(ProcessScansForm): - pass - tasks = db.get_session().query(db.Task).filter_by(round=round).order_by(db.Task.code).all() for t in tasks: setattr(GPF, f'task_{t.task_id}', wtforms.BooleanField(t.code, default=True)) - setattr(PSF, f'task_{t.task_id}', wtforms.BooleanField(t.code, default=True)) gen_form = GPF() gen_task_fields = [f for f in gen_form if f.name.startswith('task_')] - proc_form = PSF() - proc_task_fields = [f for f in proc_form if f.name.startswith('task_')] - if gen_form.validate_on_submit() and gen_form.gen_protos.data: mo.jobs.protocols.schedule_create_protocols( contest, site, g.user, @@ -1688,21 +1679,246 @@ def org_contest_protocols(ct_id: int, site_id: Optional[int] = None): flash('Výroba prototokolů zahájena.', 'success') return redirect(url_for('org_jobs')) + return render_template( + 'org_contest_protocols.html', + ctx=ctx, + gen_form=gen_form, + gen_task_fields=gen_task_fields, + ) + + +class ProcessScansForm(FlaskForm): + files = wtforms.MultipleFileField('Soubory PDF se scany', validators=[validators.required()]) + process_scans = wtforms.SubmitField('Zpracovat scany') + + +@app.route('/org/contest/c/<int:ct_id>/scans', methods=('GET', 'POST')) +@app.route('/org/contest/c/<int:ct_id>/site/<int:site_id>/scans', methods=('GET', 'POST')) +def org_contest_scans(ct_id: int, site_id: Optional[int] = None): + ctx = get_context(ct_id=ct_id, site_id=site_id) + round, contest, site = ctx.round, ctx.contest, ctx.site + assert contest + + sess = db.get_session() + + class PSF(ProcessScansForm): + pass + + tasks = sess.query(db.Task).filter_by(round=round).order_by(db.Task.code).all() + tasks_map: Dict[int,db.Task] = {} + for t in tasks: + setattr(PSF, f'task_{t.task_id}', wtforms.BooleanField(t.code, default=True)) + tasks_map[t.task_id] = t + + proc_form = PSF() + proc_task_fields = [f for f in proc_form if f.name.startswith('task_')] + if proc_form.validate_on_submit() and proc_form.process_scans.data: files = request.files.getlist(proc_form.files.name) - mo.jobs.protocols.schedule_process_scans( + job_id = mo.jobs.protocols.schedule_process_scans( contest, site, g.user, tasks=[t for t in tasks if getattr(proc_form, f'task_{t.task_id}').data], in_file_names=[f.stream.name for f in files], ) - flash('Zpracování scanů zahájeno.', 'success') - return redirect(url_for('org_jobs')) + flash('Zpracování skenů zahájeno. Vyčkejte chvíli, než budou připraveny k roztřídění, poté je roztřiďte', 'success') + return redirect(ctx.url_for('org_contest_scans_process', job_id=job_id)) + + jobs_query = sess.query(db.Job).filter_by(type=db.JobType.process_scans) + if not g.user.is_admin: + jobs_query = jobs_query.filter_by(user=g.user) + jobs = [] + for job in jobs_query.all(): + if 'contest_id' not in job.in_json or job.in_json['contest_id'] != id: + continue + if site_id is not None and ('site_id' not in job.in_json or job.in_json['site_id'] != site_id): + continue + + job.tasks = [] + for task_id in job.in_json['task_ids']: + if task_id in tasks_map: + job.tasks.append(tasks_map[task_id]) + + jobs.append(job) return render_template( - 'org_contest_protocols.html', + 'org_contest_scans.html', ctx=ctx, - gen_form=gen_form, - gen_task_fields=gen_task_fields, proc_form=proc_form, proc_task_fields=proc_task_fields, + jobs=jobs, + ) + + +class ScanProcessForm(FlaskForm): + data = wtforms.HiddenField() + save = wtforms.SubmitField() + process_all = wtforms.SubmitField() + + +@app.route('/org/contest/c/<int:ct_id>/scans/<int:job_id>', methods=('GET', 'POST')) +@app.route('/org/contest/c/<int:ct_id>/site/<int:site_id>/scans/<int:job_id>', methods=('GET', 'POST')) +def org_contest_scans_process(ct_id: int, job_id: int, site_id: Optional[int] = None): + ctx = get_context(ct_id=ct_id, site_id=site_id, right_needed=Right.upload_solutions) + contest = ctx.contest + assert contest + + sess = db.get_session() + + # Získáme job a zkontrolujeme, že je to správný job a máme na něj práva + job = sess.query(db.Job).get(job_id) + if not job or job.type != db.JobType.process_scans: + flash('Dávka naskenovaných úloh nenalezena, skeny již byly pravděpodobně zpracovány nebo smazány.') + return redirect(ctx.url_for('org_contest_scans')) + if not g.user.is_admin and g.user.user_id != job.user_id: + raise werkzeug.exceptions.Forbidden() + + pages = sess.query(db.ScanPage).filter_by(job_id=job_id).order_by('file_nr', 'page_nr').all() + tasks = sess.query(db.Task).filter(db.Task.task_id.in_(job.in_json['task_ids'])).order_by('code').all() + pion_query = sess.query(db.Participation).filter( + db.Participation.contest == contest, + db.Participation.state == db.PartState.active ) + if site_id is not None: + pion_query = pion_query.filter_by(place_id=site_id) + pions: List[db.Participation] = pion_query.options(joinedload(db.Participation.user)).all() + + # Vytvoříme si mapu stránek načtených z databáze, mapu uživatelů a úloh + p_map: Dict[Tuple[int, int], db.ScanPage] = {(p.file_nr, p.page_nr): p for p in pages} + pion_map: Dict[int, db.Participation] = {pion.user_id: pion for pion in pions} + task_map: Dict[int, db.Task] = {task.task_id: task for task in tasks} + + # Uložení stránek z GUI + self_url = ctx.url_for('org_contest_scans_process', job_id=job_id) + process_form = ScanProcessForm() + if process_form.validate_on_submit() and process_form.save.data: + data: List = json.loads(process_form.data.data) + if len(data) == 0: + flash('Žádné změny k uložení', 'info') + return redirect(self_url) + count = 0 + for p in data: + if 'file_nr' not in p or 'page_nr' not in p: + flash('Schází file_nr a page_nr, nelze zpracovat!', 'danger') + return redirect(self_url) + index = (p['file_nr'], p['page_nr']) + if 'user_id' not in p or 'task_id' not in p or 'seq_id' not in p: + flash(f'Schází user_id, task_id nebo seq_id u skenu {index}, nelze zpracovat!', 'danger') + return redirect(self_url) + + user_id = p['user_id'] + task_id = p['task_id'] + seq_id = p['seq_id'] + if seq_id is None: + seq_id = 0 + + if index not in p_map: + flash(f'Stránka {index} není v databázi, nebyla uložena', 'danger') + continue + if user_id is not None and user_id not in pion_map: + flash(f'Neexistující uživatel {user_id} u stránky {index}', 'danger') + continue + if task_id is not None and task_id not in task_map: + flash(f'Neexistující úloha {task_id} u stránky {index}', 'danger') + continue + if seq_id < -4: + flash(f'Schází číslo stránky u stránky {index}', 'danger') + continue + + pp = p_map[index] + if pp.user_id != user_id or pp.task_id != task_id or pp.seq_id != seq_id: + pp.user_id = user_id + pp.task_id = task_id + pp.seq_id = seq_id + sess.add(pp) + count += 1 + # neukládáme do databázového logu, zpracování skenů je jen dočasné + # a finální uložení založí nová řešení + sess.commit() + flash('Uloženy změny u ' + inflect_number(count, 'skenu', 'skenů', 'skenů'), 'success') + return redirect(self_url) + + # Kontrola stavu + warnings: List[str] = [] + errors: List[str] = [] + seq_map: Dict[Tuple[int, int, int], db.ScanPage] = {} + sol_map: Dict[Tuple[int, int], bool] = {} + for p in pages: + if not p.is_ok() and not p.is_empty(): + errors.append(f'Sken {p.file_nr}/{p.page_nr} není označen jako prázdný ani není správně určen!') + if p.is_ok(): + index = (p.task_id, p.user_id, p.seq_id) + if index in seq_map: + pp = seq_map[index] + t = task_map[p.task_id] + u = pion_map[p.user_id].user + errors.append( + f'Duplicita - Sken {p.file_nr}/{p.page_nr} je označený jako {p.seq_id}. stránka řešení' + +f' úlohy {t.code} {t.name} soutěžícího {u.full_name()} ale' + +f' sken {pp.file_nr}/{pp.page_nr} je označený úplně stejně!' + ) + else: + seq_map[index] = p + sol_map[(p.task_id, p.user_id)] = True + for task in tasks: + for pion in pions: + index = (task.task_id, pion.user_id) + if index not in sol_map: + warnings.append(f'Mezi skeny není řešení úlohy {task.code} {task.name} od {pion.user.full_name()}') + + if process_form.validate_on_submit() and process_form.process_all.data: + if len(errors) > 0: + flash('Nelze zpracovat, dokud kontrola vrací chyby. Nejdříve je opravte.') + return redirect(self_url) + mo.jobs.protocols.schedule_sort_scans(job_id, for_user=g.user) + flash('Skeny zařazeny pro zpracování, během několika chvil se řešení roztřídí k soutěžícím.', 'success') + return redirect(url_for('org_jobs')) + + def png_small(page: db.ScanPage) -> str: + return ctx.url_for( + 'org_contest_scans_file', job_id=job_id, + file=f'p-{page.file_nr:02d}-{page.page_nr:04d}-small.png' + ) + + def png_full(page: db.ScanPage) -> str: + return ctx.url_for( + 'org_contest_scans_file', job_id=job_id, + file=f'p-{page.file_nr:02d}-{page.page_nr:04d}-full.png' + ) + + return render_template( + 'org_contest_scans_process.html', + ctx=ctx, + job=job, + tasks=tasks, + pions=pions, + pages=pages, + process_form=process_form, + errors=errors, + warnings=warnings, + png_small=png_small, + png_full=png_full, + ) + + +@app.route('/org/contest/c/<int:ct_id>/scans/<int:job_id>/file/<file>') +@app.route('/org/contest/c/<int:ct_id>/site/<int:site_id>/scans/<int:job_id>/file/<file>') +def org_contest_scans_file(ct_id: int, job_id: int, file: str, site_id: Optional[int] = None): + get_context(ct_id=ct_id, site_id=site_id, right_needed=Right.upload_solutions) + + sess = db.get_session() + + # Získáme job a zkontrolujeme, že je to správný job a máme na něj práva + job = sess.query(db.Job).get(job_id) + if not job or job.type != db.JobType.process_scans: + raise werkzeug.exceptions.NotFound() + if not g.user.is_admin and g.user.user_id != job.user_id: + raise werkzeug.exceptions.Forbidden() + + if os.path.basename(file) != file: + raise werkzeug.exceptions.Forbidden() + + path = os.path.join(job.dir_path(), file) + if os.path.isfile(path): + return send_file(path) + else: + raise werkzeug.exceptions.NotFound() diff --git a/mo/web/org_users.py b/mo/web/org_users.py index 7b9f5c44060c2a72452d9f154c1d5ae46398f10b..7ebfb32f30d6b656d6e421a9385c7437eb753c4f 100644 --- a/mo/web/org_users.py +++ b/mo/web/org_users.py @@ -118,7 +118,6 @@ def org_users(): if schools is not None: q = rr.restrict_user_query(q, schools) - # print(str(q)) (count, q) = filter.apply_limits(q, pagesize=50) users = q.all() diff --git a/mo/web/templates/org_contest.html b/mo/web/templates/org_contest.html index 53d44d0640d2452d1cfcb878d57250e14bd6a5b5..d27f3268314a358d8d156efaf4816adef354bc7f 100644 --- a/mo/web/templates/org_contest.html +++ b/mo/web/templates/org_contest.html @@ -61,6 +61,9 @@ <a class="btn btn-primary" href='{{ ctx.url_for('org_score') }}'>Výsledky</a> {% endif %} <a class="btn btn-default" href='{{ ctx.url_for('org_contest_protocols') }}'>Protokoly</a> + {% if can_upload %} + <a class="btn btn-default" href='{{ ctx.url_for('org_contest_scans') }}'>Zpracování skenů</a> + {% endif %} {% if state == RoundState.preparing and round.seq > 1 %} <a class="btn btn-primary" href='{{ ctx.url_for('org_contest_advance') }}'>Postup z minulého kola</a> {% endif %} diff --git a/mo/web/templates/org_contest_protocols.html b/mo/web/templates/org_contest_protocols.html index cf1b8a53b846881d538f9d2b484bfebf29a254d8..9512197c2373d5b8f913edb2fe29ec58609bb6ee 100644 --- a/mo/web/templates/org_contest_protocols.html +++ b/mo/web/templates/org_contest_protocols.html @@ -35,22 +35,4 @@ formulář je opatřen unikátním QR kódem. FIXME: Dovysvětlit. {{ field(gen_form.gen_protos) }} </form> -<h3>Zpracování scanů</h3> - -<form action="" method=POST class="form form-horizontal" role="form" enctype='multipart/form-data'> - {{ proc_form.csrf_token }} - {% if proc_task_fields %} - <div class='form-group'> - <label class='control-label col-lg-3' for='{{ proc_task_fields[0].id }}'>Úlohy</label> - <div class='col-lg-7'> - {% for f in proc_task_fields %} - {{ wtf.form_field(f) }} - {% endfor %} - </div> - </div> - {% endif %} - {{ field(proc_form.files) }} - {{ field(proc_form.process_scans) }} -</form> - {% endblock %} diff --git a/mo/web/templates/org_contest_scans.html b/mo/web/templates/org_contest_scans.html new file mode 100644 index 0000000000000000000000000000000000000000..d8fc942f34f3c9a71cdd335f4ecacbf4ab757d22 --- /dev/null +++ b/mo/web/templates/org_contest_scans.html @@ -0,0 +1,75 @@ +{% extends "base.html" %} +{% import "bootstrap/wtf.html" as wtf %} + +{% block title %} +Zpracování skenů pro {{ ctx.round.name|lower }} kategorie {{ ctx.round.category }} +{% endblock %} +{% block breadcrumbs %} +{{ ctx.breadcrumbs(action="Zpracování skenů") }} +{% endblock %} +{% block body %} + +<h3>Nahrání skenů</h3> + +<p>Zde je možné nahrát PDF s naskenovanými protokoly soutěžících. Vyberte úlohy, +které naskenované PDF obsahuje, a soubor nahrajte. Po krátkém zpracování +zobrazí systém možnost roztřídit jednotlivé naskenované stránky a potvrdit +jejich přiřazení jednotlivým soutěžícím. + +{% macro field(f) %} +{{ wtf.form_field(f, form_type='horizontal', horizontal_columns=('lg', 3, 7), button_map={'gen_protos': 'primary', 'process_scans': 'primary'}) }} +{% endmacro %} + +<form action="" method=POST class="form form-horizontal" role="form" enctype='multipart/form-data'> + {{ proc_form.csrf_token }} + {% if proc_task_fields %} + <div class='form-group'> + <label class='control-label col-lg-3' for='{{ proc_task_fields[0].id }}'>Úlohy</label> + <div class='col-lg-7'> + {% for f in proc_task_fields %} + {{ wtf.form_field(f) }} + {% endfor %} + </div> + </div> + {% endif %} + {{ field(proc_form.files) }} + {{ field(proc_form.process_scans) }} +</form> + +<h3>Nahráno k roztřídění</h3> +{% if jobs %} +<p>Smazat dávky lze v <a href="{{ url_for('org_jobs') }}">přehledu všech dávek</a>.</p> +<table class=data> + <thead> + <tr> + <th>Datum + <th>Úlohy + <th>Stav + {% if g.user.is_admin %}<th>Vlastník{% endif %} + <th>Akce + </tr> + </thead> + {% for job in jobs %} + <tr class="job-{{ job.state.name }}"> + <td>{{ job.created_at|timeformat }} + <td>{% for task in job.tasks %}{{ task.code }} {% endfor %} + <td> + {% if job.state == JobState.done %} + Připraveno k roztřídění + {% else %} + {{ job.state.friendly_name() }} + {% endif %} + {% if g.user.is_admin %}<td>{{ job.user|user_link }}{% endif %} + <td><div class="btn-group"> + {% if job.state == JobState.done %} + <a class="btn btn-xs btn-primary" href="{{ url_for('org_contest_scans_process', id=contest.contest_id, site_id=site_id, job_id=job.job_id) }}">Zpracovat skeny</a> + {% endif %} + </div> + </tr> + {% endfor %} +</table> +{% else %} +<p>Žádné nahrané skeny k roztřídění.</p> +{% endif %} + +{% endblock %} diff --git a/mo/web/templates/org_contest_scans_process.html b/mo/web/templates/org_contest_scans_process.html new file mode 100644 index 0000000000000000000000000000000000000000..60d3ecfca79adc0cb00e2f80813cfebe902b6427 --- /dev/null +++ b/mo/web/templates/org_contest_scans_process.html @@ -0,0 +1,438 @@ +{% extends "base.html" %} +{% import "bootstrap/wtf.html" as wtf %} + +{% block head %} + <script src="{{ asset_url('js/autocomplete.js') }}" type="text/javascript"></script> +{% endblock %} + +{% block title %} +Třídění skenů pro {{ ctx.round.name|lower }} kategorie {{ ctx.round.category }} +{% endblock %} +{% block breadcrumbs %} +{{ ctx.breadcrumbs(action="Třídění skenů") }} +{% endblock %} +{% block body %} + +{% if job.state in [JobState.preparing, JobState.ready, JobState.running] %} +<p>Právě běží zpracování, vyčkejte prosím několik okamžiků (stránka se sama obnoví).</p> + +<script> +setTimeout(function () { location.reload(1); }, 10_000); +</script> +{% elif job.state == JobState.failed %} +<p>Zpracování selhalo, více detailů naleznete v <a href="{{ url_for('org_job', id=job.job_id) }}">detailu dávky</a>.</p> +{% else %} + +<p>Napravo můžete klikáním vybírat jednotlivé naskenované stránky a pomocí vrchních políček je přiřazovat jednotlivým úlohám a soutěžícím. Pokud +je nějaké řešení přes více stránek, musí na sebe navazovat číslování stránek. Až bude vše správně zatříděné, můžete aktuální stav uložit +tlačítkem <b>[Uložit]</b>. Poté můžete celou dávku odeslat ke zpracování pomocí <b>[Ukončit a zpracovat]</b> (řešení se uloží k řešitelům, +tuto akci nelze vzít zpět). + +{% if errors or warnings %} +<div class="collapsible"> + {% set error_count = errors | count %} + {% set warning_count = warnings | count %} + + <input type="checkbox" class="toggle" id="messages-toggle" {% if errors %}checked{% endif %}> + <label for="messages-toggle" class="toggle"> + Kontrola třídění ( + {%- if warning_count > 0 %}{{ warning_count }} varování{% endif -%} + {%- if warning_count > 0 and error_count > 0 %}, {% endif %} + {%- if error_count > 0 %}<span class="error">{{ error_count|inflected('chyba', 'chyby', 'chyb') }}</span>{% endif -%} + ) + </label> + <div class="collapsible-inner"> + <div class="alert alert-warning"> + <ul> + {% for msg in warnings %} + <li>Varování: {{ msg }} + {% endfor %} + {% for msg in errors %} + <li class="error"><b>Chyba: {{ msg }}</b> + {% endfor %} + </ul></div> + </div> + </div> +</div> +{% else %} +<p><span class="text-success">Skeny zkontrolovány, nenalezeny žádné chyby ani varování.</span></p> +{% endif %} + +<form method="post" class="btn-group pull-right"> + <input class="btn btn-primary" type="submit" name="save" value="Uložit" id="save-button" onclick="saveData()"> + {{ process_form.csrf_token }} + <input type="hidden" name="data" id="save-data"> + {% if errors %} + <input class="btn btn-primary" type="submit" value="Ukončit a zpracovat" title="Kontrola ukázala chyby, nejdříve je opravte a uložte pomocí [Uložit]" disabled> + {% else %} + <input class="btn btn-primary" type="submit" name="process_all" value="Ukončit a zpracovat" id="process-all-button" onclick="return confirm('Opravud ukončit a zpracovat? Nelze vzít zpět.');"> + {% endif %} +</form> + +<style> + #sort_wrapper { + display: grid; + grid-template-columns: auto 350px; + grid-template-rows: 100px auto; + gap: 5px; + } + #sort_wrapper div.controls { + border: 1px #bbbbbb solid; + background-color: #e5e5e5; + border-radius: 5px; + padding: 5px 10px; + } + #sort_wrapper div.scan { + position: relative; + } + #sort_wrapper div.scan #scan_loader { + position: absolute; + width: 100%; + height: 100%; + opacity: 0; + background-color: black; + transition: 0.5s; + } + #sort_wrapper div.scan img { + max-width: 100%; + max-height: 100%; + } + #sort_wrapper div.pages { + max-height: 80vh; + grid-row: span 2; + overflow-y: scroll; + position: relative; + } + #sort_wrapper div.pages table { + width: 100%; + margin: 0px; + } + #sort_wrapper div.pages table thead { + position: sticky; + top: 0px; + } + #sort_wrapper div.pages table tr { + cursor: pointer; + } + #sort_wrapper div.pages table tr.ok { background-color: lightgreen; } + #sort_wrapper div.pages table tr.empty { background-color: #aabbff; } + #sort_wrapper div.pages table tr.error { background-color: #ff7777; } + #sort_wrapper div.pages table tr.changed { background-color: yellow; } + #sort_wrapper div.pages table tr.active { + color: white; + background-color: black; + } + #sort_wrapper div.pages table tr.active.changed { color: yellow; } +</style> + +<p><b>Ovládání klávesnicí:</b> <b><code>↑</code></b> a <b><code>↓</code></b> – posun ve skenech, <b><code>e</code></b> – přepnutí se na editaci, +<b><code>[esc]</code></b> – vyskočení z editačního políčka při editaci, <b><code>r</code></b> – reset (vrácení) změn u konkrétního skenu, +<b><code>x</code></b> – nastavit stránku jako prázdnou, <b><code>f</code></b> – nastavit stránku jako pokračování minulé. + +<div id="sort_wrapper"> + <div class="controls form-horizontal"> + <span tabindex=1 onfocus="document.getElementById('seq_input').focus();"></span> + <div class="form-group"> + <label class="col-sm-2 control-label" for="user_input">Soutěžící:</label> + <div class="col-sm-10 autocomplete"> + <input tabindex=2 class="form-control" type="text" id="user_input"> + </div> + </div> + <div class="form-group"> + <label class="col-sm-2 control-label" for="task_input">Úloha:</label> + <div class="col-sm-4 autocomplete"> + <input tabindex=3 class="form-control" type="text" id="task_input"> + </div> + <label class="col-sm-2 control-label" for="seq_input">Stránka:</label> + <div class="col-sm-4 autocomplete"> + <input tabindex=4 class="form-control" type="number" min="1" id="seq_input"> + </div> + </div> + <span tabindex=5 onfocus="document.getElementById('user_input').focus();"></span> + </div> + <div class="pages"> + <table class="data"> + <thead> + <th title="Číslo skenu"># + <th>Úloha + <th>Soutěžící + <th title="Stránka">St. + </thead> + <tbody id="pages_rows"> + </tbody> + </table> + </div> + <div class="scan"> + <div id="scan_loader"></div> + <img id="scan_img"> + </div> +</div> + +<script type="text/javascript"> +var tasks = [ +{% for task in tasks %} + { id: {{ task.task_id }}, code: "{{ task.code }}", name: "{{ task.name }}" }, +{% endfor %} +]; + +var users = [ +{% for pion in pions %} + { id: {{ pion.user_id }}, name: "{{ pion.user.first_name }} {{ pion.user.last_name }}" }, +{% endfor %} +]; + +var pages = [ +{% for page in pages %} + { + file_nr: {{ page.file_nr }}, + page_nr: {{ page.page_nr }}, + user_id: {{ page.user_id if page.user_id else 'null' }}, + orig_user_id: {{ page.user_id if page.user_id else 'null' }}, + task_id: {{ page.task_id if page.task_id else 'null' }}, + orig_task_id: {{ page.task_id if page.task_id else 'null' }}, + seq_id: {{ page.seq_id }}, + orig_seq_id: {{ page.seq_id }}, + img_full: "{{ png_full(page) }}", + img_small: "{{ png_small(page) }}", + }, +{% endfor %} +]; + +var tasks_map = {}; +var tasks_autocomplete = []; +var users_map = {}; +var users_autocomplete = []; + +// Global elements +var tbody = document.getElementById('pages_rows'); +var img = document.getElementById('scan_img'); +var loader = document.getElementById('scan_loader'); +var rows = tbody.rows; +var user_input = document.getElementById('user_input'); +var task_input = document.getElementById('task_input'); +var seq_input = document.getElementById('seq_input'); +var process_all_button = document.getElementById('process-all-button'); +var save_data_field = document.getElementById('save-data'); + +PAGE_FIX = -1; +PAGE_EMPTY = -2; +PAGE_CONTINUE = -3; +PAGE_UFO = -4; + +function isChanged(p) { return (p.orig_user_id != p.user_id || p.orig_task_id != p.task_id || p.orig_seq_id != p.seq_id); } +function isOk(p) { return (p.user_id && p.task_id && p.seq_id >= 0); } +function isEmpty(p) { return (p.user_id == null && p.task_id == null && p.seq_id == PAGE_EMPTY); } +function isError(p) { return (!isEmpty(p) && !isOk(p)); } +function anyChanged(p) { return pages.some(isChanged); } + +function beforeUnload(e) { + e.preventDefault(); + return e.returnValue = "Byly provedeny editace, opuštěním stránky je ztratíte. Skutečně opustit stránku?"; +} + +function refreshButtons() { + if (anyChanged()) { + if (process_all_button) { + process_all_button.disabled = true; + process_all_button.title = "Neuložené změny, nejdříve je uložte vedlejším tlačítkem"; + } + window.onbeforeunload = beforeUnload; + } else { + if (process_all_button) { + process_all_button.disabled = false; + process_all_button.title = ''; + } + window.onbeforeunload = null; + } +} + +function saveData() { + data = pages.filter(isChanged).map(p => ({file_nr: p.file_nr, page_nr: p.page_nr, user_id: p.user_id, task_id: p.task_id, seq_id: p.seq_id})); + save_data_field.value = JSON.stringify(data); + window.onbeforeunload = null; +} + +function setRow(i) { + p = pages[i]; + row = rows[i]; + + row.innerHTML = ''; + row.className = ''; + row.title = ''; + cellScan = row.insertCell(-1); + cellTask = row.insertCell(-1); + cellUser = row.insertCell(-1); + cellSeq = row.insertCell(-1); + + cellScan.innerHTML = p.file_nr + '/' + p.page_nr; + if (p.task_id) { + task = tasks_map[p.task_id]; + cellTask.innerHTML = task.code + ' ' + task.name; + } + if (p.user_id) { + user = users_map[p.user_id]; + cellUser.innerHTML = user.name; + } + switch (p.seq_id) { + case PAGE_EMPTY: + cellSeq.innerHTML = ''; + break + case PAGE_FIX: + case PAGE_CONTINUE: + case PAGE_UFO: + cellSeq.innerHTML = '?'; + break; + default: + cellSeq.innerHTML = p.seq_id; + } + + if (isOk(p)) { + row.classList.add('ok'); + } + if (isEmpty(p)) { + row.classList.add('empty'); + row.title += "Prázdná stránka, nebude se ukládat\n"; + } + if (isError(p)) { + row.classList.add('error'); + row.title += "Neprázdná nerozpoznaná stránka, potřebuje opravit\n"; + } + + if (isChanged(p)) { + row.classList.add('changed'); + row.title += "Neuložené změny\n"; + } + refreshButtons(); +} + +var activeRow = 0; +function selectRow(i) { + page = pages[i]; + user = null; + + user_input.blur(); + + rows[activeRow].classList.remove('active'); + rows[i].classList.add('active'); + activeRow = i; + + loader.style.opacity = 0.7; + img.onload = function() { + loader.style.opacity = 0; + if (page.user_id) { + user_input.value = users_map[page.user_id].name; + } else { + user_input.value = ''; + } + + if (page.task_id) { + task = tasks_map[page.task_id]; + task_input.value = task.code + ' ' + task.name; + } else { + task_input.value = ''; + } + if (page.seq_id >= 0) { + seq_input.value = page.seq_id; + } else { + seq_input.value = ''; + } + } + img.src = page.img_full; +} + +function refreshActiveRow() { + setRow(activeRow); + selectRow(activeRow); +} + +// Activate autocomplete on two fields + prevent catching keys on third one +autocomplete(task_input, tasks_autocomplete, callback=function (id) { + pages[activeRow].task_id = parseInt(id); + refreshActiveRow(); +}); +autocomplete(user_input, users_autocomplete, callback=function (id) { + pages[activeRow].user_id = parseInt(id); + refreshActiveRow(); +}); +seq_input.addEventListener("focus", closeAllAutocomplete); +seq_input.addEventListener("keydown", function(e) { + if (e.keyCode == 27) { // escape + this.blur(); + } + e.stopPropagation(); +}); +seq_input.addEventListener("blur", function(e) { + x = parseInt(seq_input.value); + if (Number.isNaN(x)) x = 0 + pages[activeRow].seq_id = x; + refreshActiveRow(); +}); + +// Keyboard control +function checkKey(e) { + e = e || window.event; + if (e.keyCode == '38') { + if (activeRow > 0) { + selectRow(activeRow - 1); + rows[activeRow].scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + } else if (e.keyCode == '40') { + if (activeRow < pages.length - 1) { + selectRow(activeRow + 1); + rows[activeRow].scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + } else if (e.key == 'e') { + user_input.focus(); + } else if (e.key == 'x') { + page = pages[activeRow]; + page.user_id = null; + page.task_id = null; + page.seq_id = PAGE_EMPTY; + refreshActiveRow(); + } else if (e.key == 'r') { + page = pages[activeRow]; + page.user_id = page.orig_user_id; + page.task_id = page.orig_task_id; + page.seq_id = page.orig_seq_id; + refreshActiveRow(); + } else if (e.key == 'f') { + if (activeRow > 1) { + prev = pages[activeRow-1]; + page = pages[activeRow]; + if (isOk(prev)) { + page.user_id = prev.user_id; + page.task_id = prev.task_id; + page.seq_id = prev.seq_id + 1; + refreshActiveRow(); + } else { + alert('Nelze nastavit jako pokračování předchozí stránky, je nekompletní.'); + } + } + } else { + return; + } + e.preventDefault(); +} +document.onkeydown = checkKey; + +// Start everything :) + +for (task of tasks) { + tasks_map[task.id] = task; + tasks_autocomplete.push([task.id, task.code + ' ' + task.name]); +} +for (user of users) { + users_map[user.id] = user; + users_autocomplete.push([user.id, user.name]); +} +for (page of pages) { tbody.insertRow(); } +for (let i = 0; i < pages.length; i++) { + setRow(i); + rows[i].onclick = function() { selectRow(i); } +} + +selectRow(0); +</script> + +{% endif %} +{% endblock %} diff --git a/setup.py b/setup.py index 07bbeabc137f6b2f7a5ee2074fbf8d1035eb3311..ee19d0210052efc632391c8733583b815aa28199 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setuptools.setup( # Udržujte prosím seřazené 'Flask', 'Flask-WTF', + 'PyPDF2', 'WTForms', 'bcrypt', 'bleach', diff --git a/static/js/autocomplete.js b/static/js/autocomplete.js new file mode 100644 index 0000000000000000000000000000000000000000..7148dc8813b6e08b97796eb5f3cbcac3c5e7e664 --- /dev/null +++ b/static/js/autocomplete.js @@ -0,0 +1,111 @@ +// Adapted from https://www.w3schools.com/howto/howto_js_autocomplete.asp +function autocomplete(inp, arr, callback=null, max=10) { + var currentFocus; + var listCount; + + function do_autocomplete(e) { + var a, b, i, val = this.value; + closeAllLists(); + + currentFocus = -1; + a = document.createElement("DIV"); + a.setAttribute("id", this.id + "autocomplete-list"); + a.setAttribute("class", "autocomplete-items"); + this.parentNode.appendChild(a); + + listCount = 0; + for (i = 0; i < arr.length; i++) { + var key, text; + if (Array.isArray(arr[i])) { + key = arr[i][0]; text = arr[i][1]; + } else { + key = arr[i]; text = arr[i]; + } + + /*check if the item starts with the same letters as the text field value:*/ + //if (text.substr(0, val.length).toUpperCase() == val.toUpperCase()) { + index = text.toUpperCase().indexOf(val.toUpperCase()); + if (index != -1) { + listCount++; + b = document.createElement("DIV"); + b.innerHTML = text.substr(0, index) + "<strong>" + text.substr(index, val.length) + "</strong>" + text.substr(index + val.length) + b.innerHTML += "<input type='hidden' value='" + key + "'>"; + b.innerHTML += "<input type='hidden' value='" + text + "'>"; + b.addEventListener("click", function(e) { + key = this.getElementsByTagName("input")[0].value; + text = this.getElementsByTagName("input")[1].value; + inp.value = text; + if (callback) { + callback(key); + } + closeAllLists(); + }); + a.appendChild(b); + + } + if (listCount == max) break; + } + }; + + inp.addEventListener("input", do_autocomplete); + inp.addEventListener("focus", do_autocomplete); + + inp.addEventListener("keydown", function(e) { + var x = document.getElementById(this.id + "autocomplete-list"); + if (x) x = x.getElementsByTagName("div"); + if (e.keyCode == 40) { // arrow down + currentFocus++; + addActive(x); + } else if (e.keyCode == 38) { // arrow up + currentFocus--; + addActive(x); + } else if (e.keyCode == 13) { // enter + e.preventDefault(); // do not submit the form + if (currentFocus == -1 && listCount > 0) { + currentFocus++; + } + if (currentFocus > -1) { + if (x) x[currentFocus].click(); // simulate a click on the "active" item + } + + } else if (e.keyCode == 27) { // escape + inp.blur(); + closeAllLists(); + } + e.stopPropagation(); + }); + + function addActive(x) { + if (!x) return false; + removeActive(x); + if (currentFocus >= x.length) currentFocus = 0; + if (currentFocus < 0) currentFocus = (x.length - 1); + x[currentFocus].classList.add("autocomplete-active"); + } + function removeActive(x) { + for (var i = 0; i < x.length; i++) { + x[i].classList.remove("autocomplete-active"); + } + } + function closeAllLists(elmnt) { + /* close all autocomplete lists in the document, except the one passed as an argument: */ + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + if (elmnt != x[i] && elmnt != inp && (!elmnt || elmnt.parentNode != x[i].parentNode)) { + x[i].parentNode.removeChild(x[i]); + } + } + } + // close all autocomplete lists when clicked elsewhere + document.addEventListener("click", function (e) { + closeAllLists(e.target); + }); +} + + +function closeAllAutocomplete() { + var x = document.getElementsByClassName("autocomplete-items"); + for (var i = 0; i < x.length; i++) { + x[i].parentNode.removeChild(x[i]); + } +} diff --git a/static/mo.css b/static/mo.css index 6796adad0767a50499d93370f949bafc347ac262..6f0c280b5d57ceb07ff4fddf10e60408a409c26a 100644 --- a/static/mo.css +++ b/static/mo.css @@ -416,3 +416,35 @@ div.message .msg-date { font-style: italic; color: #777; } + +/* Autocomplete for inputs */ +.autocomplete { + position: relative; + display: inline-block; +} + +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + border-bottom: none; + border-top: none; + z-index: 99; + /* position the autocomplete items to be the same width as the container */ + top: 100%; + left: 0; + right: 0; + margin: 0px 15px; +} +.autocomplete-items div { + padding: 5px; + cursor: pointer; + background-color: #fff; + border-bottom: 1px solid #d4d4d4; +} +.autocomplete-items div:hover { + background-color: #e9e9e9; +} +.autocomplete-active { + background-color: DodgerBlue !important; + color: #ffffff; +}