diff --git a/docs/architecture.md b/docs/architecture.md index 5f6fa53b..919aa8c9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,6 +57,8 @@ Edition-scoped event operations. Everything hangs off `hackathon`. - Judge roster (`judge`, `judge_assignment`): applications and approval - Judging: room-based judging with Devpost import — `judging_*` tables (`judging.ts`). The older pool-dispatch tables (`judging_project`, `judge_vote`, `judge_queue`, `hackathon_result`) are retired and kept for past editions' results - Points: `hackathon_event` points, staff adjustments and the point store (`points.ts`) +- Event check-in rules: nobody is checked into two events that run at the same time, or into two less than 30 minutes apart (meals are exempt). `recordEventAttendance` enforces this for pass scans, event codes and manual check-in +- Check-in rule-break flags: each refused attempt is recorded (`hackathon_check_in_flag`, `services/check-in-flags.ts`) and shown in judging on the hacker's team, in full to organisers and as a bare marker to judges. Organisers decide whether to pull the team from prizes (`judging_entry.prize_ineligible_at`). A flag outlives the events it names (deleting one clears its id and the organiser view shows "A deleted event"). Hackers never see flags, and they stay out of public and cached payloads - Announcements and acceptance waves Hackathon participation is **open to non-members**. Membership is not a registration requirement. diff --git a/docs/operations/environment.md b/docs/operations/environment.md index 3eadf40f..84a42174 100644 --- a/docs/operations/environment.md +++ b/docs/operations/environment.md @@ -62,7 +62,7 @@ Without `DATABASE_URL`, `db` is null, sessions fall back to JWT, and tRPC proced | Variable | Default / notes | | --- | --- | -| `JUDGING_CPSAT_PYTHON` | A Python with OR-Tools (`python3` or a path). Set, the schedule section can run `packages/api/scripts/judging-schedule/cpsat.py` on the server (`judging.schedule.solveCpsat`). Unset means CP-SAT is download, run on a laptop, upload | +| `JUDGING_CPSAT_PYTHON` | A Python with OR-Tools (`python3` or a path). Set, the schedule section can run `packages/api/scripts/judging-schedule/cpsat.py` on the server (`judging.schedule.solveCpsat`), and `judging.schedule.assignJudges` uses it to pick judges. Unset means CP-SAT is download, run on a laptop, upload, and the app assigns judges itself | | `JUDGING_CPSAT_SCRIPT` | Path to `cpsat.py`. Default: found by walking up from the working directory | ## Security / proxy diff --git a/docs/operations/testing.md b/docs/operations/testing.md index 63c81425..d6f0f425 100644 --- a/docs/operations/testing.md +++ b/docs/operations/testing.md @@ -28,6 +28,7 @@ Config: `packages/api/vitest.config.ts`. Includes `src/**/*.test.ts` and `src/.i | Hackathon flow | `hackathon-flow.test.ts`, `hackathon-admin-edge.test.ts`, `hackathon-interest.test.ts` | | Payments | `stripe-payments.test.ts` | | QR / check-in | `qr-checkin.test.ts` | +| Check-in rule-break flags | `services/check-in-flags.test.ts`, `.internal-tests/check-in-flags-*.test.ts` | | Security / sanitizer / rate limit | `security.test.ts`, `resilience.test.ts` | | Judges / participants / initiatives / bootcamp / announcements | matching `*-edge.test.ts` / `bootcamp.test.ts` / `announcements.test.ts` | | Portal context | `services/portal-context.test.ts`, `routers/user/portal-context.test.ts` | diff --git a/docs/packages/api.md b/docs/packages/api.md index 8143aedf..554a0b46 100644 --- a/docs/packages/api.md +++ b/docs/packages/api.md @@ -91,7 +91,13 @@ Batch status updates allow up to **2500** ids. The global array cap in `scrubMar **Admin** (`admin.ts`): `list`, `create` (add by email), `setActive` (approve or suspend), `remove` (super admin). -Judging itself is the `judging` namespace; its shared helpers are in `services/judging/` (`room-link`, `scope`, `scoring`, `config`). `hackathon.getResults` returns published placings from both the retired `hackathon_result` table and `judging_placement`, in one shape. +Judging itself is the `judging` namespace; its shared helpers are in `services/judging/` (`room-link`, `scope`, `scoring`, `config`). `hackathon.getResults` returns published placings from both the retired `hackathon_result` table and `judging_placement`, in one shape. It leaves out entries pulled from judging (`deleted_at`) or from prizes (`prize_ineligible_at`, set by `judging.admin.setPrizeEligibility`). Check-in rule-break flags (`services/check-in-flags.ts`) reach organisers through `judging.admin.listEntries` and `judging.results.list`, and judges through `judging.desk` as a bare `flagged` boolean; no hacker-facing, public or cached procedure returns them. + +Placings rank on scores adjusted for judge bias (`services/judging/calibration.ts`): each evaluation is calibrated within its root challenge for its judge's strictness, spread and drift, on the assumption that `judging.desk.nextProject` (`services/judging/queue.ts`) hands every judge a random sample of projects. `judging_placement.raw_score` keeps the plain mean; only `judging.results.list` (organisers) returns it and the per-judge calibration, and `judging.desk` stays on raw means. An evaluation counts when it holds a rating for every rating item (`scoring.hasEveryRating`, and `scoring-sql.holdsEveryRating` where the database counts), whether or not a written response is missing, so a draft submitted at the hard end still places its project. Organiser counts, the board and hacker itinerary status, hacker feedback, and the desk's scores, judged count and shortlist all use that rule. `judging.results.list` keeps the calibration for 60 seconds per edition, keyed on the edition's evaluation count and latest save and on its ranked entries, while `judging.results.compute` always calibrates fresh. `judging.admin.reopenGrade` deletes one evaluation and the drafts for its project and challenge so another judge can grade it, and clears the edition's unpublished placings, so publishing needs a fresh compute; it is refused while results are published. + +At an untimed challenge that random sample is enforced (`services/judging/queue-store.ts`): `judging.desk.nextProject` offers only the first `QUEUE_WINDOW` (3) open projects of the judge's own queue and takes a `skip` list for a team away from its table, and a judge's first `startGrading`, `saveDraft` or `submit` on a project outside that window is refused. `judging.desk.startGrading` holds a project from the moment it is opened, with an empty draft. A table judge grades a sponsor's challenge only from a seat in a room that serves it (`scope.isSponsorChallenge`); `judging.desk.context` lists only the challenges they may pick. + +A scheduled presentation can have one assigned judge (`judging_appointment.assigned_judge_id`, set by `judging.schedule`; `services/judging/schedule/judges.ts` decides who may be assigned it). The desk never writes it and never tells a judge which other judge is assigned to, holds or graded a project. From the presentation's start until `TAKEOVER_AFTER_MS` (5 minutes) later only the assigned judge may open it (`services/judging/timing.ts`), and other judges see it as `reserved` with the time it opens. After that any judge who may grade it there can take it over, unless the assigned judge has started: their draft holds it (`grading-hold.ts`). Every claim checks this under the grading lock. `judging.desk.nextProject` offers the judge's own running assignments first; the queue window does not apply to scheduled challenges. `judging.desk.itinerary` lists the caller's assigned presentations that have not ended, soonest first, with the room and whether they are seated there. `judging.desk.releaseGrading` deletes the caller's own draft while it is still the empty one `startGrading` made, so a skipped or accidentally opened project is free again. `judging.schedule.get` gives organisers each started assignment's `judgeProgress` (grading, graded, taken over, not started). ## Club procedures (selected) diff --git a/docs/packages/db.md b/docs/packages/db.md index d928531e..31b19450 100644 --- a/docs/packages/db.md +++ b/docs/packages/db.md @@ -27,7 +27,7 @@ Files in `src/schemas/`, re-exported from `schemas/index.ts`. `drizzle.config.ts | `auth.ts` | `user`, `account`, `session`, `verificationToken` | | `members.ts` | `user_profile`, `member`, `membership_history` | | `admins.ts` | `admin` | -| `hackathons.ts` | `hackathon`, `hackathon_team`, `hackathon_participant`, `hackathon_project`, `hackathon_interest`, `hackathon_event`, `hackathon_event_attendee`, `hackathon_announcement`, `hackathon_announcement_recipient` | +| `hackathons.ts` | `hackathon`, `hackathon_team`, `hackathon_participant`, `hackathon_project`, `hackathon_interest`, `hackathon_event`, `hackathon_event_attendee`, `hackathon_check_in_flag`, `hackathon_announcement`, `hackathon_announcement_recipient` | | `judge.ts` | `judge`, `judge_assignment` (live roster); `judging_project`, `judge_vote`, `judge_queue`, `hackathon_result` (retired, kept for past editions) | | `judging.ts` | `judging_*`: config, entries and members, challenges, buildings, rooms and room links, judges, guest sessions, presence, schedule and appointments, rubric, evaluations, deliberation, placements | | `points.ts` | `hackathon_point_adjustment`, `hackathon_store_item`, `hackathon_store_purchase` | diff --git a/packages/api/scripts/judging-schedule/cpsat.py b/packages/api/scripts/judging-schedule/cpsat.py index f81afaad..f8002323 100644 --- a/packages/api/scripts/judging-schedule/cpsat.py +++ b/packages/api/scripts/judging-schedule/cpsat.py @@ -35,6 +35,36 @@ cross-building walks only the same-building break and marks the solution `relaxed`; the portal makes an organiser confirm that before saving. +Judges +------ +When the problem lists `judges`, the script goes on to the second stage: one +judge for every presentation, solved as its own CP-SAT model once the +placement is known. + + hard a judge grades only challenges they may grade, and a guest only in + their own room; + one presentation per judge per slot; + moving to another room in the same building needs `sameGap` empty + slots, to another building `crossGap`; staying in one room needs none; + at most `maxConsecutive` slots in a row, then a free slot. + goals in this order, each one never traded for a later one: + 1. every presentation has a judge (the rest is reported, not failed) + 2. sponsor presentations go to the sponsor's own judges, not to table + judges (members) + 3. the busiest judge grades as little as possible + 4. judges of one pool (same challenges, same room) grade evenly + 5. a judge repeats a room as little as possible + 6. a judge's presentations cover as much of the day as possible, as + judge-bias calibration assumes every judge sees a random sample + +The placement stage then also keeps, in every slot, the presentations of a +challenge within the number of judges who may grade it, as far as it can +(goal 0, `judgeShortfall`, ahead of the five above). + +A problem with `placements` is a schedule already made: the placement stage is +skipped and only the judges are assigned. `locked` lists assignments that must +stay as they are. Files without `judges` work as before. + Exit codes: 0 solved, 2 no schedule exists, 3 the time limit ran out before any schedule was found, 1 bad input. """ @@ -46,12 +76,16 @@ import sys import time from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field PROBLEM_FORMAT = "query-judging-schedule-problem" SOLUTION_FORMAT = "query-judging-schedule-solution" VERSION = 1 +# Same as MAX_CONSECUTIVE and DAY_PARTS in judges.ts. +MAX_CONSECUTIVE = 4 +DAY_PARTS = 4 + class InputError(Exception): pass @@ -72,6 +106,15 @@ class Task: sponsor: bool +@dataclass(frozen=True) +class Judge: + id: str + kind: str # "member" or "guest" + challenge_ids: frozenset[str] + room_id: str | None + max_consecutive: int + + @dataclass class Problem: fingerprint: str @@ -81,6 +124,13 @@ class Problem: rooms: list[Room] tasks: list[Task] hint: dict[tuple[str, str], tuple[str, int]] + judges: list[Judge] = field(default_factory=list) + # A finished schedule to assign judges to, instead of one to look for. + fixed: dict[tuple[str, str], tuple[str, int]] | None = None + # Assignments to keep: (entry id, challenge id) -> judge id. + locked: dict[tuple[str, str], str] = field(default_factory=dict) + # Assignments to start from, as `hint` is for the placement. + assignment_hint: dict[tuple[str, str], str] = field(default_factory=dict) def _require(obj: dict, key: str, kind: type, where: str): @@ -133,6 +183,63 @@ def load_problem(raw: dict) -> Problem: for p in raw.get("hint") or []: hint[(p["entryId"], p["challengeId"])] = (p["roomId"], p["slot"]) + judges = [] + for i, j in enumerate(_require(raw, "judges", list, "problem") if "judges" in raw else []): + where = f"judges[{i}]" + if not isinstance(j, dict): + raise InputError(f"{where} must be an object") + kind = _require(j, "kind", str, where) + if kind not in ("member", "guest"): + raise InputError(f"{where}.kind must be member or guest") + challenge_ids = _require(j, "challengeIds", list, where) + if not all(isinstance(c, str) for c in challenge_ids): + raise InputError(f"{where}.challengeIds must be strings") + room_id = j.get("roomId") + if room_id is not None and not isinstance(room_id, str): + raise InputError(f"{where}.roomId must be a string or null") + max_consecutive = j.get("maxConsecutive", MAX_CONSECUTIVE) + if not isinstance(max_consecutive, int) or isinstance(max_consecutive, bool) or max_consecutive < 1: + raise InputError(f"{where}.maxConsecutive must be a positive integer") + judges.append( + Judge(_require(j, "id", str, where), kind, frozenset(challenge_ids), room_id, max_consecutive) + ) + if len({j.id for j in judges}) != len(judges): + raise InputError("judge ids repeat") + + fixed = None + if raw.get("placements") is not None: + room_ids = {r.id for r in rooms} + known = {(t.entry_id, t.challenge_id) for t in tasks} + fixed = {} + for i, p in enumerate(_require(raw, "placements", list, "problem")): + where = f"placements[{i}]" + key = (_require(p, "entryId", str, where), _require(p, "challengeId", str, where)) + room_id = _require(p, "roomId", str, where) + slot = _require(p, "slot", int, where) + if key not in known or room_id not in room_ids or key in fixed: + raise InputError(f"{where} does not match a presentation and room of this problem") + if not 0 <= slot < slot_count: + raise InputError(f"{where}.slot is outside the window") + fixed[key] = (room_id, slot) + if set(fixed) != known: + raise InputError("placements must place every presentation") + if not judges: + raise InputError("placements without judges leave nothing to solve") + + locked = {} + for i, row in enumerate(_require(raw, "locked", list, "problem") if "locked" in raw else []): + where = f"locked[{i}]" + locked[(_require(row, "entryId", str, where), _require(row, "challengeId", str, where))] = _require( + row, "judgeId", str, where + ) + + assignment_hint = {} + for i, row in enumerate(_require(raw, "assignmentHint", list, "problem") if "assignmentHint" in raw else []): + where = f"assignmentHint[{i}]" + assignment_hint[(_require(row, "entryId", str, where), _require(row, "challengeId", str, where))] = _require( + row, "judgeId", str, where + ) + return Problem( fingerprint=_require(raw, "fingerprint", str, "problem"), slot_count=slot_count, @@ -141,6 +248,10 @@ def load_problem(raw: dict) -> Problem: rooms=rooms, tasks=tasks, hint=hint, + judges=judges, + fixed=fixed, + locked=locked, + assignment_hint=assignment_hint, ) @@ -205,13 +316,40 @@ def score(problem: Problem, placement: Placement) -> dict[str, int]: for (s1, b1), (s2, b2) in zip(seq, seq[1:]): changes += b1 != b2 breaks += s2 - s1 - 1 - return { + goals = { "sponsorFinish": sponsor_finish, "sponsorIdle": sponsor_idle, "finish": finish, "buildingChanges": changes, "breakSlots": breaks, } + if problem.judges: + # Ahead of the rest, as in solve(). + goals = {"judgeShortfall": shortfall(problem, placement), **goals} + return goals + + +def judge_capacity(problem: Problem) -> dict[str, int]: + """How many judges may grade each challenge that has presentations and any judge.""" + counts = defaultdict(int) + for judge in problem.judges: + for challenge in judge.challenge_ids: + counts[challenge] += 1 + return {c: counts[c] for c in sorted({t.challenge_id for t in problem.tasks}) if counts[c] > 0} + + +def shortfall(problem: Problem, placement: Placement) -> int: + """Presentations of a challenge in one slot beyond its judges, for the worst + slot of each challenge, summed. 0 when judges could cover every slot.""" + capacity = judge_capacity(problem) + per_slot = defaultdict(int) + for i, task in enumerate(problem.tasks): + per_slot[(task.challenge_id, placement[i][1])] += 1 + worst = defaultdict(int) + for (challenge, _slot), count in per_slot.items(): + if challenge in capacity: + worst[challenge] = max(worst[challenge], count - capacity[challenge]) + return sum(worst.values()) def violations(problem: Problem, placement: Placement, cross_gap: int) -> list[str]: @@ -398,7 +536,25 @@ def busiest(selected) -> int: model.add(idle == 0).only_enforce_if(used.Not()) idle_terms.append(idle) + # Judges: in any slot, a challenge's presentations should not outnumber the + # judges who may grade it. Kept as a goal, the first, rather than a rule: too + # few judges must not make a schedule impossible. Each challenge's capacity + # is its judges plus an extra, the most it is over in any slot. + overloads = [] + for challenge, judges_for in judge_capacity(problem).items(): + members = [i for i, task in enumerate(tasks) if task.challenge_id == challenge] + if len(members) <= judges_for: + continue + extra = int_var(0, len(members) - judges_for, f"judge_extra_{challenge}") + model.add_cumulative( + [model.new_fixed_size_interval_var(start[i], 1, f"at_{i}") for i in members], + [1] * len(members), + judges_for + extra, + ) + overloads.append(extra) + goals = { + "judgeShortfall": sum(overloads), "sponsorFinish": sponsor_finish, "sponsorIdle": sum(idle_terms), "finish": finish, @@ -408,6 +564,8 @@ def busiest(selected) -> int: stages = [g for g in GOALS if not (g == "sponsorIdle" and not idle_terms) and not (g == "sponsorFinish" and not any(t.sponsor for t in tasks)) and not (g in ("buildingChanges", "breakSlots") and not break_terms)] + if overloads: + stages.insert(0, "judgeShortfall") def hint_placement(placement: Placement): model.clear_hints() @@ -497,6 +655,338 @@ def model_value(goal: str, placement: Placement) -> int: return ("optimal" if all_optimal else "feasible", best) +# Assignment: task index -> judge index, or None when nobody grades it. +Assignment = dict[int, "int | None"] + +# The second stage's goals, in order. Lower is better except the last. +JUDGE_GOALS = ["unassigned", "memberOnSponsor", "maxLoad", "loadSpread", "repeats", "dayPartsCovered"] + + +def may_grade(judge: Judge, task: Task, room: Room) -> bool: + if task.challenge_id not in judge.challenge_ids: + return False + # A guest grades only in their own room, and with no room grades nowhere. + if judge.room_id is not None: + return judge.room_id == room.id + return judge.kind != "guest" + + +def day_part(problem: Problem, slot: int) -> int: + return min(DAY_PARTS - 1, slot * DAY_PARTS // problem.slot_count) + + +def judge_pools(problem: Problem) -> list[list[int]]: + """Judges who are interchangeable (same challenges, same room); only within + one of these can the load be evened out.""" + grouped = defaultdict(list) + for k, judge in enumerate(problem.judges): + grouped[(tuple(sorted(judge.challenge_ids)), judge.room_id or "")].append(k) + return list(grouped.values()) + + +def locked_tasks(problem: Problem) -> dict[int, int]: + """Task index -> judge index for the assignments to keep that name a known + presentation and judge.""" + task_index = {(t.entry_id, t.challenge_id): i for i, t in enumerate(problem.tasks)} + judge_index = {j.id: k for k, j in enumerate(problem.judges)} + return { + task_index[key]: judge_index[judge_id] + for key, judge_id in problem.locked.items() + if key in task_index and judge_id in judge_index + } + + +def judges_clash(problem: Problem, placement: Placement, i: int, q: int) -> bool: + """Whether one judge cannot grade both: at the same time, or too soon to walk + between their rooms. Staying in one room needs no walk.""" + (room_i, slot_i), (room_q, slot_q) = placement[i], placement[q] + apart = abs(slot_i - slot_q) + if apart == 0: + return True + if room_i == room_q: + return False + same = problem.rooms[room_i].building_id == problem.rooms[room_q].building_id + return apart <= (problem.same_gap if same else problem.cross_gap) + + +def assignment_score(problem: Problem, placement: Placement, assignment: Assignment) -> dict[str, float]: + """The second stage's goals for a finished assignment, from the assignment alone.""" + judges, tasks = problem.judges, problem.tasks + load = [0] * len(judges) + rooms_seen = [defaultdict(int) for _ in judges] + parts_seen = [set() for _ in judges] + unassigned = member_on_sponsor = 0 + for i, task in enumerate(tasks): + k = assignment.get(i) + if k is None: + unassigned += 1 + continue + load[k] += 1 + rooms_seen[k][placement[i][0]] += 1 + parts_seen[k].add(day_part(problem, placement[i][1])) + member_on_sponsor += task.sponsor and judges[k].kind == "member" + spread = sum( + max(load[k] for k in pool) - min(load[k] for k in pool) + for pool in judge_pools(problem) + if len(pool) > 1 + ) + graders = [k for k in range(len(judges)) if load[k]] + return { + "unassigned": unassigned, + "memberOnSponsor": member_on_sponsor, + "maxLoad": max(load, default=0), + "loadSpread": spread, + "repeats": sum(count - 1 for seen in rooms_seen for count in seen.values()), + "dayPartsCovered": sum(len(parts) for parts in parts_seen), + "avgDistinctRooms": round(sum(len(rooms_seen[k]) for k in graders) / len(graders), 2) if graders else 0, + } + + +def assignment_violations(problem: Problem, placement: Placement, assignment: Assignment) -> list[str]: + """Independent re-check of the second stage's hard rules.""" + judges, tasks, rooms = problem.judges, problem.tasks, problem.rooms + kept = locked_tasks(problem) + found = [] + mine = defaultdict(list) + for i, k in assignment.items(): + if k is None: + continue + mine[k].append(i) + if i not in kept and not may_grade(judges[k], tasks[i], rooms[placement[i][0]]): + found.append(f"judge {judges[k].id} may not grade {tasks[i].entry_id}/{tasks[i].challenge_id}") + max_gap = max(problem.same_gap, problem.cross_gap) + for k, members in mine.items(): + members.sort(key=lambda i: placement[i][1]) + for a, i in enumerate(members): + for q in members[a + 1:]: + if placement[q][1] - placement[i][1] > max_gap: + break + if judges_clash(problem, placement, i, q): + found.append(f"judge {judges[k].id} cannot be at both {tasks[i].entry_id} and {tasks[q].entry_id}") + run, previous = 0, -2 + for slot in sorted({placement[i][1] for i in members}): + run = run + 1 if slot == previous + 1 else 1 + previous = slot + if run == judges[k].max_consecutive + 1: + found.append(f"judge {judges[k].id} grades more than {judges[k].max_consecutive} slots in a row") + return found + + +def assign(problem: Problem, placement: Placement, seconds: float, workers: int, log): + """Lexicographic CP-SAT solve of who grades what, once the schedule is fixed. + Returns (status, assignment or None). A presentation nobody may grade, or + that no judge has time for, is left out rather than failing the solve.""" + from ortools.sat.python import cp_model + + judges, tasks, rooms = problem.judges, problem.tasks, problem.rooms + n = len(tasks) + slot_of = [placement[i][1] for i in range(n)] + room_of = [placement[i][0] for i in range(n)] + kept = locked_tasks(problem) + + model = cp_model.CpModel() + tracked = [] + + def int_var(lo: int, hi: int, name: str): + v = model.new_int_var(lo, hi, name) + tracked.append(v) + return v + + def flag(name: str): + v = model.new_bool_var(name) + tracked.append(v) + return v + + # A kept presentation may go to its judge alone; the rest to whoever may grade it. + eligible = [ + [kept[i]] if i in kept + else [k for k, judge in enumerate(judges) if may_grade(judge, tasks[i], rooms[room_of[i]])] + for i in range(n) + ] + pick = {(i, k): flag(f"task_{i}_judge_{k}") for i in range(n) for k in eligible[i]} + left_out = [flag(f"left_out_{i}") for i in range(n)] + for i in range(n): + model.add_exactly_one([pick[i, k] for k in eligible[i]] + [left_out[i]]) + if i in kept: + model.add(pick[i, kept[i]] == 1) + + max_gap = max(problem.same_gap, problem.cross_gap) + + def walk(room: int, other: int) -> int: + """Free slots a judge needs between grading in `room` and in `other`.""" + if room == other: + return 0 + same = rooms[room].building_id == rooms[other].building_id + return problem.same_gap if same else problem.cross_gap + + mine = [] # judge -> {task: literal} + for k, judge in enumerate(judges): + grades = {i: pick[i, k] for i in range(n) if (i, k) in pick} + mine.append(grades) + # Where this judge could be: slot -> room -> literals. Each rule below + # is a set of which at most one holds, the same as forbidding every + # pair that judges_clash() forbids, in far fewer constraints. + at = defaultdict(lambda: defaultdict(list)) + for i, lit in grades.items(): + at[slot_of[i]][room_of[i]].append(lit) + for slot, here in at.items(): + # One place at a time. + everywhere = [lit for lits in here.values() for lit in lits] + if len(everywhere) > 1: + model.add_at_most_one(everywhere) + # Time to walk: grading in a room rules out, in each of the next + # few slots, every room too far to reach by then. + for room, lits in here.items(): + for ahead in range(1, max_gap + 1): + later = at.get(slot + ahead, {}) + far = [lit for other, group in later.items() + if ahead <= walk(room, other) for lit in group] + if far: + model.add_at_most_one(lits + far) + # No more than max_consecutive slots in a row: any window one longer + # holds at most max_consecutive. + run = judge.max_consecutive + in_slot = defaultdict(list) + for i, lit in grades.items(): + in_slot[slot_of[i]].append(lit) + for first in range(problem.slot_count - run): + window = [lit for slot in range(first, first + run + 1) for lit in in_slot.get(slot, ())] + if len(window) > run: + model.add(sum(window) <= run) + + load = [] + for k, grades in enumerate(mine): + v = int_var(0, len(grades), f"load_{k}") + model.add(v == sum(grades.values())) + load.append(v) + + # Goals, each (name, expression to minimise, whether it has anything to decide). + member_on_sponsor = [ + pick[i, k] for (i, k) in pick if tasks[i].sponsor and judges[k].kind == "member" + ] + busiest = int_var(0, n, "busiest") + if load: + model.add_max_equality(busiest, load) + spreads = [] + for pool in judge_pools(problem): + if len(pool) < 2: + continue + high, low = int_var(0, n, "pool_high"), int_var(0, n, "pool_low") + for k in pool: + model.add(high >= load[k]) + model.add(low <= load[k]) + spreads.append(high - low) + rooms_used, parts_used = [], [] + for k, grades in enumerate(mine): + in_room, in_part = defaultdict(list), defaultdict(list) + for i, lit in grades.items(): + in_room[room_of[i]].append(lit) + in_part[day_part(problem, slot_of[i])].append(lit) + for lits, used in ((in_room, rooms_used), (in_part, parts_used)): + for group in lits.values(): + seen = flag(f"seen_{k}_{len(used)}") + model.add(seen <= sum(group)) + used.append(seen) + + stages = [ + ("unassigned", sum(left_out), n > 0), + ("memberOnSponsor", sum(member_on_sponsor), bool(member_on_sponsor)), + ("maxLoad", busiest, bool(load)), + ("loadSpread", sum(spreads), bool(spreads)), + ("repeats", -sum(rooms_used), bool(rooms_used)), + ("dayPartsCovered", -sum(parts_used), bool(parts_used)), + ] + stages = [(name, expression) for name, expression, decides in stages if decides] + + def new_solver(limit: float): + solver = cp_model.CpSolver() + solver.parameters.max_time_in_seconds = max(1.0, limit) + solver.parameters.num_workers = workers + solver.parameters.random_seed = 7 + # Presolve and the symmetry search cost seconds on a big event, longer + # than a stage's share, and every stage after the first starts from a + # complete assignment, so skip both. + solver.parameters.cp_model_presolve = False + solver.parameters.symmetry_level = 0 + return solver + + def take(solver) -> Assignment: + model.clear_hints() + for v in tracked: + model.add_hint(v, solver.value(v)) + return { + i: next((k for k in eligible[i] if solver.boolean_value(pick[i, k])), None) + for i in range(n) + } + + # Start from the hinted assignment where it names a pair the model has. One + # that keeps every rule is already an answer, as a good `hint` is for the + # placement, so running out of time can only fail to improve it. + judge_index = {j.id: k for k, j in enumerate(judges)} + task_index = {(t.entry_id, t.challenge_id): i for i, t in enumerate(tasks)} + hinted: Assignment = {i: kept.get(i) for i in range(n)} + for key, judge_id in problem.assignment_hint.items(): + i, k = task_index.get(key), judge_index.get(judge_id) + if i is not None and k is not None and (i, k) in pick and i not in kept: + hinted[i] = k + deadline = time.monotonic() + seconds + best: Assignment | None = None + if problem.assignment_hint: + model.clear_hints() + for (i, k), lit in pick.items(): + model.add_hint(lit, hinted[i] == k) + for i in range(n): + model.add_hint(left_out[i], hinted[i] is None) + if not assignment_violations(problem, placement, hinted): + best = hinted + + def model_value(goal: str, assignment: Assignment) -> int: + """`goal` for `assignment` as the stage's expression counts it.""" + score = assignment_score(problem, placement, assignment) + if goal == "repeats": + # Minus the rooms each judge grades in, summed: repeats less the + # presentations graded. + return score["repeats"] - (n - score["unassigned"]) + if goal == "dayPartsCovered": + return -score["dayPartsCovered"] + return score[goal] + + all_optimal = True + for position, (name, expression) in enumerate(stages): + if position == 0 and best is not None: + # The hint already reaches this; the search can only improve on it. + model.add(expression <= model_value(name, best)) + remaining = deadline - time.monotonic() + share = remaining / (len(stages) - position) + if position == 0: + share = max(share, remaining / 2) + solver = new_solver(share) + model.minimize(expression) + status = solver.solve(model) + if status not in (cp_model.OPTIMAL, cp_model.FEASIBLE): + if status == cp_model.INFEASIBLE and position == 0: + return ("infeasible", None) + if best is None: + return ("unknown", None) + all_optimal = False + log(f" {name}: no improvement within its time share, {solver.status_name(status)}") + # Hold this goal where the assignment in hand has it, so the + # stages after it cannot trade it away. + model.add(expression <= model_value(name, best)) + continue + all_optimal = all_optimal and status == cp_model.OPTIMAL + best = take(solver) + value = int(solver.objective_value) + shown = assignment_score(problem, placement, best)[name] + log(f" {name}: {shown}{'' if status == cp_model.OPTIMAL else ' (time limit)'}") + model.add(expression <= value) + + if best is None: + # Nothing to decide: every presentation is left out or settled by its kept judge. + best = {i: kept.get(i) for i in range(n)} + return ("optimal" if all_optimal else "feasible", best) + + def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("problem", help="problem.json exported from the judging console, " @@ -534,35 +1024,62 @@ def log(message: str): f"{problem.slot_count} slots (OR-Tools {ortools.__version__})") began = time.monotonic() - attempts = [(False, problem.cross_gap)] - if problem.cross_gap > problem.same_gap: - attempts.append((True, problem.same_gap)) - status, placement, relaxed = "infeasible", None, False - reasons = capacity_problems(problem, problem.same_gap) - for relaxed, cross_gap in [] if reasons else attempts: - log("relaxed: cross-building walks get the same-building break" if relaxed - else "strict: full cross-building breaks") - status, placement = solve(problem, cross_gap, args.time_limit, args.workers, log) - if placement is not None: - broken = violations(problem, placement, cross_gap) + reasons: list[str] = [] + if problem.fixed is not None: + # The schedule is decided; only the judges remain. + room_index = {r.id: i for i, r in enumerate(problem.rooms)} + placement = { + i: (room_index[problem.fixed[t.entry_id, t.challenge_id][0]], + problem.fixed[t.entry_id, t.challenge_id][1]) + for i, t in enumerate(problem.tasks) + } + status = "optimal" + else: + attempts = [(False, problem.cross_gap)] + if problem.cross_gap > problem.same_gap: + attempts.append((True, problem.same_gap)) + + reasons = capacity_problems(problem, problem.same_gap) + for relaxed, cross_gap in [] if reasons else attempts: + log("relaxed: cross-building walks get the same-building break" if relaxed + else "strict: full cross-building breaks") + status, placement = solve(problem, cross_gap, args.time_limit, args.workers, log) + if placement is not None: + broken = violations(problem, placement, cross_gap) + if broken: + # Would be a bug in the model above; never hand the portal a bad file. + log("internal check failed: " + "; ".join(broken[:5])) + return 1 + break + if status == "unknown": + # Running out of time proves nothing, so the breaks are only + # shortened once the full ones are shown to be impossible. + break + + assignment: Assignment | None = None + assignment_status = None + if placement is not None and problem.judges: + log(f"{len(problem.judges)} judges") + assignment_status, assignment = assign(problem, placement, args.time_limit, args.workers, log) + if assignment is not None: + broken = assignment_violations(problem, placement, assignment) if broken: - # Would be a bug in the model above; never hand the portal a bad file. log("internal check failed: " + "; ".join(broken[:5])) return 1 - break - if status == "unknown": - # Running out of time proves nothing, so the breaks are only - # shortened once the full ones are shown to be impossible. - break + elif problem.fixed is not None: + # Nothing else to hand back. + status = assignment_status + reasons = ["the assignments to keep break the judges' rules"] if status == "infeasible" else [] + solved = placement is not None and (problem.fixed is None or assignment is not None) solution = { "format": SOLUTION_FORMAT, "version": VERSION, "fingerprint": problem.fingerprint, - "status": status, - "relaxed": relaxed if placement is not None else False, - "objective": score(problem, placement) if placement is not None else None, + "status": status if not (solved and problem.fixed is not None) else assignment_status, + "relaxed": relaxed if solved else False, + "objective": score(problem, placement) if solved else None, "placements": [ { "entryId": problem.tasks[i].entry_id, @@ -571,7 +1088,7 @@ def log(message: str): "slot": slot, } for i, (r, slot) in sorted(placement.items()) - ] if placement is not None else [], + ] if solved else [], "reasons": reasons, "solver": { "name": "cp-sat", @@ -579,6 +1096,18 @@ def log(message: str): "seconds": round(time.monotonic() - began, 2), }, } + if assignment is not None: + solution["assignmentStatus"] = assignment_status + solution["judging"] = assignment_score(problem, placement, assignment) + solution["assignments"] = [ + { + "entryId": problem.tasks[i].entry_id, + "challengeId": problem.tasks[i].challenge_id, + "judgeId": problem.judges[k].id, + } + for i, k in sorted(assignment.items()) + if k is not None + ] text = json.dumps(solution, indent=2) + "\n" if args.output: with open(args.output, "w", encoding="utf-8") as handle: @@ -586,9 +1115,12 @@ def log(message: str): else: sys.stdout.write(text) - if placement is not None: - log(f"{status}{', relaxed' if solution['relaxed'] else ''}: " + if solved: + log(f"{solution['status']}{', relaxed' if solution['relaxed'] else ''}: " + ", ".join(f"{k} {v}" for k, v in solution["objective"].items())) + if assignment is not None: + log(f"judges ({assignment_status}): " + + ", ".join(f"{k} {v}" for k, v in solution["judging"].items())) return 0 if status == "unknown": log("no schedule found within the time limit; try a larger --time-limit") diff --git a/packages/api/scripts/judging-schedule/test_cpsat.py b/packages/api/scripts/judging-schedule/test_cpsat.py index 84ca5ae2..69b458b2 100644 --- a/packages/api/scripts/judging-schedule/test_cpsat.py +++ b/packages/api/scripts/judging-schedule/test_cpsat.py @@ -10,6 +10,7 @@ import contextlib import io +import itertools import json import random import subprocess @@ -188,16 +189,20 @@ def test_a_valid_hint_survives_running_out_of_time(self): self.assertEqual(placement, hinted) +def run_cli(problem: dict | str, *args: str): + text = problem if isinstance(problem, str) else json.dumps(problem) + return subprocess.run( + [sys.executable, "-I", str(HERE / "cpsat.py"), "-", *args], + input=text.encode("utf-8"), + capture_output=True, + timeout=120, + ) + + @unittest.skipUnless(HAVE_ORTOOLS, "OR-Tools is not installed") class CommandLineTest(unittest.TestCase): def run_script(self, problem: dict | str, *args: str): - text = problem if isinstance(problem, str) else json.dumps(problem) - return subprocess.run( - [sys.executable, "-I", str(HERE / "cpsat.py"), "-", *args], - input=text.encode("utf-8"), - capture_output=True, - timeout=120, - ) + return run_cli(problem, *args) def test_reads_the_problem_from_stdin(self): raw = small_problem(1) @@ -259,5 +264,231 @@ def test_exit_codes(self): self.assertTrue(json.loads(done.stdout)["reasons"]) +JUDGE_KEYS = cpsat.JUDGE_GOALS + + +def judge_problem(seed: int) -> dict: + """small_problem with a finished schedule and judges: two table judges (one + with a short run limit) and a guest in one sponsor room.""" + rng = random.Random(seed * 7 + 1) + raw = small_problem(seed) + free = [(r["id"], s) for r in raw["rooms"] for s in range(raw["slotCount"])] + rng.shuffle(free) + placements, taken = [], set() + for task in raw["tasks"]: + room, slot = next( + (r, s) for r, s in free + if (r, s) not in taken + and next(x for x in raw["rooms"] if x["id"] == r)["challengeId"] == task["challengeId"] + ) + taken.add((room, slot)) + placements.append( + {"entryId": task["entryId"], "challengeId": task["challengeId"], "roomId": room, "slot": slot} + ) + everything = ["general", "acme", "globex"] + raw["placements"] = placements + raw["judges"] = [ + {"id": "m1", "kind": "member", "challengeIds": everything, "roomId": None}, + {"id": "m2", "kind": "member", "challengeIds": everything, "roomId": None, + "maxConsecutive": rng.choice([1, 2, 4])}, + {"id": "gs", "kind": "guest", "challengeIds": ["acme"], "roomId": "s1"}, + ] + return raw + + +def judge_key(score: dict) -> tuple[int, ...]: + """Lower is better, in goal order; the last goal counts what is covered.""" + return tuple(-score[k] if k == "dayPartsCovered" else score[k] for k in JUDGE_KEYS) + + +def brute_force_assign(problem: cpsat.Problem, placement: cpsat.Placement): + """Best goal vector over every assignment that keeps the judges' rules.""" + rooms, tasks, judges = problem.rooms, problem.tasks, problem.judges + options = [ + [None] + [k for k, judge in enumerate(judges) + if cpsat.may_grade(judge, tasks[i], rooms[placement[i][0]])] + for i in range(len(tasks)) + ] + best = None + for combo in itertools.product(*options): + assignment = dict(enumerate(combo)) + if cpsat.assignment_violations(problem, placement, assignment): + continue + key = judge_key(cpsat.assignment_score(problem, placement, assignment)) + if best is None or key < best: + best = key + return best + + +@unittest.skipUnless(HAVE_ORTOOLS, "OR-Tools is not installed") +class JudgeTest(unittest.TestCase): + def fixed(self, raw: dict) -> tuple[cpsat.Problem, cpsat.Placement]: + problem = cpsat.load_problem(raw) + index = {r.id: i for i, r in enumerate(problem.rooms)} + placement = { + i: (index[problem.fixed[t.entry_id, t.challenge_id][0]], + problem.fixed[t.entry_id, t.challenge_id][1]) + for i, t in enumerate(problem.tasks) + } + return problem, placement + + def test_assignment_matches_brute_force(self): + for seed in range(12): + problem, placement = self.fixed(judge_problem(seed)) + with self.subTest(seed=seed): + status, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + self.assertEqual(status, "optimal") + self.assertEqual(cpsat.assignment_violations(problem, placement, assignment), []) + self.assertEqual( + judge_key(cpsat.assignment_score(problem, placement, assignment)), + brute_force_assign(problem, placement), + ) + + def test_judges_are_assigned_from_the_command_line(self): + raw = judge_problem(2) + done = run_cli(raw, "--time-limit", "10", "--workers", "1") + self.assertEqual(done.returncode, 0, done.stderr.decode()) + solution = json.loads(done.stdout) + self.assertEqual(solution["status"], solution["assignmentStatus"]) + self.assertEqual(solution["placements"], raw["placements"]) + self.assertEqual(list(solution["judging"])[:6], JUDGE_KEYS) + # One judge at most per presentation, and the rest counted as left out. + done_for = {(a["entryId"], a["challengeId"]) for a in solution["assignments"]} + self.assertEqual(len(done_for), len(solution["assignments"])) + self.assertEqual( + len(solution["assignments"]) + solution["judging"]["unassigned"], len(raw["tasks"]) + ) + + def test_locked_assignments_stay(self): + raw = judge_problem(4) + first = raw["tasks"][0] + raw["locked"] = [{"entryId": first["entryId"], "challengeId": first["challengeId"], "judgeId": "m2"}] + problem, placement = self.fixed(raw) + _, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + self.assertEqual(problem.judges[assignment[0]].id, "m2") + + def test_a_valid_hint_survives_running_out_of_time(self): + from ortools.sat.python import cp_model + + raw = judge_problem(1) + problem, placement = self.fixed(raw) + _, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + raw["assignmentHint"] = [ + { + "entryId": problem.tasks[i].entry_id, + "challengeId": problem.tasks[i].challenge_id, + "judgeId": problem.judges[k].id, + } + for i, k in assignment.items() + if k is not None + ] + problem, placement = self.fixed(raw) + # Every solve runs out of time without finding anything. + with mock.patch.object(cp_model.CpSolver, "solve", return_value=cp_model.UNKNOWN): + status, kept = cpsat.assign(problem, placement, 1, 1, quiet) + self.assertEqual(status, "feasible") + self.assertEqual(kept, assignment) + + def test_a_goal_cut_short_is_not_traded_away_by_the_next(self): + from ortools.sat.python import cp_model + + solve = cp_model.CpSolver.solve + calls = [] + + for seed in range(12): + raw = judge_problem(seed) + problem, placement = self.fixed(raw) + _, best = cpsat.assign(problem, placement, 20, 1, quiet) + raw["assignmentHint"] = [ + { + "entryId": problem.tasks[i].entry_id, + "challengeId": problem.tasks[i].challenge_id, + "judgeId": problem.judges[k].id, + } + for i, k in best.items() + if k is not None + ] + problem, placement = self.fixed(raw) + # The hint is the best there is. Whichever goal runs out of time, + # the goals after it must not make it worse to do better themselves. + for cut in (1, 2): + + def runs_out(solver, model, *args, **kwargs): + calls.append(1) + if len(calls) == cut: + return cp_model.UNKNOWN + return solve(solver, model, *args, **kwargs) + + calls.clear() + with mock.patch.object(cp_model.CpSolver, "solve", autospec=True, side_effect=runs_out): + _, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + with self.subTest(seed=seed, cut=cut): + self.assertEqual( + judge_key(cpsat.assignment_score(problem, placement, assignment)), + judge_key(cpsat.assignment_score(problem, placement, best)), + ) + + def test_a_presentation_nobody_may_grade_is_left_out_not_failed(self): + raw = judge_problem(1) + for judge in raw["judges"]: + judge["challengeIds"] = ["general"] + problem, placement = self.fixed(raw) + status, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + self.assertEqual(status, "optimal") + sponsor = [i for i, t in enumerate(problem.tasks) if t.challenge_id != "general"] + self.assertTrue(sponsor) + self.assertTrue(all(assignment[i] is None for i in sponsor)) + + def test_placement_keeps_a_challenge_within_its_judges(self): + # Three teams, each in general once, three tables, one judge for general. + # Without judges all three go at once; with one they must take turns. + raw = { + "format": cpsat.PROBLEM_FORMAT, "version": cpsat.VERSION, "fingerprint": "capacity", + "slotCount": 4, "sameGap": 0, "crossGap": 0, + "rooms": [{"id": f"g{n}", "challengeId": "general", "buildingId": "a", "sponsor": False} + for n in range(3)], + "tasks": [{"entryId": f"p{n}", "challengeId": "general", "sponsor": False} + for n in range(3)], + } + problem = cpsat.load_problem(raw) + _, together = cpsat.solve(problem, 0, 10, 1, quiet) + self.assertEqual({slot for _, slot in together.values()}, {0}) + self.assertEqual(list(cpsat.score(problem, together)), KEYS) + + raw["judges"] = [{"id": "m1", "kind": "member", "challengeIds": ["general"], "roomId": None}] + problem = cpsat.load_problem(raw) + _, turns = cpsat.solve(problem, 0, 10, 1, quiet) + self.assertEqual(sorted(slot for _, slot in turns.values()), [0, 1, 2]) + self.assertEqual(cpsat.score(problem, turns)["judgeShortfall"], 0) + + def test_too_few_judges_do_not_make_a_schedule_impossible(self): + raw = { + "format": cpsat.PROBLEM_FORMAT, "version": cpsat.VERSION, "fingerprint": "short", + "slotCount": 2, "sameGap": 0, "crossGap": 0, + "rooms": [{"id": f"g{n}", "challengeId": "general", "buildingId": "a", "sponsor": False} + for n in range(3)], + "tasks": [{"entryId": f"p{n}", "challengeId": "general", "sponsor": False} + for n in range(5)], + "judges": [{"id": "m1", "kind": "member", "challengeIds": ["general"], "roomId": None}], + } + problem = cpsat.load_problem(raw) + status, placement = cpsat.solve(problem, 0, 10, 1, quiet) + self.assertIn(status, ("optimal", "feasible")) + # Five presentations in two slots: at best three and two at a time, so + # the worst slot has two more than the one judge. + self.assertEqual(cpsat.score(problem, placement)["judgeShortfall"], 2) + + def test_rejects_bad_judges_and_placements(self): + raw = judge_problem(0) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({**raw, "judges": [{**raw["judges"][0], "kind": "robot"}]}) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({**raw, "judges": [raw["judges"][0], raw["judges"][0]]}) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({**raw, "placements": raw["placements"][:-1]}) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({k: v for k, v in raw.items() if k != "judges"}) + + if __name__ == "__main__": unittest.main() diff --git a/packages/api/src/.internal-tests/_db-tx-mock.ts b/packages/api/src/.internal-tests/_db-tx-mock.ts index 9af27af6..804f78dc 100644 --- a/packages/api/src/.internal-tests/_db-tx-mock.ts +++ b/packages/api/src/.internal-tests/_db-tx-mock.ts @@ -147,6 +147,14 @@ export const createTransactionMock = ({ if (select) { tx.select = (...selectArgs: any[]) => ({ from: (...fromArgs: any[]) => ({ + // A join is a plain read, never a locking one; the hook gets the + // joined table as a fifth argument. + innerJoin: (...joinArgs: any[]) => ({ + where: (...wArgs: any[]) => + Promise.resolve( + select("select", selectArgs, fromArgs, wArgs, joinArgs), + ), + }), // Lazy: `.for("update")` must be the thing that takes the lock, and // eagerly building the unlocked promise would run the query twice. where: (...wArgs: any[]) => { diff --git a/packages/api/src/.internal-tests/announcements.test.ts b/packages/api/src/.internal-tests/announcements.test.ts index 0f2b2d0f..39df9907 100644 --- a/packages/api/src/.internal-tests/announcements.test.ts +++ b/packages/api/src/.internal-tests/announcements.test.ts @@ -148,6 +148,7 @@ vi.mock("@query/db", () => { email: "email", sentAt: "sent_at", failedAt: "failed_at", + claimedAt: "claimed_at", }, }; }); @@ -513,6 +514,43 @@ describe("Announcements", () => { expect(res.inAppOnly).toBe(false); expect(recipientRowsOf()[0]).not.toHaveProperty("sentAt"); }); + + /** + * The counts are SQL aggregates, so the rows here are canned as the query + * would return them. What this pins is the router's side: an all-stamped, + * never-claimed announcement is reported as in-app with nothing pending, + * and `pending` is taken from the query rather than derived from + * total - sent - failed, which would read 2 here and put it under + * "Unfinished sends". + */ + it("lists an in-app announcement as published, not as an unfinished send", async () => { + asAdmin(); + mockSelectRows.mockReturnValue([ + { + id: ANNOUNCEMENT, + subject: "Registration is open", + audience: "interested", + createdAt: new Date("2026-02-01T12:00:00Z"), + total: 2, + sent: 0, + failed: 0, + pending: 0, + inAppOnly: true, + }, + ]); + + const [first] = await callerFor(ADMIN).hackathon.listAnnouncements({ + hackathonId: HACK, + }); + + expect(first).toMatchObject({ + total: 2, + sent: 0, + failed: 0, + pending: 0, + inAppOnly: true, + }); + }); }); /** diff --git a/packages/api/src/.internal-tests/assigned-judge.test.ts b/packages/api/src/.internal-tests/assigned-judge.test.ts new file mode 100644 index 00000000..c81910c7 --- /dev/null +++ b/packages/api/src/.internal-tests/assigned-judge.test.ts @@ -0,0 +1,1139 @@ +/** + * A presentation assigned to a judge at the judge desk: the assigned judge + * has it to themselves until TAKEOVER_AFTER_MS after it starts, then any judge + * who may grade it there can take it unless the assigned judge has started. + * Also what the lists and the sheet say about it, the next project a judge is + * sent to, the judge's itinerary, giving a hold back (releaseGrading), and the + * scores the desk shows. + * + * The router runs for real against a stand-in database that answers each select + * by the tables it reads and keeps every statement in order (see + * graded-once.test.ts, whose stand-in this follows). + */ +import { describe, it, expect } from "vitest"; +import { getTableName } from "drizzle-orm"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + judgingAppointments, + judgingBuildings, + judgingChallenges, + judgingConfigs, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingEvaluationDrafts, + judgingEvaluationRatings, + judgingEvaluations, + judgingJudges, + judges, + judgingPresence, + judgingRooms, + judgingRubricItems, + judgingSchedules, +} from "@query/db"; +import { judgingDeskRouter } from "../routers/judging/desk"; +import { reconcileDueDrafts } from "../services/judging/evaluation-store"; +import { GRADING_HOLD_MS } from "../services/judging/grading-hold"; +import { + DEADLINE_GRACE_MS, + TAKEOVER_AFTER_MS, +} from "../services/judging/timing"; + +const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; +const GENERAL = "0c4a8e10-2b3d-4c5e-9f60-7a8b9c0d1e2f"; +const SPONSOR = "0c4a8e11-2b3d-4c5e-9f60-7a8b9c0d1e2f"; +const ME = "1d5b9f21-3c4e-4d6f-8a70-8b9c0d1e2f3a"; +const OTHER = "1d5b9f22-3c4e-4d6f-8a70-8b9c0d1e2f3a"; +const ROOM = "2e6ca032-4d5f-4e70-9b81-9c0d1e2f3a4b"; +const ELSEWHERE = "2e6ca033-4d5f-4e70-9b81-9c0d1e2f3a4b"; +const RATING = "3f7db143-5e60-4f81-8c92-0d1e2f3a4b5c"; +const RATING_2 = "3f7db144-5e60-4f81-8c92-0d1e2f3a4b5c"; + +const E_ONE = "40e8c254-6f71-4092-9da3-1e2f3a4b5c01"; +const E_TWO = "40e8c254-6f71-4092-9da3-1e2f3a4b5c02"; +const E_THREE = "40e8c254-6f71-4092-9da3-1e2f3a4b5c03"; + +const challenge = ( + id: string, + label: string, + mode: "scheduled" | "unscheduled" | "remote", +) => ({ + id, + parentId: null, + mode, + label, + isGeneral: id === GENERAL, + isGroup: false, +}); +const general = challenge(GENERAL, "General", "unscheduled"); +const sponsor = challenge(SPONSOR, "Best Use of AI", "scheduled"); + +const entry = (id: string, title: string) => ({ + id, + title, + description: "", + submissionUrl: null, + demoLinks: [], + videoUrl: null, + technologies: [], + participantCount: 2, +}); + +const name = (table: unknown) => getTableName(table as never); +const via = (...tables: unknown[]) => tables.map(name).join(">"); + +type Rows = unknown[] | ((call: number) => unknown[]); +type Answers = Record; +type Event = { + kind: "select" | "execute" | "insert" | "update" | "delete"; + table?: string; + where?: SQL; + values?: Record; + /** A select that takes row locks. */ + locking?: boolean; +}; + +/** The nth read of a table gets the nth list. */ +const byCall = + (...lists: unknown[][]) => + (call: number) => + lists[call] ?? []; + +function fakeDb(answers: Answers) { + const events: Event[] = []; + const reads = new Map(); + const rowsFor = (key: string) => { + const call = reads.get(key) ?? 0; + reads.set(key, call + 1); + const answer = answers[key] ?? []; + return typeof answer === "function" ? answer(call) : answer; + }; + + const select = () => { + const joined: unknown[] = []; + let from: unknown; + let where: SQL | undefined; + const event: Event = { kind: "select" }; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + where: (clause: SQL) => { + where = clause; + return chain; + }, + for: () => { + event.locking = true; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (error: unknown) => unknown, + ) => { + const key = via(from, ...joined); + events.push({ ...event, table: key, where }); + return Promise.resolve(rowsFor(key)).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown) => { + joined.push(table); + return chain; + }; + } + for (const method of ["orderBy", "limit"]) { + chain[method] = () => chain; + } + return chain; + }; + const db: Record = { + select, + execute: () => { + events.push({ kind: "execute" }); + return Promise.resolve(); + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + events.push({ kind: "insert", table: via(table), values }); + const done = Promise.resolve([{ id: "new" }]); + return Object.assign(done, { returning: () => done }); + }, + }), + update: (table: unknown) => ({ + set: (values: Record) => ({ + where: (where: SQL) => { + events.push({ kind: "update", table: via(table), where, values }); + return Promise.resolve(); + }, + }), + }), + delete: (table: unknown) => ({ + where: (where: SQL) => { + events.push({ kind: "delete", table: via(table), where }); + return Promise.resolve(); + }, + }), + transaction: (run: (tx: unknown) => Promise) => run(db), + }; + return { db, events }; +} + +const rating = { + id: RATING, + kind: "rating", + label: "Impact", + description: "", + required: true, + guestVisibility: null, +}; + +const evaluations = name(judgingEvaluations); +const drafts = name(judgingEvaluationDrafts); +const entriesKey = via(judgingEntries, judgingEntryChallenges); +const presenceKey = via(judgingPresence, judgingRooms); +const slotsKey = via(judgingAppointments); +const listedSlotsKey = via(judgingAppointments, judgingRooms, judgingBuildings); + +const seatAt = (challengeId: string, roomId = ROOM) => [ + { roomId, roomName: "Table 4", challengeId }, +]; + +/** A table judge seated at the sponsor's room, with a schedule saved. */ +const world = (over: Answers = {}): Answers => ({ + [via(judges)]: [{ name: "Grace Hopper" }], + [via(judgingJudges)]: [{ id: ME, displayName: "Grace Hopper" }], + [via(judgingChallenges)]: [general, sponsor], + [via(judgingConfigs)]: [ + { state: "open", showAllScores: false, closedAt: null }, + ], + [via(judgingRubricItems)]: [rating], + [entriesKey]: [entry(E_ONE, "Tidewater")], + [via(judgingSchedules)]: [{ id: "schedule" }], + [presenceKey]: seatAt(SPONSOR), + ...over, +}); + +// A new user for each context: the desk's flood guard counts calls per user. +let callers = 0; +const memberCtx = (db: unknown) => { + const userId = `assigned-user-${(callers += 1)}`; + return { + db, + session: { user: { id: userId, name: "Grace Hopper" } }, + userId, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as never; +}; + +const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000); +const minutesAhead = (minutes: number) => + new Date(Date.now() + minutes * 60_000); +const takeoverMinutes = TAKEOVER_AFTER_MS / 60_000; + +/** A presentation for E_ONE that started `startedMinutesAgo` minutes ago. */ +const slot = ( + assignedJudgeId: string | null, + startedMinutesAgo: number, + over: Record = {}, +) => ({ + id: "slot-1", + entryId: E_ONE, + roomId: ROOM, + startsAt: minutesAgo(startedMinutesAgo), + deadlineAt: minutesAhead(10 - startedMinutesAgo), + endsAt: minutesAhead(12 - startedMinutesAgo), + assignedJudgeId, + ...over, +}); + +const reservedRefusal = + "Another judge is assigned to this presentation. If they have not started 5 minutes in, you can take it."; +const beingGraded = + "Another judge is grading this project right now. Pick another one."; +const gradedElsewhere = + "Another judge has already graded this project for Best Use of AI. Each project is graded once."; + +const wrote = (events: Event[], table: unknown) => + events.some((e) => e.kind === "insert" && e.table === name(table)); + +const save = { + hackathonId: HACK, + entryId: E_ONE, + challengeId: SPONSOR, + ratings: { [RATING]: 4 }, + responses: {}, +}; + +const grade = (answers: Answers) => { + const fake = fakeDb(answers); + const caller = judgingDeskRouter.createCaller(memberCtx(fake.db)); + return { + ...fake, + caller, + save: () => caller.saveDraft({ ...save, revision: 0 }), + start: () => + caller.startGrading({ + hackathonId: HACK, + entryId: E_ONE, + challengeId: SPONSOR, + }), + submit: () => caller.submit({ ...save, expectedRevision: 0 }), + }; +}; + +describe("claiming a presentation assigned to a judge", () => { + it("lets the assigned judge claim it as it starts", async () => { + const { save: go, events } = grade(world({ [slotsKey]: [slot(ME, 0)] })); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + it("keeps another judge off until the takeover time, on every way in", async () => { + const answers = world({ [slotsKey]: [slot(OTHER, takeoverMinutes - 1)] }); + const refused = { + code: "PRECONDITION_FAILED", + message: reservedRefusal, + }; + + for (const way of ["save", "start", "submit"] as const) { + const attempt = grade(answers); + await expect(attempt[way]()).rejects.toMatchObject(refused); + expect(wrote(attempt.events, judgingEvaluationDrafts)).toBe(false); + expect(wrote(attempt.events, judgingEvaluations)).toBe(false); + } + }); + + it("refuses another judge from the very start of the presentation", async () => { + const { save: go } = grade(world({ [slotsKey]: [slot(OTHER, 0)] })); + + await expect(go()).rejects.toMatchObject({ message: reservedRefusal }); + }); + + it("reads the assignment under the grading lock, after the judge's own draft", async () => { + // Two judges reaching for it at the takeover time queue on the lock, and + // the second then sees the first one's draft. + for (const way of ["save", "start", "submit"] as const) { + const attempt = grade(world({ [slotsKey]: [slot(OTHER, 1)] })); + await expect(attempt[way]()).rejects.toMatchObject({ + message: reservedRefusal, + }); + + const at = (match: (e: Event) => boolean) => + attempt.events.findIndex(match); + const ownDraft = at( + (e) => e.kind === "select" && e.table === drafts && !!e.locking, + ); + const lock = at((e) => e.kind === "execute"); + const assignment = at((e) => e.table === slotsKey); + expect(ownDraft).toBeGreaterThan(-1); + expect(lock).toBeGreaterThan(ownDraft); + expect(assignment).toBeGreaterThan(lock); + } + }); + + it("lets another judge take it once the takeover time has passed", async () => { + const answers = world({ [slotsKey]: [slot(OTHER, takeoverMinutes + 1)] }); + + for (const way of ["save", "start", "submit"] as const) { + const attempt = grade(answers); + await expect(attempt[way]()).resolves.toBeDefined(); + } + const { save: go, events } = grade(answers); + await go(); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + it("refuses a takeover when the assigned judge already holds a draft", async () => { + const answers = (draft: Record) => + world({ + [slotsKey]: [slot(OTHER, takeoverMinutes + 1)], + [drafts]: byCall([], [{ judgeId: OTHER, ...draft }]), + }); + // Opened with nothing saved holds until the deadline, as a saved one does. + const held = [ + { updatedAt: minutesAgo(takeoverMinutes), deadlineAt: minutesAhead(4) }, + { updatedAt: minutesAgo(1), deadlineAt: null }, + ]; + + for (const draft of held) { + for (const way of ["save", "start", "submit"] as const) { + const attempt = grade(answers(draft)); + await expect(attempt[way]()).rejects.toMatchObject({ + code: "CONFLICT", + message: beingGraded, + }); + expect(wrote(attempt.events, judgingEvaluationDrafts)).toBe(false); + } + } + }); + + it("refuses a takeover when the assigned judge already graded it", async () => { + const { save: go, events } = grade( + world({ + [slotsKey]: [slot(OTHER, takeoverMinutes + 1)], + [evaluations]: byCall([], [{ judgeId: OTHER }]), + }), + ); + + await expect(go()).rejects.toMatchObject({ + code: "CONFLICT", + message: gradedElsewhere, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("leaves a presentation nobody is assigned to open from the start", async () => { + const { save: go, events } = grade(world({ [slotsKey]: [slot(null, 1)] })); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + it("leaves a judge editing their own evaluation alone, whoever was assigned", async () => { + const { save: go } = grade( + world({ + [slotsKey]: [slot(OTHER, 1)], + [evaluations]: byCall([{ appointmentId: "slot-1" }]), + }), + ); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + }); + + it("does not care who is assigned at a challenge that is not timed", async () => { + const { save: go } = grade( + world({ + [via(judgingChallenges)]: [ + general, + { ...sponsor, mode: "unscheduled" }, + ], + [slotsKey]: [slot(OTHER, 1)], + }), + ); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + }); +}); + +describe("a first save in the write grace window", () => { + // The deadline passed five seconds ago, inside DEADLINE_GRACE_MS, so a + // write still lands. The draft has to carry the deadline, or nothing ever + // submits it for the judge. + const late = () => { + const deadlineAt = new Date(Date.now() - 5_000); + return { + deadlineAt, + answers: world({ [slotsKey]: [slot(ME, 8, { deadlineAt })] }), + }; + }; + const draftWritten = (events: Event[]) => + events.find((e) => e.kind === "insert" && e.table === drafts)?.values; + + it("stamps the presentation's deadline on the draft", async () => { + const { deadlineAt, answers } = late(); + const { save: go, events } = grade(answers); + + await expect(go()).resolves.toMatchObject({ revision: 1, deadlineAt }); + expect(draftWritten(events)).toMatchObject({ + appointmentId: "slot-1", + deadlineAt, + }); + }); + + it("stamps it when the judge opens the project", async () => { + const { deadlineAt, answers } = late(); + const { start, events } = grade(answers); + + await expect(start()).resolves.toMatchObject({ held: true, deadlineAt }); + expect(draftWritten(events)).toMatchObject({ deadlineAt }); + }); + + it("is submitted for the judge as of the deadline", async () => { + const { deadlineAt, answers } = late(); + const { save: go, events } = grade(answers); + await go(); + const saved = draftWritten(events); + expect(saved?.deadlineAt).toEqual(deadlineAt); + + const later = fakeDb( + world({ + [via(judgingEvaluationDrafts, judgingConfigs)]: [ + { id: "draft-1", ...saved }, + ], + [via(judgingJudges)]: [{ id: ME, kind: "member" }], + }), + ); + await reconcileDueDrafts( + later.db as never, + HACK, + new Date(deadlineAt.getTime() + DEADLINE_GRACE_MS), + ); + + const evaluation = later.events.find( + (e) => e.kind === "insert" && e.table === evaluations, + ); + expect(evaluation?.values).toMatchObject({ + judgeId: ME, + appointmentId: "slot-1", + submittedAt: deadlineAt, + autoSubmittedAt: deadlineAt, + }); + expect( + later.events.some((e) => e.kind === "delete" && e.table === drafts), + ).toBe(true); + }); + + it("still never times an edit of a submitted evaluation", async () => { + const { answers } = late(); + const { save: go, events } = grade({ + ...answers, + [evaluations]: byCall([{ appointmentId: "slot-1" }]), + }); + + await expect(go()).resolves.toMatchObject({ + revision: 1, + deadlineAt: null, + }); + expect(draftWritten(events)).toMatchObject({ deadlineAt: null }); + }); +}); + +describe("desk.project for a presentation assigned to a judge", () => { + const open = async (answers: Answers) => { + const { db } = fakeDb({ + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + ...answers, + }); + return judgingDeskRouter.createCaller(memberCtx(db)).project({ + hackathonId: HACK, + entryId: E_ONE, + challengeId: SPONSOR, + }); + }; + + it("is reserved for another judge, with when it opens and no name", async () => { + const startsAt = minutesAgo(2); + const sheet = await open( + world({ [slotsKey]: [slot(OTHER, 2, { startsAt })] }), + ); + + expect(sheet.grading).toBe("reserved"); + expect(sheet.access).toMatchObject({ + canEdit: false, + reason: reservedRefusal, + gradedElsewhere: false, + needsRoom: false, + ended: false, + deadlineAt: null, + }); + expect(sheet.access.opensAt).toEqual( + new Date(startsAt.getTime() + TAKEOVER_AFTER_MS), + ); + // Still there to read. + expect(sheet.entry.title).toBe("Tidewater"); + expect(JSON.stringify(sheet)).not.toContain(OTHER); + }); + + it("is open once the takeover time has passed", async () => { + const sheet = await open( + world({ [slotsKey]: [slot(OTHER, takeoverMinutes + 1)] }), + ); + + expect(sheet.grading).toBe("open"); + expect(sheet.access).toMatchObject({ canEdit: true, reason: null }); + expect(JSON.stringify(sheet)).not.toContain(OTHER); + }); + + it("is being graded, not reserved, once the assigned judge has started", async () => { + const sheet = await open( + world({ + [slotsKey]: [slot(OTHER, 2)], + [drafts]: byCall( + [], + [{ judgeId: OTHER, updatedAt: minutesAgo(1), deadlineAt: null }], + ), + }), + ); + + expect(sheet.grading).toBe("in_progress"); + expect(sheet.access).toMatchObject({ + canEdit: false, + reason: beingGraded, + gradedElsewhere: true, + }); + expect(JSON.stringify(sheet)).not.toContain(OTHER); + }); + + it("shows the assigned judge nothing new: it is theirs to write", async () => { + const sheet = await open(world({ [slotsKey]: [slot(ME, 2)] })); + + expect(sheet.grading).toBe("open"); + expect(sheet.access).toMatchObject({ + canEdit: true, + reason: null, + opensAt: null, + gradedElsewhere: false, + }); + }); + + it("is open as before where nobody is assigned", async () => { + const sheet = await open(world({ [slotsKey]: [slot(null, 2)] })); + + expect(sheet.grading).toBe("open"); + expect(sheet.access.canEdit).toBe(true); + }); +}); + +describe("desk.projects with assigned presentations", () => { + const list = async (answers: Answers, over = {}) => { + const { db } = fakeDb(answers); + return judgingDeskRouter.createCaller(memberCtx(db)).projects({ + hackathonId: HACK, + challengeId: SPONSOR, + ...over, + }); + }; + const listed = ( + entryId: string, + assignedJudgeId: string | null, + startsAt: Date, + ) => ({ + id: `slot-${entryId}`, + entryId, + roomId: ROOM, + roomName: "Table 4", + buildingName: null, + startsAt, + deadlineAt: new Date(startsAt.getTime() + 10 * 60_000), + endsAt: new Date(startsAt.getTime() + 12 * 60_000), + assignedJudgeId, + }); + const crowded = (over: Answers = {}) => + world({ + [entriesKey]: [ + entry(E_ONE, "Alpha"), + entry(E_TWO, "Bravo"), + entry(E_THREE, "Charlie"), + ], + [listedSlotsKey]: [ + listed(E_ONE, null, minutesAhead(10)), + listed(E_TWO, ME, minutesAhead(40)), + listed(E_THREE, ME, minutesAhead(20)), + ], + ...over, + }); + + it("puts the judge's assigned presentations first, in start order, marked", async () => { + const result = await list(crowded()); + + expect(result.projects.map((p) => p.id)).toEqual([E_THREE, E_TWO, E_ONE]); + expect(result.projects.map((p) => p.assignedToYou)).toEqual([ + true, + true, + false, + ]); + }); + + it("lists the rest in start order as before", async () => { + const result = await list( + crowded({ + [listedSlotsKey]: [ + listed(E_ONE, null, minutesAhead(30)), + listed(E_TWO, null, minutesAhead(10)), + listed(E_THREE, null, minutesAhead(20)), + ], + }), + ); + + expect(result.projects.map((p) => p.id)).toEqual([E_TWO, E_THREE, E_ONE]); + expect(result.projects.some((p) => p.assignedToYou)).toBe(false); + }); + + it("shows a presentation kept for another judge as reserved, after the open ones", async () => { + const started = minutesAgo(2); + const result = await list( + crowded({ + [listedSlotsKey]: [ + listed(E_ONE, OTHER, started), + listed(E_TWO, null, minutesAhead(40)), + listed(E_THREE, ME, minutesAhead(20)), + ], + }), + ); + + expect( + result.projects.map((p) => [p.id, p.grading, p.assignedToYou]), + ).toEqual([ + [E_THREE, "open", true], + [E_TWO, "open", false], + [E_ONE, "reserved", false], + ]); + const reserved = result.projects.find((p) => p.id === E_ONE); + expect(reserved?.reservedUntil).toEqual( + new Date(started.getTime() + TAKEOVER_AFTER_MS), + ); + expect( + result.projects.find((p) => p.id === E_TWO)?.reservedUntil, + ).toBeNull(); + }); + + it("opens it to the judge once the takeover time has passed", async () => { + const result = await list( + crowded({ + [listedSlotsKey]: [ + listed(E_ONE, OTHER, minutesAgo(takeoverMinutes + 1)), + listed(E_TWO, ME, minutesAhead(40)), + listed(E_THREE, ME, minutesAhead(20)), + ], + }), + ); + + expect(result.projects.find((p) => p.id === E_ONE)).toMatchObject({ + grading: "open", + reservedUntil: null, + assignedToYou: false, + }); + }); + + it("never says which judge", async () => { + const result = await list( + crowded({ + [listedSlotsKey]: [ + listed(E_ONE, OTHER, minutesAgo(2)), + listed(E_TWO, OTHER, minutesAgo(takeoverMinutes + 1)), + listed(E_THREE, ME, minutesAhead(20)), + ], + }), + ); + + expect(JSON.stringify(result)).not.toContain(OTHER); + }); +}); + +describe("desk.nextProject with assigned presentations", () => { + const POOL = [E_ONE, E_TWO, E_THREE]; + const next = async (answers: Answers) => { + const { db } = fakeDb(answers); + return judgingDeskRouter + .createCaller(memberCtx(db)) + .nextProject({ hackathonId: HACK, challengeId: SPONSOR }); + }; + const running = ( + entryId: string, + assignedJudgeId: string | null, + startedMinutesAgo: number, + ) => slot(assignedJudgeId, startedMinutesAgo, { id: entryId, entryId }); + const crowded = (slots: unknown[]) => + world({ + [entriesKey]: POOL.map((id) => ({ id })), + [slotsKey]: slots, + }); + + it("sends the judge to their own assigned presentation first", async () => { + // Whatever place the queue would give each, the assigned one leads. + for (const mine of POOL) { + const result = await next( + crowded(POOL.map((id) => running(id, id === mine ? ME : null, 2))), + ); + expect(result.entryId).toBe(mine); + } + }); + + it("then offers one open for takeover, never one still reserved", async () => { + const result = await next( + crowded([ + running(E_ONE, OTHER, 2), + running(E_TWO, OTHER, takeoverMinutes + 1), + running(E_THREE, OTHER, 1), + ]), + ); + + expect(result.entryId).toBe(E_TWO); + }); + + it("is nothing when every running presentation is reserved for others", async () => { + const result = await next( + crowded([running(E_ONE, OTHER, 2), running(E_TWO, OTHER, 1)]), + ); + + expect(result.entryId).toBeNull(); + expect(result.needsRoom).toBe(false); + }); + + it("passes over an assigned presentation the judge skipped", async () => { + const { db } = fakeDb( + crowded([ + running(E_ONE, ME, 2), + running(E_TWO, null, 2), + running(E_THREE, OTHER, 1), + ]), + ); + + const result = await judgingDeskRouter + .createCaller(memberCtx(db)) + .nextProject({ hackathonId: HACK, challengeId: SPONSOR, skip: [E_ONE] }); + + expect(result.entryId).toBe(E_TWO); + }); + + it("leaves an untimed challenge to its queue window", async () => { + const result = await next( + world({ + [via(judgingChallenges)]: [ + general, + { ...sponsor, mode: "unscheduled" }, + ], + [entriesKey]: POOL.map((id) => ({ id })), + // Assignments mean nothing where nothing is timed. + [slotsKey]: POOL.map((id) => running(id, OTHER, 1)), + }), + ); + + expect(POOL).toContain(result.entryId); + }); +}); + +describe("desk.itinerary", () => { + const row = (over: Record = {}) => ({ + id: "slot-1", + entryId: E_ONE, + challengeId: SPONSOR, + title: "Tidewater", + roomId: ROOM, + roomName: "Table 4", + roomKind: "table", + buildingName: "Klaus", + startsAt: minutesAhead(10), + deadlineAt: minutesAhead(20), + ...over, + }); + const itineraryKey = via( + judgingAppointments, + judgingEntries, + judgingRooms, + judgingBuildings, + ); + const itinerary = async (answers: Answers) => { + const fake = fakeDb(world(answers)); + const result = await judgingDeskRouter + .createCaller(memberCtx(fake.db)) + .itinerary({ hackathonId: HACK }); + return { result, ...fake }; + }; + + it("lists the judge's own presentations with where, when and what challenge", async () => { + const { result } = await itinerary({ + [itineraryKey]: [ + row(), + row({ + id: "slot-2", + entryId: E_TWO, + title: "Undertow", + challengeId: GENERAL, + roomId: ELSEWHERE, + roomName: "1116", + roomKind: "room", + buildingName: null, + }), + ], + }); + + expect(result.presentations).toHaveLength(2); + expect(result.presentations[0]).toMatchObject({ + entryId: E_ONE, + title: "Tidewater", + roomId: ROOM, + roomName: "Table 4", + roomKind: "table", + buildingName: "Klaus", + challengeId: SPONSOR, + challengeLabel: "Best Use of AI", + graded: false, + takenOver: false, + inRoom: true, + }); + expect(result.presentations[0]?.startsAt).toBeInstanceOf(Date); + expect(result.presentations[0]?.deadlineAt).toBeInstanceOf(Date); + // Seated at the sponsor's table: the other is in a room the judge is not in. + expect(result.presentations[1]).toMatchObject({ + challengeLabel: "General", + inRoom: false, + }); + }); + + it("asks only for the judge's own presentations that have not ended", async () => { + const { events } = await itinerary({ [itineraryKey]: [row()] }); + + const read = events.find((e) => e.table === itineraryKey); + const { sql, params } = new PgDialect().sqlToQuery(read?.where as SQL); + expect(sql).toContain('"assigned_judge_id" = $'); + expect(params).toContain(ME); + expect(sql).toContain('"deadline_at" > $'); + expect(sql).toContain('"deleted_at" is null'); + }); + + it("says a presentation is graded, or that another judge has taken it, without who", async () => { + const { result } = await itinerary({ + [itineraryKey]: [ + row({ id: "a", entryId: E_ONE }), + row({ id: "b", entryId: E_TWO }), + row({ id: "c", entryId: E_THREE }), + ], + [evaluations]: [{ entryId: E_ONE, challengeId: SPONSOR, judgeId: OTHER }], + [drafts]: [ + { + entryId: E_TWO, + challengeId: SPONSOR, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: minutesAhead(5), + }, + // A stale draft holds nothing. + { + entryId: E_THREE, + challengeId: SPONSOR, + judgeId: OTHER, + updatedAt: new Date(Date.now() - GRADING_HOLD_MS - 60_000), + deadlineAt: null, + }, + ], + }); + + expect( + result.presentations.map((p) => [p.entryId, p.graded, p.takenOver]), + ).toEqual([ + [E_ONE, true, false], + [E_TWO, false, true], + [E_THREE, false, false], + ]); + expect(JSON.stringify(result)).not.toContain(OTHER); + }); + + it("is empty when the judge has nothing assigned, and before a schedule is saved", async () => { + const none = await itinerary({ [itineraryKey]: [] }); + expect(none.result.presentations).toEqual([]); + + const unscheduled = await itinerary({ + [via(judgingSchedules)]: [], + [itineraryKey]: [row()], + }); + expect(unscheduled.result.presentations).toEqual([]); + }); +}); + +describe("desk.releaseGrading", () => { + const release = async (answers: Answers, input = {}) => { + const fake = fakeDb(world(answers)); + const result = await judgingDeskRouter + .createCaller(memberCtx(fake.db)) + .releaseGrading({ + hackathonId: HACK, + entryId: E_ONE, + challengeId: SPONSOR, + ...input, + }); + return { result, ...fake }; + }; + const opened = { id: "draft-1", revision: 0 }; + const deleted = (events: Event[]) => + events.some((e) => e.kind === "delete" && e.table === drafts); + + it("deletes a draft the judge opened and never saved, locking it first", async () => { + const { result, events } = await release({ [drafts]: [opened] }); + + expect(result).toEqual({ released: true }); + expect(deleted(events)).toBe(true); + const read = events.findIndex( + (e) => e.kind === "select" && e.table === drafts, + ); + expect(events[read]?.locking).toBe(true); + expect(read).toBeLessThan(events.findIndex((e) => e.kind === "delete")); + }); + + it("only touches the judge's own draft", async () => { + const { events } = await release({ [drafts]: [opened] }); + + const read = events.find((e) => e.kind === "select" && e.table === drafts); + const { params } = new PgDialect().sqlToQuery(read?.where as SQL); + expect(params).toContain(ME); + expect(params).toContain(E_ONE); + expect(params).toContain(SPONSOR); + const gone = events.find((e) => e.kind === "delete"); + expect(new PgDialect().sqlToQuery(gone?.where as SQL).params).toEqual([ + "draft-1", + ]); + }); + + it("keeps a draft the judge has saved to", async () => { + const { result, events } = await release({ + [drafts]: [{ id: "draft-1", revision: 1 }], + }); + + expect(result).toEqual({ released: false }); + expect(deleted(events)).toBe(false); + }); + + it("keeps a draft of a project the judge has graded", async () => { + const { result, events } = await release({ + [drafts]: [opened], + [evaluations]: [{ id: "evaluation-1" }], + }); + + expect(result).toEqual({ released: false }); + expect(deleted(events)).toBe(false); + }); + + it("is nothing when the judge holds no draft", async () => { + const { result, events } = await release({}); + + expect(result).toEqual({ released: false }); + expect(deleted(events)).toBe(false); + }); + + it("gives back a sponsor's project from a judge who has left its room", async () => { + const { result } = await release({ + [drafts]: [opened], + [presenceKey]: [], + }); + + expect(result).toEqual({ released: true }); + }); + + it("finds the challenge by the judge's seat when none is passed", async () => { + const { result } = await release( + { [drafts]: [opened] }, + { challengeId: undefined }, + ); + + expect(result).toEqual({ released: true }); + }); + + it("leaves a challenge of another edition alone", async () => { + const { result, events } = await release( + { [drafts]: [opened] }, + { challengeId: "0c4a8e99-2b3d-4c5e-9f60-7a8b9c0d1e2f" }, + ); + + expect(result).toEqual({ released: false }); + expect(deleted(events)).toBe(false); + }); + + it("takes only the draft's row lock, never the grading lock", async () => { + // A save racing it waits on the same row: one that lands first leaves a + // saved draft this keeps, and one that lands after starts a new draft. + // With no grading lock, it cannot deadlock with a judge's claim. + const { events } = await release({ [drafts]: [opened] }); + + expect(events.some((e) => e.kind === "execute")).toBe(false); + expect( + events + .filter((e) => e.kind === "select" && e.locking) + .map((e) => e.table), + ).toEqual([drafts]); + }); +}); + +describe("the scores the desk shows count what placings count", () => { + const MINE = "evaluation-mine"; + const two = [rating, { ...rating, id: RATING_2, label: "Polish" }]; + const own = (isComplete: boolean) => ({ + id: MINE, + revision: 1, + isComplete, + appointmentId: null, + submittedAt: minutesAgo(30), + autoSubmittedAt: minutesAgo(30), + updatedAt: minutesAgo(30), + }); + const rated = (...items: string[]) => + items.map((itemId) => ({ evaluationId: MINE, itemId, value: 4 })); + + const sheet = async (answers: Answers) => { + const { db } = fakeDb( + world({ + [via(judgingChallenges)]: [general], + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + [slotsKey]: [], + [via(judgingSchedules)]: [], + ...answers, + }), + ); + return judgingDeskRouter.createCaller(memberCtx(db)).project({ + hackathonId: HACK, + entryId: E_ONE, + challengeId: GENERAL, + }); + }; + const withEvaluation = ( + isComplete: boolean, + items: string[], + rubric = two, + ) => ({ + [via(judgingRubricItems)]: rubric, + [evaluations]: byCall( + [own(isComplete)], + [{ judgeId: ME }], + [{ id: MINE, challengeId: GENERAL }], + ), + [via(judgingEvaluationRatings)]: rated(...items), + }); + + it("scores an evaluation that rated every item though a response was left empty", async () => { + const result = await sheet(withEvaluation(false, [RATING, RATING_2])); + + expect(result.evaluation).toMatchObject({ isComplete: false, score: 4 }); + expect(result.scores?.challenge).toEqual({ score: 4, count: 1 }); + expect(result.scores?.overall).toEqual({ score: 4, count: 1 }); + }); + + it("leaves out an evaluation that is missing a rating", async () => { + const result = await sheet(withEvaluation(false, [RATING])); + + expect(result.evaluation?.score).toBeNull(); + expect(result.scores?.challenge).toEqual({ score: null, count: 0 }); + }); + + it("scores a whole evaluation as before", async () => { + const result = await sheet(withEvaluation(true, [RATING], [rating])); + + expect(result.evaluation?.score).toBe(4); + expect(result.scores?.challenge).toEqual({ score: 4, count: 1 }); + }); + + it("scores the same in the judge's submitted list", async () => { + const mine = async (items: string[]) => { + const { db } = fakeDb( + world({ + [via(judgingRubricItems)]: two, + [via(judgingEvaluations, judgingEntries)]: [], + [via(judgingEvaluations, judgingEntries, judgingChallenges)]: [ + { + id: MINE, + entryId: E_ONE, + challengeId: GENERAL, + challengeLabel: "General", + title: "Tidewater", + deletedAt: null, + revision: 1, + isComplete: false, + submittedAt: minutesAgo(30), + autoSubmittedAt: minutesAgo(30), + updatedAt: minutesAgo(30), + }, + ], + [via(judgingEvaluationRatings)]: rated(...items), + }), + ); + return judgingDeskRouter + .createCaller(memberCtx(db)) + .myEvaluations({ hackathonId: HACK }); + }; + + expect((await mine([RATING, RATING_2]))[0]?.score).toBe(4); + expect((await mine([RATING]))[0]?.score).toBeNull(); + }); +}); diff --git a/packages/api/src/.internal-tests/check-in-flags-desk.test.ts b/packages/api/src/.internal-tests/check-in-flags-desk.test.ts new file mode 100644 index 00000000..c4f2db71 --- /dev/null +++ b/packages/api/src/.internal-tests/check-in-flags-desk.test.ts @@ -0,0 +1,437 @@ +/** + * Check-in rule-break flags at the judge desk. A judge, member or guest, is + * told only that a team is flagged: one boolean on each project in the list and + * on the project being judged. The hacker, the rule and the events stay with + * organisers, so these tests read the whole serialized response for them. + * + * The router runs for real against a stand-in database that answers each + * select by the tables it reads, which is also how the tests see what the desk + * did and did not read. + */ +import { describe, it, expect } from "vitest"; +import { getTableName } from "drizzle-orm"; +import { + hackathonCheckInFlags, + hackathonEvents, + hackathonParticipants, + hackathonProjects, + judgingChallenges, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingGuestSessions, + judgingJudges, + judges, + judgingRoomLinks, + judgingRooms, + users, +} from "@query/db"; +import { judgingDeskRouter } from "../routers/judging/desk"; + +const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; +const CHALLENGE = "0c4a8e10-2b3d-4c5e-9f60-7a8b9c0d1e2f"; +const JUDGE_ID = "1d5b9f21-3c4e-4d6f-8a70-8b9c0d1e2f3a"; +const ROOM = "2e6ca032-4d5f-4e70-9b81-9c0d1e2f3a4b"; + +// The two projects on the desk. One belongs to the flagged hacker's team. +const FLAGGED_ENTRY = "3f7db143-5e60-4f81-8c92-0d1e2f3a4b5c"; +const CLEAN_ENTRY = "40e8c254-6f71-4092-9da3-1e2f3a4b5c6d"; + +// Everything an organiser would see about the flag. None of it may reach a +// judge. +const FLAG_ID = "51f9d365-7082-41a3-8eb4-2f3a4b5c6d7e"; +const PARTICIPANT_ID = "62a0e476-8193-42b4-9fc5-3a4b5c6d7e8f"; +const HACKER_USER_ID = "73b1f587-92a4-43c5-80d6-4b5c6d7e8f90"; +const FLAGGED_TEAM = "84c2a698-a3b5-44d6-91e7-5c6d7e8f90a1"; +const OTHER_TEAM = "95d3b7a9-b4c6-45e7-a2f8-6d7e8f90a1b2"; +const HACKER_EMAIL = "mia.okafor@example.com"; +const HACKER_NAME = "Mia Okafor"; +const TRIED_EVENT = "Opening Ceremony"; +const CONFLICT_EVENT = "BNY Mellon Workshop"; + +const challenges = [ + { + id: CHALLENGE, + parentId: null, + mode: "unscheduled", + label: "General", + isGeneral: true, + isGroup: false, + }, +]; + +const entry = (id: string, title: string, participantCount: number) => ({ + id, + title, + description: "", + submissionUrl: null, + demoLinks: [], + videoUrl: null, + technologies: [], + participantCount, +}); + +const flaggedEntry = entry(FLAGGED_ENTRY, "Tidewater", 3); +const cleanEntry = entry(CLEAN_ENTRY, "Quiet Garden", 2); + +/** The key a stand-in answer is filed under: the table read, then its joins. */ +const via = (from: unknown, ...joins: unknown[]) => + [from, ...joins].map((table) => getTableName(table as never)).join(">"); + +type Answers = Record; + +/** + * A database whose selects resolve from `answers`, looked up by the tables the + * query reads and defaulting to no rows. Every select is kept, so a test can + * see which tables were read and how often. + */ +function fakeDb(answers: Answers) { + const reads: { key: string; from: string; fields: string[] }[] = []; + const db = { + select: (fields: Record) => { + const joined: unknown[] = []; + let from: unknown; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (error: unknown) => unknown, + ) => { + const key = via(from, ...joined); + reads.push({ + key, + from: getTableName(from as never), + fields: Object.keys(fields ?? {}), + }); + return Promise.resolve(answers[key] ?? []).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown) => { + joined.push(table); + return chain; + }; + } + for (const method of ["where", "orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }, + }; + return { db, reads }; +} + +/** What the desk reads before the project list or the project itself. */ +const deskBasics = (): Answers => ({ + [via(judges)]: [{ name: "Grace Hopper" }], + [via(judgingJudges)]: [{ id: JUDGE_ID, displayName: "Grace Hopper" }], + [via(judgingChallenges)]: challenges, +}); + +/** + * One open flag from a hacker on FLAGGED_TEAM. The rows for the tables only an + * organiser's view reads are filled in too, so a desk that went and fetched + * them would have something to leak. + */ +const flagRows = ( + opts: { portal?: unknown[]; members?: unknown[] } = {}, +): Answers => ({ + [via(hackathonCheckInFlags, hackathonParticipants)]: [ + { + id: FLAG_ID, + participantId: PARTICIPANT_ID, + teamId: FLAGGED_TEAM, + userId: HACKER_USER_ID, + }, + ], + [via(judgingEntries, hackathonProjects)]: opts.portal ?? [ + { entryId: FLAGGED_ENTRY, teamId: FLAGGED_TEAM, submittedById: null }, + { entryId: CLEAN_ENTRY, teamId: OTHER_TEAM, submittedById: null }, + ], + [via(judgingEntryMembers, judgingEntries)]: opts.members ?? [], + [via(users)]: [{ id: HACKER_USER_ID, email: HACKER_EMAIL }], + [via(hackathonEvents)]: [ + { id: "event-a", name: TRIED_EVENT }, + { id: "event-b", name: CONFLICT_EVENT }, + ], +}); + +const memberCtx = (db: unknown) => + ({ + db, + session: { user: { id: "judge-user", name: "Grace Hopper" } }, + userId: "judge-user", + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + }) as never; + +const guestAnswers = (): Answers => ({ + [via(judgingGuestSessions, judgingRoomLinks, judgingRooms, judgingJudges)]: [ + { + sessionId: "session-1", + hackathonId: HACK, + linkId: "link-1", + roomId: ROOM, + roomChallengeId: CHALLENGE, + judgeId: JUDGE_ID, + displayName: "Guest Judge", + }, + ], + [via(judgingChallenges)]: challenges, +}); + +const guestCtx = (db: unknown) => + ({ + db, + session: null, + userId: undefined, + clientIp: "127.0.0.2", + req: { + headers: { + get: (name: string) => + name.toLowerCase() === "cookie" + ? "query_judging_guest=guest-token" + : null, + }, + }, + }) as never; + +/** Words and ids that only organisers may see. */ +const organiserOnly = [ + "email", + "userId", + FLAG_ID, + PARTICIPANT_ID, + HACKER_USER_ID, + FLAGGED_TEAM, + HACKER_EMAIL, + HACKER_NAME, + TRIED_EVENT, + CONFLICT_EVENT, + "overlap", + "too_soon", + "attempts", +]; + +const expectNothingOrganiserOnly = (response: unknown) => { + const serialized = JSON.stringify(response); + for (const secret of organiserOnly) expect(serialized).not.toContain(secret); +}; + +/** The tables a desk read took data from, for the privacy checks. */ +const tablesRead = (reads: { from: string }[]) => + new Set(reads.map((read) => read.from)); + +const flagReads = (reads: T[]) => + reads.filter((read) => read.from === getTableName(hackathonCheckInFlags)); + +const judgesUnderTest = [ + { + kind: "member judge", + answers: deskBasics, + ctx: memberCtx, + }, + { + kind: "guest judge", + answers: guestAnswers, + ctx: guestCtx, + }, +] as const; + +describe.each(judgesUnderTest)( + "check-in flags at the desk for a $kind", + ({ answers, ctx }) => { + describe("desk.projects", () => { + const listAnswers = (flags: Answers): Answers => ({ + ...answers(), + [via(judgingEntries, judgingEntryChallenges)]: [ + { id: FLAGGED_ENTRY, title: "Tidewater", participantCount: 3 }, + { id: CLEAN_ENTRY, title: "Quiet Garden", participantCount: 2 }, + ], + ...flags, + }); + + it("marks the flagged team's project true and the other false", async () => { + const { db } = fakeDb(listAnswers(flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + const flagged = Object.fromEntries( + result.projects.map((project) => [project.id, project.flagged]), + ); + expect(flagged).toEqual({ + [FLAGGED_ENTRY]: true, + [CLEAN_ENTRY]: false, + }); + }); + + it("marks a project flagged through a member row, as a Devpost entry is", async () => { + const { db } = fakeDb( + listAnswers( + flagRows({ + portal: [], + members: [{ entryId: CLEAN_ENTRY, userId: HACKER_USER_ID }], + }), + ), + ); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect( + result.projects.find((project) => project.id === CLEAN_ENTRY) + ?.flagged, + ).toBe(true); + expect( + result.projects.find((project) => project.id === FLAGGED_ENTRY) + ?.flagged, + ).toBe(false); + }); + + it("is false for every project when nobody is flagged", async () => { + const { db } = fakeDb(listAnswers({})); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect(result.projects).toHaveLength(2); + expect(result.projects.map((project) => project.flagged)).toEqual([ + false, + false, + ]); + }); + + it("adds a boolean to each project and nothing else about the flag", async () => { + const { db } = fakeDb(listAnswers(flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + for (const project of result.projects) { + expect(typeof project.flagged).toBe("boolean"); + expect(Object.keys(project).sort()).toEqual([ + "assignedToYou", + "challenges", + "flagged", + "grading", + "hasDraft", + "id", + "myEvaluation", + "participantCount", + "presentation", + "reservedUntil", + "title", + ]); + } + expectNothingOrganiserOnly(result); + }); + + it("asks for flags once for the whole list, not once per project", async () => { + const { db, reads } = fakeDb(listAnswers(flagRows())); + + await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect(flagReads(reads)).toHaveLength(1); + }); + + it("never reads the people or events behind a flag", async () => { + const { db, reads } = fakeDb(listAnswers(flagRows())); + + await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + const read = tablesRead(reads); + expect(read.has(getTableName(users))).toBe(false); + expect(read.has(getTableName(hackathonEvents))).toBe(false); + // Only the flag's own ids are selected, never a name or an email. + for (const select of flagReads(reads)) { + expect(select.fields).toEqual([ + "id", + "participantId", + "teamId", + "userId", + ]); + } + }); + + it("does not look for flags when there are no projects", async () => { + const { db, reads } = fakeDb({ ...answers(), ...flagRows() }); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect(result.projects).toEqual([]); + expect(flagReads(reads)).toHaveLength(0); + }); + }); + + describe("desk.project", () => { + const sheetAnswers = ( + projectEntry: ReturnType, + flags: Answers, + ): Answers => ({ + ...answers(), + [via(judgingEntries, judgingEntryChallenges)]: [projectEntry], + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + ...flags, + }); + + it("marks the flagged team's project true", async () => { + const { db } = fakeDb(sheetAnswers(flaggedEntry, flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: FLAGGED_ENTRY }); + + expect(result.entry.flagged).toBe(true); + }); + + it("marks another team's project false", async () => { + const { db } = fakeDb(sheetAnswers(cleanEntry, flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: CLEAN_ENTRY }); + + expect(result.entry.flagged).toBe(false); + }); + + it("is false when nobody is flagged", async () => { + const { db } = fakeDb(sheetAnswers(flaggedEntry, {})); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: FLAGGED_ENTRY }); + + expect(result.entry.flagged).toBe(false); + }); + + it("tells the judge nothing more than the boolean", async () => { + const { db, reads } = fakeDb(sheetAnswers(flaggedEntry, flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: FLAGGED_ENTRY }); + + expect(result.entry.flagged).toBe(true); + expectNothingOrganiserOnly(result); + const read = tablesRead(reads); + expect(read.has(getTableName(users))).toBe(false); + expect(read.has(getTableName(hackathonEvents))).toBe(false); + expect(flagReads(reads)).toHaveLength(1); + }); + }); + }, +); diff --git a/packages/api/src/.internal-tests/check-in-flags-organizer.test.ts b/packages/api/src/.internal-tests/check-in-flags-organizer.test.ts new file mode 100644 index 00000000..eb1b60a3 --- /dev/null +++ b/packages/api/src/.internal-tests/check-in-flags-organizer.test.ts @@ -0,0 +1,783 @@ +/** + * Check-in flags in the organiser's judging console: the project list carries + * each project's flags and whether it was pulled from prizes, can be narrowed + * to flagged projects, and the header counts open flags. Two writes close the + * loop: dismissing a flag and pulling (or returning) a project from prizes. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way judge-edge.test.ts does. It keeps + * the rows of the tables a write touches and applies the conditions the router + * asks for, so a missing edition condition lets another edition's row through + * and fails the test, rather than passing on a canned answer. The flag + * service has its own tests; here it is a stub, which also lets these tests + * see exactly what the router asks it for. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + auditLogs, + hackathonCheckInFlags, + hackathons, + judgingConfigs, + judgingEntries, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; +import type * as CheckInFlags from "../services/check-in-flags"; + +const mockFlagsForEntries = vi.fn(); +const mockFlaggedIds = vi.fn(); +const mockOpenCount = vi.fn(); + +vi.mock("../services/check-in-flags", async (importActual) => ({ + ...(await importActual()), + checkInFlagsForEntries: (...args: unknown[]) => mockFlagsForEntries(...args), + flaggedEntryIds: (...args: unknown[]) => mockFlaggedIds(...args), + openCheckInFlagCount: (...args: unknown[]) => mockOpenCount(...args), +})); + +const HACK_A = "11111111-1111-4111-8111-111111111111"; +const HACK_B = "22222222-2222-4222-8222-222222222222"; +const ENTRY_1 = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1"; +const ENTRY_2 = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa2"; +const ENTRY_3 = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa3"; +const ENTRY_OTHER = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1"; +const FLAG_1 = "cccccccc-cccc-4ccc-8ccc-ccccccccccc1"; +const FLAG_OTHER = "dddddddd-dddd-4ddd-8ddd-ddddddddddd1"; +const NO_SUCH = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); + +type Row = Record; +type SelectCall = { + fields: Record | undefined; + table: unknown; + where: unknown; +}; + +const camel = (name: string) => + name.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase()); + +const TERM = /"[a-z_]+"\."([a-z_]+)" = \$(\d+)/g; + +/** + * A row filter from a where made only of `column = value` terms joined by + * `and`. Anything else throws, so the fake never quietly ignores a condition + * it does not understand. + */ +function matcher(where: unknown) { + const { sql: text, params } = render(where); + const terms = [...text.matchAll(TERM)].map( + (match) => [camel(match[1]!), params[Number(match[2]) - 1]] as const, + ); + if (text.replace(TERM, "").replace(/[()\s]|and/g, "")) { + throw new Error(`The fake database cannot evaluate: ${text}`); + } + return (row: Row) => terms.every(([key, value]) => row[key] === value); +} + +// --------------------------------------------------------------------------- +// The stand-in database + +let adminRow: Row | undefined; +const store = new Map(); +const selects: SelectCall[] = []; +const updates: { table: unknown; values: Row }[] = []; +const inserts: { table: unknown; values: Row }[] = []; +// Answers a select outright; undefined hands it to the stored rows. +let route: ((call: SelectCall) => unknown[] | undefined) | null = null; + +const matching = (table: unknown, where: unknown) => { + const rows = store.get(table) ?? []; + return where ? rows.filter(matcher(where)) : [...rows]; +}; + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: (fields?: Record) => { + const call: SelectCall = { fields, table: null, where: null }; + selects.push(call); + const chain: any = { + then: (onOk: any, onErr: any) => + Promise.resolve(route?.(call) ?? matching(call.table, call.where)).then( + onOk, + onErr, + ), + }; + chain.from = (table: unknown) => { + call.table = table; + return chain; + }; + chain.where = (where: unknown) => { + call.where = where; + return chain; + }; + for (const method of [ + "innerJoin", + "leftJoin", + "groupBy", + "orderBy", + "limit", + "offset", + "for", + ]) { + chain[method] = () => chain; + } + return chain; + }, + insert: (table: unknown) => ({ + values: (values: Row) => { + inserts.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + // The config row already exists, so a create-if-missing finds nothing + // to return and reads the stored one. + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [values], + }); + }, + }), + update: (table: unknown) => ({ + set: (values: Row) => ({ + where: (where: unknown) => { + const hit = matching(table, where); + for (const row of hit) Object.assign(row, values); + updates.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => hit.map((row) => ({ ...row })), + }); + }, + }), + }), + delete: () => ({ where: async () => [] }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string | undefined) => + appRouter.createCaller({ + db: fakeDb, + session: userId ? { user: { id: userId } } : null, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); + +const admin = () => callerAs("admin_user"); + +const audits = () => + inserts.filter((write) => write.table === auditLogs).map((w) => w.values); + +const flagView = ( + over: Partial = {}, +): CheckInFlags.CheckInFlagView => ({ + id: FLAG_1, + participantName: "Ada Lovelace", + rule: "overlap", + eventName: "BNY Workshop", + conflictEventName: "Full Sail Fireside", + method: "code", + attempts: 3, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T19:41:00Z"), + dismissed: false, + ...over, +}); + +const entryRow = (id: string, over: Row = {}) => ({ + id, + source: "portal", + sourceProjectId: null, + title: `Project ${id.slice(-1)}`, + submissionUrl: null, + description: "", + demoLinks: [], + videoUrl: null, + technologies: [], + universities: [], + prizeCategories: [], + participantCount: 2, + submittedAt: null, + deletedAt: null, + prizeIneligible: false, + createdAt: new Date("2027-02-06T12:00:00Z"), + ...over, +}); + +beforeEach(() => { + vi.clearAllMocks(); + cache.clear(); + selects.length = 0; + updates.length = 0; + inserts.length = 0; + route = null; + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + mockFlagsForEntries.mockReset().mockResolvedValue(new Map()); + mockFlaggedIds.mockReset().mockResolvedValue(new Set()); + mockOpenCount.mockReset().mockResolvedValue(0); + + store.clear(); + store.set(hackathons, [ + { id: HACK_A, name: "Hacklytics 2027", startDate: null, endDate: null }, + { id: HACK_B, name: "Hacklytics 2026", startDate: null, endDate: null }, + ]); + store.set(judgingConfigs, [ + { + hackathonId: HACK_A, + state: "draft", + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: false, + emergencyMode: false, + claimsStartedAt: null, + }, + { + hackathonId: HACK_B, + state: "draft", + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: false, + emergencyMode: false, + claimsStartedAt: null, + }, + ]); + store.set(hackathonCheckInFlags, [ + { + id: FLAG_1, + hackathonId: HACK_A, + dismissedAt: null, + dismissedById: null, + }, + { + id: FLAG_OTHER, + hackathonId: HACK_B, + dismissedAt: null, + dismissedById: null, + }, + ]); + store.set(judgingEntries, [ + { + id: ENTRY_1, + hackathonId: HACK_A, + prizeIneligibleAt: null, + prizeIneligibleById: null, + }, + { + id: ENTRY_OTHER, + hackathonId: HACK_B, + prizeIneligibleAt: null, + prizeIneligibleById: null, + }, + ]); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.listEntries", () => { + /** The page of projects the database returns, whatever filter it was given. */ + const wirePage = (rows: Row[], liveIds: string[] = rows.map((r) => r.id)) => { + route = (call) => { + if (call.table !== judgingEntries) return []; + if (call.fields && "value" in call.fields) + return [{ value: rows.length }]; + if (call.fields && "title" in call.fields) return rows; + return liveIds.map((id) => ({ id })); + }; + }; + + const pageQuery = () => + selects.find((call) => call.fields && "title" in call.fields)!; + const totalQuery = () => + selects.find((call) => call.fields && "value" in call.fields)!; + + it("puts each project's flags and its prize status on the project", async () => { + wirePage([entryRow(ENTRY_1, { prizeIneligible: true }), entryRow(ENTRY_2)]); + const flag = flagView(); + mockFlagsForEntries.mockResolvedValue(new Map([[ENTRY_1, [flag]]])); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + }); + + expect(result.entries.map((e) => e.id)).toEqual([ENTRY_1, ENTRY_2]); + expect(result.entries[0]).toMatchObject({ + checkInFlags: [flag], + prizeIneligible: true, + }); + expect(result.entries[1]).toMatchObject({ + checkInFlags: [], + prizeIneligible: false, + }); + // Asked for this page's projects only, in this edition. + expect(mockFlagsForEntries).toHaveBeenCalledTimes(1); + expect(mockFlagsForEntries).toHaveBeenCalledWith(fakeDb, HACK_A, [ + ENTRY_1, + ENTRY_2, + ]); + }); + + it("reads prize status from the entry's prize_ineligible_at column", async () => { + wirePage([entryRow(ENTRY_1)]); + + await admin().judging.admin.listEntries({ hackathonId: HACK_A }); + + expect(render(pageQuery().fields!.prizeIneligible).sql).toBe( + '"judging_entry"."prize_ineligible_at" is not null', + ); + }); + + it("does not look up flags for an empty page", async () => { + wirePage([]); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + }); + + expect(result.entries).toEqual([]); + expect(mockFlagsForEntries).not.toHaveBeenCalled(); + }); + + it("applies no flag condition unless asked", async () => { + wirePage([entryRow(ENTRY_1)]); + + await admin().judging.admin.listEntries({ hackathonId: HACK_A }); + await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: false, + }); + + expect(mockFlaggedIds).not.toHaveBeenCalled(); + for (const call of selects.filter((c) => c.fields && "title" in c.fields)) { + expect(render(call.where).sql).not.toMatch(/ in \(/); + } + }); + + it("narrows the page and the total to projects with an open flag", async () => { + wirePage([entryRow(ENTRY_2)], [ENTRY_1, ENTRY_2, ENTRY_3]); + mockFlaggedIds.mockResolvedValue(new Set([ENTRY_2])); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: true, + }); + + // Open flags are looked for among this edition's live projects. + expect(mockFlaggedIds).toHaveBeenCalledWith(fakeDb, HACK_A, [ + ENTRY_1, + ENTRY_2, + ENTRY_3, + ]); + const liveRead = selects.find( + (call) => + call.table === judgingEntries && + call.fields && + !("title" in call.fields) && + !("value" in call.fields), + )!; + const live = render(liveRead.where); + expect(live.sql).toContain('"judging_entry"."deleted_at" is null'); + expect(live.params).toContain(HACK_A); + + // The same condition sits in the query that pages and the one that counts, + // so the total is the flagged total and not the edition's. + for (const call of [pageQuery(), totalQuery()]) { + const { sql, params } = render(call.where); + expect(sql).toMatch(/"judging_entry"\."id" in \(\$\d+\)/); + expect(params).toContain(ENTRY_2); + expect(params).not.toContain(ENTRY_1); + expect(params).not.toContain(ENTRY_3); + } + expect(result.total).toBe(1); + }); + + it("matches nothing when no project has an open flag", async () => { + wirePage([], [ENTRY_1, ENTRY_2]); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: true, + }); + + for (const call of [pageQuery(), totalQuery()]) { + expect(render(call.where).sql).toMatch(/ and false\)$/); + } + expect(result).toMatchObject({ total: 0, entries: [] }); + }); + + it("combines the flag filter with the others", async () => { + wirePage([], [ENTRY_1]); + mockFlaggedIds.mockResolvedValue(new Set([ENTRY_1])); + + await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: true, + source: "portal", + status: "live", + }); + + const { sql, params } = render(pageQuery().where); + expect(sql).toContain('"judging_entry"."source" = $'); + expect(sql).toMatch(/"judging_entry"\."id" in \(/); + expect(params).toEqual(expect.arrayContaining([HACK_A, "portal", ENTRY_1])); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.overview", () => { + const overview = () => + admin().judging.admin.overview({ hackathonId: HACK_A }); + + // Every other count reads from tables this test leaves empty. + beforeEach(() => { + route = (call) => + call.table === hackathons || call.table === judgingConfigs + ? undefined + : []; + }); + + it("counts the open check-in flags and nothing about them", async () => { + mockOpenCount.mockResolvedValue(4); + + const result = await overview(); + + expect(result.counts.checkInFlags).toEqual({ open: 4 }); + expect(mockOpenCount).toHaveBeenCalledWith(fakeDb, HACK_A); + }); + + it("reports zero when there are none", async () => { + const result = await overview(); + + expect(result.counts.checkInFlags).toEqual({ open: 0 }); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.dismissCheckInFlag", () => { + const flagRow = (id: string) => + store.get(hackathonCheckInFlags)!.find((row) => row.id === id)!; + + it("dismisses a flag and records who", async () => { + const result = await admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: true, + }); + + expect(result).toEqual({ dismissed: true }); + expect(flagRow(FLAG_1)).toMatchObject({ + dismissedAt: expect.any(Date), + dismissedById: "admin_user", + }); + expect(updates).toHaveLength(1); + expect(updates[0]!.table).toBe(hackathonCheckInFlags); + expect(Object.keys(updates[0]!.values).sort()).toEqual([ + "dismissedAt", + "dismissedById", + ]); + expect(audits()).toEqual([ + expect.objectContaining({ + userId: "admin_user", + action: "judging.dismissCheckInFlag", + resourceId: FLAG_1, + severity: "info", + metadata: { hackathonId: HACK_A }, + }), + ]); + }); + + it("opens a dismissed flag again", async () => { + Object.assign(flagRow(FLAG_1), { + dismissedAt: new Date(), + dismissedById: "someone", + }); + + const result = await admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: false, + }); + + expect(result).toEqual({ dismissed: false }); + expect(flagRow(FLAG_1)).toMatchObject({ + dismissedAt: null, + dismissedById: null, + }); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.reopenCheckInFlag", + resourceId: FLAG_1, + }), + ]); + }); + + it("refuses a flag that belongs to another hackathon", async () => { + await expect( + admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_OTHER, + dismissed: true, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(flagRow(FLAG_OTHER).dismissedAt).toBeNull(); + expect(audits()).toEqual([]); + }); + + it("refuses a flag that does not exist", async () => { + await expect( + admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: NO_SUCH, + dismissed: true, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(audits()).toEqual([]); + }); + + it("leaves the cached public results alone", async () => { + cache.set(`hackathon:${HACK_A}:results`, [{ id: "placing" }], 60); + + await admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: true, + }); + + expect(cache.get(`hackathon:${HACK_A}:results`)).toEqual([ + { id: "placing" }, + ]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.setPrizeEligibility", () => { + const entry = (id: string) => + store.get(judgingEntries)!.find((row) => row.id === id)!; + const resultsKey = (hackathonId: string) => + `hackathon:${hackathonId}:results`; + + it("pulls a project from prizes and clears the cached results at once", async () => { + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + cache.set(resultsKey(HACK_B), [{ id: "last year" }], 60); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: false, + }); + + expect(result).toEqual({ changed: true }); + expect(entry(ENTRY_1)).toMatchObject({ + prizeIneligibleAt: expect.any(Date), + prizeIneligibleById: "admin_user", + }); + expect(updates).toHaveLength(1); + expect(updates[0]!.table).toBe(judgingEntries); + expect(cache.get(resultsKey(HACK_A))).toBeNull(); + // Only this edition's results. + expect(cache.get(resultsKey(HACK_B))).toEqual([{ id: "last year" }]); + expect(audits()).toEqual([ + expect.objectContaining({ + userId: "admin_user", + action: "judging.setPrizeEligibility", + resourceId: ENTRY_1, + severity: "warn", + metadata: { hackathonId: HACK_A, eligible: false }, + }), + ]); + }); + + it("puts a project back in the running and clears the cached results", async () => { + Object.assign(entry(ENTRY_1), { + prizeIneligibleAt: new Date(), + prizeIneligibleById: "someone", + }); + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: true, + }); + + expect(result).toEqual({ changed: true }); + expect(entry(ENTRY_1)).toMatchObject({ + prizeIneligibleAt: null, + prizeIneligibleById: null, + }); + expect(cache.get(resultsKey(HACK_A))).toBeNull(); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.setPrizeEligibility", + severity: "warn", + metadata: { hackathonId: HACK_A, eligible: true }, + }), + ]); + }); + + it("does nothing when the project is already in that state", async () => { + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: true, + }); + + expect(result).toEqual({ changed: false }); + expect(updates).toEqual([]); + expect(audits()).toEqual([]); + expect(cache.get(resultsKey(HACK_A))).toEqual([{ id: "placing" }]); + }); + + it("does not move the pull date when asked to pull again", async () => { + const pulledAt = new Date("2027-02-06T20:00:00Z"); + Object.assign(entry(ENTRY_1), { + prizeIneligibleAt: pulledAt, + prizeIneligibleById: "first_admin", + }); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: false, + }); + + expect(result).toEqual({ changed: false }); + expect(entry(ENTRY_1)).toMatchObject({ + prizeIneligibleAt: pulledAt, + prizeIneligibleById: "first_admin", + }); + }); + + it("refuses a project that belongs to another hackathon", async () => { + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + cache.set(resultsKey(HACK_B), [{ id: "last year" }], 60); + + await expect( + admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_OTHER, + eligible: false, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(entry(ENTRY_OTHER).prizeIneligibleAt).toBeNull(); + expect(updates).toEqual([]); + expect(audits()).toEqual([]); + expect(cache.get(resultsKey(HACK_A))).toEqual([{ id: "placing" }]); + expect(cache.get(resultsKey(HACK_B))).toEqual([{ id: "last year" }]); + }); + + it("refuses a project that does not exist", async () => { + await expect( + admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: NO_SUCH, + eligible: false, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(updates).toEqual([]); + }); + + it("refuses an edition that does not exist", async () => { + await expect( + admin().judging.admin.setPrizeEligibility({ + hackathonId: NO_SUCH, + entryId: ENTRY_1, + eligible: false, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(entry(ENTRY_1).prizeIneligibleAt).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +describe("who may use them", () => { + const writes = (caller: ReturnType) => [ + () => + caller.judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: true, + }), + () => + caller.judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: false, + }), + ]; + const reads = (caller: ReturnType) => [ + () => caller.judging.admin.listEntries({ hackathonId: HACK_A }), + () => caller.judging.admin.overview({ hackathonId: HACK_A }), + ]; + + const untouched = () => { + expect(updates).toEqual([]); + expect( + store + .get(hackathonCheckInFlags)! + .every((row) => row.dismissedAt === null), + ).toBe(true); + expect( + store.get(judgingEntries)!.every((row) => row.prizeIneligibleAt === null), + ).toBe(true); + expect(mockFlagsForEntries).not.toHaveBeenCalled(); + expect(mockOpenCount).not.toHaveBeenCalled(); + }; + + it("refuses someone who is not an admin, reads and writes alike", async () => { + adminRow = undefined; + const caller = callerAs("hacker_user"); + + for (const run of [...reads(caller), ...writes(caller)]) { + await expect(run()).rejects.toMatchObject({ code: "FORBIDDEN" }); + } + untouched(); + }); + + it("refuses a volunteer, who only staffs the scanner", async () => { + adminRow = { userId: "vol_user", isActive: true, role: "volunteer" }; + const caller = callerAs("vol_user"); + + for (const run of [...reads(caller), ...writes(caller)]) { + await expect(run()).rejects.toMatchObject({ code: "FORBIDDEN" }); + } + untouched(); + }); + + it("refuses someone who is not signed in", async () => { + const caller = callerAs(undefined); + + for (const run of [...reads(caller), ...writes(caller)]) { + await expect(run()).rejects.toMatchObject({ code: "UNAUTHORIZED" }); + } + untouched(); + }); + + it("lets a bug tester read but not write", async () => { + adminRow = { userId: "qa_user", isActive: true, role: "bug_tester" }; + const caller = callerAs("qa_user"); + route = (call) => + call.table === hackathons || call.table === judgingConfigs + ? undefined + : []; + + for (const run of reads(caller)) await run(); + for (const run of writes(caller)) { + await expect(run()).rejects.toMatchObject({ code: "FORBIDDEN" }); + } + + expect(updates).toEqual([]); + expect(audits()).toEqual([]); + }); +}); diff --git a/packages/api/src/.internal-tests/check-in-flags-results.test.ts b/packages/api/src/.internal-tests/check-in-flags-results.test.ts new file mode 100644 index 00000000..bd001ea2 --- /dev/null +++ b/packages/api/src/.internal-tests/check-in-flags-results.test.ts @@ -0,0 +1,1381 @@ +/** + * Check-in flags meeting results: a team an organiser pulled from prizes is not + * ranked by judging.results.compute, judging.results.list tells the organiser + * how many flags are open on each placed team, and the public hackathon.getResults + * drops a pulled team at once without ever carrying flag data. + * + * It also covers what calibration adds to those procedures: compute stores the + * judge-adjusted score with the raw mean beside it, the organiser's list returns + * both and a per-judge summary, and the public payload carries neither the raw + * score nor anything about a judge. + * + * It covers which evaluations count (every rating, whether or not a written + * response is missing, as with a draft submitted at the deadline), and the + * calibration the list keeps for a minute, which compute never reads. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way judge-edge.test.ts does. Its + * selects answer by the table they read from, and the entries query applies + * the filters it is given, so a filter taken out of the router changes the + * result instead of passing unnoticed. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + hackathonCheckInFlags, + hackathonEvents, + hackathons, + judgingAppointments, + judgingChallenges, + judgingConfigs, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingEvaluationDrafts, + judgingEvaluationRatings, + judgingEvaluationResponses, + judgingEvaluations, + judgingJudges, + judgingPlacements, + judgingRubricItems, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; + +const HACK = "11111111-1111-4111-8111-111111111111"; +const ADMIN_ROW = { userId: "admin_user", isActive: true, role: "admin" }; + +const render = (condition: unknown) => + new PgDialect().sqlToQuery(condition as never); + +type Query = { + table: unknown; + wheres: unknown[]; + /** The columns the select asked for, to tell two reads of one table apart. */ + fields?: Record; +}; + +const mockSelect = vi.fn<(query: Query) => unknown[]>(); +const mockFindMany = vi.fn(); +const inserts: { table: unknown; values: any }[] = []; +const deletes: unknown[] = []; + +const settled = (value: T) => Object.assign(Promise.resolve(value), {}); + +const selectChain = (fields?: Record) => { + const query: Query = { table: null, wheres: [], fields }; + const chain: any = { + then: (onOk: any, onErr: any) => + Promise.resolve(mockSelect(query)).then(onOk, onErr), + }; + chain.from = (table: unknown) => { + query.table = table; + return chain; + }; + chain.where = (condition: unknown) => { + query.wheres.push(condition); + return chain; + }; + for (const method of [ + "innerJoin", + "leftJoin", + "groupBy", + "orderBy", + "limit", + "offset", + "for", + ]) { + chain[method] = () => chain; + } + return chain; +}; + +const fakeDb: any = { + query: { + admins: { findFirst: async () => ADMIN_ROW }, + hackathons: { + findFirst: async () => ({ id: HACK, status: "completed" }), + }, + hackathonResults: { findMany: async () => [] }, + judgingPlacements: { + findMany: (...args: unknown[]) => mockFindMany(...args), + }, + }, + select: (fields?: Record) => selectChain(fields), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), + insert: (table: unknown) => ({ + values: (values: unknown) => { + inserts.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => [{ id: "row" }], + onConflictDoNothing: () => ({ returning: async () => [{ id: "row" }] }), + }); + }, + }), + delete: (table: unknown) => ({ + where: () => { + deletes.push(table); + return settled(undefined); + }, + }), +}; + +const adminCaller = () => + appRouter.createCaller({ + db: fakeDb, + session: { user: { id: "admin_user" } }, + userId: "admin_user", + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); + +const anonCaller = () => + appRouter.createCaller({ + db: fakeDb, + session: null, + userId: undefined, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); + +const placementsInserted = () => + inserts + .filter((write) => write.table === judgingPlacements) + .flatMap((write) => write.values as any[]); + +beforeEach(() => { + vi.clearAllMocks(); + cache.clear(); + inserts.length = 0; + deletes.length = 0; + mockSelect.mockReset(); + mockFindMany.mockReset(); +}); + +// --------------------------------------------------------------------------- +describe("judging.results.compute", () => { + const entries = [ + { id: "entry-a", deletedAt: null, prizeIneligibleAt: null }, + // Pulled from prizes. Its judges scored it highest of the three. + { id: "entry-b", deletedAt: null, prizeIneligibleAt: new Date() }, + { id: "entry-c", deletedAt: null, prizeIneligibleAt: null }, + ]; + const evaluationRow = ( + id: string, + entryId: string, + judgeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + }); + // One judge marked all three. With the pulled team left out of the + // calibration she has two grades, a mean of 3.5 and the pool's own spread, so + // nothing about her scores moves; with it counted, they would. + const scored = [ + evaluationRow("eval-a", "entry-a", "judge-1", 0), + evaluationRow("eval-b", "entry-b", "judge-1", 1), + evaluationRow("eval-c", "entry-c", "judge-1", 2), + ]; + const ratings = [ + { evaluationId: "eval-a", value: 4 }, + { evaluationId: "eval-b", value: 5 }, + { evaluationId: "eval-c", value: 3 }, + ]; + + let entryWheres: unknown[] = []; + + const wire = (stored = entries) => { + entryWheres = []; + mockSelect.mockImplementation(({ table, wheres }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + // Nothing is due, so reconcileDueDrafts returns at once. + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + // Nothing is published yet. + if (table === judgingPlacements) return [{ n: 0 }]; + if (table === judgingEntries) { + entryWheres = wheres; + // What the database would hand back for the conditions asked for. + const asked = wheres.map((where) => render(where).sql).join(" "); + return stored.filter( + (entry) => + (!asked.includes('"deleted_at" is null') || !entry.deletedAt) && + (!asked.includes('"prize_ineligible_at" is null') || + !entry.prizeIneligibleAt), + ); + } + if (table === judgingEntryChallenges) return []; + if (table === judgingChallenges) return []; + if (table === judgingEvaluations) return scored; + if (table === judgingEvaluationRatings) return ratings; + // One rating item, which every evaluation here rated. + if (table === judgingRubricItems) return [{ n: 1 }]; + return []; + }); + }; + + it("leaves a team pulled from prizes out of the ranking", async () => { + wire(); + + const result = await adminCaller().judging.results.compute({ + hackathonId: HACK, + }); + + const placed = placementsInserted(); + expect(placed.map((row) => row.entryId)).not.toContain("entry-b"); + // The next team moves up: entry-a is 1st on 4.00 and entry-c 2nd on 3.00, + // although entry-b's 5.00 would have taken first place. + expect( + placed.map((row) => [row.entryId, row.scope, row.placement, row.score]), + ).toEqual([ + ["entry-a", "overall", 1, "4.00"], + ["entry-c", "overall", 2, "3.00"], + ]); + expect(placed.map((row) => row.rawScore)).toEqual(["4.00", "3.00"]); + expect(result.placements).toBe(2); + }); + + it("still counts the pulled team's evaluations as scored", async () => { + wire(); + + const result = await adminCaller().judging.results.compute({ + hackathonId: HACK, + }); + + // Judges keep scoring it; only the placing is withheld. + expect(result.evaluations).toBe(3); + }); + + it("asks for live, prize-eligible entries of the edition", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + const query = render(entryWheres[0]); + expect(query.params).toContain(HACK); + expect(query.sql).toContain('"deleted_at" is null'); + expect(query.sql).toContain('"prize_ineligible_at" is null'); + }); + + it("ranks everyone when nobody is pulled", async () => { + wire(entries.map((entry) => ({ ...entry, prizeIneligibleAt: null }))); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + expect(placementsInserted().map((row) => row.entryId)).toEqual([ + "entry-b", + "entry-a", + "entry-c", + ]); + }); +}); + +describe("judging.results.compute with judge calibration", () => { + const entries = ["e1", "e2", "e3", "e4"].map((id) => ({ + id, + deletedAt: null, + prizeIneligibleAt: null, + })); + const evaluation = ( + id: string, + entryId: string, + judgeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + }); + + // A strict judge marked e1 and e2, a lenient one e3 and e4. Raw, e2 and e3 + // tie on 4.00 and the lenient judge's e4 runs away with first. + const wire = () => + mockSelect.mockImplementation(({ table }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + if (table === judgingPlacements) return [{ n: 0 }]; + if (table === judgingEntries) return entries; + if (table === judgingEvaluations) { + return [ + evaluation("v1", "e1", "strict", 0), + evaluation("v2", "e2", "strict", 1), + evaluation("v3", "e3", "lenient", 2), + evaluation("v4", "e4", "lenient", 3), + ]; + } + if (table === judgingEvaluationRatings) { + return [ + { evaluationId: "v1", value: 3 }, + { evaluationId: "v2", value: 4 }, + { evaluationId: "v3", value: 4 }, + { evaluationId: "v4", value: 5 }, + ]; + } + if (table === judgingRubricItems) return [{ n: 1 }]; + return []; + }); + + it("stores the judge-adjusted score with the raw mean beside it", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + // Pool mean 4. Two grades each gives an offset of 2/5 of the 0.5 gap: the + // strict judge is lifted 0.2, the lenient one lowered 0.2. Both spread + // their marks by the same 0.5, so neither is stretched. + expect( + placementsInserted().map((row) => [ + row.entryId, + row.placement, + row.score, + row.rawScore, + ]), + ).toEqual([ + ["e4", 1, "4.80", "5.00"], + ["e2", 2, "4.20", "4.00"], + ["e3", 3, "3.80", "4.00"], + ["e1", 4, "3.20", "3.00"], + ]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.results.compute with grades submitted at the deadline", () => { + const entries = ["entry-a", "entry-b", "entry-c"].map((id) => ({ + id, + deletedAt: null, + prizeIneligibleAt: null, + })); + const evaluation = ( + id: string, + entryId: string, + minute: number, + isComplete: boolean, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId: "judge-1", + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + isComplete, + }); + // A rubric of two rating items. entry-a's only grade is an auto-submit: both + // ratings, a required response left empty, so isComplete is false. entry-b's + // stopped after one rating. entry-c's is whole. + const stored = [ + evaluation("eval-a", "entry-a", 0, false), + evaluation("eval-b", "entry-b", 1, false), + evaluation("eval-c", "entry-c", 2, true), + ]; + const ratings = [ + { evaluationId: "eval-a", value: 4 }, + { evaluationId: "eval-a", value: 4 }, + { evaluationId: "eval-b", value: 5 }, + { evaluationId: "eval-c", value: 3 }, + { evaluationId: "eval-c", value: 3 }, + ]; + let evaluationWheres: unknown[] = []; + let ratingWheres: unknown[] = []; + + const wire = () => { + evaluationWheres = []; + ratingWheres = []; + mockSelect.mockImplementation(({ table, wheres }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + if (table === judgingPlacements) return [{ n: 0 }]; + if (table === judgingEntries) return entries; + if (table === judgingRubricItems) return [{ n: 2 }]; + if (table === judgingEvaluations) { + evaluationWheres = wheres; + return stored; + } + if (table === judgingEvaluationRatings) { + ratingWheres = wheres; + return ratings; + } + return []; + }); + }; + + it("places a project whose only grade is such an auto-submit", async () => { + wire(); + + const result = await adminCaller().judging.results.compute({ + hackathonId: HACK, + }); + + const placed = placementsInserted(); + expect( + placed.map((row) => [row.entryId, row.scope, row.placement, row.score]), + ).toEqual([ + ["entry-a", "overall", 1, "4.00"], + ["entry-c", "overall", 2, "3.00"], + ]); + expect(placed.map((row) => row.evaluationCount)).toEqual([1, 1]); + expect(result.placements).toBe(2); + }); + + it("leaves a grade short of a rating out, and does not count it as ranked", async () => { + wire(); + + const result = await adminCaller().judging.results.compute({ + hackathonId: HACK, + }); + + expect(placementsInserted().map((row) => row.entryId)).not.toContain( + "entry-b", + ); + expect(result.evaluations).toBe(2); + }); + + it("does not filter evaluations or ratings on isComplete", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + expect(evaluationWheres.length).toBeGreaterThan(0); + expect(ratingWheres.length).toBeGreaterThan(0); + for (const where of [...evaluationWheres, ...ratingWheres]) { + expect(render(where).sql).not.toContain("is_complete"); + expect(render(where).params).toContain(HACK); + } + }); + + it("asks the rubric how many rating items an evaluation must have", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + const rubric = mockSelect.mock.calls.find( + ([query]) => query.table === judgingRubricItems, + )![0]; + expect(render(rubric.wheres[0]).params).toEqual([HACK, "rating"]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.results.compute beside the list's calibration cache", () => { + const entries = ["e1", "e2", "e3", "e4"].map((id) => ({ + id, + deletedAt: null, + prizeIneligibleAt: null, + })); + const evaluation = ( + id: string, + entryId: string, + judgeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + }); + const stored = [ + evaluation("v1", "e1", "strict", 0), + evaluation("v2", "e2", "strict", 1), + evaluation("v3", "e3", "lenient", 2), + evaluation("v4", "e4", "lenient", 3), + ]; + let ratings = [ + { evaluationId: "v1", value: 3 }, + { evaluationId: "v2", value: 4 }, + { evaluationId: "v3", value: 4 }, + { evaluationId: "v4", value: 5 }, + ]; + const computedAt = new Date("2027-02-07T09:00:00Z"); + + beforeEach(() => { + ratings = [ + { evaluationId: "v1", value: 3 }, + { evaluationId: "v2", value: 4 }, + { evaluationId: "v3", value: 4 }, + { evaluationId: "v4", value: 5 }, + ]; + }); + + // The same edition answers both procedures. Placements come back as rows for + // the list and as a published count for compute, which asks for `n`. + const wire = () => + mockSelect.mockImplementation(({ table, fields }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + if (table === judgingPlacements) { + return fields && "n" in fields + ? [{ n: 0 }] + : [ + { + id: "p1", + scope: "overall", + scopeLabel: "Overall", + placement: 1, + score: "4.00", + evaluationCount: 1, + publishedAt: null, + computedAt, + entryId: "e1", + title: "Project e1", + deletedAt: null, + prizeIneligibleAt: null, + }, + ]; + } + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + if (table === judgingEntries) { + return fields && "latest" in fields + ? [{ n: 4, latest: new Date("2027-02-06T09:00:00Z") }] + : entries; + } + if (table === judgingEvaluations) { + if (fields && "latest" in fields) { + return [{ n: 4, latest: new Date("2027-02-06T11:00:00Z") }]; + } + return fields && "n" in fields ? [{ n: 0 }] : stored; + } + if (table === judgingRubricItems) return [{ n: 1 }]; + if (table === judgingEvaluationRatings) return ratings; + if (table === judgingJudges) { + return [ + { id: "strict", displayName: "Sam Strict", kind: "member" }, + { id: "lenient", displayName: "Lee Lenient", kind: "member" }, + ]; + } + if (table === judgingChallenges) { + return [{ id: "ch-1", label: "General" }]; + } + return []; + }); + + const ratingReads = () => + mockSelect.mock.calls.filter( + ([query]) => query.table === judgingEvaluationRatings, + ).length; + + it("ranks on the evaluations as they stand, whatever the list has cached", async () => { + wire(); + const getOrSet = vi.spyOn(cache, "getOrSet"); + + // The list fills the cache, and a second poll is answered from it. + await adminCaller().judging.results.list({ hackathonId: HACK }); + await adminCaller().judging.results.list({ hackathonId: HACK }); + expect(ratingReads()).toBe(1); + expect(getOrSet).toHaveBeenCalledTimes(2); + + // A rating changes under it, with nothing the cache key could see. + ratings = ratings.map((rating) => + rating.evaluationId === "v4" ? { ...rating, value: 1 } : rating, + ); + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + // Compute read the ratings itself and did not touch the cache. + expect(ratingReads()).toBe(2); + expect(getOrSet).toHaveBeenCalledTimes(2); + // v4 is the lenient judge's lowest now, so e4 ranks last. + expect(placementsInserted().at(-1)?.entryId).toBe("e4"); + getOrSet.mockRestore(); + }); + + it("reads the ratings again on every compute", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + expect(ratingReads()).toBe(2); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.results.list", () => { + // What the calibration cache key is read from: the edition's evaluations and + // its ranked entries, each a count and a latest save. A test changes one + // between two lists to stand for a save, a reopen or a pulled team. + const stamps = { + evaluations: { n: 4, latest: new Date("2027-02-06T11:00:00Z") } as Record< + string, + unknown + >, + entries: { n: 6, latest: new Date("2027-02-06T09:00:00Z") } as Record< + string, + unknown + >, + }; + beforeEach(() => { + stamps.evaluations = { n: 4, latest: new Date("2027-02-06T11:00:00Z") }; + stamps.entries = { n: 6, latest: new Date("2027-02-06T09:00:00Z") }; + }); + + const COMPUTED_AT = new Date("2027-02-07T09:00:00Z"); + const placement = ( + id: string, + entryId: string, + over: Record = {}, + ) => ({ + id, + scope: "overall", + scopeLabel: "Overall", + placement: 1, + score: "4.00", + evaluationCount: 2, + publishedAt: null, + computedAt: COMPUTED_AT, + entryId, + title: `Project ${entryId}`, + deletedAt: null, + prizeIneligibleAt: null, + ...over, + }); + + const flagRow = (id: string, over: Record = {}) => ({ + id, + participantId: `part-${id}`, + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attempts: 1, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T18:00:00Z"), + dismissedAt: null, + teamId: "team-1", + userId: `user-${id}`, + firstName: "Secret", + lastName: `Hacker${id}`, + accountName: null, + email: `${id}@example.com`, + ...over, + }); + + const wire = (opts: { + placements: unknown[]; + flags?: unknown[]; + portal?: unknown[]; + /** Live entries, as the calibration summary reads them. */ + live?: unknown[]; + /** Complete evaluations, as the calibration summary reads them. */ + evaluations?: unknown[]; + ratings?: unknown[]; + /** Rating items on the rubric. The fixtures rate one each unless said. */ + ratingItems?: number; + judges?: unknown[]; + challenges?: unknown[]; + }) => { + mockSelect.mockImplementation(({ table, fields }) => { + if (table === judgingPlacements) return opts.placements; + // The cache key reads a count and a latest time, the stale count reads + // a count alone, and the calibration summary reads many columns. + if (table === judgingEvaluations) { + if (fields && "latest" in fields) return [stamps.evaluations]; + return fields && "n" in fields ? [{ n: 0 }] : (opts.evaluations ?? []); + } + if (table === judgingRubricItems) return [{ n: opts.ratingItems ?? 1 }]; + if (table === judgingEvaluationRatings) return opts.ratings ?? []; + if (table === judgingJudges) return opts.judges ?? []; + if (table === judgingChallenges) return opts.challenges ?? []; + if (table === hackathonCheckInFlags) return opts.flags ?? []; + if (table === hackathonEvents) { + return [ + { id: "event-1", name: "Opening Workshop" }, + { id: "event-2", name: "Sponsor Talk" }, + ]; + } + if (table === judgingEntries) { + if (fields && "latest" in fields) return [stamps.entries]; + return opts.portal ?? opts.live ?? []; + } + if (table === judgingEntryMembers) return []; + return []; + }); + }; + + const rowsOf = (data: { scopes: { placements: { entryId: string }[] }[] }) => + data.scopes.flatMap((scope) => scope.placements); + + it("counts the open flags on each placed team and skips dismissed ones", async () => { + wire({ + placements: [ + placement("p1", "entry-a"), + placement("p2", "entry-b", { placement: 2 }), + ], + // Team one has two open flags and a cleared one; entry-b has none. + flags: [ + flagRow("f1"), + flagRow("f2", { participantId: "part-f1b" }), + flagRow("f3", { dismissedAt: new Date("2027-02-06T20:00:00Z") }), + ], + portal: [{ entryId: "entry-a", teamId: "team-1", submittedById: null }], + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const byEntry = Object.fromEntries( + rowsOf(data).map((row) => [row.entryId, row]), + ) as Record; + expect(byEntry["entry-a"].openCheckInFlags).toBe(2); + expect(byEntry["entry-b"].openCheckInFlags).toBe(0); + }); + + it("says which placed teams are pulled from prizes", async () => { + wire({ + placements: [ + placement("p1", "entry-a"), + placement("p2", "entry-b", { + placement: 2, + prizeIneligibleAt: new Date("2027-02-07T10:00:00Z"), + }), + ], + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const byEntry = Object.fromEntries( + rowsOf(data).map((row) => [row.entryId, row]), + ) as Record; + expect(byEntry["entry-a"].prizeIneligible).toBe(false); + expect(byEntry["entry-b"].prizeIneligible).toBe(true); + expect(byEntry["entry-a"].openCheckInFlags).toBe(0); + }); + + it("carries a count and a flag, never who or why", async () => { + wire({ + placements: [placement("p1", "entry-a")], + flags: [flagRow("f1")], + portal: [{ entryId: "entry-a", teamId: "team-1", submittedById: null }], + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const text = JSON.stringify(data); + expect(text).not.toContain("Secret"); + expect(text).not.toContain("example.com"); + expect(text).not.toContain("Opening Workshop"); + expect(text).not.toContain("Sponsor Talk"); + expect(Object.keys(rowsOf(data)[0]!).sort()).toEqual([ + "entryId", + "evaluationCount", + "id", + "openCheckInFlags", + "placement", + "prizeIneligible", + "rawScore", + "score", + "title", + "withdrawn", + ]); + }); + + it("returns the raw score beside the adjusted one, and null on older rows", async () => { + wire({ + placements: [ + placement("p1", "entry-a", { score: "4.20", rawScore: "3.90" }), + placement("p2", "entry-b", { placement: 2, rawScore: null }), + ], + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const byEntry = Object.fromEntries( + rowsOf(data).map((row) => [row.entryId, row]), + ) as Record; + expect(byEntry["entry-a"].score).toBe(4.2); + expect(byEntry["entry-a"].rawScore).toBe(3.9); + expect(byEntry["entry-b"].rawScore).toBeNull(); + }); + + describe("judge calibration summary", () => { + const graded = ( + id: string, + entryId: string, + judgeId: string, + challengeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId, + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + }); + const challenges = [ + { id: "ch-sponsor", label: "Sponsor Prize" }, + { id: "ch-general", label: "General" }, + ]; + const judges = [ + { id: "strict", displayName: "Sam Strict", kind: "member" }, + { id: "lenient", displayName: "Lee Lenient", kind: "guest" }, + { id: "sponsor-guest", displayName: "Gus Guest", kind: "guest" }, + ]; + // Six table grades by two judges, and two by the sponsor's own guest. + const table = [ + ["g1", "strict", 3], + ["g2", "strict", 3], + ["g3", "strict", 4], + ["g4", "lenient", 4], + ["g5", "lenient", 5], + ["g6", "lenient", 5], + ] as const; + const evaluations = [ + ...table.map(([id, judgeId], i) => + graded(id, `entry-${i}`, judgeId, "ch-general", i), + ), + graded("s1", "entry-0", "sponsor-guest", "ch-sponsor", 10), + graded("s2", "entry-1", "sponsor-guest", "ch-sponsor", 11), + ]; + const ratings = [ + ...table.map(([id, , value]) => ({ evaluationId: id, value })), + { evaluationId: "s1", value: 2 }, + { evaluationId: "s2", value: 2 }, + ]; + const live = Array.from({ length: 6 }, (_, i) => ({ id: `entry-${i}` })); + + it("is computed on read, per challenge, one row for each judge", async () => { + wire({ + placements: [placement("p1", "entry-0")], + evaluations, + ratings, + judges, + challenges, + live, + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + expect(data.calibration.map((c) => c.challenge)).toEqual([ + "Sponsor Prize", + "General", + ]); + const [sponsor, general] = data.calibration; + // The sponsor's guest is compared with no one at the tables. + expect(sponsor!.judges).toHaveLength(1); + expect(sponsor!.judges[0]).toMatchObject({ + judgeName: "Gus Guest", + kind: "guest", + grades: 2, + offset: 0, + spread: 1, + drift: 0, + }); + expect(general!.judges.map((j) => [j.judgeName, j.grades])).toEqual([ + ["Lee Lenient", 3], + ["Sam Strict", 3], + ]); + const [lee, sam] = general!.judges; + expect(lee!.offset).toBeGreaterThan(0); + expect(sam!.offset).toBeLessThan(0); + // Rounded to two places. + for (const judge of general!.judges) { + for (const value of [judge.offset, judge.spread, judge.drift]) { + expect(value).toBe(Math.round(value * 100) / 100); + } + } + }); + + it("leaves a team pulled from prizes out of the summary", async () => { + wire({ + placements: [placement("p1", "entry-0")], + evaluations, + ratings, + judges, + challenges, + // Only the first four teams are live and prize-eligible. + live: live.slice(0, 4), + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const general = data.calibration.find((c) => c.challenge === "General"); + expect(general!.judges.map((j) => j.grades)).toEqual([1, 3]); + }); + + it("is empty before anything is computed", async () => { + wire({ placements: [], evaluations, ratings, judges, challenges, live }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + expect(data.calibration).toEqual([]); + }); + + it("counts a grade submitted at the deadline with every rating, and not one short of a rating", async () => { + // Two rating items. g1 is an auto-submit: all its ratings, a response + // left empty, so the evaluation is incomplete. g3 stopped after one. + const two = [ + { ...evaluations[0]!, isComplete: false }, + evaluations[1]!, + evaluations[2]!, + ]; + const rated = [ + { evaluationId: "g1", value: 3 }, + { evaluationId: "g1", value: 3 }, + { evaluationId: "g2", value: 3 }, + { evaluationId: "g2", value: 3 }, + { evaluationId: "g3", value: 4 }, + ]; + wire({ + placements: [placement("p1", "entry-0")], + evaluations: two, + ratings: rated, + ratingItems: 2, + judges, + challenges, + live, + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const general = data.calibration.find((c) => c.challenge === "General"); + expect(general!.judges.map((j) => [j.judgeName, j.grades])).toEqual([ + ["Sam Strict", 2], + ]); + }); + + describe("cache", () => { + const list = () => + adminCaller().judging.results.list({ hackathonId: HACK }); + // The heavy read: every rating of the edition. + const calibrations = () => + mockSelect.mock.calls.filter( + ([query]) => query.table === judgingEvaluationRatings, + ).length; + const setUp = () => + wire({ + placements: [placement("p1", "entry-0")], + evaluations, + ratings, + judges, + challenges, + live, + }); + + it("calibrates once for polls that find nothing changed", async () => { + setUp(); + + const first = await list(); + const second = await list(); + const third = await list(); + + expect(calibrations()).toBe(1); + expect(second.calibration).toEqual(first.calibration); + expect(third.calibration).toEqual(first.calibration); + expect(first.calibration.length).toBeGreaterThan(0); + }); + + it("calibrates again when an evaluation is added or deleted", async () => { + setUp(); + await list(); + + stamps.evaluations = { ...stamps.evaluations, n: 5 }; + await list(); + expect(calibrations()).toBe(2); + + // Two grades reopened, the newest among them: fewer, and an earlier + // latest save. + stamps.evaluations = { + n: 3, + latest: new Date("2027-02-06T10:30:00Z"), + }; + await list(); + expect(calibrations()).toBe(3); + }); + + it("calibrates again when an evaluation is edited", async () => { + setUp(); + await list(); + + // Same count, saved later. + stamps.evaluations = { + ...stamps.evaluations, + latest: new Date("2027-02-06T11:05:00Z"), + }; + await list(); + + expect(calibrations()).toBe(2); + }); + + it("calibrates again when a team is pulled from prizes or put back", async () => { + setUp(); + await list(); + + stamps.entries = { n: 5, latest: new Date("2027-02-07T08:00:00Z") }; + await list(); + expect(calibrations()).toBe(2); + + // Back in, a moment later: the count is as it was, the time is not. + stamps.entries = { n: 6, latest: new Date("2027-02-07T08:01:00Z") }; + await list(); + expect(calibrations()).toBe(3); + }); + + it("keys the result by edition and by what it was computed from", async () => { + setUp(); + const getOrSet = vi.spyOn(cache, "getOrSet"); + + await list(); + + expect(getOrSet).toHaveBeenCalledTimes(1); + const [key, , ttl] = getOrSet.mock.calls[0]!; + const latest = (stamps.evaluations.latest as Date).getTime(); + const entries = (stamps.entries.latest as Date).getTime(); + expect(key).toBe( + `judging:${HACK}:calibration:4:${latest}:6:${entries}`, + ); + expect(ttl).toBe(60); + getOrSet.mockRestore(); + }); + + it("does not share a result between editions", async () => { + setUp(); + await list(); + + await adminCaller().judging.results.list({ + hackathonId: "99999999-9999-4999-8999-999999999999", + }); + + expect(calibrations()).toBe(2); + }); + + it("keeps the result for a minute and no longer", async () => { + setUp(); + const getOrSet = vi.spyOn(cache, "getOrSet"); + await list(); + const key = getOrSet.mock.calls[0]![0]; + getOrSet.mockRestore(); + + // Only the clock the cache reads is moved: the router's own rate + // limiter keeps real time. + const clock = vi.spyOn(Date, "now"); + const now = Date.now(); + try { + clock.mockReturnValue(now + 59_000); + expect(cache.has(key)).toBe(true); + clock.mockReturnValue(now + 61_000); + expect(cache.has(key)).toBe(false); + } finally { + clock.mockRestore(); + } + }); + }); + }); + + it("reads no flags when nothing is placed", async () => { + wire({ placements: [] }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + expect(data.scopes).toEqual([]); + const tables = mockSelect.mock.calls.map(([query]) => query.table); + expect(tables).not.toContain(hackathonCheckInFlags); + }); +}); + +// --------------------------------------------------------------------------- +describe("the organiser's evaluation views with a grade short of a response", () => { + const APPOINTMENT = "33333333-3333-4333-8333-333333333333"; + const at = new Date("2027-02-06T11:00:00Z"); + // A rubric of two rating items. Judge A's grade was submitted at the + // deadline with both ratings and a response left empty. B's is whole. C's + // stopped after one rating. + const evaluations = [ + { + id: "eval-a", + entryId: "entry-1", + title: "Project 1", + challengeId: "ch-1", + challenge: "General", + judgeId: "judge-a", + judgeName: "Ada Auto", + kind: "member", + appointmentId: APPOINTMENT, + isComplete: false, + revision: 1, + submittedAt: at, + autoSubmittedAt: at, + updatedAt: at, + }, + { + id: "eval-b", + entryId: "entry-1", + title: "Project 1", + challengeId: "ch-1", + challenge: "General", + judgeId: "judge-b", + judgeName: "Bea Whole", + kind: "member", + appointmentId: APPOINTMENT, + isComplete: true, + revision: 1, + submittedAt: at, + autoSubmittedAt: null, + updatedAt: at, + }, + { + id: "eval-c", + entryId: "entry-1", + title: "Project 1", + challengeId: "ch-1", + challenge: "General", + judgeId: "judge-c", + judgeName: "Cy Short", + kind: "member", + appointmentId: APPOINTMENT, + isComplete: false, + revision: 1, + submittedAt: at, + autoSubmittedAt: at, + updatedAt: at, + }, + ]; + const rating = ( + evaluationId: string, + item: "one" | "two", + value: number, + ) => ({ + evaluationId, + itemId: `item-${item}`, + label: item === "one" ? "Impact" : "Craft", + value, + }); + const ratings = [ + rating("eval-a", "one", 5), + rating("eval-a", "two", 3), + rating("eval-b", "one", 4), + rating("eval-b", "two", 4), + rating("eval-c", "one", 2), + ]; + + const wire = () => + mockSelect.mockImplementation(({ table, fields }) => { + if (table === judgingEvaluationDrafts) return []; + if (table === judgingRubricItems) return [{ n: 2 }]; + if (table === judgingEvaluationRatings) return ratings; + if (table === judgingEvaluationResponses) return []; + if (table === judgingAppointments) { + return [ + { + id: APPOINTMENT, + entryId: "entry-1", + challengeId: "ch-1", + startsAt: at, + endsAt: at, + title: "Project 1", + challenge: "General", + room: "Table 1", + building: null, + }, + ]; + } + if (table === judgingEvaluations) { + // The "elsewhere" check reads the id alone; the rest read the judge. + return fields && "judgeName" in fields ? evaluations : []; + } + return []; + }); + + describe("appointmentResults", () => { + const results = () => + adminCaller().judging.results.appointmentResults({ + hackathonId: HACK, + appointmentId: APPOINTMENT, + }); + + it("counts a grade with every rating, whether or not it is complete", async () => { + wire(); + + const data = await results(); + + const byName = Object.fromEntries( + data.judges.map((judge) => [judge.judgeName, judge]), + ); + expect(byName["Ada Auto"]).toMatchObject({ + status: "incomplete", + autoSubmitted: true, + counted: true, + score: 4, + }); + expect(byName["Ada Auto"]!.ratings.map((r) => r.value)).toEqual([5, 3]); + expect(byName["Bea Whole"]).toMatchObject({ + status: "complete", + counted: true, + score: 4, + }); + }); + + it("leaves a grade short of a rating out of the score and the average", async () => { + wire(); + + const data = await results(); + + const short = data.judges.find((judge) => judge.judgeName === "Cy Short"); + expect(short).toMatchObject({ counted: false, score: null, ratings: [] }); + expect(data.countedCount).toBe(2); + expect(data.average).toBe(4); + }); + }); + + describe("evaluationAudit", () => { + it("says which evaluations the placings count", async () => { + wire(); + + const rows = await adminCaller().judging.results.evaluationAudit({ + hackathonId: HACK, + }); + + expect(rows.map((row) => [row.id, row.isComplete, row.counted])).toEqual([ + ["eval-a", false, true], + ["eval-b", true, true], + ["eval-c", false, false], + ]); + }); + }); +}); + +// --------------------------------------------------------------------------- +describe("hackathon.getResults", () => { + const placementRow = (id: string, over: Record = {}) => ({ + id, + scope: "overall", + scopeLabel: "Overall", + placement: 1, + score: "4.25", + // Stored beside the score; it must never leave in the public payload. + rawScore: "3.75", + entry: { + id: `entry-${id}`, + title: `Project ${id}`, + submissionUrl: null, + demoLinks: [], + deletedAt: null, + prizeIneligibleAt: null, + sourceProject: null, + ...over, + }, + }); + + const wire = (rows: unknown[]) => { + mockFindMany.mockImplementation(async () => rows); + }; + + it("hides a team pulled from prizes after the placings went out", async () => { + wire([ + placementRow("a"), + placementRow("b", { + prizeIneligibleAt: new Date("2027-02-07T10:00:00Z"), + }), + placementRow("c", { deletedAt: new Date("2027-02-07T10:00:00Z") }), + ]); + + const results = await anonCaller().hackathon.getResults({ + hackathonId: HACK, + }); + + expect(results.map((row) => row.projectName)).toEqual(["Project a"]); + }); + + it("reads the column it filters on", async () => { + wire([]); + + await anonCaller().hackathon.getResults({ hackathonId: HACK }); + + const args = mockFindMany.mock.calls[0]![0]; + expect(args.with.entry.columns).toMatchObject({ + deletedAt: true, + prizeIneligibleAt: true, + }); + }); + + it("carries no flag data, and reads none", async () => { + wire([ + placementRow("a"), + placementRow("b", { + prizeIneligibleAt: new Date("2027-02-07T10:00:00Z"), + }), + ]); + + const results = await anonCaller().hackathon.getResults({ + hackathonId: HACK, + }); + + expect(Object.keys(results[0]!).sort()).toEqual([ + "demoUrl", + "githubUrl", + "id", + "placement", + "projectName", + "scope", + "scopeLabel", + "score", + "submissionUrl", + "teamName", + ]); + const text = JSON.stringify(results).toLowerCase(); + expect(text).not.toContain("flag"); + expect(text).not.toContain("ineligible"); + expect(text).not.toContain("3.75"); + expect(text).not.toContain("calibrat"); + // The cached payload is built from placements alone: no select, so no + // flag table, ran for it. + expect(mockSelect).not.toHaveBeenCalled(); + }); + + it("serves a pulled team's absence from the cache too", async () => { + wire([ + placementRow("a"), + placementRow("b", { + prizeIneligibleAt: new Date("2027-02-07T10:00:00Z"), + }), + ]); + + await anonCaller().hackathon.getResults({ hackathonId: HACK }); + const cached = await anonCaller().hackathon.getResults({ + hackathonId: HACK, + }); + + expect(mockFindMany).toHaveBeenCalledTimes(1); + expect(cached.map((row) => row.projectName)).toEqual(["Project a"]); + }); +}); diff --git a/packages/api/src/.internal-tests/evaluation-counts.test.ts b/packages/api/src/.internal-tests/evaluation-counts.test.ts new file mode 100644 index 00000000..44773b28 --- /dev/null +++ b/packages/api/src/.internal-tests/evaluation-counts.test.ts @@ -0,0 +1,363 @@ +/** + * Counts of evaluations that organisers and hackers read agree with placings: + * an evaluation counts once it holds a rating for every rating item, whether + * or not a written response was left empty, which is what an auto-submit at + * the deadline often leaves. The organiser's overview and project list, the + * hacker's feedback and the status the itinerary reports all read it that way, + * and none reads the judge's own "submitted whole" flag. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way check-in-flags-organizer.test.ts + * does. It answers each select by the table it reads from, so what these tests + * look at is the SQL each router asks the database to apply (the rule is + * checked against scoring.hasEveryRating in services/judging/scoring-sql.test.ts) + * and what the router makes of the rows it is given. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + hackathonParticipants, + hackathons, + judgingAppointments, + judgingChallenges, + judgingConfigs, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingEvaluationResponses, + judgingEvaluations, + judgingPresence, + judgingRooms, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; +import type * as Store from "../services/judging/evaluation-store"; + +// Nothing is due on any read here, so there is nothing to reconcile. +vi.mock("../services/judging/evaluation-store", async (importActual) => ({ + ...(await importActual()), + reconcileDueDrafts: async () => undefined, +})); + +const HACK = "11111111-1111-4111-8111-111111111111"; +const ENTRY = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1"; +const GENERAL = "cccccccc-cccc-4ccc-8ccc-ccccccccccc1"; +const ROOM = "dddddddd-dddd-4ddd-8ddd-ddddddddddd1"; +const SLOT = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1"; +const EVAL_1 = "ffffffff-ffff-4fff-8fff-fffffffffff1"; +const EVAL_2 = "ffffffff-ffff-4fff-8fff-fffffffffff2"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); +/** The SQL with its spacing and parameter numbers evened out. */ +const flat = (condition: unknown) => + render(condition) + .sql.replace(/\s+/g, " ") + .replace(/\( /g, "(") + .replace(/ \)/g, ")") + .replace(/\$\d+/g, "$?"); + +// scoring.hasEveryRating, as SQL: the evaluation's ratings are at least as +// many as the rubric's rating items. +const HOLDS_EVERY_RATING = + '(select count(*) from "judging_evaluation_rating" as r where r."evaluation_id" = "judging_evaluation"."id") >= (select count(*) from "judging_rubric_item" as i where i."hackathon_id" = $? and i."kind" = \'rating\')'; + +type Row = Record; +type SelectCall = { + fields: Record | undefined; + table: unknown; + where: unknown; +}; + +let adminRow: Row | undefined; +const selects: SelectCall[] = []; +// The rows each table answers with, whatever condition it was asked for. +const rows = new Map(); + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: (fields?: Record) => { + const call: SelectCall = { fields, table: null, where: null }; + selects.push(call); + const chain: any = { + then: (onOk: any, onErr: any) => + Promise.resolve(rows.get(call.table) ?? []).then(onOk, onErr), + }; + chain.from = (table: unknown) => { + call.table = table; + return chain; + }; + chain.where = (where: unknown) => { + call.where = where; + return chain; + }; + for (const method of [ + "innerJoin", + "leftJoin", + "groupBy", + "orderBy", + "limit", + "offset", + "for", + ]) { + chain[method] = () => chain; + } + return chain; + }, + insert: () => ({ + values: () => + Object.assign(Promise.resolve(undefined), { + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [{ id: "row" }], + }), + }), + delete: () => ({ where: async () => [] }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string) => + appRouter.createCaller({ + db: fakeDb, + session: { user: { id: userId } }, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); +const admin = () => callerAs("admin_user"); +const hacker = () => callerAs("hacker_user"); + +const callsOn = (table: unknown) => + selects.filter((call) => call.table === table); + +const at = (time: string) => new Date(`2027-02-06T${time}:00Z`); + +const config = (over: Row = {}) => ({ + hackathonId: HACK, + state: "open", + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: true, + emergencyMode: false, + claimsStartedAt: null, + ...over, +}); + +beforeEach(() => { + cache.clear(); + selects.length = 0; + rows.clear(); + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + rows.set(hackathons, [{ id: HACK, name: "Hacklytics 2027" }]); + rows.set(judgingConfigs, [config()]); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.overview", () => { + it("counts the evaluations that hold every rating, not the ones submitted whole", async () => { + rows.set(judgingEvaluations, [{ total: 5, complete: 3 }]); + + const { counts } = await admin().judging.admin.overview({ + hackathonId: HACK, + }); + + expect(counts.evaluations).toMatchObject({ total: 5, complete: 3 }); + const aggregate = callsOn(judgingEvaluations).find( + (call) => call.fields && "complete" in call.fields, + )!; + const text = flat(aggregate.fields!.complete); + expect(text).toContain(`filter (where ${HOLDS_EVERY_RATING})`); + expect(text).not.toContain("is_complete"); + // For this edition's rubric. + expect(render(aggregate.fields!.complete).params).toEqual([HACK]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.listEntries", () => { + it("counts each project's evaluations that hold every rating", async () => { + rows.set(judgingEntries, [ + { + id: ENTRY, + title: "Tidewater", + value: 1, + deletedAt: null, + prizeIneligible: false, + }, + ]); + rows.set(judgingEvaluations, [{ entryId: ENTRY, value: 2 }]); + + const page = await admin().judging.admin.listEntries({ hackathonId: HACK }); + + expect(page.entries[0]).toMatchObject({ + id: ENTRY, + completeEvaluations: 2, + }); + const [counted] = callsOn(judgingEvaluations); + const text = flat(counted!.where); + expect(text).toContain(HOLDS_EVERY_RATING); + expect(text).not.toContain("is_complete"); + expect(render(counted!.where).params).toEqual([ENTRY, HACK]); + }); +}); + +// --------------------------------------------------------------------------- +describe("hackathon participant reads", () => { + const ratedResponse = ( + evaluationId: string, + label: string, + value: string, + ) => ({ + evaluationId, + label, + displayOrder: 1, + value, + }); + + beforeEach(() => { + rows.set(hackathonParticipants, [{ status: "checked_in" }]); + rows.set(judgingEntryMembers, [ + { + memberId: "m1", + entryId: ENTRY, + title: "Tidewater", + participantCount: 2, + deletedAt: null, + }, + ]); + rows.set(judgingChallenges, [ + { + id: GENERAL, + parentId: null, + mode: "scheduled", + label: "General", + displayOrder: 0, + isGeneral: true, + isGroup: false, + }, + ]); + }); + + describe("judging.hacker.feedback", () => { + it("asks only for member evaluations that hold every rating", async () => { + rows.set(judgingEvaluations, []); + + await hacker().judging.hacker.feedback({ hackathonId: HACK }); + + const [read] = callsOn(judgingEvaluations); + const text = flat(read!.where); + expect(text).toContain(HOLDS_EVERY_RATING); + expect(text).not.toContain("is_complete"); + // Of this project, in this edition, by a member judge. + expect(render(read!.where).params).toEqual( + expect.arrayContaining([HACK, ENTRY, "member"]), + ); + }); + + it("shows the written responses that exist and leaves out the ones left empty", async () => { + // The first auto-submitted at the deadline with every rating and one + // response; the second has none written at all. + rows.set(judgingEvaluations, [ + { id: EVAL_1, challengeId: GENERAL, submittedAt: at("15:00") }, + { id: EVAL_2, challengeId: GENERAL, submittedAt: at("15:10") }, + ]); + rows.set(judgingEvaluationResponses, [ + ratedResponse(EVAL_1, "Strengths", " Clear demo "), + { ...ratedResponse(EVAL_1, "Next steps", ""), displayOrder: 2 }, + ]); + + const feedback = await hacker().judging.hacker.feedback({ + hackathonId: HACK, + }); + + expect(feedback).toEqual({ + available: true, + items: [ + { + challenge: "General", + responses: [{ label: "Strengths", value: "Clear demo" }], + }, + ], + }); + }); + + it("asks for public responses only", async () => { + rows.set(judgingEvaluations, [ + { id: EVAL_1, challengeId: GENERAL, submittedAt: at("15:00") }, + ]); + + await hacker().judging.hacker.feedback({ hackathonId: HACK }); + + const [read] = callsOn(judgingEvaluationResponses); + expect(flat(read!.where)).toContain( + '"judging_evaluation_response"."is_public" = $?', + ); + expect(render(read!.where).params).toContain(true); + }); + }); + + describe("judging.hacker.itinerary", () => { + beforeEach(() => { + rows.set(judgingEntries, [{ id: ENTRY, title: "Tidewater" }]); + rows.set(judgingEntryChallenges, [{ challengeId: GENERAL }]); + rows.set(judgingRooms, [ + { + id: ROOM, + challengeId: GENERAL, + name: "Table 1", + kind: "table", + building: null, + displayOrder: 0, + archivedAt: null, + }, + ]); + rows.set(judgingAppointments, [ + { + id: SLOT, + challengeId: GENERAL, + roomId: ROOM, + startsAt: at("10:00"), + endsAt: at("10:10"), + }, + ]); + rows.set(judgingPresence, []); + }); + + const evaluation = (ratingsComplete: boolean) => ({ + judgeId: "judge-1", + challengeId: GENERAL, + appointmentId: SLOT, + ratingsComplete, + }); + const statusOfSlot = async () => { + const itinerary = await hacker().judging.hacker.itinerary({ + hackathonId: HACK, + }); + return itinerary.appointments.map((a) => a.status); + }; + + it("reads whether an evaluation holds every rating, not whether it was submitted whole", async () => { + rows.set(judgingEvaluations, [evaluation(true)]); + + await statusOfSlot(); + + const [read] = callsOn(judgingEvaluations); + expect(flat(read!.fields!.ratingsComplete)).toBe(HOLDS_EVERY_RATING); + expect(read!.fields).not.toHaveProperty("isComplete"); + expect(render(read!.fields!.ratingsComplete).params).toEqual([HACK]); + }); + + it("calls a presentation judged once its evaluation holds every rating", async () => { + rows.set(judgingEvaluations, [evaluation(true)]); + await expect(statusOfSlot()).resolves.toEqual(["complete"]); + }); + + it("calls it presented while an evaluation is short of a rating", async () => { + rows.set(judgingEvaluations, [evaluation(false)]); + await expect(statusOfSlot()).resolves.toEqual(["incomplete"]); + }); + }); +}); diff --git a/packages/api/src/.internal-tests/graded-once.test.ts b/packages/api/src/.internal-tests/graded-once.test.ts new file mode 100644 index 00000000..3fb75423 --- /dev/null +++ b/packages/api/src/.internal-tests/graded-once.test.ts @@ -0,0 +1,1983 @@ +/** + * Each project is graded once per challenge, by one judge, at the judge desk: + * the refusals on a draft save and a submit, what the project list and sheet + * report, the next project a judge is sent to (its queue window and skips), + * opening a project to hold it, and who may grade which challenge. + * + * The router runs for real against a stand-in database that answers each select + * by the tables it reads and keeps every statement in order, which is how the + * tests see the grading lock being taken before a write. + */ +import { describe, it, expect } from "vitest"; +import { getTableName } from "drizzle-orm"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + hackathons, + judgingAppointments, + judgingBuildings, + judgingChallenges, + judgingConfigs, + judgingDeliberationEntries, + judgingDeliberationSections, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingEvaluationDrafts, + judgingEvaluations, + judgingGuestSessions, + judgingJudges, + judges, + judgingPresence, + judgingRoomLinks, + judgingRooms, + judgingRubricItems, + judgingSchedules, +} from "@query/db"; +import { judgingDeskRouter } from "../routers/judging/desk"; +import { GRADING_HOLD_MS } from "../services/judging/grading-hold"; +import { QUEUE_WINDOW, queueOrder } from "../services/judging/queue"; + +const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; +const GENERAL = "0c4a8e10-2b3d-4c5e-9f60-7a8b9c0d1e2f"; +const SPONSOR = "0c4a8e11-2b3d-4c5e-9f60-7a8b9c0d1e2f"; +const ME = "1d5b9f21-3c4e-4d6f-8a70-8b9c0d1e2f3a"; +const OTHER = "1d5b9f22-3c4e-4d6f-8a70-8b9c0d1e2f3a"; +const ROOM = "2e6ca032-4d5f-4e70-9b81-9c0d1e2f3a4b"; +const RATING = "3f7db143-5e60-4f81-8c92-0d1e2f3a4b5c"; +const SECTION = "5a9fd365-7082-41a3-8eb4-2f3a4b5c6d01"; + +const E_OPEN = "40e8c254-6f71-4092-9da3-1e2f3a4b5c01"; +const E_MINE = "40e8c254-6f71-4092-9da3-1e2f3a4b5c02"; +const E_GRADED = "40e8c254-6f71-4092-9da3-1e2f3a4b5c03"; +const E_HELD = "40e8c254-6f71-4092-9da3-1e2f3a4b5c04"; +const E_STALE = "40e8c254-6f71-4092-9da3-1e2f3a4b5c05"; + +const challenge = ( + id: string, + label: string, + mode: "scheduled" | "unscheduled" | "remote", +) => ({ + id, + parentId: null, + mode, + label, + isGeneral: id === GENERAL, + isGroup: false, +}); +const general = challenge(GENERAL, "General", "unscheduled"); +const sponsor = challenge(SPONSOR, "Best Use of AI", "scheduled"); + +const entry = (id: string, title: string) => ({ + id, + title, + description: "", + submissionUrl: null, + demoLinks: [], + videoUrl: null, + technologies: [], + participantCount: 2, +}); + +const name = (table: unknown) => getTableName(table as never); +const via = (...tables: unknown[]) => tables.map(name).join(">"); + +type Rows = unknown[] | ((call: number) => unknown[]); +type Answers = Record; +type Event = { + kind: "select" | "execute" | "insert" | "update" | "delete"; + table?: string; + where?: SQL; + values?: Record; + /** What a select asked for. */ + fields?: Record; +}; + +/** The nth read of a table gets the nth list. */ +const byCall = + (...lists: unknown[][]) => + (call: number) => + lists[call] ?? []; + +function fakeDb(answers: Answers) { + const events: Event[] = []; + const reads = new Map(); + const rowsFor = (key: string) => { + const call = reads.get(key) ?? 0; + reads.set(key, call + 1); + const answer = answers[key] ?? []; + return typeof answer === "function" ? answer(call) : answer; + }; + + const select = (fields?: Record) => { + const joined: unknown[] = []; + let from: unknown; + let where: SQL | undefined; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + where: (clause: SQL) => { + where = clause; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (error: unknown) => unknown, + ) => { + const key = via(from, ...joined); + events.push({ kind: "select", table: key, where, fields }); + return Promise.resolve(rowsFor(key)).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown) => { + joined.push(table); + return chain; + }; + } + for (const method of ["orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }; + + const db: Record = { + select, + execute: () => { + events.push({ kind: "execute" }); + return Promise.resolve(); + }, + insert: (table: unknown) => ({ + values: (values: Record) => { + events.push({ kind: "insert", table: via(table), values }); + const done = Promise.resolve([{ id: "new" }]); + return Object.assign(done, { returning: () => done }); + }, + }), + update: (table: unknown) => ({ + set: (values: Record) => ({ + where: (where: SQL) => { + events.push({ kind: "update", table: via(table), where, values }); + return Promise.resolve(); + }, + }), + }), + delete: (table: unknown) => ({ + where: (where: SQL) => { + events.push({ kind: "delete", table: via(table), where }); + return Promise.resolve(); + }, + }), + transaction: (run: (tx: unknown) => Promise) => run(db), + }; + return { db, events }; +} + +const rating = { + id: RATING, + kind: "rating", + label: "Impact", + description: "", + required: true, + guestVisibility: null, +}; + +/** What the desk reads for a member judge at an edition with one challenge. */ +const memberWorld = (over: Answers = {}): Answers => ({ + [via(judges)]: [{ name: "Grace Hopper" }], + [via(judgingJudges)]: [{ id: ME, displayName: "Grace Hopper" }], + [via(judgingChallenges)]: [general], + [via(judgingConfigs)]: [ + { state: "open", showAllScores: false, closedAt: null }, + ], + [via(judgingRubricItems)]: [rating], + [via(judgingEntries, judgingEntryChallenges)]: [entry(E_OPEN, "Tidewater")], + ...over, +}); + +/** A guest judge whose room serves `roomChallengeId`. */ +const guestWorld = (over: Answers = {}): Answers => ({ + [via(judgingGuestSessions, judgingRoomLinks, judgingRooms, judgingJudges)]: [ + { + sessionId: "session-1", + hackathonId: HACK, + linkId: "link-1", + roomId: ROOM, + roomChallengeId: SPONSOR, + judgeId: ME, + displayName: "Guest Judge", + }, + ], + [via(judgingGuestSessions, judgingRoomLinks, judgingRooms)]: [{ id: "s" }], + [via(judgingChallenges)]: [general, sponsor], + [via(judgingConfigs)]: [ + { state: "open", showAllScores: false, closedAt: null }, + ], + [via(judgingRubricItems)]: [rating], + [via(judgingEntries, judgingEntryChallenges)]: [entry(E_OPEN, "Tidewater")], + ...over, +}); + +// A new user for each context: the desk's flood guard counts calls per user, +// and this file makes more than one user's worth. +let callers = 0; +const memberCtx = (db: unknown) => { + const userId = `judge-user-${(callers += 1)}`; + return { + db, + session: { user: { id: userId, name: "Grace Hopper" } }, + userId, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as never; +}; + +const guestCtx = (db: unknown) => + ({ + db, + session: null, + userId: undefined, + clientIp: "127.0.0.2", + req: { + headers: { + get: (header: string) => + header.toLowerCase() === "cookie" + ? "query_judging_guest=guest-token" + : null, + }, + }, + }) as never; + +const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000); +const minutesAhead = (minutes: number) => + new Date(Date.now() + minutes * 60_000); + +const evaluations = name(judgingEvaluations); +const drafts = name(judgingEvaluationDrafts); + +const kinds = (events: Event[]) => events.map((e) => e.kind); +const wrote = (events: Event[], table: unknown) => + events.some((e) => e.kind === "insert" && e.table === name(table)); +const firstRead = (events: Event[], table: string) => + events.findIndex((e) => e.kind === "select" && e.table === table); + +const gradedElsewhere = + "Another judge has already graded this project for General. Each project is graded once."; +const beingGraded = + "Another judge is grading this project right now. Pick another one."; + +const sponsorRefusal = + "Sponsor judges grade Best Use of AI. Sit in its room to grade it."; +const queueRefusal = + "Grade the projects your queue offers so every judge sees a random mix. Use Grade next project."; + +const save = { + hackathonId: HACK, + entryId: E_OPEN, + challengeId: GENERAL, + ratings: { [RATING]: 4 }, + responses: {}, +}; + +describe("saveDraft: a project is graded once", () => { + const run = (answers: Answers) => { + const fake = fakeDb(answers); + return { + ...fake, + save: () => + judgingDeskRouter + .createCaller(memberCtx(fake.db)) + .saveDraft({ ...save, revision: 0 }), + }; + }; + + it("saves a draft of a project nobody has, after taking the grading lock", async () => { + const { save: go, events } = run(memberWorld()); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + expect(kinds(events).indexOf("execute")).toBeGreaterThan(-1); + expect(kinds(events).indexOf("execute")).toBeLessThan( + kinds(events).indexOf("insert"), + ); + // The judge's own draft is locked first, so the locks cannot deadlock + // with a deadline submission. + expect(firstRead(events, drafts)).toBeLessThan( + kinds(events).indexOf("execute"), + ); + }); + + it("refuses a project another judge already graded", async () => { + const { save: go, events } = run( + memberWorld({ [evaluations]: byCall([], [{ judgeId: OTHER }]) }), + ); + + await expect(go()).rejects.toMatchObject({ + code: "CONFLICT", + message: gradedElsewhere, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a project another judge is grading right now", async () => { + const { save: go, events } = run( + memberWorld({ + [drafts]: byCall( + [], + [{ judgeId: OTHER, updatedAt: minutesAgo(1), deadlineAt: null }], + ), + }), + ); + + await expect(go()).rejects.toMatchObject({ + code: "CONFLICT", + message: beingGraded, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses while another judge's presentation draft is before its deadline", async () => { + const { save: go } = run( + memberWorld({ + [drafts]: byCall( + [], + [ + { + judgeId: OTHER, + updatedAt: minutesAgo(60), + deadlineAt: minutesAhead(3), + }, + ], + ), + }), + ); + + await expect(go()).rejects.toMatchObject({ message: beingGraded }); + }); + + it("takes over a draft the other judge left alone past the hold", async () => { + const { save: go, events } = run( + memberWorld({ + [drafts]: byCall( + [], + [ + { + judgeId: OTHER, + updatedAt: new Date(Date.now() - GRADING_HOLD_MS - 60_000), + deadlineAt: null, + }, + ], + ), + }), + ); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + it("leaves a judge editing their own evaluation alone", async () => { + // Old duplicates: another judge graded it too. The judge's own is theirs. + const { save: go, events } = run( + memberWorld({ + [evaluations]: byCall([{ appointmentId: null }], [{ judgeId: OTHER }]), + }), + ); + + await expect(go()).resolves.toMatchObject({ revision: 1 }); + expect(kinds(events)).not.toContain("execute"); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); +}); + +describe("submit: a project is graded once", () => { + const run = (answers: Answers) => { + const fake = fakeDb(answers); + return { + ...fake, + submit: () => + judgingDeskRouter + .createCaller(memberCtx(fake.db)) + .submit({ ...save, expectedRevision: 0 }), + }; + }; + + it("submits a project nobody has graded, after taking the grading lock", async () => { + const { submit, events } = run(memberWorld()); + + await expect(submit()).resolves.toMatchObject({ isComplete: true }); + + expect(wrote(events, judgingEvaluations)).toBe(true); + expect(kinds(events).indexOf("execute")).toBeGreaterThan(-1); + expect(kinds(events).indexOf("execute")).toBeLessThan( + kinds(events).indexOf("insert"), + ); + // The judge's own draft and evaluation are locked first. + expect(firstRead(events, drafts)).toBeLessThan( + kinds(events).indexOf("execute"), + ); + expect(firstRead(events, evaluations)).toBeLessThan( + kinds(events).indexOf("execute"), + ); + }); + + it("refuses a project another judge already graded", async () => { + const { submit, events } = run( + memberWorld({ [evaluations]: byCall([], [{ judgeId: OTHER }]) }), + ); + + await expect(submit()).rejects.toMatchObject({ + code: "CONFLICT", + message: gradedElsewhere, + }); + expect(wrote(events, judgingEvaluations)).toBe(false); + }); + + it("refuses a project another judge is grading right now", async () => { + const { submit, events } = run( + memberWorld({ + [drafts]: byCall( + [], + [{ judgeId: OTHER, updatedAt: minutesAgo(2), deadlineAt: null }], + ), + }), + ); + + await expect(submit()).rejects.toMatchObject({ message: beingGraded }); + expect(wrote(events, judgingEvaluations)).toBe(false); + }); + + it("lets a judge update their own evaluation", async () => { + const { db, events } = fakeDb( + memberWorld({ + [evaluations]: byCall( + [ + { + id: "mine", + revision: 1, + appointmentId: null, + autoSubmittedAt: null, + }, + ], + [{ judgeId: OTHER }], + ), + }), + ); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .submit({ ...save, expectedRevision: 1 }), + ).resolves.toMatchObject({ isComplete: true }); + expect(kinds(events)).not.toContain("execute"); + }); +}); + +describe("desk.projects: each project's grading state", () => { + const list = async (answers: Answers, ctx = memberCtx) => { + const { db } = fakeDb(answers); + return judgingDeskRouter.createCaller(ctx(db)).projects({ + hackathonId: HACK, + inMyRoomOnly: false, + hideJudged: false, + }); + }; + + const crowded = () => + memberWorld({ + [via(judgingEntries, judgingEntryChallenges)]: [ + entry(E_GRADED, "Alpha"), + entry(E_HELD, "Bravo"), + entry(E_MINE, "Charlie"), + entry(E_STALE, "Delta"), + entry(E_OPEN, "Echo"), + ], + [evaluations]: [ + { entryId: E_MINE, judgeId: ME, revision: 1, counted: true }, + { entryId: E_GRADED, judgeId: OTHER, revision: 1, counted: true }, + ], + [drafts]: [ + { + entryId: E_HELD, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: E_STALE, + judgeId: OTHER, + updatedAt: minutesAgo(60), + deadlineAt: null, + }, + ], + }); + + it("reports mine, graded, in progress and open", async () => { + const result = await list(crowded()); + + expect( + Object.fromEntries(result.projects.map((p) => [p.id, p.grading])), + ).toEqual({ + [E_MINE]: "mine", + [E_GRADED]: "graded", + [E_HELD]: "in_progress", + [E_STALE]: "open", + [E_OPEN]: "open", + }); + }); + + it("lists open projects first, then those being graded, then graded ones", async () => { + const result = await list(crowded()); + + expect(result.projects.map((p) => p.grading)).toEqual([ + "mine", + "open", + "open", + "in_progress", + "graded", + ]); + }); + + it("keeps a judge's own draft and evaluation to themselves", async () => { + const result = await list(crowded()); + + const held = result.projects.find((p) => p.id === E_HELD); + expect(held?.hasDraft).toBe(false); + expect(held?.myEvaluation).toBeNull(); + const graded = result.projects.find((p) => p.id === E_GRADED); + expect(graded?.myEvaluation).toBeNull(); + expect(result.judgedCount).toBe(1); + }); + + it("does not call a project the judge only opened a saved draft", async () => { + const mine = (entryId: string, revision: number) => ({ + entryId, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision, + }); + const result = await list( + memberWorld({ + [via(judgingEntries, judgingEntryChallenges)]: [ + entry(E_OPEN, "Alpha"), + entry(E_HELD, "Bravo"), + ], + [drafts]: [mine(E_OPEN, 0), mine(E_HELD, 2)], + }), + ); + + const hasDraft = Object.fromEntries( + result.projects.map((p) => [p.id, p.hasDraft]), + ); + expect(hasDraft).toEqual({ [E_OPEN]: false, [E_HELD]: true }); + expect(result.projects.map((p) => p.grading)).toEqual(["mine", "mine"]); + }); + + it("never says which other judge", async () => { + const result = await list(crowded()); + + expect(JSON.stringify(result)).not.toContain(OTHER); + }); + + it("tells a guest the same, for their own room", async () => { + const result = await list( + guestWorld({ + [evaluations]: [ + { entryId: E_OPEN, judgeId: OTHER, revision: 1, counted: true }, + ], + }), + guestCtx, + ); + + expect(result.challenge.id).toBe(SPONSOR); + expect(result.projects.map((p) => p.grading)).toEqual(["graded"]); + expect(JSON.stringify(result)).not.toContain(OTHER); + }); + + it("calls a project judged once its evaluation counts, as placings do", async () => { + const answers = memberWorld({ + [via(judgingEntries, judgingEntryChallenges)]: [ + entry(E_MINE, "Alpha"), + entry(E_OPEN, "Bravo"), + ], + // Alpha holds every rating, a written response left empty when time + // ran out. Bravo lacks a rating. + [evaluations]: [ + { entryId: E_MINE, judgeId: ME, revision: 1, counted: true }, + { entryId: E_OPEN, judgeId: ME, revision: 1, counted: false }, + ], + }); + const { db, events } = fakeDb(answers); + const shown = await judgingDeskRouter.createCaller(memberCtx(db)).projects({ + hackathonId: HACK, + inMyRoomOnly: false, + hideJudged: false, + }); + + expect(shown.judgedCount).toBe(1); + expect( + Object.fromEntries( + shown.projects.map((p) => [p.id, p.myEvaluation?.counted]), + ), + ).toEqual({ [E_MINE]: true, [E_OPEN]: false }); + // The database applies scoring.hasEveryRating's rule, not the judge's + // "answered everything" flag. + const read = events.find( + (e) => e.table === evaluations && e.fields && "counted" in e.fields, + ); + expect(read?.fields).not.toHaveProperty("isComplete"); + expect( + new PgDialect().sqlToQuery(read?.fields?.counted as SQL).sql, + ).toContain('"judging_evaluation_rating"'); + + const hidden = await judgingDeskRouter + .createCaller(memberCtx(fakeDb(answers).db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false, hideJudged: true }); + expect(hidden.projects.map((p) => p.id)).toEqual([E_OPEN]); + }); +}); + +describe("desk.deliberation.add: what can go on a shortlist", () => { + it("takes a project once the judge's evaluation counts, as placings do", async () => { + const { db, events } = fakeDb( + memberWorld({ [via(judgingDeliberationSections)]: [{ id: SECTION }] }), + ); + + await expect( + judgingDeskRouter.createCaller(memberCtx(db)).deliberation.add({ + hackathonId: HACK, + sectionId: SECTION, + entryId: E_OPEN, + }), + ).rejects.toMatchObject({ code: "PRECONDITION_FAILED" }); + const read = events.find( + (e) => e.table === via(judgingEvaluations, judgingEntries), + ); + const clause = new PgDialect().sqlToQuery(read?.where as SQL).sql; + expect(clause).toContain('"judging_evaluation_rating"'); + expect(clause).not.toContain("is_complete"); + expect(wrote(events, judgingDeliberationEntries)).toBe(false); + }); +}); + +describe("desk.project: whether the judge may write", () => { + const open = async ( + entryId: string, + over: Answers, + ctx = memberCtx, + challengeId: string | undefined = undefined, + ) => { + const { db } = fakeDb( + memberWorld({ + [via(judgingEntries, judgingEntryChallenges)]: [ + entry(entryId, "Tidewater"), + ], + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + ...over, + }), + ); + return judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId, challengeId }); + }; + + it("is open to write on a project nobody has", async () => { + const sheet = await open(E_OPEN, {}); + + expect(sheet.grading).toBe("open"); + expect(sheet.access).toMatchObject({ + canEdit: true, + reason: null, + gradedElsewhere: false, + }); + }); + + it("is read-only on a project another judge graded", async () => { + const sheet = await open(E_GRADED, { + [evaluations]: byCall([], [{ judgeId: OTHER }]), + }); + + expect(sheet.grading).toBe("graded"); + expect(sheet.access).toMatchObject({ + canEdit: false, + reason: gradedElsewhere, + gradedElsewhere: true, + needsRoom: false, + ended: false, + opensAt: null, + deadlineAt: null, + }); + // Still there to read. + expect(sheet.entry.title).toBe("Tidewater"); + expect(sheet.rubric).toHaveLength(1); + expect(sheet.evaluation).toBeNull(); + expect(JSON.stringify(sheet)).not.toContain(OTHER); + }); + + it("is read-only on a project another judge is grading", async () => { + const sheet = await open(E_HELD, { + [drafts]: byCall( + [], + [{ judgeId: OTHER, updatedAt: minutesAgo(1), deadlineAt: null }], + ), + }); + + expect(sheet.grading).toBe("in_progress"); + expect(sheet.access).toMatchObject({ + canEdit: false, + reason: beingGraded, + gradedElsewhere: true, + needsRoom: false, + }); + expect(sheet.draft).toBeNull(); + expect(JSON.stringify(sheet)).not.toContain(OTHER); + }); + + it("stays the judge's own project when another judge also graded it", async () => { + const sheet = await open(E_MINE, { + [evaluations]: byCall( + [ + { + id: "mine", + revision: 1, + isComplete: true, + appointmentId: null, + submittedAt: minutesAgo(30), + autoSubmittedAt: null, + updatedAt: minutesAgo(30), + }, + ], + [{ judgeId: OTHER }, { judgeId: ME }], + ), + }); + + expect(sheet.grading).toBe("mine"); + expect(sheet.access.canEdit).toBe(true); + }); + + it("does not hide that judging is closed behind the grading", async () => { + const sheet = await open(E_GRADED, { + [via(judgingConfigs)]: [ + { state: "closed", showAllScores: false, closedAt: new Date() }, + ], + [evaluations]: byCall([], [{ judgeId: OTHER }]), + }); + + expect(sheet.access.canEdit).toBe(false); + expect(sheet.access.reason).toBe( + "Judging is closed. Your submitted evaluations are read-only.", + ); + }); +}); + +describe("desk.nextProject", () => { + const next = async ( + answers: Answers, + input: { challengeId?: string } = {}, + ctx = memberCtx, + ) => { + const fake = fakeDb(answers); + const result = await judgingDeskRouter + .createCaller(ctx(fake.db)) + .nextProject({ hackathonId: HACK, ...input }); + return { result, ...fake }; + }; + + const pool = [E_OPEN, E_MINE, E_GRADED, E_HELD, E_STALE]; + const crowded = (over: Answers = {}) => + memberWorld({ + [via(judgingEntries, judgingEntryChallenges)]: pool.map((id) => ({ + id, + })), + [evaluations]: [ + { entryId: E_MINE, judgeId: ME }, + { entryId: E_GRADED, judgeId: OTHER }, + ], + [drafts]: [ + { + entryId: E_HELD, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: E_STALE, + judgeId: OTHER, + updatedAt: minutesAgo(60), + deadlineAt: null, + }, + ], + ...over, + }); + + it("sends the judge to the first open project in their own order", async () => { + const { result } = await next(crowded()); + + const open = [E_OPEN, E_STALE]; + expect(result.entryId).toBe(queueOrder(ME, GENERAL, open)[0]); + expect(result.challenge).toMatchObject({ id: GENERAL, label: "General" }); + expect(result.needsRoom).toBe(false); + }); + + it("sends the same judge to the same project each time", async () => { + const first = await next(crowded()); + const again = await next(crowded()); + + expect(again.result.entryId).toBe(first.result.entryId); + }); + + it("is nothing when every project is graded or held", async () => { + const { result } = await next( + crowded({ + [via(judgingEntries, judgingEntryChallenges)]: [ + { id: E_MINE }, + { id: E_GRADED }, + { id: E_HELD }, + ], + }), + ); + + expect(result.entryId).toBeNull(); + expect(result.needsRoom).toBe(false); + }); + + it("is nothing when the challenge has no projects", async () => { + const { result } = await next( + memberWorld({ [via(judgingEntries, judgingEntryChallenges)]: [] }), + ); + + expect(result.entryId).toBeNull(); + }); + + it("leaves out withdrawn projects, and still grades prize-ineligible ones", async () => { + const { events } = await next(crowded()); + + const read = events.find( + (e) => e.table === via(judgingEntries, judgingEntryChallenges), + ); + const clause = new PgDialect().sqlToQuery(read?.where as SQL).sql; + expect(clause).toContain('"deleted_at" is null'); + expect(clause).not.toContain("prize_ineligible"); + }); + + describe("at a scheduled challenge", () => { + const seat = (challengeId: string) => [ + { roomId: ROOM, roomName: "Table 4", challengeId }, + ]; + const scheduled = (over: Answers = {}) => + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [via(judgingPresence, judgingRooms)]: seat(SPONSOR), + ...over, + }); + const slot = (entryId: string, startsAt: Date, deadlineAt: Date) => ({ + id: `slot-${entryId}`, + entryId, + startsAt, + deadlineAt, + }); + + it("sends the judge to a presentation running in their room", async () => { + const { result, events } = await next( + scheduled({ + [via(judgingEntries, judgingEntryChallenges)]: [ + { id: E_OPEN }, + { id: E_STALE }, + { id: E_HELD }, + ], + [via(judgingAppointments)]: [ + slot(E_OPEN, minutesAgo(30), minutesAgo(20)), + slot(E_STALE, minutesAgo(5), minutesAhead(5)), + slot(E_HELD, minutesAhead(20), minutesAhead(30)), + ], + }), + { challengeId: SPONSOR }, + ); + + // Over, running, and not started: only the running one is on now. + expect(result.entryId).toBe(E_STALE); + const read = events.find((e) => e.table === name(judgingAppointments)); + const clause = new PgDialect().sqlToQuery(read?.where as SQL).sql; + expect(clause).toContain('"room_id" = $'); + }); + + it("is nothing when no presentation in the room is running", async () => { + const { result } = await next( + scheduled({ + [via(judgingEntries, judgingEntryChallenges)]: [{ id: E_OPEN }], + [via(judgingAppointments)]: [ + slot(E_OPEN, minutesAhead(10), minutesAhead(20)), + ], + }), + { challengeId: SPONSOR }, + ); + + expect(result.entryId).toBeNull(); + expect(result.needsRoom).toBe(false); + }); + + it("skips a running presentation another judge already graded", async () => { + const { result } = await next( + scheduled({ + [via(judgingEntries, judgingEntryChallenges)]: [ + { id: E_OPEN }, + { id: E_STALE }, + ], + [via(judgingAppointments)]: [ + slot(E_OPEN, minutesAgo(5), minutesAhead(5)), + slot(E_STALE, minutesAgo(5), minutesAhead(5)), + ], + [evaluations]: [{ entryId: E_OPEN, judgeId: OTHER }], + }), + { challengeId: SPONSOR }, + ); + + expect(result.entryId).toBe(E_STALE); + }); + + // A table judge can only reach a scheduled challenge that is General's: + // a sponsor's is refused before the room matters (see "who may grade + // which challenge"). + const timedGeneral = { ...general, mode: "scheduled" as const }; + + it("asks a judge seated elsewhere to join the room first", async () => { + const { result, events } = await next( + scheduled({ [via(judgingChallenges)]: [timedGeneral, sponsor] }), + { challengeId: GENERAL }, + ); + + expect(result.entryId).toBeNull(); + expect(result.needsRoom).toBe(true); + expect( + events.some( + (e) => e.table === via(judgingEntries, judgingEntryChallenges), + ), + ).toBe(false); + }); + + it("asks a judge with no seat to join the room first", async () => { + const { result } = await next( + scheduled({ + [via(judgingChallenges)]: [timedGeneral, sponsor], + [via(judgingPresence, judgingRooms)]: [], + }), + { challengeId: GENERAL }, + ); + + expect(result.needsRoom).toBe(true); + }); + }); +}); + +describe("who may grade which challenge", () => { + it("keeps a sponsor's guest judge on their room's challenge when saving", async () => { + const { db, events } = fakeDb(guestWorld()); + + await expect( + judgingDeskRouter + .createCaller(guestCtx(db)) + .saveDraft({ ...save, challengeId: GENERAL, revision: 0 }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + expect(kinds(events)).not.toContain("execute"); + }); + + it("keeps a sponsor's guest judge on their room's challenge when submitting", async () => { + const { db, events } = fakeDb(guestWorld()); + + await expect( + judgingDeskRouter + .createCaller(guestCtx(db)) + .submit({ ...save, challengeId: GENERAL, expectedRevision: 0 }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(wrote(events, judgingEvaluations)).toBe(false); + }); + + it("keeps a guest from opening or queueing another challenge", async () => { + const { db } = fakeDb(guestWorld()); + const caller = judgingDeskRouter.createCaller(guestCtx(db)); + + await expect( + caller.project({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: GENERAL, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + await expect( + caller.nextProject({ hackathonId: HACK, challengeId: GENERAL }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + await expect( + caller.projects({ hackathonId: HACK, challengeId: GENERAL }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it("queues a guest in their room's challenge without being told which", async () => { + const { db } = fakeDb(guestWorld()); + + const result = await judgingDeskRouter + .createCaller(guestCtx(db)) + .nextProject({ hackathonId: HACK }); + + expect(result.challenge.id).toBe(SPONSOR); + expect(result.entryId).toBe(E_OPEN); + }); + + it("lets a guest grade their own room's challenge", async () => { + const { db, events } = fakeDb(guestWorld()); + + await expect( + judgingDeskRouter + .createCaller(guestCtx(db)) + .saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + describe("a table judge and a scheduled sponsor challenge", () => { + const sheet = (seatedAt: string) => + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [via(judgingAppointments)]: [ + { + id: "slot", + roomId: ROOM, + startsAt: minutesAgo(5), + deadlineAt: minutesAhead(5), + endsAt: minutesAhead(10), + }, + ], + [via(judgingPresence, judgingRooms)]: [ + { roomId: ROOM, roomName: "Table 4", challengeId: seatedAt }, + ], + }); + + it("cannot grade it from a table that judges another challenge", async () => { + const { db, events } = fakeDb(sheet(GENERAL)); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).rejects.toMatchObject({ code: "FORBIDDEN", message: sponsorRefusal }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(fakeDb(sheet(GENERAL)).db)) + .submit({ ...save, challengeId: SPONSOR, expectedRevision: 0 }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it("still needs the room for a scheduled General", async () => { + const world = memberWorld({ + ...sheet(SPONSOR), + [via(judgingChallenges)]: [{ ...general, mode: "scheduled" }, sponsor], + }); + const { db, events } = fakeDb(world); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .saveDraft({ ...save, challengeId: GENERAL, revision: 0 }), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: + "Join the room or table that judges this challenge to evaluate this project.", + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("can grade it when seated at the room that judges it", async () => { + const { db, events } = fakeDb(sheet(SPONSOR)); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + }); +}); + +/** More projects than the queue window, with ids the desk accepts as input. */ +const eid = (n: number) => + `40e8c254-6f71-4092-9da3-1e2f3a4b5c${String(n).padStart(2, "0")}`; +const POOL = Array.from({ length: 8 }, (_, i) => eid(i + 10)); + +const entriesKey = via(judgingEntries, judgingEntryChallenges); +const presenceKey = via(judgingPresence, judgingRooms); +const seatAt = (challengeId: string) => [ + { roomId: ROOM, roomName: "Table 4", challengeId }, +]; + +describe("sponsor challenges belong to sponsor judges", () => { + const MLH = "0c4a8e12-2b3d-4c5e-9f60-7a8b9c0d1e2f"; + const MONGO = "0c4a8e13-2b3d-4c5e-9f60-7a8b9c0d1e2f"; + const mlh = { + ...challenge(MLH, "MLH Challenges", "unscheduled"), + isGroup: true, + }; + const mongo = { + ...challenge(MONGO, "Best Use of MongoDB", "unscheduled"), + parentId: MLH, + }; + + const world = (seatedAt: string | null, over: Answers = {}) => + memberWorld({ + [via(judgingChallenges)]: [general, sponsor, mlh, mongo], + [presenceKey]: seatedAt ? seatAt(seatedAt) : [], + ...over, + }); + + it("refuses a table judge who is not seated in the sponsor's room, on every call that picks or writes", async () => { + for (const seatedAt of [null, GENERAL, MLH]) { + const { db, events } = fakeDb(world(seatedAt)); + const caller = judgingDeskRouter.createCaller(memberCtx(db)); + const refused = { code: "FORBIDDEN", message: sponsorRefusal }; + + await expect( + caller.projects({ hackathonId: HACK, challengeId: SPONSOR }), + ).rejects.toMatchObject(refused); + await expect( + caller.project({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: SPONSOR, + }), + ).rejects.toMatchObject(refused); + await expect( + caller.nextProject({ hackathonId: HACK, challengeId: SPONSOR }), + ).rejects.toMatchObject(refused); + await expect( + caller.saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).rejects.toMatchObject(refused); + await expect( + caller.submit({ ...save, challengeId: SPONSOR, expectedRevision: 0 }), + ).rejects.toMatchObject(refused); + await expect( + caller.startGrading({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: SPONSOR, + }), + ).rejects.toMatchObject(refused); + + expect(kinds(events)).not.toContain("execute"); + expect(kinds(events)).not.toContain("insert"); + } + }); + + it("names the group when the judge asked for a prize inside it", async () => { + const { db } = fakeDb(world(GENERAL)); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK, challengeId: MONGO }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "Sponsor judges grade MLH Challenges. Sit in its room to grade it.", + }); + }); + + it("lets a table judge in once they are seated in the room that serves it", async () => { + const { db, events } = fakeDb(world(SPONSOR)); + const caller = judgingDeskRouter.createCaller(memberCtx(db)); + + const listed = await caller.projects({ + hackathonId: HACK, + challengeId: SPONSOR, + inMyRoomOnly: false, + }); + expect(listed.challenge.id).toBe(SPONSOR); + await expect( + caller.nextProject({ hackathonId: HACK, challengeId: SPONSOR }), + ).resolves.toMatchObject({ entryId: E_OPEN }); + await expect( + caller.saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + it("lets a table judge grade a prize inside a group from a seat at the group's room", async () => { + const { db } = fakeDb(world(MLH)); + + const listed = await judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK, challengeId: MONGO }); + expect(listed.challenge.id).toBe(MLH); + }); + + it("keeps General open to a table judge wherever they sit", async () => { + for (const seatedAt of [null, GENERAL, SPONSOR]) { + const { db } = fakeDb(world(seatedAt)); + const listed = await judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK, challengeId: GENERAL }); + expect(listed.challenge.id).toBe(GENERAL); + } + }); + + it("defaults an unseated table judge to General", async () => { + const { db } = fakeDb(world(null)); + + const listed = await judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK }); + + expect(listed.challenge.id).toBe(GENERAL); + }); + + describe("the challenges the desk offers", () => { + const offered = async (answers: Answers, ctx = memberCtx) => { + const { db } = fakeDb({ + [via(hackathons)]: [{ id: HACK, name: "Hacklytics" }], + ...answers, + }); + const context = await judgingDeskRouter + .createCaller(ctx(db)) + .context({ hackathonId: HACK }); + return context.challenges.map((c) => c.id); + }; + + it("is General alone for a table judge with no seat or another's", async () => { + expect(await offered(world(null))).toEqual([GENERAL]); + expect(await offered(world(GENERAL))).toEqual([GENERAL]); + }); + + it("adds the sponsor's challenge, and what is below it, from its room", async () => { + expect(await offered(world(SPONSOR))).toEqual([GENERAL, SPONSOR]); + expect(await offered(world(MLH))).toEqual([GENERAL, MLH, MONGO]); + }); + + it("is still only the room's challenge for a guest", async () => { + expect(await offered(guestWorld(), guestCtx)).toEqual([SPONSOR]); + }); + }); +}); + +describe("a judge's first claim is limited to their queue window", () => { + const order = queueOrder(ME, GENERAL, POOL); + const world = (over: Answers = {}) => + memberWorld({ + [entriesKey]: POOL.map((id) => entry(id, "Project")), + ...over, + }); + const firstSave = (entryId: string) => ({ ...save, entryId, revision: 0 }); + const desk = (answers: Answers) => { + const fake = fakeDb(answers); + return { + ...fake, + caller: judgingDeskRouter.createCaller(memberCtx(fake.db)), + }; + }; + + it("saves a draft of any of the first projects the queue offers", async () => { + for (const entryId of order.slice(0, QUEUE_WINDOW)) { + const { caller, events } = desk(world()); + + await expect(caller.saveDraft(firstSave(entryId))).resolves.toMatchObject( + { revision: 1 }, + ); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + } + }); + + it("refuses a first save of a project further down, after the grading lock and before any write", async () => { + const { caller, events } = desk(world()); + + await expect( + caller.saveDraft(firstSave(order[QUEUE_WINDOW])), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: queueRefusal, + }); + expect(kinds(events)).toContain("execute"); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a first submission of a project further down", async () => { + const { caller, events } = desk(world()); + + await expect( + caller.submit({ ...firstSave(order[QUEUE_WINDOW]), expectedRevision: 0 }), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: queueRefusal, + }); + expect(wrote(events, judgingEvaluations)).toBe(false); + }); + + it("lets a judge submit a project in the window without ever saving a draft", async () => { + const { caller } = desk(world()); + + await expect( + caller.submit({ ...firstSave(order[0]), expectedRevision: 0 }), + ).resolves.toMatchObject({ isComplete: true }); + }); + + it("slides the window past projects that are graded or held", async () => { + const held = (entryId: string, minutes: number) => ({ + entryId, + judgeId: OTHER, + updatedAt: minutesAgo(minutes), + deadlineAt: null, + }); + const { caller } = desk( + world({ + [evaluations]: byCall([], [], [{ entryId: order[0], judgeId: OTHER }]), + [drafts]: byCall([], [], [held(order[1], 1), held(order[2], 2)]), + }), + ); + + await expect( + caller.saveDraft(firstSave(order[QUEUE_WINDOW])), + ).resolves.toMatchObject({ revision: 1 }); + }); + + it("leaves a project the judge already holds alone, however far down", async () => { + const { caller } = desk( + world({ + [drafts]: byCall([{ id: "mine", revision: 0, deadlineAt: null }]), + }), + ); + + await expect(caller.saveDraft(firstSave(order[7]))).resolves.toMatchObject({ + revision: 1, + }); + }); + + it("leaves a project the judge already graded alone, however far down", async () => { + const { caller, events } = desk( + world({ [evaluations]: byCall([{ appointmentId: null }]) }), + ); + + await expect(caller.saveDraft(firstSave(order[7]))).resolves.toMatchObject({ + revision: 1, + }); + expect(kinds(events)).not.toContain("execute"); + }); + + it("asks nothing of a scheduled challenge: the schedule decides", async () => { + const { caller, events } = desk( + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [via(judgingAppointments)]: [ + { + id: "slot", + roomId: ROOM, + startsAt: minutesAgo(5), + deadlineAt: minutesAhead(5), + endsAt: minutesAhead(10), + }, + ], + [presenceKey]: seatAt(SPONSOR), + [entriesKey]: POOL.map((id) => entry(id, "Project")), + }), + ); + + await expect( + caller.saveDraft({ + ...firstSave(queueOrder(ME, SPONSOR, POOL)[7]), + challengeId: SPONSOR, + }), + ).resolves.toMatchObject({ revision: 1 }); + // Only the check that the project is entered read the projects. + expect(events.filter((e) => e.table === entriesKey)).toHaveLength(1); + }); +}); + +describe("desk.nextProject: the queue window and skips", () => { + const order = queueOrder(ME, GENERAL, POOL); + const next = async ( + input: { challengeId?: string; skip?: string[] } = {}, + over: Answers = {}, + ) => { + const { db } = fakeDb( + memberWorld({ [entriesKey]: POOL.map((id) => ({ id })), ...over }), + ); + return judgingDeskRouter + .createCaller(memberCtx(db)) + .nextProject({ hackathonId: HACK, ...input }); + }; + + it("offers the first project of the judge's queue", async () => { + const result = await next(); + + expect(result.entryId).toBe(order[0]); + expect(result.allSkipped).toBe(false); + }); + + it("passes over skipped projects, one at a time, within the window", async () => { + expect((await next({ skip: [order[0]] })).entryId).toBe(order[1]); + expect((await next({ skip: [order[0], order[1]] })).entryId).toBe(order[2]); + }); + + it("says everything is skipped once the whole window is, and never reaches past it", async () => { + const result = await next({ skip: order.slice(0, QUEUE_WINDOW) }); + + expect(result.entryId).toBeNull(); + expect(result.allSkipped).toBe(true); + expect(result.needsRoom).toBe(false); + }); + + it("ignores a skip outside the window", async () => { + expect((await next({ skip: [order[6]] })).entryId).toBe(order[0]); + }); + + it("does not call a challenge with nothing open skipped", async () => { + const held = POOL.map((entryId) => ({ + entryId, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + })); + + const result = await next({ skip: [order[0]] }, { [drafts]: held }); + + expect(result.entryId).toBeNull(); + expect(result.allSkipped).toBe(false); + }); + + it("slides the window as other judges grade projects", async () => { + const result = await next( + { skip: order.slice(0, QUEUE_WINDOW - 1) }, + { [evaluations]: [{ entryId: order[0], judgeId: OTHER }] }, + ); + + // order[0] is graded, so the window is order[1..3]; skipping order[0] + // and order[1] leaves order[2]. + expect(result.entryId).toBe(order[2]); + const all = await next( + { skip: order.slice(1, QUEUE_WINDOW + 1) }, + { [evaluations]: [{ entryId: order[0], judgeId: OTHER }] }, + ); + expect(all.entryId).toBeNull(); + expect(all.allSkipped).toBe(true); + }); + + it("keeps a project the judge opened and skipped inside the window", async () => { + // Opening a project holds it for the judge (an empty draft); skipping it + // must not let them reach past the window. + const opened = order.slice(0, QUEUE_WINDOW).map((entryId) => ({ + entryId, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision: 0, + })); + + expect((await next({}, { [drafts]: opened.slice(0, 1) })).entryId).toBe( + order[0], + ); + expect( + (await next({ skip: [order[0]] }, { [drafts]: opened.slice(0, 1) })) + .entryId, + ).toBe(order[1]); + const all = await next( + { skip: order.slice(0, QUEUE_WINDOW) }, + { [drafts]: opened }, + ); + expect(all.entryId).toBeNull(); + expect(all.allSkipped).toBe(true); + }); + + it("takes at most 20 skips, and only project ids", async () => { + const many = Array.from({ length: 21 }, (_, i) => eid(i + 30)); + await expect(next({ skip: many })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + await expect(next({ skip: ["not-an-id"] })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + await expect(next({ skip: many.slice(0, 20) })).resolves.toMatchObject({ + entryId: order[0], + }); + }); + + it("offers every running presentation at a scheduled challenge, not just a window", async () => { + const ids = POOL.slice(0, 5); + const sponsorOrder = queueOrder(ME, SPONSOR, ids); + const running = ids.map((entryId) => ({ + id: `slot-${entryId}`, + entryId, + startsAt: minutesAgo(5), + deadlineAt: minutesAhead(5), + })); + const result = await next( + { challengeId: SPONSOR, skip: sponsorOrder.slice(0, 4) }, + { + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [presenceKey]: seatAt(SPONSOR), + [entriesKey]: ids.map((id) => ({ id })), + [via(judgingAppointments)]: running, + }, + ); + + expect(result.entryId).toBe(sponsorOrder[4]); + }); +}); + +describe("desk.projects: open projects in the judge's queue order", () => { + const titled = [ + ["Alpha", eid(40)], + ["Bravo", eid(41)], + ["Charlie", eid(42)], + ["Delta", eid(43)], + ["Echo", eid(44)], + ["Foxtrot", eid(45)], + ["Golf", eid(46)], + ["Hotel", eid(47)], + ].map(([title, id]) => entry(id, title)); + // Alpha is the judge's own, Bravo is held by another judge, Charlie was + // graded by another judge, and the rest are open. + const [MINE, HELD, GRADED, ...OPEN] = titled.map((e) => e.id); + + const list = async ( + over: Answers = {}, + input: { sort?: "scheduled" | "title"; challengeId?: string } = {}, + ) => { + const { db } = fakeDb( + memberWorld({ + [entriesKey]: titled, + [drafts]: [ + { + entryId: MINE, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: HELD, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + ], + [evaluations]: [ + { entryId: GRADED, judgeId: OTHER, revision: 1, counted: true }, + ], + ...over, + }), + ); + return judgingDeskRouter.createCaller(memberCtx(db)).projects({ + hackathonId: HACK, + inMyRoomOnly: false, + hideJudged: false, + ...input, + }); + }; + + it("lists the judge's own first, then open projects in their queue order", async () => { + const result = await list(); + const queue = queueOrder(ME, GENERAL, OPEN); + // The test only means something if the queue is not the title order. + expect(queue).not.toEqual(OPEN); + + expect(result.projects.map((p) => p.id)).toEqual([ + MINE, + ...queue, + HELD, + GRADED, + ]); + expect(result.projects.map((p) => p.grading)).toEqual([ + "mine", + ...queue.map(() => "open"), + "in_progress", + "graded", + ]); + }); + + it("puts first the open project that Grade next project goes to", async () => { + const result = await list(); + const { db } = fakeDb( + memberWorld({ + [entriesKey]: titled.map((e) => ({ id: e.id })), + [drafts]: [ + { + entryId: MINE, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: HELD, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + ], + [evaluations]: [{ entryId: GRADED, judgeId: OTHER }], + }), + ); + const next = await judgingDeskRouter + .createCaller(memberCtx(db)) + .nextProject({ hackathonId: HACK }); + + expect(result.projects.find((p) => p.grading === "open")?.id).toBe( + next.entryId, + ); + }); + + it("keeps title order when the judge asks for it", async () => { + const result = await list({}, { sort: "title" }); + + expect( + result.projects.filter((p) => p.grading === "open").map((p) => p.title), + ).toEqual(["Delta", "Echo", "Foxtrot", "Golf", "Hotel"]); + }); + + it("keeps presentation order at a scheduled challenge", async () => { + const slot = (entryId: string, minutes: number) => ({ + id: `slot-${entryId}`, + entryId, + roomId: ROOM, + roomName: "Table 4", + buildingName: null, + startsAt: minutesAhead(minutes), + deadlineAt: minutesAhead(minutes + 10), + endsAt: minutesAhead(minutes + 12), + }); + const [delta, echo, foxtrot] = OPEN; + const result = await list( + { + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [presenceKey]: seatAt(SPONSOR), + [via(judgingAppointments, judgingRooms, judgingBuildings)]: [ + slot(foxtrot, 10), + slot(echo, 20), + slot(delta, 30), + ], + }, + { challengeId: SPONSOR }, + ); + + expect(result.projects.map((p) => p.title)).toEqual([ + "Foxtrot", + "Echo", + "Delta", + "Alpha", + "Golf", + "Hotel", + "Bravo", + "Charlie", + ]); + }); +}); + +describe("desk.startGrading: the hold starts when a project is opened", () => { + const start = (entryId = E_OPEN, challengeId = GENERAL) => ({ + hackathonId: HACK, + entryId, + challengeId, + }); + const desk = (answers: Answers, ctx = memberCtx) => { + const fake = fakeDb(answers); + return { + ...fake, + caller: judgingDeskRouter.createCaller(ctx(fake.db)), + }; + }; + const insertedDraft = (events: Event[]) => + events.find((e) => e.kind === "insert" && e.table === drafts)?.values; + + it("creates an empty draft after taking the grading lock", async () => { + const { caller, events } = desk(memberWorld()); + + const result = await caller.startGrading(start()); + + expect(result).toEqual({ held: true, revision: 0, deadlineAt: null }); + expect(insertedDraft(events)).toMatchObject({ + hackathonId: HACK, + judgeId: ME, + entryId: E_OPEN, + challengeId: GENERAL, + ratings: {}, + responses: {}, + revision: 0, + appointmentId: null, + deadlineAt: null, + }); + expect(kinds(events).indexOf("execute")).toBeGreaterThan(-1); + expect(kinds(events).indexOf("execute")).toBeLessThan( + kinds(events).indexOf("insert"), + ); + }); + + it("picks the challenge a table judge sits at when none is named", async () => { + const { caller, events } = desk( + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [presenceKey]: seatAt(SPONSOR), + }), + ); + + await caller.startGrading({ hackathonId: HACK, entryId: E_OPEN }); + + expect(insertedDraft(events)).toMatchObject({ challengeId: SPONSOR }); + }); + + it("touches a draft the judge already has, keeping its answers and revision", async () => { + const { caller, events } = desk( + memberWorld({ + [drafts]: byCall([{ id: "mine", revision: 3, deadlineAt: null }]), + }), + ); + + const result = await caller.startGrading(start()); + + expect(result).toEqual({ held: true, revision: 3, deadlineAt: null }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + const touched = events.find( + (e) => e.kind === "update" && e.table === drafts, + ); + expect(touched?.values).toHaveProperty("updatedAt"); + expect(touched?.values).not.toHaveProperty("revision"); + expect(touched?.values).not.toHaveProperty("ratings"); + expect(touched?.values).not.toHaveProperty("responses"); + // A draft they hold needs no queue check. + expect(events.filter((e) => e.table === entriesKey)).toHaveLength(1); + }); + + it("does not make the editor's first save stale", async () => { + // The sheet opened with no draft and sends revision 0; the empty draft + // holds revision 0, so that save goes through as an update of it. + const { caller, events } = desk( + memberWorld({ + [drafts]: byCall([{ id: "new", revision: 0, deadlineAt: null }]), + }), + ); + + await expect( + caller.saveDraft({ ...save, revision: 0 }), + ).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + expect(events.some((e) => e.kind === "update" && e.table === drafts)).toBe( + true, + ); + }); + + it("refuses a project another judge already graded", async () => { + const { caller, events } = desk( + memberWorld({ [evaluations]: byCall([], [{ judgeId: OTHER }]) }), + ); + + await expect(caller.startGrading(start())).rejects.toMatchObject({ + code: "CONFLICT", + message: gradedElsewhere, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a project another judge is grading right now", async () => { + const { caller, events } = desk( + memberWorld({ + [drafts]: byCall( + [], + [{ judgeId: OTHER, updatedAt: minutesAgo(1), deadlineAt: null }], + ), + }), + ); + + await expect(caller.startGrading(start())).rejects.toMatchObject({ + code: "CONFLICT", + message: beingGraded, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a project outside the judge's queue window", async () => { + const order = queueOrder(ME, GENERAL, POOL); + const { caller, events } = desk( + memberWorld({ [entriesKey]: POOL.map((id) => entry(id, "Project")) }), + ); + + await expect( + caller.startGrading(start(order[QUEUE_WINDOW])), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: queueRefusal, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + + await expect( + desk( + memberWorld({ [entriesKey]: POOL.map((id) => entry(id, "Project")) }), + ).caller.startGrading(start(order[0])), + ).resolves.toMatchObject({ held: true }); + }); + + it("leaves the judge's own graded project alone", async () => { + const { caller, events } = desk( + memberWorld({ [evaluations]: byCall([{ id: "mine" }]) }), + ); + + await expect(caller.startGrading(start())).resolves.toMatchObject({ + held: false, + }); + expect(kinds(events)).not.toContain("execute"); + expect(kinds(events)).not.toContain("insert"); + expect(kinds(events)).not.toContain("update"); + }); + + it("is refused while judging is not open", async () => { + const { caller, events } = desk( + memberWorld({ + [via(judgingConfigs)]: [ + { state: "draft", showAllScores: false, closedAt: null }, + ], + }), + ); + + await expect(caller.startGrading(start())).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("holds a guest's own room's project the same way", async () => { + const { caller, events } = desk(guestWorld(), guestCtx); + + await caller.startGrading({ hackathonId: HACK, entryId: E_OPEN }); + + expect(insertedDraft(events)).toMatchObject({ + challengeId: SPONSOR, + ratings: {}, + revision: 0, + }); + }); + + describe("for a scheduled presentation", () => { + const presenting = (startsAt: Date, deadlineAt: Date) => + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [via(judgingAppointments)]: [ + { + id: "slot", + roomId: ROOM, + startsAt, + deadlineAt, + endsAt: minutesAhead(10), + }, + ], + [presenceKey]: seatAt(SPONSOR), + }); + + it("stamps the draft with the presentation and its deadline, as a save does", async () => { + const deadline = minutesAhead(5); + const { caller, events } = desk(presenting(minutesAgo(5), deadline)); + + const result = await caller.startGrading(start(E_OPEN, SPONSOR)); + + expect(result.deadlineAt).toEqual(deadline); + expect(insertedDraft(events)).toMatchObject({ + challengeId: SPONSOR, + appointmentId: "slot", + deadlineAt: deadline, + revision: 0, + }); + }); + + it("stamps the slot's deadline once it has just passed, inside the write grace", async () => { + const passed = new Date(Date.now() - 5_000); + // Within the grace a write still gets, the slot still gives its + // deadline, so the draft is submitted for the judge when it is due. + const { caller, events } = desk({ + ...presenting(minutesAgo(10), passed), + [drafts]: byCall([ + { id: "mine", revision: 2, deadlineAt: minutesAhead(2) }, + ]), + }); + + const result = await caller.startGrading(start(E_OPEN, SPONSOR)); + + expect(result).toEqual({ held: true, revision: 2, deadlineAt: passed }); + const touched = events.find( + (e) => e.kind === "update" && e.table === drafts, + ); + expect(touched?.values).toMatchObject({ deadlineAt: passed }); + }); + + it("does not start before the presentation does", async () => { + const { caller, events } = desk( + presenting(minutesAhead(10), minutesAhead(20)), + ); + + await expect( + caller.startGrading(start(E_OPEN, SPONSOR)), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: + "This project has not presented yet. You can judge it once its presentation starts.", + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + }); + + describe("a second judge after the first opens a project", () => { + const secondJudge = (held: Event["values"]) => + memberWorld({ + [via(judgingJudges)]: [{ id: OTHER, displayName: "Alan Turing" }], + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + [drafts]: [ + { + entryId: held?.entryId, + judgeId: held?.judgeId, + updatedAt: held?.updatedAt, + deadlineAt: held?.deadlineAt, + }, + ], + }); + + it("sees the project in progress, read-only, and cannot take it", async () => { + const first = desk(memberWorld()); + await first.caller.startGrading(start()); + const held = insertedDraft(first.events); + expect(held).toBeDefined(); + + const { db } = fakeDb(secondJudge(held)); + const caller = judgingDeskRouter.createCaller(memberCtx(db)); + + const listed = await caller.projects({ + hackathonId: HACK, + inMyRoomOnly: false, + }); + expect(listed.projects.map((p) => p.grading)).toEqual(["in_progress"]); + + const sheet = await caller.project({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: GENERAL, + }); + expect(sheet.grading).toBe("in_progress"); + expect(sheet.access).toMatchObject({ + canEdit: false, + reason: beingGraded, + gradedElsewhere: true, + }); + + const { db: db2, events } = fakeDb(secondJudge(held)); + const second = judgingDeskRouter.createCaller(memberCtx(db2)); + await expect(second.startGrading(start())).rejects.toMatchObject({ + code: "CONFLICT", + message: beingGraded, + }); + await expect( + second.saveDraft({ ...save, revision: 0 }), + ).rejects.toMatchObject({ code: "CONFLICT", message: beingGraded }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + }); +}); diff --git a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts index 3ab02749..5612dc42 100644 --- a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts +++ b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts @@ -1,11 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { appRouter } from "../root"; +import { errorFormatter } from "../trpc"; import { cache } from "../middleware/cache"; import { db, hackathons, hackathonParticipants, hackathonEventAttendees, + hackathonCheckInFlags, } from "@query/db"; // Fully mock the DB at the file level. vi.mock factories are hoisted and @@ -19,6 +21,11 @@ const mockDelete = vi.fn(); const mockCount = vi.fn(); /** maxParticipants as the capacity check reads it; undefined = uncapped. */ const mockMaxParticipants = vi.fn<() => number | undefined>(() => undefined); +/** + * A participant's earlier check-ins as recordEventAttendance reads them: + * attendance joined to its event. Empty unless a test says otherwise. + */ +const mockEarlierAttendance = vi.fn<() => unknown[]>(() => []); // Acceptance emails are sent through a dynamic import; intercept it so we can // see exactly who a mass-approval actually mailed. const mockSendAcceptanceEmail = vi.fn(); @@ -111,7 +118,7 @@ vi.mock("@query/db", () => { select: vi.fn().mockImplementation(() => ({ from: (tbl: any) => { const count = mockCount(tbl?._t) ?? 0; - return thenable([ + const node = thenable([ { count, n: count, @@ -119,6 +126,10 @@ vi.mock("@query/db", () => { tbl?._t === "hackathons" ? mockMaxParticipants() : undefined, }, ]); + if (tbl?._t === "hackathonEventAttendees") { + node.innerJoin = () => thenable(mockEarlierAttendance()); + } + return node; }, })), }, @@ -186,6 +197,16 @@ vi.mock("@query/db", () => { participantId: "participant_id", checkedInAt: "checked_in_at", }, + hackathonCheckInFlags: { + _t: "hackathonCheckInFlags", + id: "id", + hackathonId: "hackathon_id", + participantId: "participant_id", + eventId: "event_id", + conflictEventId: "conflict_event_id", + attempts: "attempts", + lastAttemptAt: "last_attempt_at", + }, members: { _t: "members", id: "id", @@ -292,6 +313,7 @@ describe("Hackathon admin management edge cases", () => { cache.clear(); mockCount.mockReturnValue(0); mockMaxParticipants.mockImplementation(() => undefined); + mockEarlierAttendance.mockReturnValue([]); mockFindMany.mockReturnValue([]); mockInsert.mockReturnValue([{}]); mockUpdate.mockReturnValue([{}]); @@ -546,6 +568,25 @@ describe("Hackathon admin management edge cases", () => { ).rejects.toThrow(refused); }); + // A banned hacker can keep a registration, so the code check-in is a way + // of taking part that has to stop at the gate, before any lookup. + it("refuses a banned hacker's event code check-in", async () => { + const caller = participant(true); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: expect.stringMatching(/can't take part in Hacklytics/), + }); + + expect(mockFindFirst).not.toHaveBeenCalledWith( + "hackathonParticipants", + expect.anything(), + ); + expect(mockInsert).not.toHaveBeenCalled(); + }); + it("lets everyone else through the ban gate", async () => { const caller = participant(false); const result = await caller.hackathon @@ -1274,6 +1315,354 @@ describe("Hackathon admin management edge cases", () => { }); }); + // ===================================================================== + // Organisers' rules for bad actors: nobody is in two events that run at the + // same time, or in two within 30 minutes. They live in recordEventAttendance, + // so every way of writing attendance is held to them. + describe("Event check-in rules", () => { + const MINUTE = 60 * 1000; + + const person = { + id: PART_A1, + hackathonId: HACK_A, + registrationStatus: "checked_in", + firstName: "Ada", + lastName: "Lovelace", + user: { name: "Ada Lovelace", email: "ada@example.com" }, + }; + const fireside = { + id: EVENT_A, + hackathonId: HACK_A, + name: "Full Sail Fireside", + type: "sponsor_session", + points: 10, + selfCheckIn: true, + startTime: new Date(Date.now() - HOUR), + endTime: new Date(Date.now() + HOUR), + }; + + // A check-in for another event, as the attendance read returns it. + const earlier = (event: Record, minutesAgo: number) => ({ + event: { + id: "99999999-9999-4999-8999-999999999999", + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 5 * HOUR), + endTime: new Date(Date.now() - 4 * HOUR), + ...event, + }, + checkedInAt: new Date(Date.now() - minutesAgo * MINUTE), + }); + + // The hacker's own session, past the ban gate, with the event live. + const hackerCaller = () => { + mockFindFirst.mockImplementation((table: string) => { + if (table === "users") return { email: "ada@example.com" }; + if (table === "hackathonParticipants") return person; + if (table === "hackathonEvents") return fireside; + return undefined; + }); + return appRouter.createCaller(createMockCtx("rules-hacker")); + }; + + const attendanceWrites = () => + mockInsert.mock.calls.filter((c) => c[1][0] === hackathonEventAttendees); + + // Each refused attempt leaves a flag for organisers; the values are the + // third element of the mock's recorded call. + const flagWrites = () => + mockInsert.mock.calls.filter((c) => c[1][0] === hackathonCheckInFlags); + + // The event the fixtures' earlier check-ins are for. + const BNY = "99999999-9999-4999-8999-999999999999"; + + it("refuses a code check-in into an event that overlaps one already attended", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([ + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 20, + ), + ]); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "You're already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. You can check into one event at a time.", + }); + expect(attendanceWrites()).toHaveLength(0); + }); + + it("lets a code check-in through when nothing overlaps and the last was long ago", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([earlier({}, 120)]); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).resolves.toMatchObject({ eventName: "Full Sail Fireside" }); + expect(attendanceWrites()).toHaveLength(1); + }); + + it("refuses a pass scan within 30 minutes of the last check-in", async () => { + const caller = adminCaller({ + hackathonParticipants: { ...person, registrationStatus: "approved" }, + hackathonEvents: fireside, + hackathonEventAttendees: undefined, + }); + mockEarlierAttendance.mockReturnValue([earlier({}, 10)]); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "Ada Lovelace checked into BNY Workshop 10 minutes ago. Event check-ins have to be at least 30 minutes apart. If that was a mistake, remove them from BNY Workshop first.", + }); + // The whole scan rolls back: no attendance, and no arrival either. + expect(attendanceWrites()).toHaveLength(0); + expect(mockUpdate).not.toHaveBeenCalled(); + }); + + it("refuses a manual check-in into an event that overlaps one already attended", async () => { + const caller = adminCaller({ + hackathonParticipants: person, + hackathonEvents: fireside, + }); + mockEarlierAttendance.mockReturnValue([ + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 45, + ), + ]); + + await expect( + caller.hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", + }); + expect(attendanceWrites()).toHaveLength(0); + }); + + describe("flags for organisers", () => { + const overlapping = () => + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 20, + ); + + it("records a refused code check-in against the hacker", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([overlapping()]); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(flagWrites()).toHaveLength(1); + expect(flagWrites()[0]![2][0]).toEqual({ + hackathonId: HACK_A, + participantId: PART_A1, + eventId: EVENT_A, + conflictEventId: BNY, + rule: "overlap", + method: "code", + attemptedById: "rules-hacker", + attempts: 1, + createdAt: expect.any(Date), + lastAttemptAt: expect.any(Date), + }); + }); + + it("records a refused pass scan, with the staff member who scanned", async () => { + const caller = adminCaller({ + hackathonParticipants: { ...person, registrationStatus: "approved" }, + hackathonEvents: fireside, + hackathonEventAttendees: undefined, + }); + mockEarlierAttendance.mockReturnValue([earlier({}, 10)]); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(flagWrites()).toHaveLength(1); + expect(flagWrites()[0]![2][0]).toMatchObject({ + hackathonId: HACK_A, + participantId: PART_A1, + eventId: EVENT_A, + conflictEventId: BNY, + rule: "too_soon", + method: "scan", + attemptedById: ADMIN_USER, + }); + }); + + it("records a refused manual check-in", async () => { + const caller = adminCaller({ + hackathonParticipants: person, + hackathonEvents: fireside, + }); + mockEarlierAttendance.mockReturnValue([overlapping()]); + + await expect( + caller.hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(flagWrites()).toHaveLength(1); + expect(flagWrites()[0]![2][0]).toMatchObject({ + participantId: PART_A1, + rule: "overlap", + method: "manual", + attemptedById: ADMIN_USER, + }); + }); + + it("records nothing for a check-in that goes through", async () => { + mockEarlierAttendance.mockReturnValue([earlier({}, 120)]); + + await hackerCaller().points.me.checkIn({ + hackathonId: HACK_A, + code: "ABC234", + }); + await adminCaller({ + hackathonParticipants: person, + hackathonEvents: fireside, + }).hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }); + + expect(attendanceWrites()).toHaveLength(2); + expect(flagWrites()).toHaveLength(0); + }); + + it("records nothing for a duplicate scan", async () => { + const caller = adminCaller({ + hackathonParticipants: { ...person, registrationStatus: "approved" }, + hackathonEvents: fireside, + hackathonEventAttendees: { id: "already" }, + }); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ code: "CONFLICT" }); + + expect(flagWrites()).toHaveLength(0); + }); + + it("gives the hacker the same refusal when the flag cannot be written", async () => { + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([overlapping()]); + mockInsert.mockImplementation((_op, [table]) => { + if (table === hackathonCheckInFlags) throw new Error("db is down"); + return [{}]; + }); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "You're already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. You can check into one event at a time.", + }); + expect(consoleError).toHaveBeenCalled(); + consoleError.mockRestore(); + }); + + it("keeps the flag out of what the hacker is sent", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([overlapping()]); + + const error: any = await caller.points.me + .checkIn({ hackathonId: HACK_A, code: "ABC234" }) + .catch((e: unknown) => e); + + // The record rides on the error server side only; the formatter is what + // builds the response. + const response = JSON.stringify( + errorFormatter({ + shape: { + message: error.message, + code: -32003, + data: { + code: "FORBIDDEN", + httpStatus: 403, + path: "points.me.checkIn", + }, + }, + error, + }), + ); + expect(response).not.toContain(PART_A1); + expect(response).not.toContain(BNY); + expect(response).not.toContain("CheckInRuleBreak"); + expect(response).not.toContain("conflictEventId"); + }); + }); + + it("does not hold a meal to the rules", async () => { + const caller = adminCaller({ + hackathonParticipants: person, + hackathonEvents: { ...fireside, name: "Dinner", type: "meal" }, + }); + mockEarlierAttendance.mockReturnValue([ + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 5, + ), + ]); + + await expect( + caller.hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).resolves.toMatchObject({ eventName: "Dinner" }); + expect(attendanceWrites()).toHaveLength(1); + }); + }); + // ===================================================================== describe("Chasing unconfirmed seats", () => { const whereOf = () => diff --git a/packages/api/src/.internal-tests/qr-checkin.test.ts b/packages/api/src/.internal-tests/qr-checkin.test.ts index d3e9af21..c402667a 100644 --- a/packages/api/src/.internal-tests/qr-checkin.test.ts +++ b/packages/api/src/.internal-tests/qr-checkin.test.ts @@ -1,7 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { appRouter } from "../root"; import { cache } from "../middleware/cache"; -import { db, events, eventCheckIns, hackathonEventAttendees } from "@query/db"; +import { + db, + events, + eventCheckIns, + hackathonCheckInFlags, + hackathonEventAttendees, +} from "@query/db"; // The real rule: a hardcoded "2026-fall" would fail every January. import { currentTerm } from "@query/db/services/membership"; import { __onRollback } from "./_db-tx-mock"; @@ -21,6 +27,9 @@ const mockInsert = vi.fn(); const mockUpdate = vi.fn(); const mockDelete = vi.fn(); const mockSelect = vi.fn(); +// A participant's earlier check-ins, which recordEventAttendance reads as +// attendance joined to its event. Empty unless a test says otherwise. +const mockEarlierAttendance = vi.fn<() => unknown[]>(() => []); vi.mock("@query/db", async () => { const { createTransactionMock } = await import("./_db-tx-mock"); @@ -188,6 +197,12 @@ vi.mock("@query/db", async () => { eventId: "event_id", participantId: "participant_id", }, + hackathonCheckInFlags: { + participantId: "participant_id", + eventId: "event_id", + conflictEventId: "conflict_event_id", + attempts: "attempts", + }, members: { id: "id", userId: "user_id", hackathonId: "hackathon_id" }, events: { id: "id", @@ -261,7 +276,10 @@ describe("QR check-in", () => { cache.clear(); // Aggregate reads default to "nothing there"; the concurrency tests below // override this to hand back the live row a FOR UPDATE would have locked. - mockSelect.mockReturnValue([{ count: 0 }]); + mockSelect.mockImplementation((_op, _sel, _from, _where, joined) => + joined ? mockEarlierAttendance() : [{ count: 0 }], + ); + mockEarlierAttendance.mockReturnValue([]); }); const createMockCtx = (userId?: string) => @@ -898,6 +916,110 @@ describe("QR check-in", () => { expect(err.message).toMatch(/is already checked into/); }); + it("refuses a scan into an event that runs at the same time as one already attended", async () => { + // Named and typed outright: the default fixture reads as a meal, and + // meals are exempt. + const caller = scannerCtx({ + event: hackEvent({ + name: "Full Sail Fireside", + type: "sponsor_session", + endTime: new Date(Date.now() + HOUR), + }), + }); + mockEarlierAttendance.mockReturnValue([ + { + event: { + id: CLUB_EVENT, + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 2 * HOUR), + endTime: new Date(Date.now() + 2 * HOUR), + }, + checkedInAt: new Date(Date.now() - 90 * 60 * 1000), + }, + ]); + + const err: any = await caller.hackathon + .scanParticipantPass({ + hackathonId: HACK_A, + eventId: HACK_EVENT, + participantId: PARTICIPANT, + }) + .catch((e: unknown) => e); + + expect(err.code).toBe("FORBIDDEN"); + expect(err.message).toBe( + "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", + ); + // No attendance and no arrival, but the attempt is on record for + // organisers: it is the one write that survives the refusal. + const inserts = mockInsert.mock.calls as any[][]; + expect( + inserts.filter((c) => c[1][0] === hackathonEventAttendees), + ).toHaveLength(0); + const flags = inserts.filter((c) => c[1][0] === hackathonCheckInFlags); + expect(flags).toHaveLength(1); + expect(flags[0]![2][0]).toMatchObject({ + hackathonId: HACK_A, + participantId: PARTICIPANT, + eventId: HACK_EVENT, + conflictEventId: CLUB_EVENT, + rule: "overlap", + method: "scan", + attemptedById: "admin_user_id", + attempts: 1, + }); + expect(mockUpdate).not.toHaveBeenCalled(); + }); + + it("writes the flag after the refused scan has rolled back, not inside it", async () => { + const caller = scannerCtx({ + event: hackEvent({ + name: "Full Sail Fireside", + type: "sponsor_session", + endTime: new Date(Date.now() + HOUR), + }), + }); + mockEarlierAttendance.mockReturnValue([ + { + event: { + id: CLUB_EVENT, + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 2 * HOUR), + endTime: new Date(Date.now() + 2 * HOUR), + }, + checkedInAt: new Date(Date.now() - 90 * 60 * 1000), + }, + ]); + // A write made through a transaction registers an undo that runs when the + // refusal throws; one made on the pool registers nothing. + let flagRolledBack = false; + mockInsert.mockImplementation((_op, [table]) => { + if (table === hackathonCheckInFlags) { + __onRollback(() => { + flagRolledBack = true; + }); + } + return [{}]; + }); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: HACK_EVENT, + participantId: PARTICIPANT, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect( + (mockInsert.mock.calls as any[][]).filter( + (c) => c[1][0] === hackathonCheckInFlags, + ), + ).toHaveLength(1); + expect(flagRolledBack).toBe(false); + }); + // BUG: scanParticipantPass matches on (id, hackathonId) only // (admin.ts:287-298) and never compares startTime/endTime to now, so one // pass of the scanner can check a participant into the whole weekend. diff --git a/packages/api/src/.internal-tests/reopen-grade.test.ts b/packages/api/src/.internal-tests/reopen-grade.test.ts new file mode 100644 index 00000000..218ec8bc --- /dev/null +++ b/packages/api/src/.internal-tests/reopen-grade.test.ts @@ -0,0 +1,463 @@ +/** + * judging.admin.reopenGrade: an organiser deletes one judge's evaluation, and + * any draft for the same project and challenge, so the project is open for + * another judge. The edition's draft placings were ranked with that grade, so + * they go in the same transaction and results have to be computed again + * before they can be published. Refused once results are published. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way check-in-flags-organizer.test.ts + * does. It applies the equality conditions the router asks for, so a missing + * edition condition lets another edition's evaluation through and fails the + * test. It keeps every statement in order, which is how the tests see the + * drafts being locked before the project's grading lock and both before a + * delete. + */ +import { beforeEach, describe, expect, it } from "vitest"; +import { getTableConfig } from "drizzle-orm/pg-core"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + auditLogs, + hackathons, + judgingConfigs, + judgingEvaluationDrafts, + judgingEvaluationRatings, + judgingEvaluationResponses, + judgingEvaluationRevisions, + judgingEvaluations, + judgingPlacements, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; + +const HACK_A = "11111111-1111-4111-8111-111111111111"; +const HACK_B = "22222222-2222-4222-8222-222222222222"; +const ENTRY = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1"; +const CHALLENGE = "cccccccc-cccc-4ccc-8ccc-ccccccccccc1"; +const JUDGE = "dddddddd-dddd-4ddd-8ddd-ddddddddddd1"; +const EVAL = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1"; +const EVAL_OTHER_EDITION = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee2"; +const DRAFT_1 = "ffffffff-ffff-4fff-8fff-fffffffffff1"; +const DRAFT_2 = "ffffffff-ffff-4fff-8fff-fffffffffff2"; +const NO_SUCH = "99999999-9999-4999-8999-999999999999"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); + +type Row = Record; +type Event = { + kind: "select" | "execute" | "insert" | "delete" | "update"; + table?: unknown; + where?: unknown; + /** A select that takes row locks. */ + locking?: boolean; +}; + +const camel = (name: string) => + name.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase()); +const TERM = /"[a-z_]+"\."([a-z_]+)" = \$(\d+)/g; + +/** A row filter from a where of `column = value` terms joined by and. */ +function matcher(where: unknown) { + const { sql: text, params } = render(where); + const terms = [...text.matchAll(TERM)].map( + (match) => [camel(match[1]!), params[Number(match[2]) - 1]] as const, + ); + if (text.replace(TERM, "").replace(/[()\s]|and/g, "")) { + throw new Error(`The fake database cannot evaluate: ${text}`); + } + return (row: Row) => terms.every(([key, value]) => row[key] === value); +} + +let adminRow: Row | undefined; +let published = false; +const store = new Map(); +const events: Event[] = []; + +const rowsOf = (table: unknown, where: unknown) => { + const rows = store.get(table) ?? []; + return where ? rows.filter(matcher(where)) : [...rows]; +}; + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: () => { + const event: Event = { kind: "select" }; + events.push(event); + const chain: any = { + then: (onOk: any, onErr: any) => { + // Counts of placements: those that carry a publish time (the published + // check), or every one of the edition's (what publish finds to publish). + const placings = () => { + const { sql: text, params } = render(event.where); + const count = text.includes('"published_at" is not null') + ? published + ? 3 + : 0 + : (store.get(judgingPlacements) ?? []).filter((row) => + params.includes(row.hackathonId), + ).length; + return [{ value: count, n: count }]; + }; + const rows = + event.table === judgingPlacements + ? placings() + : rowsOf(event.table, event.where); + return Promise.resolve(rows).then(onOk, onErr); + }, + }; + chain.from = (table: unknown) => { + event.table = table; + return chain; + }; + chain.where = (where: unknown) => { + event.where = where; + return chain; + }; + chain.for = () => { + event.locking = true; + return chain; + }; + for (const method of ["innerJoin", "leftJoin", "orderBy", "limit"]) { + chain[method] = () => chain; + } + return chain; + }, + execute: async () => { + events.push({ kind: "execute" }); + }, + insert: (table: unknown) => ({ + values: (values: Row) => { + events.push({ kind: "insert", table, where: values }); + return Object.assign(Promise.resolve(undefined), { + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [values], + }); + }, + }), + update: (table: unknown) => ({ + set: () => ({ + where: (where: unknown) => { + events.push({ kind: "update", table, where }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => [], + }); + }, + }), + }), + delete: (table: unknown) => ({ + where: (where: unknown) => { + events.push({ kind: "delete", table, where }); + // The placings the delete takes: this edition's that carry no publish + // time, which the fake reads off the condition's parameters. + const hit = + table === judgingPlacements + ? (store.get(table) ?? []).filter( + (row) => + render(where).params.includes(row.hackathonId) && + row.publishedAt === null, + ) + : []; + if (table === judgingPlacements) { + store.set( + table, + (store.get(table) ?? []).filter((row) => !hit.includes(row)), + ); + } + return Object.assign(Promise.resolve([]), { + returning: async () => hit.map((row) => ({ id: row.id })), + }); + }, + }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string | undefined) => + appRouter.createCaller({ + db: fakeDb, + session: userId ? { user: { id: userId } } : null, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); + +const admin = () => callerAs("admin_user"); +const reopen = ( + over: Partial<{ hackathonId: string; evaluationId: string }> = {}, +) => + admin().judging.admin.reopenGrade({ + hackathonId: HACK_A, + evaluationId: EVAL, + ...over, + }); + +// Without the audit log's own housekeeping, which prunes old rows. +const deletes = () => + events.filter( + (event) => event.kind === "delete" && event.table !== auditLogs, + ); +const audits = () => + events + .filter((event) => event.kind === "insert" && event.table === auditLogs) + .map((event) => event.where as Row); + +const configRow = (hackathonId: string, state = "open") => ({ + hackathonId, + state, + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: false, + emergencyMode: false, + claimsStartedAt: null, +}); + +beforeEach(() => { + cache.clear(); + events.length = 0; + published = false; + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + store.clear(); + store.set(hackathons, [ + { id: HACK_A, name: "Hacklytics 2027", startDate: null, endDate: null }, + { id: HACK_B, name: "Hacklytics 2026", startDate: null, endDate: null }, + ]); + store.set(judgingConfigs, [configRow(HACK_A), configRow(HACK_B)]); + store.set(judgingEvaluations, [ + { + id: EVAL, + hackathonId: HACK_A, + entryId: ENTRY, + challengeId: CHALLENGE, + judgeId: JUDGE, + revision: 2, + isComplete: false, + }, + { + id: EVAL_OTHER_EDITION, + hackathonId: HACK_B, + entryId: ENTRY, + challengeId: CHALLENGE, + judgeId: JUDGE, + revision: 1, + isComplete: true, + }, + ]); + store.set(judgingPlacements, [ + { id: "p1", hackathonId: HACK_A, scope: "overall", publishedAt: null }, + { id: "p2", hackathonId: HACK_A, scope: CHALLENGE, publishedAt: null }, + { id: "p3", hackathonId: HACK_B, scope: "overall", publishedAt: null }, + ]); + store.set(judgingEvaluationDrafts, [ + { id: DRAFT_1, entryId: ENTRY, challengeId: CHALLENGE }, + { id: DRAFT_2, entryId: ENTRY, challengeId: CHALLENGE }, + { id: "elsewhere", entryId: NO_SUCH, challengeId: CHALLENGE }, + ]); +}); + +describe("judging.admin.reopenGrade", () => { + it("deletes the evaluation and every draft for the project and challenge", async () => { + await expect(reopen()).resolves.toEqual({ + reopened: true, + drafts: 2, + placings: 2, + }); + + const [draftDelete, evaluationDelete] = deletes(); + expect(deletes()).toHaveLength(3); + expect(draftDelete!.table).toBe(judgingEvaluationDrafts); + expect(render(draftDelete!.where).params).toEqual([DRAFT_1, DRAFT_2]); + expect(evaluationDelete!.table).toBe(judgingEvaluations); + expect(render(evaluationDelete!.where).params).toEqual([EVAL]); + }); + + it("deletes an evaluation with no draft beside it", async () => { + store.set(judgingEvaluationDrafts, []); + + await expect(reopen()).resolves.toMatchObject({ + reopened: true, + drafts: 0, + }); + + expect(deletes().map((event) => event.table)).toEqual([ + judgingEvaluations, + judgingPlacements, + ]); + }); + + it("locks the drafts, then the project's grading lock, before it deletes", async () => { + await reopen(); + + const draftLock = events.findIndex( + (event) => + event.kind === "select" && + event.table === judgingEvaluationDrafts && + event.locking, + ); + const gradingLock = events.findIndex((event) => event.kind === "execute"); + const firstDelete = events.findIndex((event) => event.kind === "delete"); + const configLock = events.findIndex( + (event) => + event.kind === "select" && + event.table === judgingConfigs && + event.locking, + ); + expect(configLock).toBeGreaterThan(-1); + expect(draftLock).toBeGreaterThan(configLock); + expect(gradingLock).toBeGreaterThan(draftLock); + expect(firstDelete).toBeGreaterThan(gradingLock); + }); + + it("deletes the edition's draft placings once the grade is gone, and no other edition's", async () => { + await expect(reopen()).resolves.toMatchObject({ placings: 2 }); + + const [, evaluationDelete, placingDelete] = deletes(); + expect(evaluationDelete!.table).toBe(judgingEvaluations); + expect(placingDelete!.table).toBe(judgingPlacements); + const query = render(placingDelete!.where); + expect(query.params).toEqual([HACK_A]); + expect(query.sql).toContain('"judging_placement"."hackathon_id" = $1'); + expect(query.sql).toContain('"published_at" is null'); + expect(store.get(judgingPlacements)!.map((row) => row.id)).toEqual(["p3"]); + }); + + it("leaves nothing to publish until results are computed again", async () => { + const publish = () => + admin().judging.results.publish({ hackathonId: HACK_A }); + + // The draft placings as computed, with the grade in them. + await expect(publish()).resolves.toMatchObject({ publishedAt: expect.any(Date) }); + + await reopen(); + + await expect(publish()).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: expect.stringContaining( + "Reopening a grade clears the draft placings, so compute again after one.", + ), + }); + }); + + it("reports none cleared when results were never computed", async () => { + store.set(judgingPlacements, []); + + await expect(reopen()).resolves.toMatchObject({ + reopened: true, + placings: 0, + }); + }); + + it("refuses once results are published, and deletes nothing", async () => { + published = true; + + await expect(reopen()).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: + "Results are published. Unpublish them first, then reopen the grade and compute results again.", + }); + + expect(deletes()).toEqual([]); + expect(audits()).toEqual([]); + }); + + it("reads published from the placings of this edition", async () => { + await reopen(); + + const placings = events.find( + (event) => event.kind === "select" && event.table === judgingPlacements, + )!; + const query = render(placings.where); + expect(query.params).toContain(HACK_A); + expect(query.sql).toContain('"published_at" is not null'); + }); + + it("is allowed while judging is closed", async () => { + store.set(judgingConfigs, [configRow(HACK_A, "closed")]); + + await expect(reopen()).resolves.toMatchObject({ reopened: true }); + + expect(deletes()).toHaveLength(3); + }); + + it("does not reach an evaluation of another edition", async () => { + await expect( + reopen({ evaluationId: EVAL_OTHER_EDITION }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(deletes()).toEqual([]); + }); + + it("says so when the evaluation is already gone", async () => { + await expect(reopen({ evaluationId: NO_SUCH })).rejects.toMatchObject({ + code: "NOT_FOUND", + message: "That evaluation is gone. Refresh the list.", + }); + }); + + it("is audited as a critical action, with what was deleted", async () => { + await reopen(); + + expect(audits()).toEqual([ + expect.objectContaining({ + userId: "admin_user", + action: "judging.reopenGrade", + resourceId: EVAL, + severity: "critical", + metadata: { + hackathonId: HACK_A, + entryId: ENTRY, + challengeId: CHALLENGE, + judgeId: JUDGE, + revision: 2, + wasComplete: false, + draftsDeleted: 2, + placingsDeleted: 2, + }, + }), + ]); + }); + + it("is for admins only", async () => { + adminRow = undefined; + await expect(reopen()).rejects.toMatchObject({ code: "FORBIDDEN" }); + + adminRow = { userId: "admin_user", isActive: true, role: "bug_tester" }; + cache.clear(); + await expect(reopen()).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(deletes()).toEqual([]); + }); + + it("rejects ids that are not uuids", async () => { + await expect(reopen({ evaluationId: "not-a-uuid" })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + }); +}); + +describe("what goes with an evaluation", () => { + const foreignKeys = (table: Parameters[0]) => + getTableConfig(table).foreignKeys.map((key) => { + const reference = key.reference(); + return { + to: reference.foreignTable, + columns: reference.columns.map((column) => column.name), + onDelete: key.onDelete, + }; + }); + + it.each([ + ["ratings", judgingEvaluationRatings], + ["responses", judgingEvaluationResponses], + ["revisions", judgingEvaluationRevisions], + ])("deletes its %s by cascade", (_, table) => { + expect(foreignKeys(table)).toContainEqual({ + to: judgingEvaluations, + columns: ["evaluation_id"], + onDelete: "cascade", + }); + }); +}); diff --git a/packages/api/src/.internal-tests/schedule-judges.test.ts b/packages/api/src/.internal-tests/schedule-judges.test.ts new file mode 100644 index 00000000..94fcd334 --- /dev/null +++ b/packages/api/src/.internal-tests/schedule-judges.test.ts @@ -0,0 +1,1094 @@ +/** + * Judges on the saved schedule: saving gives every presentation a judge as far + * as there are judges, the board shows who grades what and where judges run + * short, "assign judges" picks again for what has not started, and an + * organiser can set one appointment's judge by hand until it starts. + * + * The schema is the real one and so is the scheduling code; only the database + * handle is a stand-in, handed to the router through the context the way + * check-in-flags-organizer.test.ts does. It keeps the rows of the tables the + * router reads and records what it writes. The CP-SAT runner is a stub, so a + * test decides what the script "printed" and sees exactly what it was asked. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + auditLogs, + hackathons, + judgingAppointments, + judgingChallenges, + judgingConfigs, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingGuestSessions, + judgingJudges, + judgingPresence, + judgingRooms, + judgingSchedules, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; +import { assignJudges } from "../services/judging/schedule/judges"; +import type * as Store from "../services/judging/evaluation-store"; +import type * as Runner from "../services/judging/schedule/cpsat-runner"; + +const mockAvailable = vi.fn(); +const mockRun = vi.fn(); + +// Nothing is due on any board read here, so there is nothing to reconcile. +vi.mock("../services/judging/evaluation-store", async (importActual) => ({ + ...(await importActual()), + reconcileDueDrafts: async () => undefined, +})); +vi.mock("../services/judging/schedule/cpsat-runner", async (importActual) => ({ + ...(await importActual()), + cpsatAvailable: () => mockAvailable(), + runCpsat: (...args: unknown[]) => mockRun(...args), +})); + +const id = (n: number) => + `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const HACK = id(1); +const GENERAL = id(10); +const ACME = id(11); +const G1 = id(20); +const G2 = id(21); +const A1 = id(22); +const A2 = id(23); +const E1 = id(30); +const E2 = id(31); +const E3 = id(32); +const APPT = [id(40), id(41), id(42), id(43)] as const; +const [P1, P2, P3, P4] = APPT; +const M1 = id(50); +const M2 = id(51); +const GS = id(52); +const NOROOM = id(53); +const SCHEDULE = id(60); + +// Slots are 10 minutes from 14:00. It is 15:00, so P1 (14:00) is over and the +// rest (15:30 and 16:00) are still to come. +const NOW = new Date("2027-02-06T15:00:00Z"); +const at = (time: string) => new Date(`2027-02-06T${time}:00Z`); + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); + +type Row = Record; +type Write = { table: unknown; values: Row | Row[]; where?: unknown }; + +const camel = (name: string) => + name.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase()); +const TERM = /"[a-z_]+"\."([a-z_]+)" = \$(\d+)/g; + +/** A row filter from a where of `column = value` terms, or null for anything else. */ +function matcher(where: unknown) { + const { sql: text, params } = render(where); + const terms = [...text.matchAll(TERM)].map( + (match) => [camel(match[1]!), params[Number(match[2]) - 1]] as const, + ); + if (text.replace(TERM, "").replace(/[()\s]|and/g, "")) return null; + return (row: Row) => terms.every(([key, value]) => row[key] === value); +} + +let adminRow: Row | undefined; +const store = new Map(); +const inserts: Write[] = []; +const updates: Write[] = []; +// Counts reads of a table, so a test can change what a later read returns. +const reads = new Map(); +let onRead: ((table: unknown, count: number) => Row[] | undefined) | null = + null; + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: () => { + let table: unknown = null; + let where: unknown = null; + const chain: any = { + then: (ok: any, err: any) => { + const count = (reads.get(table) ?? 0) + 1; + reads.set(table, count); + const rows = onRead?.(table, count) ?? store.get(table) ?? []; + const keep = where ? matcher(where) : null; + return Promise.resolve(keep ? rows.filter(keep) : [...rows]).then( + ok, + err, + ); + }, + }; + chain.from = (from: unknown) => { + table = from; + return chain; + }; + chain.where = (condition: unknown) => { + where = condition; + return chain; + }; + for (const method of ["innerJoin", "leftJoin", "orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }, + insert: (table: unknown) => ({ + values: (values: Row | Row[]) => { + inserts.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [values], + }); + }, + }), + update: (table: unknown) => ({ + set: (values: Row) => ({ + where: (where: unknown) => { + const keep = matcher(where); + const hit = keep ? (store.get(table) ?? []).filter(keep) : []; + for (const row of hit) Object.assign(row, values); + updates.push({ table, values, where }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => hit.map((row) => ({ ...row })), + }); + }, + }), + }), + delete: () => ({ where: async () => [] }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string | undefined) => + appRouter.createCaller({ + db: fakeDb, + session: userId ? { user: { id: userId } } : null, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); +const admin = () => callerAs("admin_user"); + +const audits = () => + inserts + .filter((write) => write.table === auditLogs) + .map((write) => write.values as Row); + +/** The appointments an update touched, by the ids in its where. */ +const touched = (write: Write) => + render(write.where).params.filter((p) => APPT.includes(p as never)); + +/** Which judge each appointment was left with by the judge updates. */ +function judgeAfter() { + const result = new Map(); + for (const write of updates) { + if (write.table !== judgingAppointments) continue; + if (!("assignedJudgeId" in (write.values as Row))) continue; + for (const apptId of touched(write)) { + result.set(apptId as string, (write.values as Row).assignedJudgeId); + } + } + return result; +} + +const appointment = (apptId: string, over: Row): Row => ({ + id: apptId, + scheduleId: SCHEDULE, + hackathonId: HACK, + revision: 0, + assignedJudgeId: null, + entryDeletedAt: null, + deadlineAt: null, + ...over, +}); +const times = (start: string) => ({ + startsAt: at(start), + deadlineAt: at(start), + endsAt: new Date(at(start).getTime() + 10 * 60_000), +}); + +// Over (P1), then P2 and P3 at once, then P4 for the sponsor. +const appointments = (): Row[] => [ + appointment(P1, { + entryId: E1, + challengeId: GENERAL, + roomId: G1, + entryTitle: "Project 1", + ...times("14:00"), + }), + appointment(P2, { + entryId: E2, + challengeId: GENERAL, + roomId: G1, + entryTitle: "Project 2", + ...times("15:30"), + }), + appointment(P3, { + entryId: E3, + challengeId: GENERAL, + roomId: G2, + entryTitle: "Project 3", + ...times("15:30"), + }), + appointment(P4, { + entryId: E1, + challengeId: ACME, + roomId: A1, + entryTitle: "Project 1", + ...times("16:00"), + }), +]; + +const judge = (judgeId: string, kind: string, displayName: string) => ({ + id: judgeId, + kind, + userId: kind === "member" ? `user-${displayName}` : null, + displayName, +}); + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + vi.clearAllMocks(); + cache.clear(); + inserts.length = 0; + updates.length = 0; + reads.clear(); + onRead = null; + mockAvailable.mockReturnValue(false); + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + + store.clear(); + // Every table the router reads by edition holds that edition's rows. + const set = store.set.bind(store); + store.set = (table, rows) => + set( + table, + table === hackathons || + table === judgingSchedules || + table === judgingConfigs + ? rows + : rows.map((row) => ({ hackathonId: HACK, ...row })), + ); + store.set(hackathons, [ + { id: HACK, name: "Hacklytics 2027", startDate: null, endDate: null }, + ]); + store.set(judgingConfigs, [{ hackathonId: HACK, state: "open" }]); + store.set(judgingSchedules, [ + { + id: SCHEDULE, + hackathonId: HACK, + windowStart: at("14:00"), + windowEnd: at("17:00"), + setupMinutes: 2, + judgingMinutes: 6, + teardownMinutes: 2, + sameBuildingBreakMinutes: 10, + crossBuildingBreakMinutes: 20, + reducedBreaksAcknowledgedAt: null, + firstResultAt: null, + createdById: "admin_user", + createdAt: NOW, + }, + ]); + const challenge = (cid: string, label: string, general: boolean) => ({ + id: cid, + label, + parentId: null, + mode: "scheduled", + isGeneral: general, + isGroup: false, + }); + store.set(judgingChallenges, [ + challenge(GENERAL, "General", true), + challenge(ACME, "Acme", false), + ]); + const room = ( + rid: string, + name: string, + challengeId: string, + building: string, + ) => ({ + id: rid, + name, + kind: "table", + challengeId, + buildingId: building, + buildingName: building, + archivedAt: null, + }); + store.set(judgingRooms, [ + room(G1, "Table G1", GENERAL, "ENG"), + room(G2, "Table G2", GENERAL, "ENG"), + room(A1, "Acme 1", ACME, "HEC"), + room(A2, "Acme 2", ACME, "HEC"), + ]); + store.set(judgingEntryChallenges, [ + { entryId: E1, challengeId: GENERAL, title: "Project 1" }, + { entryId: E2, challengeId: GENERAL, title: "Project 2" }, + { entryId: E3, challengeId: GENERAL, title: "Project 3" }, + { entryId: E1, challengeId: ACME, title: "Project 1" }, + ]); + store.set(judgingAppointments, appointments()); + store.set(judgingEvaluations, []); + store.set(judgingEvaluationDrafts, []); + store.set(judgingPresence, []); + store.set(judgingJudges, [ + judge(M1, "member", "Ada"), + judge(M2, "member", "Bea"), + judge(GS, "guest", "Grace"), + judge(NOROOM, "guest", "Nobody"), + ]); + store.set(judgingGuestSessions, [ + { judgeId: GS, roomId: A1, createdAt: NOW }, + ]); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +/** Gives an appointment a judge in the stored rows. */ +const give = (apptId: string, judgeId: string) => { + store + .get(judgingAppointments)! + .find((a) => a.id === apptId)!.assignedJudgeId = judgeId; +}; + +// --------------------------------------------------------------------------- +describe("judging.schedule.get", () => { + it("shows each appointment's judge, each judge's load and where judges run short", async () => { + // Only Ada and Grace have joined: two general presentations at 15:30 need two. + store.set(judgingJudges, [ + judge(M1, "member", "Ada"), + judge(GS, "guest", "Grace"), + ]); + give(P2, M1); + give(P4, GS); + + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + + expect( + board.appointments.map((a) => [ + a.id, + a.assignedJudgeName, + a.canAssignJudge, + ]), + ).toEqual([ + [P1, null, false], + [P2, "Ada", true], + [P3, null, true], + [P4, "Grace", true], + ]); + expect(board.judging).toMatchObject({ + judges: 2, + assigned: 2, + unassigned: 1, + loads: [ + { judgeId: M1, name: "Ada", kind: "member", load: 1, available: true }, + { + judgeId: GS, + name: "Grace", + kind: "guest", + roomName: "Acme 1", + load: 1, + }, + ], + warnings: [ + { + slot: 9, + challengeId: GENERAL, + challengeLabel: "General", + presentations: 2, + judgesFree: 1, + message: "Slot 10: 2 presentations in General, only 1 judge free", + }, + ], + }); + expect(board.judging!.warnings[0]!.startsAt).toEqual(at("15:30")); + }); + + it("lists a judge who holds appointments but can no longer take any", async () => { + store.set(judgingJudges, [ + judge(M1, "member", "Ada"), + judge(GS, "guest", "Grace"), + ]); + // Grace's session was revoked: she has no room, but P4 is still hers. + store.set(judgingGuestSessions, []); + give(P4, GS); + + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(board.judging!.loads).toEqual([ + expect.objectContaining({ judgeId: M1, load: 0, available: true }), + expect.objectContaining({ + judgeId: GS, + name: "Grace", + load: 1, + available: false, + }), + ]); + }); + + it("says nothing of judges before a schedule is saved", async () => { + store.set(judgingSchedules, []); + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(board.judging).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.get: where each assigned judge has got", () => { + const evaluation = ( + judgeId: string, + entryId: string, + challengeId: string, + appointmentId: string | null, + ratingsComplete = true, + ) => ({ judgeId, entryId, challengeId, appointmentId, ratingsComplete }); + const draft = ( + judgeId: string, + entryId: string, + challengeId: string, + appointmentId: string | null, + updatedAt = NOW, + ) => ({ judgeId, entryId, challengeId, appointmentId, updatedAt }); + + // The clock stays at 15:00: the rate limiter reads it, and a clock that + // runs backwards between tests starves its bucket. Presentations start + // earlier instead. + const startAt = (apptId: string, time: string) => { + store.get(judgingAppointments)!.find((a) => a.id === apptId)!.startsAt = + at(time); + }; + + const progress = async () => { + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + return Object.fromEntries( + board.appointments.map((a) => [a.id, a.judgeProgress]), + ); + }; + + it("tells graded, grading, taken over and not started apart, and leaves the rest blank", async () => { + // It is 15:00: P1 is over, P2 and P3 began long enough ago for the + // takeover time to have passed and P4 is still to come. + startAt(P2, "14:10"); + startAt(P3, "14:20"); + give(P1, M1); + give(P2, M1); + give(P3, M2); + give(P4, GS); + store.set(judgingEvaluations, [ + // Ada graded P1. + evaluation(M1, E1, GENERAL, P1), + // Bea graded P2, which was Ada's. + evaluation(M2, E2, GENERAL, P2), + ]); + + await expect(progress()).resolves.toEqual({ + [P1]: "graded", + [P2]: "taken_over", + // Nobody has begun it and Bea had 5 minutes. + [P3]: "not_started", + // Not started yet. + [P4]: null, + }); + + // A draft is a start. + store.set(judgingEvaluations, []); + store.set(judgingEvaluationDrafts, [ + draft(M2, E3, GENERAL, P3), + draft(M2, E2, GENERAL, P2), + ]); + await expect(progress()).resolves.toMatchObject({ + [P1]: "not_started", + // Ada's, with Bea's draft on it. + [P2]: "taken_over", + // Bea's own. + [P3]: "grading", + }); + }); + + it("gives the assigned judge five minutes from the start before it flags them", async () => { + give(P2, M1); + startAt(P2, "14:56"); + await expect(progress()).resolves.toMatchObject({ [P2]: null }); + startAt(P2, "14:55"); + await expect(progress()).resolves.toMatchObject({ [P2]: "not_started" }); + }); + + it("does not count a draft left without its deadline once the presentation is over", async () => { + // P1's deadline was 14:00. Ada only opened it, so her empty draft was + // kept without its deadline and will never be submitted. + give(P1, M1); + store.set(judgingEvaluationDrafts, [ + { ...draft(M1, E1, GENERAL, P1, at("13:58")), deadlineAt: null }, + ]); + await expect(progress()).resolves.toMatchObject({ [P1]: "not_started" }); + + // Still carrying it: due, and about to be submitted for her. + store.set(judgingEvaluationDrafts, [ + { ...draft(M1, E1, GENERAL, P1, at("13:58")), deadlineAt: at("14:00") }, + ]); + await expect(progress()).resolves.toMatchObject({ [P1]: "grading" }); + }); + + it("matches evaluations and drafts to the presentation the way the board does", async () => { + startAt(P2, "14:10"); + startAt(P3, "14:20"); + give(P2, M1); + give(P3, M1); + store.set(judgingEvaluations, [ + // Names P2, whatever its challenge. + evaluation(M1, E2, ACME, P2), + // Names no slot: the project and a challenge under P3's root. + evaluation(M1, E3, GENERAL, null), + ]); + // Another project's evaluation, and a slot P3 was moved from. + store.set(judgingEvaluationDrafts, [ + draft(M2, E2, GENERAL, P3), + draft(M2, E1, GENERAL, null), + ]); + + await expect(progress()).resolves.toMatchObject({ + [P2]: "graded", + [P3]: "graded", + }); + }); + + it("leaves a presentation with no judge, or whose project is gone, blank", async () => { + startAt(P2, "14:10"); + startAt(P3, "14:20"); + give(P2, M1); + store.get(judgingAppointments)!.find((a) => a.id === P2)!.entryDeletedAt = + NOW; + store.set(judgingEvaluations, [evaluation(M2, E1, GENERAL, P1)]); + + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + expect( + board.appointments.map((a) => [a.id, a.removed, a.judgeProgress]), + ).toEqual([ + // Graded by someone, but nobody was assigned. + [P1, false, null], + [P2, true, null], + [P3, false, null], + [P4, false, null], + ]); + }); + + it("reads a presentation's status from evaluations that hold every rating", async () => { + store.set(judgingEvaluations, [evaluation(M1, E1, GENERAL, P1, false)]); + const short = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(short.appointments[0]).toMatchObject({ + id: P1, + status: "incomplete", + evaluationCount: 1, + }); + + store.set(judgingEvaluations, [evaluation(M1, E1, GENERAL, P1, true)]); + const whole = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(whole.appointments[0]).toMatchObject({ id: P1, status: "complete" }); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.assignJudges", () => { + const run = () => + admin().judging.schedule.assignJudges({ hackathonId: HACK }); + + it("gives every appointment that has not started a judge who may grade it", async () => { + const result = await run(); + + expect(result).toMatchObject({ + solver: "in-app", + changed: 3, + assigned: 3, + unassigned: 0, + warnings: [], + }); + const after = judgeAfter(); + // The sponsor presentation goes to the sponsor's own judge. + expect(after.get(P4)).toBe(GS); + // The two at 15:30 are in different rooms at once: two judges. + expect(new Set([after.get(P2), after.get(P3)])).toEqual(new Set([M1, M2])); + // The one that is over is left as it is. + expect(after.has(P1)).toBe(false); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.schedule.assignJudges", + resourceId: HACK, + metadata: expect.objectContaining({ + solver: "in-app", + changed: 3, + unassigned: 0, + }), + }), + ]); + }); + + it("keeps the judge of an appointment a judge has begun, and plans around it", async () => { + give(P2, M1); + // Ada has a draft on P2: it stays hers, so P3 goes to Bea. + store.set(judgingEvaluationDrafts, [ + { entryId: E2, challengeId: GENERAL, appointmentId: null }, + ]); + + const result = await run(); + + expect(result).toMatchObject({ kept: 1, changed: 2 }); + const after = judgeAfter(); + expect(after.has(P2)).toBe(false); + expect(after.get(P3)).toBe(M2); + }); + + it("leaves appointments alone that already have the judge it would choose", async () => { + await run(); + const first = judgeAfter(); + updates.length = 0; + for (const [apptId, judgeId] of first) give(apptId, judgeId!); + + const again = await run(); + expect(again.changed).toBe(0); + expect(updates.filter((u) => u.table === judgingAppointments)).toEqual([]); + }); + + it("runs CP-SAT when the server has it and takes its assignments", async () => { + mockAvailable.mockReturnValue(true); + mockRun.mockImplementation(async (problem: Row) => ({ + status: "solved", + log: [], + solution: JSON.stringify({ + format: "query-judging-schedule-solution", + version: 1, + fingerprint: problem.fingerprint, + status: "optimal", + relaxed: false, + placements: problem.placements, + // Bea at 15:30 in G1 and Ada in G2: the in-app order is the other way. + assignments: [ + { entryId: E2, challengeId: GENERAL, judgeId: M2 }, + { entryId: E3, challengeId: GENERAL, judgeId: M1 }, + { entryId: E1, challengeId: ACME, judgeId: GS }, + ], + }), + })); + + const result = await admin().judging.schedule.assignJudges({ + hackathonId: HACK, + seconds: 30, + }); + + expect(result.solver).toBe("cp-sat"); + expect(judgeAfter().get(P2)).toBe(M2); + expect(judgeAfter().get(P3)).toBe(M1); + // It was asked for the schedule as it stands, with the judges. + const [problem, options] = mockRun.mock.calls[0]!; + expect(options).toMatchObject({ seconds: 30 }); + expect(problem.placements).toHaveLength(3); + expect(problem.judges.map((j: Row) => j.id)).toEqual([M1, M2, GS]); + }); + + it("falls back to the in-app assigner when CP-SAT fails or answers against the rules", async () => { + mockAvailable.mockReturnValue(true); + mockRun.mockResolvedValueOnce({ + status: "failed", + reason: "no python", + log: [], + }); + expect((await run()).solver).toBe("in-app"); + + // Ada in two rooms at once. + mockRun.mockImplementationOnce(async (problem: Row) => ({ + status: "solved", + log: [], + solution: JSON.stringify({ + format: "query-judging-schedule-solution", + version: 1, + fingerprint: problem.fingerprint, + status: "optimal", + relaxed: false, + placements: problem.placements, + assignments: [ + { entryId: E2, challengeId: GENERAL, judgeId: M1 }, + { entryId: E3, challengeId: GENERAL, judgeId: M1 }, + ], + }), + })); + expect((await run()).solver).toBe("in-app"); + }); + + it("warns of each slot where judges run short", async () => { + store.set(judgingJudges, [judge(M1, "member", "Ada")]); + const result = await run(); + expect(result.unassigned).toBe(1); + expect(result.warnings.map((w) => w.message)).toEqual([ + "Slot 10: 2 presentations in General, only 1 judge free", + ]); + }); + + it("refuses when no judge can be assigned yet", async () => { + store.set(judgingJudges, [judge(NOROOM, "guest", "Nobody")]); + await expect(run()).rejects.toMatchObject({ code: "PRECONDITION_FAILED" }); + }); + + it("refuses without a saved schedule", async () => { + store.set(judgingSchedules, []); + await expect(run()).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("refuses anyone but an admin", async () => { + adminRow = undefined; + await expect( + callerAs("a_hacker").judging.schedule.assignJudges({ hackathonId: HACK }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(updates).toEqual([]); + }); + + it("refuses to write when the judges changed while it worked", async () => { + // The first read of judges sees three; the read inside the transaction two. + onRead = (table, count) => + table === judgingJudges && count > 1 + ? [judge(M1, "member", "Ada"), judge(GS, "guest", "Grace")] + : undefined; + await expect(run()).rejects.toMatchObject({ code: "CONFLICT" }); + expect(updates.filter((u) => u.table === judgingAppointments)).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.setAssignedJudge", () => { + const set = (appointmentId: string, judgeId: string | null) => + admin().judging.schedule.setAssignedJudge({ + hackathonId: HACK, + appointmentId, + judgeId, + }); + + it("sets the judge of an appointment that has not started, and audits it", async () => { + const result = await set(P2, M1); + + expect(result).toEqual({ + appointmentId: P2, + fromJudgeId: null, + toJudgeId: M1, + }); + expect(judgeAfter().get(P2)).toBe(M1); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.schedule.setJudge", + resourceId: P2, + metadata: expect.objectContaining({ toJudgeId: M1 }), + }), + ]); + }); + + it("clears a judge with null", async () => { + give(P2, M1); + await set(P2, null); + expect(judgeAfter().get(P2)).toBeNull(); + }); + + it("refuses a judge who may not grade it: a guest outside their room", async () => { + await expect(set(P2, GS)).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: "Grace cannot grade General at Table G1.", + }); + // Their own room is fine. + await expect(set(P4, GS)).resolves.toMatchObject({ toJudgeId: GS }); + }); + + it("refuses a judge who is already grading something else then", async () => { + give(P3, M1); + await expect(set(P2, M1)).rejects.toMatchObject({ + code: "CONFLICT", + message: "Ada is already grading Project 3 at that time.", + }); + await expect(set(P2, M2)).resolves.toMatchObject({ toJudgeId: M2 }); + }); + + it("refuses a judge who is not one of the edition's, or a guest with no room", async () => { + await expect(set(P2, id(99))).rejects.toMatchObject({ code: "NOT_FOUND" }); + await expect(set(P2, NOROOM)).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("refuses once the presentation has started or a judge has begun on it", async () => { + await expect(set(P1, M1)).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + }); + + store.set(judgingEvaluationDrafts, [ + { entryId: E2, challengeId: GENERAL, appointmentId: P2 }, + ]); + await expect(set(P2, M1)).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + }); + expect(judgeAfter().size).toBe(0); + }); + + it("refuses an appointment that does not exist", async () => { + await expect(set(id(98), M1)).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("refuses anyone but an admin", async () => { + adminRow = undefined; + await expect( + callerAs("a_hacker").judging.schedule.setAssignedJudge({ + hackathonId: HACK, + appointmentId: P2, + judgeId: M1, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(updates).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.judgeChoices", () => { + it("lists the judges who may grade it, free ones first, with those booked marked", async () => { + give(P3, M1); + give(P4, M2); + + const result = await admin().judging.schedule.judgeChoices({ + hackathonId: HACK, + appointmentId: P2, + }); + + expect(result.appointment).toMatchObject({ + id: P2, + challengeLabel: "General", + roomName: "Table G1", + canAssignJudge: true, + }); + expect(result.choices).toEqual([ + expect.objectContaining({ + judgeId: M2, + name: "Bea", + load: 1, + booked: false, + }), + expect.objectContaining({ + judgeId: M1, + name: "Ada", + load: 1, + booked: true, + bookedWith: "Project 3", + }), + ]); + }); + + it("offers a sponsor presentation to the sponsor's judge and the members", async () => { + const result = await admin().judging.schedule.judgeChoices({ + hackathonId: HACK, + appointmentId: P4, + }); + expect(result.choices.map((c) => c.judgeId).sort()).toEqual( + [GS, M1, M2].sort(), + ); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.move", () => { + const staffed = () => + store.set( + judgingPresence, + [G1, G2, A1, A2].map((roomId) => ({ + judgeId: M1, + roomId, + joinedAt: at("14:00"), + lastSeenAt: NOW, + leftAt: null, + })), + ); + const move = (appointmentId: string, roomId: string, startsAt: Date) => + admin().judging.schedule.move({ + hackathonId: HACK, + appointmentId, + revision: 0, + roomId, + startsAt, + }); + const written = () => updates.filter((u) => u.table === judgingAppointments); + + it("clears a guest's assignment when the presentation moves to another room", async () => { + staffed(); + give(P4, GS); + + const moved = await move(P4, A2, at("16:00")); + + expect(moved.clearedJudgeId).toBe(GS); + expect(written()[0]!.values).toMatchObject({ + roomId: A2, + assignedJudgeId: null, + }); + }); + + it("clears a member's assignment when they would be in two places at the new time", async () => { + staffed(); + give(P2, M1); + give(P4, M1); + + // P4 is at 16:00; moving P2 there puts Ada in two rooms at once. + const moved = await move(P2, G2, at("16:00")); + + expect(moved.clearedJudgeId).toBe(M1); + expect(written()[0]!.values).toMatchObject({ assignedJudgeId: null }); + }); + + it("keeps the judge when they can still grade it at the new time", async () => { + staffed(); + give(P2, M1); + + const moved = await move(P2, G2, at("15:40")); + + expect(moved.clearedJudgeId).toBeNull(); + expect("assignedJudgeId" in (written()[0]!.values as Row)).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +describe("saving a schedule", () => { + const timing = { + windowStart: at("16:00"), + windowEnd: at("19:00"), + setupMinutes: 2, + judgingMinutes: 6, + teardownMinutes: 2, + sameBuildingBreakMinutes: 10, + crossBuildingBreakMinutes: 20, + }; + const rowsWritten = () => + inserts + .filter((write) => write.table === judgingAppointments) + .flatMap((write) => write.values as Row[]); + + beforeEach(() => { + // Nothing saved yet. + store.set(judgingSchedules, []); + store.set(judgingAppointments, []); + }); + + it("gives every presentation a judge as far as there are judges", async () => { + const preview = await admin().judging.schedule.preview({ + hackathonId: HACK, + timing, + }); + if (!preview.ok) throw new Error("the inventory fits the window"); + + const saved = await admin().judging.schedule.save({ + hackathonId: HACK, + timing, + candidateKey: preview.candidateKey, + }); + + expect(saved).toMatchObject({ appointments: 4, judged: 4 }); + const rows = rowsWritten(); + expect(rows).toHaveLength(4); + // Grace judges Acme's presentation; the three general ones are Ada's and Bea's. + const judgeOf = (challengeId: string) => + rows + .filter((r) => r.challengeId === challengeId) + .map((r) => r.assignedJudgeId); + expect(judgeOf(ACME)).toEqual([GS]); + expect(judgeOf(GENERAL).every((j) => j === M1 || j === M2)).toBe(true); + expect(audits()[0]).toMatchObject({ + action: "judging.schedule.save", + metadata: expect.objectContaining({ judged: 4 }), + }); + }); + + it("still saves when no judge has joined: the presentations have none yet", async () => { + store.set(judgingJudges, []); + const preview = await admin().judging.schedule.preview({ + hackathonId: HACK, + timing, + }); + if (!preview.ok) throw new Error("the inventory fits the window"); + + const saved = await admin().judging.schedule.save({ + hackathonId: HACK, + timing, + candidateKey: preview.candidateKey, + }); + + expect(saved).toMatchObject({ appointments: 4, judged: 0 }); + expect(rowsWritten().every((r) => r.assignedJudgeId === null)).toBe(true); + }); + + it("puts the judges in the CP-SAT problem, and saves the assignments CP-SAT chose", async () => { + const problem = await admin().judging.schedule.cpsatProblem({ + hackathonId: HACK, + timing, + }); + const file = JSON.parse(problem.contents); + expect(problem.judges).toBe(3); + expect(file.judges.map((j: Row) => j.id)).toEqual([M1, M2, GS]); + expect(file.judges.find((j: Row) => j.id === GS)).toMatchObject({ + kind: "guest", + roomId: A1, + challengeIds: [ACME], + }); + + // The in-app schedule with its judges, the two general judges swapped. + const preview = await admin().judging.schedule.preview({ + hackathonId: HACK, + timing, + }); + if (!preview.ok) throw new Error("the inventory fits the window"); + const placements = preview.appointments.map((a) => ({ + entryId: a.entryId, + challengeId: a.challengeId, + roomId: a.roomId, + slot: (a.startsAt.getTime() - timing.windowStart.getTime()) / 600_000, + })); + const inputs = { + rooms: file.rooms, + tasks: file.tasks, + slots: file.slotCount, + sameGap: file.sameGap, + crossGap: file.crossGap, + }; + const inApp = assignJudges(inputs, placements, file.judges).assignment; + const swapped = inApp.map((a) => ({ + ...a, + judgeId: a.judgeId === M1 ? M2 : a.judgeId === M2 ? M1 : a.judgeId, + })); + const solution = JSON.stringify({ + format: "query-judging-schedule-solution", + version: 1, + fingerprint: file.fingerprint, + status: "feasible", + relaxed: false, + placements, + assignments: swapped, + }); + + const checked = await admin().judging.schedule.previewCpsat({ + hackathonId: HACK, + timing, + solution, + }); + expect(checked.judging).toEqual({ + judges: 3, + presentations: 4, + unassigned: 0, + source: "cp-sat", + }); + + await admin().judging.schedule.saveCpsat({ + hackathonId: HACK, + timing, + solution, + candidateKey: checked.candidateKey, + }); + const chosen = new Map( + swapped.map((a) => [`${a.entryId}:${a.challengeId}`, a.judgeId]), + ); + for (const row of rowsWritten()) { + expect(row.assignedJudgeId).toBe( + chosen.get(`${row.entryId}:${row.challengeId}`), + ); + } + expect(audits()[0]).toMatchObject({ + metadata: expect.objectContaining({ solver: "cp-sat", judges: "cp-sat" }), + }); + }); +}); diff --git a/packages/api/src/middleware/cache.ts b/packages/api/src/middleware/cache.ts index 295926a7..8e9326ce 100644 --- a/packages/api/src/middleware/cache.ts +++ b/packages/api/src/middleware/cache.ts @@ -228,6 +228,10 @@ export const CacheKeys = { judge: (userId: string) => `judge:${userId}`, projectLeader: (userId: string) => `project-leader:${userId}`, portalContext: (userId: string) => `user:${userId}:portal`, + // judging.results.list's per-judge calibration; the caller adds what it was + // computed from, so a changed evaluation reads a new key. + judgingCalibration: (hackathonId: string) => + `judging:${hackathonId}:calibration`, } as const; export const invalidatePortalContext = (userId: string) => { diff --git a/packages/api/src/middleware/procedures.ts b/packages/api/src/middleware/procedures.ts index 67392057..db6ce2f7 100644 --- a/packages/api/src/middleware/procedures.ts +++ b/packages/api/src/middleware/procedures.ts @@ -195,7 +195,7 @@ export const hackathonBanCacheKey = (email: string) => // Refuses people banned from hackathons. Goes on the procedures where someone // takes part — registering, the interest list, teams, submitting, applying to -// judge — and not on reads or on the ways out (withdraw, leave), so a banned +// judge, event-code check-in — and not on reads or on the ways out (withdraw, leave), so a banned // person can still see and undo what they already did. The ban is by email, // so it holds across providers and new accounts on the same address. Cached // 60s, the negative answer too; ban and unban clear the key. diff --git a/packages/api/src/routers/hackathon/admin.ts b/packages/api/src/routers/hackathon/admin.ts index a8b951d7..4737c9f8 100644 --- a/packages/api/src/routers/hackathon/admin.ts +++ b/packages/api/src/routers/hackathon/admin.ts @@ -14,6 +14,7 @@ import { recordEventAttendance, searchParticipants, } from "../../services/points"; +import { flaggingRuleBreaks } from "../../services/check-in-flags"; import { CacheKeys } from "../../middleware/cache"; import { MASS_EMAIL_BATCH } from "../../services/email-limits"; import { @@ -272,14 +273,16 @@ async function checkInParticipant( // 4. Record attendance. The unique index is the real guard, and // recordEventAttendance turns a racing scanner's 23505 into the same - // conflict as the read above. + // conflict as the read above. It also refuses an event that overlaps, or + // comes too soon after, one this person is already in; that refusal ends the + // scan before the promotion below. // Attendance and the roster promotion are one transaction. Split, a promotion // failing after the attendance row committed left the scan half-done — and // the retry hits the duplicate guard, so that attendee could never submit. // Rolling the attendance back makes a rescan the fix. let promotedToCheckedIn = false; - await ctx.db.transaction(async (tx) => { + const attendAndPromote = () => ctx.db.transaction(async (tx) => { await recordEventAttendance(tx, attendance); // The first scan is the check-in. `checked_in` previously had no writer @@ -306,6 +309,10 @@ async function checkInParticipant( promotedToCheckedIn = !!promoted; }); + // A refusal for the check-in rules rolls that transaction back, so the flag + // for organisers is written here, on the pool, once it has. + await flaggingRuleBreaks(ctx.db, input.hackathonId, attendAndPromote); + // A scan changes one event's attendee count and, on the first one, this // attendee's roster row. This runs at every door all weekend, so eviction // stays narrow — the old blanket pattern wiped every cached registration. @@ -1313,7 +1320,14 @@ export const hackathonAdminRouter = createTRPCRouter({ eq(hackathonEvents.id, input.eventId), eq(hackathonEvents.hackathonId, input.hackathonId), ), - columns: { id: true, name: true, points: true, endTime: true }, + columns: { + id: true, + name: true, + type: true, + points: true, + startTime: true, + endTime: true, + }, }), ]); @@ -1349,12 +1363,17 @@ export const hackathonAdminRouter = createTRPCRouter({ }); } - const { pointsAwarded } = await recordEventAttendance(db, { - event, - participant: { id: participant.id, name }, - method: "manual", - actorId: ctx.userId, - }); + const { pointsAwarded } = await flaggingRuleBreaks( + db, + input.hackathonId, + () => + recordEventAttendance(db, { + event, + participant: { id: participant.id, name }, + method: "manual", + actorId: ctx.userId, + }), + ); ctx.cache.delete(`hackathon:${input.hackathonId}:events`); diff --git a/packages/api/src/routers/hackathon/announce.ts b/packages/api/src/routers/hackathon/announce.ts index a837e3bf..a37feafa 100644 --- a/packages/api/src/routers/hackathon/announce.ts +++ b/packages/api/src/routers/hackathon/announce.ts @@ -171,20 +171,28 @@ export const hackathonAnnounceRouter = createTRPCRouter({ // Announcements for this edition and how far each got, so a reopened tab can // pick an unfinished send back up instead of starting a duplicate. + // + // `sentAt` alone does not mean "emailed": an in-app post is born with it + // stamped. Only sendBatch sets `claimedAt`, so `sent` counts rows with both, + // and `inAppOnly` is an announcement whose every row is stamped but never + // claimed. `pending` is counted directly, not total - sent - failed, which + // would leave in-app rows looking like an unfinished send. listAnnouncements: isAdmin .input(z.object({ hackathonId: z.string().uuid() })) .query(async ({ ctx, input }) => { const db = ctx.db as DrizzleDB; - const rows = await db + return await db .select({ id: hackathonAnnouncements.id, subject: hackathonAnnouncements.subject, audience: hackathonAnnouncements.audience, createdAt: hackathonAnnouncements.createdAt, total: sql`count(${hackathonAnnouncementRecipients.id})::int`, - sent: sql`count(${hackathonAnnouncementRecipients.sentAt})::int`, + sent: sql`count(${hackathonAnnouncementRecipients.id}) filter (where ${hackathonAnnouncementRecipients.sentAt} is not null and ${hackathonAnnouncementRecipients.claimedAt} is not null)::int`, failed: sql`count(${hackathonAnnouncementRecipients.failedAt})::int`, + pending: sql`count(${hackathonAnnouncementRecipients.id}) filter (where ${hackathonAnnouncementRecipients.sentAt} is null and ${hackathonAnnouncementRecipients.failedAt} is null)::int`, + inAppOnly: sql`count(${hackathonAnnouncementRecipients.id}) > 0 and count(${hackathonAnnouncementRecipients.id}) filter (where ${hackathonAnnouncementRecipients.sentAt} is not null and ${hackathonAnnouncementRecipients.claimedAt} is null) = count(${hackathonAnnouncementRecipients.id})`, }) .from(hackathonAnnouncements) .leftJoin( @@ -198,11 +206,6 @@ export const hackathonAnnounceRouter = createTRPCRouter({ .groupBy(hackathonAnnouncements.id) .orderBy(desc(hackathonAnnouncements.createdAt)) .limit(50); - - return rows.map((row) => ({ - ...row, - pending: row.total - row.sent - row.failed, - })); }), // Freezes a message and its audience. Sends nothing. The recipient rows are diff --git a/packages/api/src/routers/hackathon/content.ts b/packages/api/src/routers/hackathon/content.ts index 6511a785..e5cee72c 100644 --- a/packages/api/src/routers/hackathon/content.ts +++ b/packages/api/src/routers/hackathon/content.ts @@ -223,6 +223,7 @@ export const hackathonContentRouter = createTRPCRouter({ submissionUrl: true, demoLinks: true, deletedAt: true, + prizeIneligibleAt: true, }, with: { sourceProject: { @@ -251,9 +252,14 @@ export const hackathonContentRouter = createTRPCRouter({ score: toScore(row.weightedScore), })), // A project an organiser pulled after placings went out leaves the - // public list at once, not at the next recompute. + // public list at once, not at the next recompute. So does a team + // pulled from prizes. Only the timestamp is read, to filter on; why it + // was pulled never reaches this cached, public payload. ...placements - .filter((row) => row.entry && !row.entry.deletedAt) + .filter( + (row) => + row.entry && !row.entry.deletedAt && !row.entry.prizeIneligibleAt, + ) .map((row) => ({ id: row.id, scope: row.scope, diff --git a/packages/api/src/routers/judging/admin.ts b/packages/api/src/routers/judging/admin.ts index 241d4637..408911f0 100644 --- a/packages/api/src/routers/judging/admin.ts +++ b/packages/api/src/routers/judging/admin.ts @@ -17,6 +17,7 @@ import { sql, } from "drizzle-orm"; import { + hackathonCheckInFlags, hackathonProjects, judgingAnnouncements, judgingAppointments, @@ -29,6 +30,7 @@ import { judgingEvaluationDrafts, judgingEvaluations, judgingGuestSessions, + judgingPlacements, judgingPresence, judgingRoomLinks, judgingRooms, @@ -45,6 +47,8 @@ import { ensureJudgingConfig, } from "../../services/judging/config"; import type { JudgingDb } from "../../services/judging/config"; +import { lockGrading } from "../../services/judging/evaluation-store"; +import { outerColumn } from "../../services/judging/outer-column"; import { effectiveMode, sameLabel } from "../../services/judging/scope"; import { assertStructureEditable, @@ -90,6 +94,13 @@ import { stateChangeProblem, } from "../../services/judging/checklist"; import { notifyItineraryPosted } from "../../services/judging/notify"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; +import { + checkInFlagsForEntries, + flaggedEntryIds, + openCheckInFlagCount, +} from "../../services/check-in-flags"; +import type { CheckInFlagView } from "../../services/check-in-flags"; // Organiser setup and live control of room judging: getting projects in, // shaping challenges, rooms and the rubric, the switches flipped on judging @@ -151,6 +162,25 @@ const evictResults = ( hackathonId: string, ) => cache.delete(`hackathon:${hackathonId}:results`); +/** Live projects of the edition that carry a check-in flag nobody dismissed. */ +async function entriesWithOpenFlags(db: JudgingDb, hackathonId: string) { + const live = await db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .where( + and( + eq(judgingEntries.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + ), + ); + const flagged = await flaggedEntryIds( + db, + hackathonId, + live.map((row) => row.id), + ); + return [...flagged]; +} + // Each of these finds the row's edition, takes the edition lock, then reads // the row again so the checks see it as it stands under the lock. @@ -285,6 +315,7 @@ export const judgingAdminRouter = createTRPCRouter({ [drafts], [announcements], [seated], + openCheckInFlags, ] = await Promise.all([ db .select({ @@ -322,7 +353,7 @@ export const judgingAdminRouter = createTRPCRouter({ active: sql`count(*) filter (where ${judgingRooms.archivedAt} is null)::int`, tables: sql`count(*) filter (where ${judgingRooms.archivedAt} is null and ${judgingRooms.kind} = 'table')::int`, archived: sql`count(*) filter (where ${judgingRooms.archivedAt} is not null)::int`, - linked: sql`count(*) filter (where ${judgingRooms.archivedAt} is null and exists (select 1 from ${judgingRoomLinks} where ${judgingRoomLinks.roomId} = ${judgingRooms.id} and ${judgingRoomLinks.revokedAt} is null))::int`, + linked: sql`count(*) filter (where ${judgingRooms.archivedAt} is null and exists (select 1 from ${judgingRoomLinks} where ${judgingRoomLinks.roomId} = ${outerColumn(judgingRooms, judgingRooms.id)} and ${judgingRoomLinks.revokedAt} is null))::int`, }) .from(judgingRooms) .where(eq(judgingRooms.hackathonId, id)), @@ -340,7 +371,7 @@ export const judgingAdminRouter = createTRPCRouter({ windowEnd: judgingSchedules.windowEnd, firstResultAt: judgingSchedules.firstResultAt, createdAt: judgingSchedules.createdAt, - appointments: sql`(select count(*)::int from ${judgingAppointments} where ${judgingAppointments.scheduleId} = ${judgingSchedules.id})`, + appointments: sql`(select count(*)::int from ${judgingAppointments} where ${judgingAppointments.scheduleId} = ${outerColumn(judgingSchedules, judgingSchedules.id)})`, }) .from(judgingSchedules) .where(eq(judgingSchedules.hackathonId, id)) @@ -348,7 +379,8 @@ export const judgingAdminRouter = createTRPCRouter({ db .select({ total: count(), - complete: sql`count(*) filter (where ${judgingEvaluations.isComplete})::int`, + // The ones placings count: every rating in, written response or not. + complete: sql`count(*) filter (where ${holdsEveryRating(id)})::int`, }) .from(judgingEvaluations) .where(eq(judgingEvaluations.hackathonId, id)), @@ -375,6 +407,7 @@ export const judgingAdminRouter = createTRPCRouter({ gte(judgingPresence.lastSeenAt, seatedSince), ), ), + openCheckInFlagCount(db, id), ]); const hasEvaluations = (evaluations?.total ?? 0) + (drafts?.total ?? 0) > 0; @@ -422,6 +455,9 @@ export const judgingAdminRouter = createTRPCRouter({ }, announcements: announcements?.total ?? 0, judgesSeated: seated?.total ?? 0, + // Only how many wait for a decision. Who and why stay in the project + // list, which is organiser-only. + checkInFlags: { open: openCheckInFlags }, }, schedule: schedule ?? null, locks: { @@ -541,6 +577,8 @@ export const judgingAdminRouter = createTRPCRouter({ source: z.enum(["all", "devpost", "portal"]).default("all"), status: z.enum(["live", "removed", "all"]).default("live"), challengeId: uuid.optional(), + // Only projects with a check-in flag nobody has dismissed. + flagged: z.boolean().optional(), page: z.number().int().min(1).default(1), pageSize: z.number().int().min(1).max(200).default(50), }), @@ -549,6 +587,12 @@ export const judgingAdminRouter = createTRPCRouter({ const db = ctx.db as DrizzleDB; const pattern = input.search ? `%${escapeLike(input.search)}%` : null; + // Settled before the query so the filter narrows the page and the total + // together instead of thinning out a page afterwards. + const flaggedIds = input.flagged + ? await entriesWithOpenFlags(db, input.hackathonId) + : null; + const where = and( eq(judgingEntries.hackathonId, input.hackathonId), input.source === "all" @@ -559,6 +603,11 @@ export const judgingAdminRouter = createTRPCRouter({ : input.status === "removed" ? isNotNull(judgingEntries.deletedAt) : undefined, + flaggedIds + ? flaggedIds.length > 0 + ? inArray(judgingEntries.id, flaggedIds) + : sql`false` + : undefined, input.challengeId ? exists( db @@ -611,6 +660,7 @@ export const judgingAdminRouter = createTRPCRouter({ participantCount: judgingEntries.participantCount, submittedAt: judgingEntries.submittedAt, deletedAt: judgingEntries.deletedAt, + prizeIneligible: sql`${judgingEntries.prizeIneligibleAt} is not null`, createdAt: judgingEntries.createdAt, }) .from(judgingEntries) @@ -621,9 +671,9 @@ export const judgingAdminRouter = createTRPCRouter({ ]); const ids = rows.map((row) => row.id); - const [members, memberships, appointments, evaluations] = + const [members, memberships, appointments, evaluations, flags] = ids.length === 0 - ? [[], [], [], []] + ? [[], [], [], [], new Map()] : await Promise.all([ db .select({ @@ -671,10 +721,13 @@ export const judgingAdminRouter = createTRPCRouter({ .where( and( inArray(judgingEvaluations.entryId, ids), - eq(judgingEvaluations.isComplete, true), + holdsEveryRating(input.hackathonId), ), ) .groupBy(judgingEvaluations.entryId), + // Names and events of other people: organiser-only, which this + // procedure is, and never part of a cached payload. + checkInFlagsForEntries(db, input.hackathonId, ids), ]); const countFor = (list: { entryId: string; value: number }[], id: string) => @@ -694,6 +747,7 @@ export const judgingAdminRouter = createTRPCRouter({ .map(({ entryId: _entryId, ...membership }) => membership), appointments: countFor(appointments, row.id), completeEvaluations: countFor(evaluations, row.id), + checkInFlags: flags.get(row.id) ?? [], })), }; }), @@ -935,6 +989,99 @@ export const judgingAdminRouter = createTRPCRouter({ return { restored: true }; }), + /** + * Clears a check-in flag an organiser has looked into, or opens it again. + * The flag stays on record either way; judges stop seeing the marker while it + * is dismissed. The edition is part of the match so a flag id from another + * edition cannot be touched from this one. + */ + dismissCheckInFlag: isAdmin + .input(hackathonInput.extend({ flagId: uuid, dismissed: z.boolean() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const [flag] = await db + .update(hackathonCheckInFlags) + .set( + input.dismissed + ? { dismissedAt: new Date(), dismissedById: ctx.userId } + : { dismissedAt: null, dismissedById: null }, + ) + .where( + and( + eq(hackathonCheckInFlags.id, input.flagId), + eq(hackathonCheckInFlags.hackathonId, input.hackathonId), + ), + ) + .returning({ id: hackathonCheckInFlags.id }); + if (!flag) throw notFound("That flag is gone. Refresh the list."); + + await recordAdminAction(db, { + userId: ctx.userId, + action: input.dismissed + ? "judging.dismissCheckInFlag" + : "judging.reopenCheckInFlag", + resourceId: flag.id, + metadata: { hackathonId: input.hackathonId }, + }); + return { dismissed: input.dismissed }; + }), + + /** + * Pulls a project from prizes, or puts it back. Judges still score it and + * the room schedule is untouched; only placements and public results leave + * it out, so the cached results go at once. + */ + setPrizeEligibility: isAdmin + .input(hackathonInput.extend({ entryId: uuid, eligible: z.boolean() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const changed = await db.transaction(async (tx) => { + await lockEdition(tx, input.hackathonId); + const [entry] = await tx + .select({ prizeIneligibleAt: judgingEntries.prizeIneligibleAt }) + .from(judgingEntries) + .where( + and( + eq(judgingEntries.id, input.entryId), + eq(judgingEntries.hackathonId, input.hackathonId), + ), + ) + .limit(1); + if (!entry) throw notFound("That project is gone. Refresh the list."); + if ((entry.prizeIneligibleAt === null) === input.eligible) return false; + + const now = new Date(); + await tx + .update(judgingEntries) + .set( + input.eligible + ? { + prizeIneligibleAt: null, + prizeIneligibleById: null, + updatedAt: now, + } + : { + prizeIneligibleAt: now, + prizeIneligibleById: ctx.userId, + updatedAt: now, + }, + ) + .where(eq(judgingEntries.id, input.entryId)); + return true; + }); + if (!changed) return { changed: false }; + + evictResults(ctx.cache, input.hackathonId); + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.setPrizeEligibility", + resourceId: input.entryId, + severity: "warn", + metadata: { hackathonId: input.hackathonId, eligible: input.eligible }, + }); + return { changed: true }; + }), + // --------------------------------------------------------------------- // Challenges @@ -2395,6 +2542,131 @@ export const judgingAdminRouter = createTRPCRouter({ return { cleared: !!cleared }; }), + /** + * Deletes one evaluation and any draft for the same project and challenge, so + * the project is open for another judge to grade (grading-hold.ts). The + * ratings, responses and revisions go with the evaluation by cascade. Allowed + * while judging is closed, as setPrizeEligibility is: closing freezes judges' + * writes, not the organiser's decisions on a project. No judge can grade the + * project until judging is open again. + * + * The draft placings were ranked with the grade in them, in its challenge + * and overall, and a judge's calibration moved with it, so every draft + * placing is deleted in the same transaction. Publishing needs a compute + * after the project is graded again; until then there is nothing to publish. + * + * Refused while results are published: the public list was ranked on it. + */ + reopenGrade: isAdmin + .input(hackathonInput.extend({ evaluationId: uuid })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const reopened = await db.transaction(async (tx) => { + // Takes the config lock, which holds off every judge write and a + // publish while the grade is deleted. + await lockEdition(tx, input.hackathonId); + + const [published] = await tx + .select({ value: count() }) + .from(judgingPlacements) + .where( + and( + eq(judgingPlacements.hackathonId, input.hackathonId), + isNotNull(judgingPlacements.publishedAt), + ), + ); + if ((published?.value ?? 0) > 0) { + throw refuse( + "Results are published. Unpublish them first, then reopen the grade and compute results again.", + ); + } + + const [evaluation] = await tx + .select({ + entryId: judgingEvaluations.entryId, + challengeId: judgingEvaluations.challengeId, + judgeId: judgingEvaluations.judgeId, + revision: judgingEvaluations.revision, + isComplete: judgingEvaluations.isComplete, + }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.id, input.evaluationId), + eq(judgingEvaluations.hackathonId, input.hackathonId), + ), + ) + .limit(1); + if (!evaluation) { + throw notFound("That evaluation is gone. Refresh the list."); + } + + // The drafts first, then the project's grading lock: the order the + // deadline submission takes them in, so the two cannot deadlock. + const drafts = await tx + .select({ id: judgingEvaluationDrafts.id }) + .from(judgingEvaluationDrafts) + .where( + and( + eq(judgingEvaluationDrafts.entryId, evaluation.entryId), + eq(judgingEvaluationDrafts.challengeId, evaluation.challengeId), + ), + ) + .for("update"); + await lockGrading(tx, evaluation.entryId, evaluation.challengeId); + + if (drafts.length > 0) { + await tx.delete(judgingEvaluationDrafts).where( + inArray( + judgingEvaluationDrafts.id, + drafts.map((draft) => draft.id), + ), + ); + } + await tx + .delete(judgingEvaluations) + .where(eq(judgingEvaluations.id, input.evaluationId)); + const placings = await tx + .delete(judgingPlacements) + .where( + and( + eq(judgingPlacements.hackathonId, input.hackathonId), + isNull(judgingPlacements.publishedAt), + ), + ) + .returning({ id: judgingPlacements.id }); + + return { + ...evaluation, + drafts: drafts.length, + placings: placings.length, + }; + }); + + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.reopenGrade", + resourceId: input.evaluationId, + severity: "critical", + metadata: { + hackathonId: input.hackathonId, + entryId: reopened.entryId, + challengeId: reopened.challengeId, + judgeId: reopened.judgeId, + revision: reopened.revision, + wasComplete: reopened.isComplete, + draftsDeleted: reopened.drafts, + placingsDeleted: reopened.placings, + }, + }); + return { + reopened: true, + drafts: reopened.drafts, + /** Draft placings cleared with it: compute again once it is regraded. */ + placings: reopened.placings, + }; + }), + // --------------------------------------------------------------------- // Resets. Each takes the hackathon's name typed out. diff --git a/packages/api/src/routers/judging/desk.ts b/packages/api/src/routers/judging/desk.ts index 849147db..dc0edf02 100644 --- a/packages/api/src/routers/judging/desk.ts +++ b/packages/api/src/routers/judging/desk.ts @@ -1,6 +1,16 @@ import { z } from "zod"; import { TRPCError } from "@trpc/server"; -import { and, asc, desc, eq, ilike, inArray, isNull, or } from "drizzle-orm"; +import { + and, + asc, + desc, + eq, + gt, + ilike, + inArray, + isNull, + or, +} from "drizzle-orm"; import { hackathons, judgeAssignments, @@ -24,6 +34,7 @@ import { } from "@query/db"; import type { DrizzleDB } from "@query/db"; import type { JudgingTx as Tx } from "../../services/judging/config"; +import { flaggedEntryIds } from "../../services/check-in-flags"; import { createTRPCRouter, publicProcedure } from "../../trpc"; import { isUniqueViolation } from "../../middleware/db-errors"; import { @@ -36,6 +47,7 @@ import type { Judge } from "../../services/judging/principal"; import { assertGuestSeated, assertJudgingOpen, + claimGrading, evaluatedChallenge, evaluationAccess, guestScope, @@ -43,20 +55,43 @@ import { loadChallenges, loadEntryInChallenge, loadRubric, + memberScope, reconcileDueDrafts, seatOf, writeEvaluation, } from "../../services/judging/evaluation-store"; +import { + DEADLINE_GRACE_MS, + reservedUntil as reservedUntilOf, +} from "../../services/judging/timing"; import type { ChallengeRow } from "../../services/judging/evaluation-store"; +import { + gradingRefusal, + gradingStateOf, + gradingStates, +} from "../../services/judging/grading-hold"; +import type { GradingState } from "../../services/judging/grading-hold"; +import { + QUEUE_WINDOW, + nextToGrade, + queueOrder, + queueWindow, +} from "../../services/judging/queue"; +import { + assertQueueOffers, + loadQueue, +} from "../../services/judging/queue-store"; import { checkAnswers, responseVisibility, } from "../../services/judging/evaluation"; import { evaluationScore, + hasEveryRating, overallScore, scopedScore, } from "../../services/judging/scoring"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; import { effectiveMode, rootOf, sameLabel } from "../../services/judging/scope"; import { groupSections, @@ -270,6 +305,20 @@ async function openRoomId(db: DrizzleDB, judgeId: string) { return seat?.roomId ?? null; } +/** + * A member's seat, for the rule that a sponsor's challenge is graded from its + * room. A guest is locked to their room's challenge and needs no seat. + */ +function memberSeat( + db: DrizzleDB, + judge: Judge, + challenges: readonly ChallengeRow[], +) { + return judge.kind === "member" + ? seatOf(db, judge.judgeId, challenges) + : Promise.resolve(null); +} + /** The track a member judge asked for when they applied, or null. */ async function preferredTrackOf( db: DrizzleDB, @@ -502,6 +551,7 @@ const deliberationRouter = createTRPCRouter({ ), // Only a project the judge has finished: a shortlist ranks what they saw. + // Finished is what placings count: every rating in, written response or not. add: judgeProcedure .input( z.object({ @@ -528,7 +578,7 @@ const deliberationRouter = createTRPCRouter({ and( eq(judgingEvaluations.judgeId, ctx.judge.judgeId), eq(judgingEvaluations.entryId, input.entryId), - eq(judgingEvaluations.isComplete, true), + holdsEveryRating(input.hackathonId), isNull(judgingEntries.deletedAt), ), ) @@ -693,10 +743,12 @@ export const judgingDeskRouter = createTRPCRouter({ const seat = judge ? await seatOf(db, judge.judgeId, challenges) : null; const roomId = principal.kind === "guest" ? principal.roomId : (seat?.roomId ?? null); + // A guest sees their room's challenge; a table judge sees General's and, + // from a seat in its room, a sponsor's. const visible = principal.kind === "guest" ? guestScope(challenges, principal.roomChallengeId) - : null; + : memberScope(challenges, seat); return { kind: principal.kind, @@ -709,7 +761,7 @@ export const judgingDeskRouter = createTRPCRouter({ showAllScores: principal.kind === "member" && switches.showAllScores, room: roomId ? await roomSummary(db, roomId, challenges) : null, challenges: challenges - .filter((c) => !visible || visible.has(c.id)) + .filter((c) => visible.has(c.id)) .map((c) => ({ id: c.id, label: c.label, @@ -918,6 +970,10 @@ export const judgingDeskRouter = createTRPCRouter({ // The projects to judge for one challenge. By default: in presentation // order, only those presenting in the judge's room, not yet judged by them. + // Each says whether it is still open to grade: a project is graded once, so + // one another judge graded or holds is listed after the open ones, as is + // one still reserved for the judge assigned to it, with when it opens. + // Presentations assigned to this judge come first, marked as theirs. projects: judgeProcedure .input( z.object({ @@ -1009,6 +1065,7 @@ export const judgingDeskRouter = createTRPCRouter({ startsAt: judgingAppointments.startsAt, deadlineAt: judgingAppointments.deadlineAt, endsAt: judgingAppointments.endsAt, + assignedJudgeId: judgingAppointments.assignedJudgeId, }) .from(judgingAppointments) .innerJoin( @@ -1027,35 +1084,75 @@ export const judgingDeskRouter = createTRPCRouter({ ), ) : []; - const mine = await db + // Every judge's, to tell which projects someone has: a judge is only + // ever shown their own. + const evaluations = await db .select({ entryId: judgingEvaluations.entryId, + judgeId: judgingEvaluations.judgeId, revision: judgingEvaluations.revision, - isComplete: judgingEvaluations.isComplete, + // Every rating in, as placings count it. + counted: holdsEveryRating(input.hackathonId), }) .from(judgingEvaluations) .where( and( - eq(judgingEvaluations.judgeId, judge.judgeId), eq(judgingEvaluations.challengeId, challenge.id), inArray(judgingEvaluations.entryId, entryIds), ), ); const drafts = await db - .select({ entryId: judgingEvaluationDrafts.entryId }) + .select({ + entryId: judgingEvaluationDrafts.entryId, + judgeId: judgingEvaluationDrafts.judgeId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + revision: judgingEvaluationDrafts.revision, + }) .from(judgingEvaluationDrafts) .where( and( - eq(judgingEvaluationDrafts.judgeId, judge.judgeId), eq(judgingEvaluationDrafts.challengeId, challenge.id), inArray(judgingEvaluationDrafts.entryId, entryIds), ), ); const tags = await tagsOf(db, judge, challenges, entryIds); + // One yes or no per project and nothing else. Who was flagged and why + // is for organisers; this list is polled by every judge, guests included. + const flagged = await flaggedEntryIds(db, input.hackathonId, entryIds); const presentationOf = new Map(presentations.map((p) => [p.entryId, p])); - const evaluationOf = new Map(mine.map((e) => [e.entryId, e])); - const drafted = new Set(drafts.map((d) => d.entryId)); + const evaluationOf = new Map( + evaluations + .filter((e) => e.judgeId === judge.judgeId) + .map((e) => [e.entryId, e]), + ); + // A draft the judge has saved to. Opening a project makes an empty one + // (startGrading), and that is not a draft saved. + const drafted = new Set( + drafts + .filter((d) => d.judgeId === judge.judgeId && d.revision > 0) + .map((d) => d.entryId), + ); + // A scheduled challenge's presentation assigned to another judge is kept + // for them until its takeover time. Never says which judge. + const now = new Date(); + const timed = !!schedule && mode === "scheduled"; + const reserved = new Map(); + if (timed) { + for (const presentation of presentations) { + const until = reservedUntilOf(presentation, judge.judgeId, now); + if (until) reserved.set(presentation.entryId, until); + } + } + const grading = gradingStates( + judge.judgeId, + entryIds, + evaluations, + drafts, + now, + reserved, + ); const inView = entries.filter( (entry) => @@ -1064,16 +1161,15 @@ export const judgingDeskRouter = createTRPCRouter({ presentationOf.get(entry.id)?.roomId !== seat?.roomId ), ); - // An incomplete evaluation (time ran out on it) still has work to do, so - // only finished ones are hidden. + // Judged means the evaluation counts, as placings count it: every rating + // in, written response or not. One missing a rating is not hidden. const judgedCount = inView.filter( - (entry) => evaluationOf.get(entry.id)?.isComplete, + (entry) => evaluationOf.get(entry.id)?.counted, ).length; const projects = inView .filter( - (entry) => - !(input.hideJudged && evaluationOf.get(entry.id)?.isComplete), + (entry) => !(input.hideJudged && evaluationOf.get(entry.id)?.counted), ) .map((entry) => { const presentation = presentationOf.get(entry.id); @@ -1081,6 +1177,7 @@ export const judgingDeskRouter = createTRPCRouter({ return { ...entry, challenges: tags.get(entry.id) ?? [], + flagged: flagged.has(entry.id), presentation: presentation ? { id: presentation.id, @@ -1095,17 +1192,58 @@ export const judgingDeskRouter = createTRPCRouter({ myEvaluation: evaluation ? { revision: evaluation.revision, - isComplete: evaluation.isComplete, + counted: evaluation.counted, } : null, hasDraft: drafted.has(entry.id), + grading: grading.get(entry.id) ?? ("open" as const), + // When a reserved project opens to this judge. + reservedUntil: + grading.get(entry.id) === "reserved" + ? (reserved.get(entry.id) ?? null) + : null, + assignedToYou: + timed && presentation?.assignedJudgeId === judge.judgeId, }; }); + // What the judge can still do comes first, then projects another judge + // is grading or is assigned, then those another judge graded. + const standing = (state: GradingState) => + state === "graded" + ? 2 + : state === "in_progress" || state === "reserved" + ? 1 + : 0; const byTitle = (a: { title: string }, b: { title: string }) => a.title.localeCompare(b.title); + // Nothing is timed here, so open projects follow the judge's own queue, + // the order Grade next project walks: the natural pick is the fair one. + const queued = + input.sort === "scheduled" && (!schedule || mode !== "scheduled") + ? new Map( + queueOrder( + judge.judgeId, + challenge.id, + projects.filter((p) => p.grading === "open").map((p) => p.id), + ).map((id, place) => [id, place]), + ) + : null; projects.sort((a, b) => { - if (input.sort === "scheduled") { + const open = standing(a.grading) - standing(b.grading); + if (open !== 0) return open; + if (queued) { + // The judge's own come first, then the open ones in queue order. + const mine = + Number(b.grading === "mine") - Number(a.grading === "mine"); + if (mine !== 0) return mine; + const at = queued.get(a.id); + const bt = queued.get(b.id); + if (at !== undefined && bt !== undefined) return at - bt; + } else if (input.sort === "scheduled") { + // Their own assigned presentations lead, in start order. + const assigned = Number(b.assignedToYou) - Number(a.assignedToYou); + if (assigned !== 0) return assigned; const at = a.presentation?.startsAt.getTime() ?? Infinity; const bt = b.presentation?.startsAt.getTime() ?? Infinity; if (at !== bt) return at - bt; @@ -1116,6 +1254,106 @@ export const judgingDeskRouter = createTRPCRouter({ return { ...base, judgedCount, projects }; }), + // The presentations assigned to this judge that have not ended, soonest + // first: what they are to judge and where to go for it. Only the judge's + // own; whether another judge graded or took one over is told without saying + // who. A project has no table of its own here: a table is a room whose kind + // says so. + itinerary: judgeProcedure.query(async ({ ctx, input }) => { + const db = ctx.db; + const { judge } = ctx; + await reconcileDueDrafts(db, input.hackathonId); + + const now = new Date(); + const [schedule] = await db + .select({ id: judgingSchedules.id }) + .from(judgingSchedules) + .where(eq(judgingSchedules.hackathonId, input.hackathonId)) + .limit(1); + if (!schedule) return { presentations: [], serverNow: now }; + + const rows = await db + .select({ + id: judgingAppointments.id, + entryId: judgingAppointments.entryId, + challengeId: judgingAppointments.challengeId, + title: judgingEntries.title, + roomId: judgingRooms.id, + roomName: judgingRooms.name, + roomKind: judgingRooms.kind, + buildingName: judgingBuildings.name, + startsAt: judgingAppointments.startsAt, + deadlineAt: judgingAppointments.deadlineAt, + }) + .from(judgingAppointments) + .innerJoin( + judgingEntries, + eq(judgingEntries.id, judgingAppointments.entryId), + ) + .innerJoin(judgingRooms, eq(judgingRooms.id, judgingAppointments.roomId)) + .leftJoin( + judgingBuildings, + eq(judgingBuildings.id, judgingRooms.buildingId), + ) + .where( + and( + eq(judgingAppointments.scheduleId, schedule.id), + eq(judgingAppointments.assignedJudgeId, judge.judgeId), + gt(judgingAppointments.deadlineAt, now), + isNull(judgingEntries.deletedAt), + ), + ) + .orderBy(asc(judgingAppointments.startsAt)); + if (rows.length === 0) return { presentations: [], serverNow: now }; + + const challenges = await loadChallenges(db, input.hackathonId); + const labelOf = new Map(challenges.map((c) => [c.id, c.label])); + const roomId = + judge.kind === "guest" + ? judge.roomId + : ((await seatOf(db, judge.judgeId, challenges))?.roomId ?? null); + + const entryIds = [...new Set(rows.map((row) => row.entryId))]; + const evaluations = await db + .select({ + entryId: judgingEvaluations.entryId, + challengeId: judgingEvaluations.challengeId, + judgeId: judgingEvaluations.judgeId, + }) + .from(judgingEvaluations) + .where(inArray(judgingEvaluations.entryId, entryIds)); + const drafts = await db + .select({ + entryId: judgingEvaluationDrafts.entryId, + challengeId: judgingEvaluationDrafts.challengeId, + judgeId: judgingEvaluationDrafts.judgeId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + }) + .from(judgingEvaluationDrafts) + .where(inArray(judgingEvaluationDrafts.entryId, entryIds)); + + return { + presentations: rows.map((row) => { + const same = (task: { entryId: string; challengeId: string }) => + task.entryId === row.entryId && task.challengeId === row.challengeId; + const graders = evaluations.filter(same); + return { + ...row, + challengeLabel: labelOf.get(row.challengeId) ?? "", + // Graded by anyone: nothing is left to do for it. + graded: graders.length > 0, + // Another judge started it after the takeover time. + takenOver: + gradingStateOf(judge.judgeId, graders, drafts.filter(same), now) === + "in_progress", + inRoom: row.roomId === roomId, + }; + }), + serverNow: now, + }; + }), + // One project as the evaluation editor needs it: the project (names only // for the team), the rubric, the judge's draft or evaluation, whether they // can write now and against what deadline, and scores once they may see @@ -1150,6 +1388,8 @@ export const judgingDeskRouter = createTRPCRouter({ asc(judgingEntryMembers.createdAt), ); const tags = await tagsOf(db, judge, challenges, [entry.id]); + // A bare marker, as in the project list: never who or why. + const flagged = await flaggedEntryIds(db, input.hackathonId, [entry.id]); const rubric = await loadRubric(db, input.hackathonId); const switches = await judgingSwitches(db, input.hackathonId); @@ -1187,18 +1427,62 @@ export const judgingDeskRouter = createTRPCRouter({ ) .limit(1); - const { timing, presentation } = await evaluationAccess(db, { - hackathonId: input.hackathonId, - judgeId: judge.judgeId, - entryId: entry.id, - challenge, - challenges, - evaluation: evaluation ?? null, - }); + // Every judge's, narrowed to what says who has the project. + const graders = await db + .select({ judgeId: judgingEvaluations.judgeId }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.entryId, entry.id), + eq(judgingEvaluations.challengeId, challenge.id), + ), + ); + const holders = await db + .select({ + judgeId: judgingEvaluationDrafts.judgeId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + }) + .from(judgingEvaluationDrafts) + .where( + and( + eq(judgingEvaluationDrafts.entryId, entry.id), + eq(judgingEvaluationDrafts.challengeId, challenge.id), + ), + ); + const { timing, presentation, reservedUntil } = await evaluationAccess( + db, + { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: entry.id, + challenge, + challenges, + evaluation: evaluation ?? null, + }, + ); + const grading = gradingStateOf( + judge.judgeId, + graders, + holders, + new Date(), + reservedUntil, + ); + // Another judge's project stays readable; it cannot be written to. + // One reserved for another judge is shut by the timing, which says when + // it opens. + const gradedElsewhere = gradingRefusal(grading, challenge.label); const presentationRoom = presentation ? await roomSummary(db, presentation.roomId, challenges) : null; + // An evaluation counts once it rated every rating item, as placings + // count it: a draft submitted at the deadline with a response left + // empty still does. + const ratingItems = rubric.filter( + (item) => item.kind === "rating", + ).length; + // A judge sees how a project is scoring once they have judged it // themselves, or (members only) when organisers show every score. // Guests never see the overall score across challenges. @@ -1209,27 +1493,25 @@ export const judgingDeskRouter = createTRPCRouter({ overall: ReturnType | null; } | null = null; if (scoresVisible) { - const complete = await db + const submitted = await db .select({ id: judgingEvaluations.id, challengeId: judgingEvaluations.challengeId, }) .from(judgingEvaluations) - .where( - and( - eq(judgingEvaluations.entryId, entry.id), - eq(judgingEvaluations.isComplete, true), - ), - ); + .where(eq(judgingEvaluations.entryId, entry.id)); const answers = await answersOf( db, - complete.map((e) => e.id), + submitted.map((e) => e.id), ); - const scored = complete.map((e) => ({ - challengeId: e.challengeId, - isComplete: true, - ratings: Object.values(answers.get(e.id)?.ratings ?? {}), - })); + const scored = submitted.map((e) => { + const ratings = Object.values(answers.get(e.id)?.ratings ?? {}); + return { + challengeId: e.challengeId, + ratingsComplete: hasEveryRating(ratings, ratingItems), + ratings, + }; + }); scores = { challenge: scopedScore(scored, challenge.id), overall: judge.kind === "member" ? overallScore(scored) : null, @@ -1246,6 +1528,7 @@ export const judgingDeskRouter = createTRPCRouter({ ...entry, members: members.map((member) => member.name), challenges: tags.get(entry.id) ?? [], + flagged: flagged.has(entry.id), }, challenge: { id: challenge.id, @@ -1275,28 +1558,43 @@ export const judgingDeskRouter = createTRPCRouter({ updatedAt: evaluation.updatedAt, ratings: mine?.ratings ?? {}, responses: mine?.responses ?? {}, - score: evaluation.isComplete + score: hasEveryRating( + Object.values(mine?.ratings ?? {}), + ratingItems, + ) ? evaluationScore(Object.values(mine?.ratings ?? {})) : null, } : null, draft: draft ?? null, + // Where the project stands for this judge: theirs, graded by another + // judge, being graded by one, reserved for the judge assigned to it, + // or open. Never which judge. + grading, access: { judgingState: switches.state, - canEdit: open && timing.canEdit, + canEdit: open && timing.canEdit && !gradedElsewhere, reason: !open ? switches.state === "closed" ? CLOSED : NOT_OPEN_YET - : timing.canEdit - ? null - : timing.reason, - deadlineAt: timing.canEdit ? timing.deadlineAt : null, - opensAt: timing.canEdit ? null : timing.opensAt, + : (gradedElsewhere ?? (timing.canEdit ? null : timing.reason)), + // Shut because another judge graded the project or is grading it. + gradedElsewhere: gradedElsewhere !== null, + deadlineAt: + timing.canEdit && !gradedElsewhere ? timing.deadlineAt : null, + opensAt: timing.canEdit || gradedElsewhere ? null : timing.opensAt, + // Shut because the presentation's deadline passed: scores are final. + ended: open && !gradedElsewhere && !timing.canEdit && timing.ended, // Shut only because the judge is not in the room that serves this // challenge: a slot exists and has no start to wait for. needsRoom: - open && !timing.canEdit && !timing.opensAt && presentation !== null, + open && + !gradedElsewhere && + !timing.canEdit && + !timing.opensAt && + !timing.ended && + presentation !== null, }, presentation: presentation && presentationRoom @@ -1313,6 +1611,274 @@ export const judgingDeskRouter = createTRPCRouter({ }; }), + // The next project for this judge to grade, in the challenge they pick or + // the one they sit at. Each judge walks the projects in an order of their own + // (services/judging/queue.ts), so judges at one table rarely reach for the + // same one. Only open projects: not graded, not held by another judge. At an + // untimed challenge only the first QUEUE_WINDOW of them are on offer, and + // `skip` passes over projects the judge turned down (a team away from its + // table). At a scheduled challenge, only presentations running now in the + // judge's room. + nextProject: judgeProcedure + .input( + z.object({ + challengeId: uuid.optional(), + skip: z.array(uuid).max(20).default([]), + }), + ) + .query(async ({ ctx, input }) => { + const db = ctx.db; + const { judge } = ctx; + await reconcileDueDrafts(db, input.hackathonId); + + const challenges = await loadChallenges(db, input.hackathonId); + const seat = await seatOf(db, judge.judgeId, challenges); + const challenge = evaluatedChallenge( + judge, + challenges, + input.challengeId, + seat, + ); + const mode = modeOf(challenges, challenge); + const found = ( + entryId: string | null, + needsRoom = false, + allSkipped = false, + ) => ({ + challenge: { id: challenge.id, label: challenge.label, mode }, + entryId, + needsRoom, + allSkipped, + }); + + const queue = await loadQueue(db, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + challenge, + seat, + filterId: input.challengeId, + }); + if (queue.needsRoom) return found(null, true); + + // The schedule decides who sees what at a scheduled challenge. + const window = queue.timed ? Infinity : QUEUE_WINDOW; + const entryId = nextToGrade( + judge.judgeId, + challenge.id, + queue.candidates, + { skip: input.skip, window }, + ); + const allSkipped = + entryId === null && + queueWindow(judge.judgeId, challenge.id, queue.candidates, window) + .length > 0; + return found(entryId, false, allSkipped); + }), + + // Opening a project to grade it. Runs the checks a first save would (the + // project is free, the queue offers it) and, when it passes, creates or + // touches the judge's draft with no answers needed, so the hold starts as + // the judge opens the project rather than at their first save. The draft + // keeps saveDraft's deadline and revision, so the editor carries on from it. + startGrading: judgeProcedure + .input(z.object({ entryId: uuid, challengeId: uuid.optional() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db; + const { judge } = ctx; + await reconcileDueDrafts(db, input.hackathonId); + + const challenges = await loadChallenges(db, input.hackathonId); + const challenge = evaluatedChallenge( + judge, + challenges, + input.challengeId, + await memberSeat(db, judge, challenges), + ); + + try { + return await db.transaction(async (tx) => { + await assertJudgingOpen(tx, input.hackathonId); + await assertGuestSeated(tx, judge); + await loadEntryInChallenge(tx, { + hackathonId: input.hackathonId, + entryId: input.entryId, + challenge, + }); + + const [draft] = await tx + .select({ + id: judgingEvaluationDrafts.id, + revision: judgingEvaluationDrafts.revision, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + }) + .from(judgingEvaluationDrafts) + .where( + taskOf( + judgingEvaluationDrafts, + judge.judgeId, + input.entryId, + challenge.id, + ), + ) + .for("update") + .limit(1); + const [evaluation] = await tx + .select({ id: judgingEvaluations.id }) + .from(judgingEvaluations) + .where( + taskOf( + judgingEvaluations, + judge.judgeId, + input.entryId, + challenge.id, + ), + ) + .limit(1); + // Their own graded project is theirs to edit; there is nothing to hold. + if (evaluation) { + return { + held: false, + revision: draft?.revision ?? 0, + deadlineAt: draft?.deadlineAt ?? null, + }; + } + + await claimGrading(tx, { + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + }); + if (!draft) { + await assertQueueOffers(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + }); + } + + const { timing } = await evaluationAccess(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + evaluation: null, + graceMs: DEADLINE_GRACE_MS, + }); + if (!timing.canEdit) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: timing.reason, + }); + } + + // As saveDraft stamps a draft that is not an edit of an evaluation. + const deadlineAt = timing.deadlineAt ?? draft?.deadlineAt ?? null; + const now = new Date(); + if (draft) { + await tx + .update(judgingEvaluationDrafts) + .set({ + appointmentId: timing.appointmentId, + deadlineAt, + updatedAt: now, + }) + .where(eq(judgingEvaluationDrafts.id, draft.id)); + } else { + await tx.insert(judgingEvaluationDrafts).values({ + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challengeId: challenge.id, + ratings: {}, + responses: {}, + appointmentId: timing.appointmentId, + deadlineAt, + revision: 0, + updatedAt: now, + }); + } + + return { held: true, revision: draft?.revision ?? 0, deadlineAt }; + }); + } catch (error) { + if (isUniqueViolation(error)) { + throw new TRPCError({ + code: "CONFLICT", + message: + "This project was just opened from another window. Reload to pick up the latest.", + }); + } + throw error; + } + }), + + // Gives back a project the judge opened (startGrading) and never saved to: + // deletes their own draft while it is still revision 0, and only when they + // have no evaluation of the project. A saved draft is work and stays. Called + // when the judge skips a project or leaves its sheet unsaved, so a skipped + // project is not held for GRADING_HOLD_MS and an accidental open of a + // scheduled presentation is not held until its deadline. It touches only the + // judge's own draft, so it needs no seat or sponsor check. + releaseGrading: judgeProcedure + .input(z.object({ entryId: uuid, challengeId: uuid.optional() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db; + const { judge } = ctx; + + const challenges = await loadChallenges(db, input.hackathonId); + const challengeId = input.challengeId + ? rootOf(challenges, input.challengeId)?.id + : evaluatedChallenge( + judge, + challenges, + undefined, + await memberSeat(db, judge, challenges), + ).id; + if (!challengeId) return { released: false }; + + return db.transaction(async (tx) => { + const [draft] = await tx + .select({ + id: judgingEvaluationDrafts.id, + revision: judgingEvaluationDrafts.revision, + }) + .from(judgingEvaluationDrafts) + .where( + taskOf( + judgingEvaluationDrafts, + judge.judgeId, + input.entryId, + challengeId, + ), + ) + .for("update") + .limit(1); + if (!draft || draft.revision > 0) return { released: false }; + + const [evaluation] = await tx + .select({ id: judgingEvaluations.id }) + .from(judgingEvaluations) + .where( + taskOf( + judgingEvaluations, + judge.judgeId, + input.entryId, + challengeId, + ), + ) + .limit(1); + if (evaluation) return { released: false }; + + await tx + .delete(judgingEvaluationDrafts) + .where(eq(judgingEvaluationDrafts.id, draft.id)); + return { released: true }; + }); + }), + // Autosave. `revision` is the draft revision the editor last saw (0 for // none); a save from a stale window is refused rather than overwriting. // The draft takes the presentation's deadline, and is submitted for the @@ -1337,7 +1903,7 @@ export const judgingDeskRouter = createTRPCRouter({ judge, challenges, input.challengeId, - null, + await memberSeat(db, judge, challenges), ); const rubric = await loadRubric(db, input.hackathonId); const check = checkAnswers(rubric, input, judge.kind, "strict"); @@ -1385,12 +1951,32 @@ export const judgingDeskRouter = createTRPCRouter({ ) .limit(1); + // A project is graded once: a judge who has not graded it yet + // needs it free. Their own draft is locked above, ahead of this. + if (!evaluation) { + await claimGrading(tx, { + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + }); + // Starting a project means taking one their queue offers. + if (!draft) { + await assertQueueOffers(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + }); + } + } + if ((draft?.revision ?? 0) !== input.revision) { throw new TRPCError({ code: "CONFLICT", message: evaluation && !draft - ? "Time ran out, so your answers were submitted for you. Reload to see them; you can still edit." + ? "Time ran out, so your saved answers were submitted for you. Reload to see them." : "Your answers changed in another window. Reload to pick up the latest before you keep going.", }); } @@ -1402,6 +1988,7 @@ export const judgingDeskRouter = createTRPCRouter({ challenge, challenges, evaluation: evaluation ?? null, + graceMs: DEADLINE_GRACE_MS, }); if (!timing.canEdit) { throw new TRPCError({ @@ -1479,7 +2066,7 @@ export const judgingDeskRouter = createTRPCRouter({ judge, challenges, input.challengeId, - null, + await memberSeat(db, judge, challenges), ); const rubric = await loadRubric(db, input.hackathonId); const check = checkAnswers(rubric, input, judge.kind, "strict"); @@ -1505,7 +2092,7 @@ export const judgingDeskRouter = createTRPCRouter({ // The draft first: a deadline submission holding it finishes // before this reads the evaluation, and then sees the result. - await tx + const [draft] = await tx .select({ id: judgingEvaluationDrafts.id }) .from(judgingEvaluationDrafts) .where( @@ -1536,12 +2123,32 @@ export const judgingDeskRouter = createTRPCRouter({ .for("update") .limit(1); + // A project is graded once: a first submission needs it free. The + // judge's own draft and evaluation are locked above, ahead of this. + if (!existing) { + await claimGrading(tx, { + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + }); + // Starting a project means taking one their queue offers. + if (!draft) { + await assertQueueOffers(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + }); + } + } + if ((existing?.revision ?? 0) !== input.expectedRevision) { throw new TRPCError({ code: "CONFLICT", message: existing?.autoSubmittedAt && input.expectedRevision === 0 - ? "Time ran out, so your answers were submitted for you. Reload to see them; you can still edit." + ? "Time ran out, so your saved answers were submitted for you. Reload to see them." : "This evaluation changed in another window. Reload to see the latest before you submit.", }); } @@ -1553,6 +2160,7 @@ export const judgingDeskRouter = createTRPCRouter({ challenge, challenges, evaluation: existing ?? null, + graceMs: DEADLINE_GRACE_MS, }); if (!timing.canEdit) { throw new TRPCError({ @@ -1643,15 +2251,20 @@ export const judgingDeskRouter = createTRPCRouter({ db, rows.map((row) => row.id), ); + // The score shows once the evaluation counts, as placings count it. + const ratingItems = (await loadRubric(db, input.hackathonId)).filter( + (item) => item.kind === "rating", + ).length; return rows.map(({ deletedAt, ...row }) => { const mine = answers.get(row.id) ?? { ratings: {}, responses: {} }; + const ratings = Object.values(mine.ratings); return { ...row, withdrawn: deletedAt !== null, ratings: mine.ratings, responses: mine.responses, - score: row.isComplete - ? evaluationScore(Object.values(mine.ratings)) + score: hasEveryRating(ratings, ratingItems) + ? evaluationScore(ratings) : null, }; }); diff --git a/packages/api/src/routers/judging/hacker.ts b/packages/api/src/routers/judging/hacker.ts index 8c0f2d54..458dcba2 100644 --- a/packages/api/src/routers/judging/hacker.ts +++ b/packages/api/src/routers/judging/hacker.ts @@ -60,6 +60,7 @@ import type { Itinerary, VisibleChallenge, } from "../../services/judging/itinerary"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; const hackathonInput = z.object({ hackathonId: z.string().uuid("Invalid hackathon ID"), @@ -642,7 +643,7 @@ export const judgingHackerRouter = createTRPCRouter({ judgeId: judgingEvaluations.judgeId, challengeId: judgingEvaluations.challengeId, appointmentId: judgingEvaluations.appointmentId, - isComplete: judgingEvaluations.isComplete, + ratingsComplete: holdsEveryRating(input.hackathonId), }) .from(judgingEvaluations) .where( @@ -694,9 +695,9 @@ export const judgingHackerRouter = createTRPCRouter({ return { ...state, entry, challenges: visible, ...itinerary }; }), - // Written feedback on the caller's project: public responses from member - // judges' complete evaluations. No judge names and no scores. Hidden in - // emergency mode. + // Written feedback on the caller's project: public responses that exist, from + // member judges' evaluations that hold every rating (the ones placings + // count). No judge names and no scores. Hidden in emergency mode. feedback: protectedProcedure .input(hackathonInput) .query(async ({ ctx, input }) => { @@ -725,7 +726,7 @@ export const judgingHackerRouter = createTRPCRouter({ and( eq(judgingEvaluations.hackathonId, input.hackathonId), eq(judgingEvaluations.entryId, held.entryId), - eq(judgingEvaluations.isComplete, true), + holdsEveryRating(input.hackathonId), eq(judgingJudges.kind, "member"), ), ); diff --git a/packages/api/src/routers/judging/results.ts b/packages/api/src/routers/judging/results.ts index 82f60d6c..78b9bab1 100644 --- a/packages/api/src/routers/judging/results.ts +++ b/packages/api/src/routers/judging/results.ts @@ -10,6 +10,7 @@ import { inArray, isNotNull, isNull, + max, ne, or, } from "drizzle-orm"; @@ -33,15 +34,23 @@ import type { DrizzleDB } from "@query/db"; import { createTRPCRouter } from "../../trpc"; import { isAdmin } from "../../middleware/procedures"; import { recordAdminAction } from "../../middleware/audit"; +import { CacheKeys } from "../../middleware/cache"; +import type { CacheService } from "../../middleware/cache"; import { assertHackathonExists } from "../../services/judging/config"; import { reconcileDueDrafts } from "../../services/judging/evaluation-store"; import type { JudgingDb } from "../../services/judging/config"; import { lockJudgingConfig } from "../../services/judging/claims"; +import { calibrate } from "../../services/judging/calibration"; +import { checkInFlagsForEntries } from "../../services/check-in-flags"; import { notifyResultsPublished, resultsPublishedBefore, } from "../../services/judging/notify"; -import { evaluationScore, meanOrNull } from "../../services/judging/scoring"; +import { + evaluationScore, + hasEveryRating, + meanOrNull, +} from "../../services/judging/scoring"; import { OVERALL_SCOPE, computePlacements, @@ -82,12 +91,193 @@ async function ratingsOf(db: JudgingDb, evaluationIds: string[]) { ); } +/** How many rating items the rubric has: an evaluation counts once it rates them all. */ +async function ratingItemCount(db: JudgingDb, hackathonId: string) { + const [row] = await db + .select({ n: count() }) + .from(judgingRubricItems) + .where( + and( + eq(judgingRubricItems.hackathonId, hackathonId), + eq(judgingRubricItems.kind, "rating"), + ), + ); + return row?.n ?? 0; +} + +/** + * Every evaluation in the edition with its ratings, and whether it rated every + * rating item: what compute ranks and the judge calibration summary reads. + * Scoring and calibration count the ones that did, whether or not a written + * response was left empty, which is how a draft submitted at the hard end reads. + * The ratings are joined rather than listed by id, since an edition's + * evaluations run to thousands. + */ +async function evaluationsForScoring(db: JudgingDb, hackathonId: string) { + const evaluations = await db + .select({ + id: judgingEvaluations.id, + entryId: judgingEvaluations.entryId, + challengeId: judgingEvaluations.challengeId, + judgeId: judgingEvaluations.judgeId, + submittedAt: judgingEvaluations.submittedAt, + }) + .from(judgingEvaluations) + .where(eq(judgingEvaluations.hackathonId, hackathonId)); + const ratingRows = await db + .select({ + evaluationId: judgingEvaluationRatings.evaluationId, + value: judgingEvaluationRatings.value, + }) + .from(judgingEvaluationRatings) + .innerJoin( + judgingEvaluations, + eq(judgingEvaluations.id, judgingEvaluationRatings.evaluationId), + ) + .where(eq(judgingEvaluations.hackathonId, hackathonId)); + const ratingItems = await ratingItemCount(db, hackathonId); + const ratings = groupRatings(ratingRows); + return evaluations.map((evaluation) => { + const values = ratings.get(evaluation.id) ?? []; + return { + ...evaluation, + ratings: values, + ratingsComplete: hasEveryRating(values, ratingItems), + }; + }); +} + +// Two decimals, and never a negative zero. +const round2 = (value: number) => Math.round(value * 100) / 100 || 0; + +// The list polls every 10 seconds; calibrating on each poll reads and sorts the +// whole edition's evaluations, so the result is kept for this long. +const CALIBRATION_TTL_SECONDS = 60; + +/** The entries compute ranks: live and still eligible for prizes. */ +const rankedEntries = (hackathonId: string) => + and( + eq(judgingEntries.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + isNull(judgingEntries.prizeIneligibleAt), + ); + +/** + * What a calibration was computed from, as a cache key suffix: how many + * evaluations the edition has and when one was last saved, and the same two for + * its ranked entries. A new, edited or deleted evaluation (a reopened grade is + * one), or an entry pulled from prizes, withdrawn or restored, moves one of the + * four, so the next read calibrates afresh. The one pair that cancels out is a + * grade deleted while a draft is submitted as of an earlier deadline, both + * between two polls; the TTL holds that to a minute. Two aggregates; nothing is + * listed. + */ +async function calibrationFingerprint(db: JudgingDb, hackathonId: string) { + const [[evaluations], [entries]] = await Promise.all([ + db + .select({ n: count(), latest: max(judgingEvaluations.updatedAt) }) + .from(judgingEvaluations) + .where(eq(judgingEvaluations.hackathonId, hackathonId)), + db + .select({ n: count(), latest: max(judgingEntries.updatedAt) }) + .from(judgingEntries) + .where(rankedEntries(hackathonId)), + ]); + return [ + evaluations?.n ?? 0, + evaluations?.latest?.getTime() ?? 0, + entries?.n ?? 0, + entries?.latest?.getTime() ?? 0, + ].join(":"); +} + +/** + * How each judge's scores are being adjusted, grouped by challenge: the + * organiser's window onto what compute does. Read from the current evaluations + * of live, prize-eligible entries, the same ones compute calibrates. Judges and + * hackers never get this; only the organiser's results list returns it. + * + * The calibration is cached per edition (see calibrationFingerprint). compute + * does not read it: a ranking is always calibrated from the evaluations as they + * stand. + */ +async function judgeCalibrations( + db: JudgingDb, + cache: CacheService, + hackathonId: string, +) { + const key = `${CacheKeys.judgingCalibration(hackathonId)}:${await calibrationFingerprint(db, hackathonId)}`; + const calibrated = await cache.getOrSet( + key, + async () => { + const [evaluations, entries] = await Promise.all([ + evaluationsForScoring(db, hackathonId), + db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .where(rankedEntries(hackathonId)), + ]); + const live = new Set(entries.map((entry) => entry.id)); + return calibrate( + evaluations.filter((evaluation) => live.has(evaluation.entryId)), + ).judges; + }, + CALIBRATION_TTL_SECONDS, + ); + if (calibrated.length === 0) return []; + + const [judges, challenges] = await Promise.all([ + db + .select({ + id: judgingJudges.id, + displayName: judgingJudges.displayName, + kind: judgingJudges.kind, + }) + .from(judgingJudges) + .where(eq(judgingJudges.hackathonId, hackathonId)), + db + .select({ id: judgingChallenges.id, label: judgingChallenges.label }) + .from(judgingChallenges) + .where(eq(judgingChallenges.hackathonId, hackathonId)) + .orderBy( + asc(judgingChallenges.displayOrder), + asc(judgingChallenges.label), + ), + ]); + const judgeOf = new Map(judges.map((judge) => [judge.id, judge])); + + return challenges + .map((challenge) => ({ + challengeId: challenge.id, + challenge: challenge.label, + judges: calibrated + .filter( + (row) => row.challengeId === challenge.id && judgeOf.has(row.judgeId), + ) + .map((row) => { + const judge = judgeOf.get(row.judgeId)!; + return { + judgeId: row.judgeId, + judgeName: judge.displayName, + kind: judge.kind, + grades: row.evaluations, + offset: round2(row.offset), + spread: round2(row.spread), + drift: round2(row.drift), + }; + }) + .sort((a, b) => a.judgeName.localeCompare(b.judgeName)), + })) + .filter((challenge) => challenge.judges.length > 0); +} + // Computing and publishing placements, and the organiser's view of what each // judge submitted. export const judgingResultsRouter = createTRPCRouter({ - // Replaces the draft placings with a fresh ranking from every complete - // evaluation. Refused while results are published: recomputing then would - // either change the public list under people's feet or silently pull it. + // Replaces the draft placings with a fresh ranking from every evaluation that + // holds every rating. Refused while results are published: recomputing then + // would either change the public list under people's feet or silently pull + // it. compute: isAdmin.input(hackathonInput).mutation(async ({ ctx, input }) => { const db = ctx.db as DrizzleDB; await assertHackathonExists(db, input.hackathonId); @@ -115,6 +305,10 @@ export const judgingResultsRouter = createTRPCRouter({ }); } + // Judges still score a team an organiser pulled from prizes, so its + // evaluations stay as they are; it just cannot place. Leaving it out of + // the entry set is enough, because computePlacements ignores + // evaluations of any entry it was not given. const entries = await tx .select({ id: judgingEntries.id }) .from(judgingEntries) @@ -122,6 +316,7 @@ export const judgingResultsRouter = createTRPCRouter({ and( eq(judgingEntries.hackathonId, input.hackathonId), isNull(judgingEntries.deletedAt), + isNull(judgingEntries.prizeIneligibleAt), ), ); const memberships = await tx @@ -139,48 +334,16 @@ export const judgingResultsRouter = createTRPCRouter({ asc(judgingChallenges.displayOrder), asc(judgingChallenges.label), ); - const evaluations = await tx - .select({ - id: judgingEvaluations.id, - entryId: judgingEvaluations.entryId, - challengeId: judgingEvaluations.challengeId, - isComplete: judgingEvaluations.isComplete, - }) - .from(judgingEvaluations) - .where( - and( - eq(judgingEvaluations.hackathonId, input.hackathonId), - eq(judgingEvaluations.isComplete, true), - ), - ); - // Joined rather than listed by id: an edition's evaluations run to - // thousands, and the filter is the same one as above. - const ratingRows = await tx - .select({ - evaluationId: judgingEvaluationRatings.evaluationId, - value: judgingEvaluationRatings.value, - }) - .from(judgingEvaluationRatings) - .innerJoin( - judgingEvaluations, - eq(judgingEvaluations.id, judgingEvaluationRatings.evaluationId), - ) - .where( - and( - eq(judgingEvaluations.hackathonId, input.hackathonId), - eq(judgingEvaluations.isComplete, true), - ), - ); - const ratings = groupRatings(ratingRows); + const evaluations = await evaluationsForScoring(tx, input.hackathonId); + // Each evaluation is calibrated for its judge before it is scored. + // Always from the evaluations as they stand, never from the cache that + // the list keeps for its calibration summary. const placements = computePlacements({ entryIds: entries.map((entry) => entry.id), memberships, challenges, - evaluations: evaluations.map((evaluation) => ({ - ...evaluation, - ratings: ratings.get(evaluation.id) ?? [], - })), + evaluations, }); await tx @@ -195,6 +358,7 @@ export const judgingResultsRouter = createTRPCRouter({ scopeLabel: placement.scopeLabel, placement: placement.placement, score: formatScore(placement.score), + rawScore: formatScore(placement.rawScore), evaluationCount: placement.evaluationCount, computedAt: now, })), @@ -204,7 +368,9 @@ export const judgingResultsRouter = createTRPCRouter({ return { placements: placements.length, scopes: new Set(placements.map((placement) => placement.scope)).size, - evaluations: evaluations.length, + evaluations: evaluations.filter( + (evaluation) => evaluation.ratingsComplete, + ).length, }; }); @@ -234,7 +400,7 @@ export const judgingResultsRouter = createTRPCRouter({ throw new TRPCError({ code: "PRECONDITION_FAILED", message: - "There is nothing to publish yet. Compute results first, once judges have submitted.", + "There is nothing to publish yet. Compute results first, once judges have submitted. Reopening a grade clears the draft placings, so compute again after one.", }); } const rows = await tx @@ -305,12 +471,14 @@ export const judgingResultsRouter = createTRPCRouter({ scopeLabel: judgingPlacements.scopeLabel, placement: judgingPlacements.placement, score: judgingPlacements.score, + rawScore: judgingPlacements.rawScore, evaluationCount: judgingPlacements.evaluationCount, publishedAt: judgingPlacements.publishedAt, computedAt: judgingPlacements.computedAt, entryId: judgingEntries.id, title: judgingEntries.title, deletedAt: judgingEntries.deletedAt, + prizeIneligibleAt: judgingEntries.prizeIneligibleAt, }) .from(judgingPlacements) .innerJoin( @@ -346,6 +514,19 @@ export const judgingResultsRouter = createTRPCRouter({ evaluationsSinceCompute = since?.n ?? 0; } + // An organiser decides these teams' prizes, so each row says how many + // flags are still open. The flag detail (who, which events) stays in the + // Projects section; this payload carries the count and nothing else. + const flags = await checkInFlagsForEntries(db, input.hackathonId, [ + ...new Set(rows.map((row) => row.entryId)), + ]); + const openFlags = new Map( + [...flags].map(([entryId, views]) => [ + entryId, + views.filter((view) => !view.dismissed).length, + ]), + ); + const scopes = new Map< string, { @@ -356,10 +537,17 @@ export const judgingResultsRouter = createTRPCRouter({ entryId: string; title: string; placement: number; + /** Adjusted for judge bias; this is what the placing ranks on. */ score: number | null; + /** The plain mean of the same evaluations. Null on older rows. */ + rawScore: number | null; evaluationCount: number; /** Pulled from judging since; getResults already hides it. */ withdrawn: boolean; + /** Check-in flags nobody has dismissed. Only the count leaves here. */ + openCheckInFlags: number; + /** Pulled from prizes; getResults hides it and a recompute drops it. */ + prizeIneligible: boolean; }[]; } >(); @@ -375,8 +563,11 @@ export const judgingResultsRouter = createTRPCRouter({ title: row.title, placement: row.placement, score: toScore(row.score), + rawScore: toScore(row.rawScore), evaluationCount: row.evaluationCount, withdrawn: !!row.deletedAt, + openCheckInFlags: openFlags.get(row.entryId) ?? 0, + prizeIneligible: !!row.prizeIneligibleAt, }); scopes.set(row.scope, scope); } @@ -385,6 +576,9 @@ export const judgingResultsRouter = createTRPCRouter({ computedAt, publishedAt, evaluationsSinceCompute, + calibration: computedAt + ? await judgeCalibrations(db, ctx.cache, input.hackathonId) + : [], scopes: [...scopes.values()].sort( (a, b) => Number(b.scope === OVERALL_SCOPE) - @@ -395,7 +589,8 @@ export const judgingResultsRouter = createTRPCRouter({ }), // One presentation's results: each judge who wrote or started an - // evaluation, their ratings and responses once complete, and the mean. + // evaluation, their ratings and responses once every rating is in, and the + // mean. appointmentResults: isAdmin .input(hackathonInput.extend({ appointmentId: z.string().uuid() })) .query(async ({ ctx, input }) => { @@ -485,8 +680,8 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingEvaluationDrafts.appointmentId, appointment.id), ), ), - // Complete evaluations of the same project and challenge outside this - // slot: an untimed visit, or a slot it was moved from. + // Evaluations of the same project and challenge outside this slot: an + // untimed visit, or a slot it was moved from. db .select({ id: judgingEvaluations.id }) .from(judgingEvaluations) @@ -495,7 +690,6 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingEvaluations.hackathonId, input.hackathonId), eq(judgingEvaluations.entryId, appointment.entryId), eq(judgingEvaluations.challengeId, appointment.challengeId), - eq(judgingEvaluations.isComplete, true), or( isNull(judgingEvaluations.appointmentId), ne(judgingEvaluations.appointmentId, appointment.id), @@ -505,10 +699,8 @@ export const judgingResultsRouter = createTRPCRouter({ .limit(1), ]); - const completeIds = evaluations - .filter((evaluation) => evaluation.isComplete) - .map((evaluation) => evaluation.id); - const [ratings, responses] = completeIds.length + const evaluationIds = evaluations.map((evaluation) => evaluation.id); + const [ratings, responses, ratingItems] = evaluationIds.length ? await Promise.all([ db .select({ @@ -523,7 +715,7 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingRubricItems.id, judgingEvaluationRatings.itemId), ) .where( - inArray(judgingEvaluationRatings.evaluationId, completeIds), + inArray(judgingEvaluationRatings.evaluationId, evaluationIds), ) .orderBy(asc(judgingRubricItems.displayOrder)), db @@ -540,18 +732,34 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingRubricItems.id, judgingEvaluationResponses.itemId), ) .where( - inArray(judgingEvaluationResponses.evaluationId, completeIds), + inArray(judgingEvaluationResponses.evaluationId, evaluationIds), ) .orderBy(asc(judgingRubricItems.displayOrder)), + ratingItemCount(db, input.hackathonId), ]) - : [[], []]; + : [[], [], 0]; + + // An evaluation counts once it rated every item, whether or not it + // answered every written question. + const counted = new Set( + evaluations + .filter((evaluation) => + hasEveryRating( + ratings + .filter((rating) => rating.evaluationId === evaluation.id) + .map((rating) => rating.value), + ratingItems, + ), + ) + .map((evaluation) => evaluation.id), + ); const judged = new Set( evaluations.map((evaluation) => evaluation.judgeId), ); const judges = [ ...evaluations.map((evaluation) => { - const own = evaluation.isComplete + const own = counted.has(evaluation.id) ? ratings.filter((rating) => rating.evaluationId === evaluation.id) : []; return { @@ -563,6 +771,7 @@ export const judgingResultsRouter = createTRPCRouter({ ? ("complete" as const) : ("incomplete" as const), autoSubmitted: !!evaluation.autoSubmittedAt, + counted: counted.has(evaluation.id), updatedAt: evaluation.updatedAt, score: evaluationScore(own.map((rating) => rating.value)), ratings: own.map(({ itemId, label, value }) => ({ @@ -570,7 +779,7 @@ export const judgingResultsRouter = createTRPCRouter({ label, value, })), - responses: evaluation.isComplete + responses: counted.has(evaluation.id) ? responses .filter((response) => response.evaluationId === evaluation.id) .map(({ itemId, label, value, isPublic }) => ({ @@ -592,6 +801,7 @@ export const judgingResultsRouter = createTRPCRouter({ evaluationId: null, status: "drafting" as const, autoSubmitted: false, + counted: false, updatedAt: draft.updatedAt, score: null, ratings: [], @@ -599,15 +809,15 @@ export const judgingResultsRouter = createTRPCRouter({ })), ].sort((a, b) => a.judgeName.localeCompare(b.judgeName)); - const complete = judges.filter((judge) => judge.status === "complete"); + const counting = judges.filter((judge) => judge.counted); return { appointment, average: meanOrNull( - complete + counting .map((judge) => judge.score) .filter((score): score is number => score !== null), ), - completeCount: complete.length, + countedCount: counting.length, hasResultsElsewhere: elsewhere.length > 0, judges, }; @@ -666,12 +876,15 @@ export const judgingResultsRouter = createTRPCRouter({ db, rows.map((row) => row.id), ); + const ratingItems = await ratingItemCount(db, input.hackathonId); return rows.map(({ autoSubmittedAt, ...row }) => ({ ...row, autoSubmitted: !!autoSubmittedAt, autoSubmittedAt, score: evaluationScore(ratings.get(row.id) ?? []), + // Whether placings count it: it rated every item. + counted: hasEveryRating(ratings.get(row.id) ?? [], ratingItems), })); }), diff --git a/packages/api/src/routers/judging/schedule.ts b/packages/api/src/routers/judging/schedule.ts index 1b34a501..dd707178 100644 --- a/packages/api/src/routers/judging/schedule.ts +++ b/packages/api/src/routers/judging/schedule.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { TRPCError } from "@trpc/server"; -import { and, asc, eq, isNull, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import { judgingAppointments, judgingBuildings, @@ -8,7 +8,10 @@ import { judgingConfigs, judgingEntries, judgingEntryChallenges, + judgingEvaluationDrafts, judgingEvaluations, + judgingGuestSessions, + judgingJudges, judgingPresence, judgingRooms, judgingSchedules, @@ -37,9 +40,31 @@ import type { import { aamoghCPSolver } from "../../services/judging/schedule/aamogh-cp-solver"; import { CPSAT_COMMAND, + cpsatAssignmentProblem, cpsatProblem, + readCpsatAssignments, readCpsatSolution, } from "../../services/judging/schedule/cpsat-bridge"; +import { + assignJudges, + describeShortfall, + judgeShortfalls, + mayGrade, +} from "../../services/judging/schedule/judges"; +import type { + JudgeAssignment, + JudgeInput, +} from "../../services/judging/schedule/judges"; +import { + judgeChanges, + judgeChoices, + judgePlanOf, + rosterOf, +} from "../../services/judging/schedule/judge-plan"; +import type { + Roster, + SavedAppointment, +} from "../../services/judging/schedule/judge-plan"; import { CPSAT_SECONDS, CPSAT_UNAVAILABLE, @@ -47,11 +72,14 @@ import { runCpsat, } from "../../services/judging/schedule/cpsat-runner"; import { reconcileDueDrafts } from "../../services/judging/evaluation-store"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; import { moveChoices } from "../../services/judging/schedule/moves"; import { staffedRoomIds, withStatus, } from "../../services/judging/schedule/board"; +import { judgeProgressById } from "../../services/judging/schedule/judge-progress"; +import type { JudgeProgress } from "../../services/judging/schedule/judge-progress"; import { appointmentTimes, firstSlotFrom, @@ -131,6 +159,7 @@ async function readInventory(db: JudgingDb, hackathonId: string) { parentId: judgingChallenges.parentId, mode: judgingChallenges.mode, isGeneral: judgingChallenges.isGeneral, + isGroup: judgingChallenges.isGroup, }) .from(judgingChallenges) .where(eq(judgingChallenges.hackathonId, hackathonId)), @@ -171,6 +200,49 @@ async function readInventory(db: JudgingDb, hackathonId: string) { type Inventory = Awaited>; +/** The judges an edition can give presentations to, with every judge's name. */ +async function readRoster( + db: JudgingDb, + hackathonId: string, + inventory: Inventory, +): Promise { + const [judges, sessions] = await Promise.all([ + db + .select({ + id: judgingJudges.id, + kind: judgingJudges.kind, + userId: judgingJudges.userId, + displayName: judgingJudges.displayName, + }) + .from(judgingJudges) + .where(eq(judgingJudges.hackathonId, hackathonId)), + db + .select({ + judgeId: judgingGuestSessions.judgeId, + roomId: judgingGuestSessions.roomId, + createdAt: judgingGuestSessions.createdAt, + }) + .from(judgingGuestSessions) + .where( + and( + eq(judgingGuestSessions.hackathonId, hackathonId), + isNull(judgingGuestSessions.revokedAt), + ), + ), + ]); + return rosterOf({ + hackathonId, + judges, + seats: sessions.flatMap((s) => + s.judgeId + ? [{ judgeId: s.judgeId, roomId: s.roomId, createdAt: s.createdAt }] + : [], + ), + challenges: inventory.challenges, + rooms: selectPlan(inventory).rooms, + }); +} + function namesOf(inventory: Inventory) { const labels = new Map(inventory.challenges.map((c) => [c.id, c.label])); const titles = new Map(inventory.memberships.map((m) => [m.entryId, m.title])); @@ -319,7 +391,13 @@ async function writeSchedule( db: DrizzleDB, userId: string, input: { hackathonId: string; timing: ScheduleTiming }, - schedule: { placements: Placement[]; relaxed: boolean; fingerprint: string }, + schedule: { + placements: Placement[]; + relaxed: boolean; + fingerprint: string; + /** The judge of each presentation, as far as there are judges. */ + assignments: readonly JudgeAssignment[]; + }, ) { return db.transaction(async (tx) => { await ensureJudgingConfig(tx, input.hackathonId); @@ -351,12 +429,28 @@ async function writeSchedule( }) .returning({ id: judgingSchedules.id }); + // A judge removed since the assignment was made has no appointments left. + const judges = new Set( + ( + await tx + .select({ id: judgingJudges.id }) + .from(judgingJudges) + .where(eq(judgingJudges.hackathonId, input.hackathonId)) + ).map((judge) => judge.id), + ); + const judgeOf = new Map( + schedule.assignments + .filter((a) => judges.has(a.judgeId)) + .map((a) => [taskKey(a.entryId, a.challengeId), a.judgeId]), + ); + const rows = schedule.placements.map((p) => ({ scheduleId: row!.id, hackathonId: input.hackathonId, entryId: p.entryId, challengeId: p.challengeId, roomId: p.roomId, + assignedJudgeId: judgeOf.get(taskKey(p.entryId, p.challengeId)) ?? null, ...appointmentTimes(input.timing, p.slot), })); // Postgres takes at most 65535 parameters per statement. @@ -364,14 +458,26 @@ async function writeSchedule( await tx.insert(judgingAppointments).values(rows.slice(i, i + 1000)); } - return { scheduleId: row!.id, appointments: rows.length }; + return { + scheduleId: row!.id, + appointments: rows.length, + judged: rows.filter((r) => r.assignedJudgeId).length, + }; }); } -/** The uploaded CP-SAT solution, checked against the edition as it stands. */ -function readUpload(inventory: Inventory, timing: ScheduleTiming, solution: string) { +/** + * The uploaded CP-SAT solution, checked against the edition as it stands, + * with its judges checked against `judges` as they are now. + */ +function readUpload( + inventory: Inventory, + timing: ScheduleTiming, + solution: string, + judges: readonly JudgeInput[], +) { const { inputs, fingerprint } = planOf(inventory, timing); - const read = readCpsatSolution(inputs, fingerprint, solution); + const read = readCpsatSolution(inputs, fingerprint, solution, judges); if (!read.ok) { throw new TRPCError({ code: "BAD_REQUEST", message: read.reason }); } @@ -383,6 +489,21 @@ function readUpload(inventory: Inventory, timing: ScheduleTiming, solution: stri }; } +/** What a CP-SAT preview says about the judges. */ +function describeJudging( + read: { judging: { source: "cp-sat" | "in-app"; metrics: { unassigned: number } } }, + judges: readonly JudgeInput[], + presentations: number, +) { + return { + judges: judges.length, + presentations, + unassigned: read.judging.metrics.unassigned, + // Who worked the assignment out. + source: read.judging.source, + }; +} + /** The saved schedule with everything the board and the move pickers need. */ async function loadBoard( db: JudgingDb, @@ -404,6 +525,7 @@ async function loadBoard( deadlineAt: judgingAppointments.deadlineAt, endsAt: judgingAppointments.endsAt, revision: judgingAppointments.revision, + assignedJudgeId: judgingAppointments.assignedJudgeId, entryTitle: judgingEntries.title, entryDeletedAt: judgingEntries.deletedAt, }) @@ -422,7 +544,7 @@ async function loadBoard( entryId: judgingEvaluations.entryId, challengeId: judgingEvaluations.challengeId, appointmentId: judgingEvaluations.appointmentId, - isComplete: judgingEvaluations.isComplete, + ratingsComplete: holdsEveryRating(hackathonId), }) .from(judgingEvaluations) .where(eq(judgingEvaluations.hackathonId, hackathonId)) @@ -445,18 +567,15 @@ async function loadBoard( rootOf(inventory.challenges, c.id)?.id ?? null, ]), ); + const rootIdOf = (id: string) => roots.get(id) ?? null; return { schedule, inventory, plan: selectPlan(inventory), names: namesOf(inventory), - appointments: withStatus( - booked, - evaluations, - presence, - (id) => roots.get(id) ?? null, - now, - ), + evaluations, + rootIdOf, + appointments: withStatus(booked, evaluations, presence, rootIdOf, now), staffed: staffedRoomIds(presence, now), }; } @@ -464,6 +583,186 @@ async function loadBoard( type Board = Awaited>; type SavedSchedule = NonNullable; +/** + * The board's judges: who can be given presentations, and the saved + * appointments the way the assigner takes them. An appointment is open to a + * new judge while it has not started and nobody has begun an evaluation. + */ +async function loadJudging(db: JudgingDb, board: Board, schedule: SavedSchedule) { + const hackathonId = schedule.hackathonId; + const [roster, drafts] = await Promise.all([ + readRoster(db, hackathonId, board.inventory), + db + .select({ + judgeId: judgingEvaluationDrafts.judgeId, + entryId: judgingEvaluationDrafts.entryId, + challengeId: judgingEvaluationDrafts.challengeId, + appointmentId: judgingEvaluationDrafts.appointmentId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + }) + .from(judgingEvaluationDrafts) + .where(eq(judgingEvaluationDrafts.hackathonId, hackathonId)), + ]); + + const timing = timingOf(schedule); + const planned = new Set( + board.plan.tasks.map((t) => taskKey(t.entryId, t.challengeId)), + ); + const rootOfChallenge = (id: string) => + rootOf(board.inventory.challenges, id)?.id ?? null; + const started = (a: Board["appointments"][number]) => + drafts.some( + (d) => + d.entryId === a.entryId && + (d.appointmentId + ? d.appointmentId === a.id + : rootOfChallenge(d.challengeId) === a.challengeId), + ); + + const saved: SavedAppointment[] = board.appointments.flatMap((a) => { + // Saved appointments sit on the grid; one that does not has no slot to compare. + const slot = slotAt(timing, a.startsAt); + if (slot === null) return []; + return [ + { + id: a.id, + entryId: a.entryId, + challengeId: a.challengeId, + roomId: a.roomId, + slot, + assignedJudgeId: a.assignedJudgeId, + removed: + !!a.entryDeletedAt || !planned.has(taskKey(a.entryId, a.challengeId)), + open: a.status === "future" && !started(a), + }, + ]; + }); + + return { + roster, + saved, + drafts, + plan: planOfSaved(board, timing, saved, roster), + timing, + }; +} + +/** The saved appointments as the judge assigner takes them. */ +function planOfSaved( + board: Board, + timing: ScheduleTiming, + saved: readonly SavedAppointment[], + roster: Roster, +) { + return judgePlanOf({ + rooms: board.inventory.rooms, + sponsorChallengeIds: new Set( + board.inventory.challenges.filter((c) => !c.isGeneral).map((c) => c.id), + ), + grid: gridOf(timing), + appointments: saved, + judgeIds: new Set(roster.judges.map((j) => j.id)), + }); +} + +type Judging = Awaited>; + +/** What the board shows about judges: loads, and where there are too few. */ +function describeBoardJudging(judging: Judging, board: Board, now: Date) { + const { roster, saved, plan, timing } = judging; + const live = saved.filter((a) => !a.removed); + const rooms = new Map(board.inventory.rooms.map((r) => [r.id, r.name])); + + const loads = new Map(); + for (const a of live) { + if (a.assignedJudgeId) { + loads.set(a.assignedJudgeId, (loads.get(a.assignedJudgeId) ?? 0) + 1); + } + } + const available = new Map(roster.judges.map((j) => [j.id, j])); + const listed = new Set([...available.keys(), ...loads.keys()]); + + const from = firstSlotFrom(timing, now); + return { + judges: roster.judges.length, + assigned: live.filter((a) => a.assignedJudgeId).length, + // Still waiting for a judge: ones that have started or are over cannot + // be given one. + unassigned: live.filter((a) => a.open && !a.assignedJudgeId).length, + loads: [...listed] + .map((id) => { + const judge = available.get(id); + return { + judgeId: id, + name: roster.names.get(id) ?? "Removed judge", + kind: judge?.kind ?? null, + roomName: judge?.roomId ? (rooms.get(judge.roomId) ?? null) : null, + // Cannot take presentations: a guest with no room or table. + available: !!judge, + load: loads.get(id) ?? 0, + }; + }) + .sort( + (a, b) => + (a.kind === "guest" ? 1 : 0) - (b.kind === "guest" ? 1 : 0) || + a.name.localeCompare(b.name), + ), + warnings: judgeShortfalls( + plan.inputs, + plan.placements, + roster.judges, + plan.current, + ) + .filter((w) => w.slot >= from) + .map((w) => { + const challengeLabel = board.names.label(w.challengeId); + return { + slot: w.slot, + startsAt: slotStartsAt(timing, w.slot), + challengeId: w.challengeId, + challengeLabel, + presentations: w.presentations, + judgesFree: w.judgesFree, + message: describeShortfall(w, challengeLabel), + }; + }), + }; +} + +/** + * Who should grade the open presentations: CP-SAT when this server runs it + * and its answer passes the checks, the in-app assigner otherwise. + */ +async function makeAssignments( + judging: Judging, + options: { seconds: number; signal?: AbortSignal }, +) { + const { plan, roster } = judging; + if (cpsatAvailable()) { + const run = await runCpsat( + cpsatAssignmentProblem(plan.inputs, plan.placements, roster.judges, plan.keep), + options, + ); + if (run.status === "solved") { + const read = readCpsatAssignments( + plan.inputs, + plan.placements, + roster.judges, + plan.keep, + run.solution, + ); + if (read.ok) { + return { assignments: read.assignments, solver: "cp-sat" as const }; + } + } + } + const made = assignJudges(plan.inputs, plan.placements, roster.judges, { + keep: plan.keep, + }); + return { assignments: made.assignment, solver: "in-app" as const }; +} + /** * Every free future slot `task` can take, best first. Only rooms with a * judge seen in the last 15 minutes are offered: a slot in an empty room is @@ -562,6 +861,12 @@ const noSchedule = () => message: "No schedule is saved for this hackathon. Preview one and save it first.", }); +const noJudgeNeeded = () => + new TRPCError({ + code: "PRECONDITION_FAILED", + message: "That project is no longer entered, so it needs no judge.", + }); + const scheduleExists = () => new TRPCError({ code: "CONFLICT", @@ -638,7 +943,7 @@ export const judgingScheduleRouter = createTRPCRouter({ // Solved before the transaction so no lock is held while it computes. const inventory = await readInventory(db, input.hackathonId); - const { result, key, fingerprint } = solveInventory( + const { inputs, result, key, fingerprint } = solveInventory( inventory, input.timing, ); @@ -662,10 +967,15 @@ export const judgingScheduleRouter = createTRPCRouter({ }); } + // Judges first come with the schedule, as far as there are any yet; + // assignJudges picks up the rest once more have joined. + const roster = await readRoster(db, input.hackathonId, inventory); const written = await writeSchedule(db, ctx.userId, input, { placements: result.placements, relaxed: result.relaxed, fingerprint, + assignments: assignJudges(inputs, result.placements, roster.judges) + .assignment, }); const saved = { ...written, @@ -736,10 +1046,12 @@ export const judgingScheduleRouter = createTRPCRouter({ const inventory = await readInventory(db, input.hackathonId); const { inputs, result, fingerprint } = solveInventory(inventory, input.timing); + const roster = await readRoster(db, input.hackathonId, inventory); const problem = cpsatProblem( inputs, fingerprint, result.ok ? result.placements : undefined, + roster.judges, ); return { fileName: "problem.json", @@ -747,6 +1059,7 @@ export const judgingScheduleRouter = createTRPCRouter({ command: CPSAT_COMMAND, presentations: inputs.tasks.length, hinted: result.ok, + judges: roster.judges.length, }; }), @@ -766,12 +1079,19 @@ export const judgingScheduleRouter = createTRPCRouter({ if (await findSchedule(db, input.hackathonId)) throw scheduleExists(); const inventory = await readInventory(db, input.hackathonId); - const { inputs, read, key } = readUpload(inventory, input.timing, input.solution); + const roster = await readRoster(db, input.hackathonId, inventory); + const { inputs, read, key } = readUpload( + inventory, + input.timing, + input.solution, + roster.judges, + ); return { ...describePlan(read, inputs, namesOf(inventory), input.timing), candidateKey: key, // CP-SAT's own word: "optimal" means it proved nothing beats this. solverStatus: read.status, + judging: describeJudging(read, roster.judges, inputs.tasks.length), }; }), @@ -792,10 +1112,12 @@ export const judgingScheduleRouter = createTRPCRouter({ assertWindowAhead(input.timing); const inventory = await readInventory(db, input.hackathonId); + const roster = await readRoster(db, input.hackathonId, inventory); const { read, key, fingerprint } = readUpload( inventory, input.timing, input.solution, + roster.judges, ); if (key !== input.candidateKey) { throw new TRPCError({ @@ -815,12 +1137,14 @@ export const judgingScheduleRouter = createTRPCRouter({ placements: read.placements, relaxed: read.relaxed, fingerprint, + assignments: read.judging.assignments, }); const saved = { ...written, relaxed: read.relaxed, reducedBreaks: read.reducedBreaks.length, solver: "cp-sat", + judges: read.judging.source, }; await recordAdminAction(db, { @@ -873,8 +1197,14 @@ export const judgingScheduleRouter = createTRPCRouter({ const inventory = await readInventory(db, input.hackathonId); const { inputs, result, fingerprint } = solveInventory(inventory, input.timing); + const roster = await readRoster(db, input.hackathonId, inventory); const run = await runCpsat( - cpsatProblem(inputs, fingerprint, result.ok ? result.placements : undefined), + cpsatProblem( + inputs, + fingerprint, + result.ok ? result.placements : undefined, + roster.judges, + ), { seconds: input.seconds, signal }, ); if (run.status === "failed") { @@ -889,11 +1219,17 @@ export const judgingScheduleRouter = createTRPCRouter({ } // A proof of infeasibility is refused here with its reasons; a schedule // has to pass the same checks as an upload. - const { read, key } = readUpload(inventory, input.timing, run.solution); + const { read, key } = readUpload( + inventory, + input.timing, + run.solution, + roster.judges, + ); return { ...describePlan(read, inputs, namesOf(inventory), input.timing), candidateKey: key, solverStatus: read.status, + judging: describeJudging(read, roster.judges, inputs.tasks.length), solution: run.solution, }; }), @@ -914,12 +1250,31 @@ export const judgingScheduleRouter = createTRPCRouter({ await reconcileDueDrafts(db, input.hackathonId, now); const board = await loadBoard(db, input.hackathonId, now); const { schedule, names } = board; + const judging = schedule ? await loadJudging(db, board, schedule) : null; const planned = new Set(board.plan.tasks.map((t) => taskKey(t.entryId, t.challengeId))); const booked = new Set( board.appointments.map((a) => taskKey(a.entryId, a.challengeId)), ); const scheduledRooms = new Set(board.plan.rooms.map((r) => r.id)); + // Its judge can be chosen anew: it has not started and nobody has + // begun on it. + const assignable = new Set( + judging?.saved.filter((s) => s.open && !s.removed).map((s) => s.id), + ); + // Deleted, or no longer in this challenge, since the schedule was saved. + const isRemoved = (a: Board["appointments"][number]) => + !!a.entryDeletedAt || !planned.has(taskKey(a.entryId, a.challengeId)); + // Where each assigned judge has got on a presentation that has started. + const progress = judging + ? judgeProgressById( + board.appointments.filter((a) => !isRemoved(a)), + board.evaluations, + judging.drafts, + board.rootIdOf, + now, + ) + : new Map(); return { serverNow: now, @@ -952,9 +1307,7 @@ export const judgingScheduleRouter = createTRPCRouter({ id: a.id, entryId: a.entryId, entryTitle: a.entryTitle, - // Deleted, or no longer in this challenge, since the schedule was saved. - removed: - !!a.entryDeletedAt || !planned.has(taskKey(a.entryId, a.challengeId)), + removed: isRemoved(a), challengeId: a.challengeId, challengeLabel: names.label(a.challengeId), roomId: a.roomId, @@ -966,7 +1319,14 @@ export const judgingScheduleRouter = createTRPCRouter({ status: a.status, canMove: a.canMove, evaluationCount: a.evaluationCount, + assignedJudgeId: a.assignedJudgeId, + assignedJudgeName: a.assignedJudgeId + ? (judging?.roster.names.get(a.assignedJudgeId) ?? "Removed judge") + : null, + canAssignJudge: assignable.has(a.id), + judgeProgress: progress.get(a.id) ?? null, })), + judging: judging ? describeBoardJudging(judging, board, now) : null, unscheduled: schedule ? board.plan.tasks .filter((t) => !booked.has(taskKey(t.entryId, t.challengeId))) @@ -1074,6 +1434,31 @@ export const judgingScheduleRouter = createTRPCRouter({ input, timingOf(board.schedule), ); + + // Its judge stays only if they may grade in the new room and are free + // at the new time; otherwise the organiser assigns judges again. + let clearsJudge = false; + if (appointment.assignedJudgeId) { + const judging = await loadJudging(tx, board, board.schedule); + const judge = judging.roster.judges.find( + (j) => j.id === appointment.assignedJudgeId, + ); + const slot = slotAt(judging.timing, choice.startsAt); + clearsJudge = + !judge || + slot === null || + !mayGrade(judge, { + challengeId: appointment.challengeId, + roomId: choice.roomId, + }) || + judging.saved.some( + (other) => + other.id !== appointment.id && + !other.removed && + other.assignedJudgeId === judge.id && + other.slot === slot, + ); + } const [updated] = await tx .update(judgingAppointments) .set({ @@ -1082,6 +1467,7 @@ export const judgingScheduleRouter = createTRPCRouter({ deadlineAt: choice.deadlineAt, endsAt: choice.endsAt, revision: sql`${judgingAppointments.revision} + 1`, + ...(clearsJudge ? { assignedJudgeId: null } : {}), }) .where( and( @@ -1100,6 +1486,7 @@ export const judgingScheduleRouter = createTRPCRouter({ toRoomId: choice.roomId, toStartsAt: choice.startsAt, reducedBreaks: choice.reducedBreaks, + clearedJudgeId: clearsJudge ? appointment.assignedJudgeId : null, }; }); @@ -1113,6 +1500,264 @@ export const judgingScheduleRouter = createTRPCRouter({ return moved; }), + /** + * Who could grade an appointment instead: the judges who may grade its + * challenge in its room, least loaded first, with those already grading + * something else at that time marked. + */ + judgeChoices: isAdmin + .input(z.object({ hackathonId: uuid, appointmentId: uuid })) + .query(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const now = new Date(); + const board = await loadBoard(db, input.hackathonId, now); + const appointment = board.appointments.find( + (a) => a.id === input.appointmentId, + ); + if (!board.schedule || !appointment) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "That appointment no longer exists. Reload the board.", + }); + } + const judging = await loadJudging(db, board, board.schedule); + const saved = judging.saved.find((s) => s.id === appointment.id); + if (!saved || saved.removed) throw noJudgeNeeded(); + + const titles = new Map(board.appointments.map((a) => [a.id, a.entryTitle])); + const loads = new Map(); + for (const s of judging.saved) { + if (s.assignedJudgeId && !s.removed) { + loads.set(s.assignedJudgeId, (loads.get(s.assignedJudgeId) ?? 0) + 1); + } + } + const kinds = new Map(judging.roster.judges.map((j) => [j.id, j.kind])); + return { + appointment: { + id: appointment.id, + entryTitle: appointment.entryTitle, + challengeLabel: board.names.label(appointment.challengeId), + ...board.names.room(appointment.roomId), + startsAt: appointment.startsAt, + assignedJudgeId: appointment.assignedJudgeId, + canAssignJudge: saved.open, + }, + choices: judgeChoices(judging.roster, saved, judging.saved) + .map((choice) => ({ + judgeId: choice.judgeId, + name: judging.roster.names.get(choice.judgeId) ?? "Removed judge", + kind: kinds.get(choice.judgeId)!, + load: loads.get(choice.judgeId) ?? 0, + bookedWith: choice.bookedWith ? (titles.get(choice.bookedWith) ?? null) : null, + booked: choice.bookedWith !== null, + })) + .sort( + (a, b) => + Number(a.booked) - Number(b.booked) || + a.load - b.load || + a.name.localeCompare(b.name), + ), + }; + }), + + /** + * Sets, or with null clears, the judge of one appointment, until it starts. + * The judge has to be one who may grade it there and is not already + * grading something else at that time. + */ + setAssignedJudge: isAdmin + .input( + z.object({ + hackathonId: uuid, + appointmentId: uuid, + judgeId: uuid.nullable(), + }), + ) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + + const changed = await db.transaction(async (tx) => { + const now = new Date(); + const board = await loadBoard(tx, input.hackathonId, now, true); + const appointment = board.appointments.find( + (a) => a.id === input.appointmentId, + ); + if (!board.schedule || !appointment) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "That appointment no longer exists. Reload the board.", + }); + } + const judging = await loadJudging(tx, board, board.schedule); + const saved = judging.saved.find((s) => s.id === appointment.id); + if (!saved || saved.removed) throw noJudgeNeeded(); + if (!saved.open) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: + "This presentation has started, or a judge has begun on it, so its judge stays.", + }); + } + + if (input.judgeId) { + const judge = judging.roster.judges.find((j) => j.id === input.judgeId); + if (!judge) { + throw new TRPCError({ + code: "NOT_FOUND", + message: + "That judge cannot be assigned. A guest judge needs a room or table of their own.", + }); + } + const name = judging.roster.names.get(judge.id) ?? "That judge"; + if (!mayGrade(judge, saved)) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `${name} cannot grade ${board.names.label(saved.challengeId)} at ${board.names.room(saved.roomId).roomName}.`, + }); + } + const booked = judgeChoices(judging.roster, saved, judging.saved).find( + (c) => c.judgeId === judge.id, + )?.bookedWith; + if (booked) { + throw new TRPCError({ + code: "CONFLICT", + message: `${name} is already grading ${board.appointments.find((a) => a.id === booked)?.entryTitle ?? "another project"} at that time.`, + }); + } + } + + await tx + .update(judgingAppointments) + .set({ assignedJudgeId: input.judgeId }) + .where(eq(judgingAppointments.id, appointment.id)); + return { + appointmentId: appointment.id, + fromJudgeId: appointment.assignedJudgeId, + toJudgeId: input.judgeId, + }; + }); + + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.schedule.setJudge", + resourceId: changed.appointmentId, + metadata: { hackathonId: input.hackathonId, ...changed }, + }); + + return changed; + }), + + /** + * Gives every appointment that has not started, and that no judge has + * begun on, a judge again. The others keep theirs. CP-SAT does it when + * this server runs it, otherwise the in-app assigner. Warns of each slot + * where there are too few judges. + */ + assignJudges: isAdmin + .input( + z.object({ + hackathonId: uuid, + seconds: z + .number() + .int() + .min(CPSAT_SECONDS.min) + .max(CPSAT_SECONDS.max) + .default(CPSAT_SECONDS.default), + }), + ) + .mutation(async ({ ctx, input, signal }) => { + const db = ctx.db as DrizzleDB; + await assertHackathonExists(db, input.hackathonId); + + // Worked out before the transaction, so no lock is held while CP-SAT + // runs, and checked against the board again inside it. + const first = await loadBoard(db, input.hackathonId, new Date()); + if (!first.schedule) throw noSchedule(); + const before = await loadJudging(db, first, first.schedule); + if (before.roster.judges.length === 0) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: + "No judge can be assigned yet. Judges appear here once they have joined a room or table.", + }); + } + const made = await makeAssignments(before, { + seconds: input.seconds, + signal, + }); + + const result = await db.transaction(async (tx) => { + const now = new Date(); + const board = await loadBoard(tx, input.hackathonId, now, true); + if (!board.schedule) throw noSchedule(); + const judging = await loadJudging(tx, board, board.schedule); + const stale = (a: Judging) => + JSON.stringify([a.saved, a.roster.judges]); + if (stale(judging) !== stale(before)) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Appointments or judges changed while judges were being assigned. Assign judges again.", + }); + } + + const changes = judgeChanges(judging.saved, made.assignments); + // One update per judge, however many appointments they gain. + const byJudge = new Map(); + for (const change of changes) { + const list = byJudge.get(change.assignedJudgeId); + if (list) list.push(change.id); + else byJudge.set(change.assignedJudgeId, [change.id]); + } + for (const [judgeId, ids] of byJudge) { + for (let i = 0; i < ids.length; i += 1000) { + await tx + .update(judgingAppointments) + .set({ assignedJudgeId: judgeId }) + .where(inArray(judgingAppointments.id, ids.slice(i, i + 1000))); + } + } + + const chosen = new Map(changes.map((c) => [c.id, c.assignedJudgeId])); + const saved = judging.saved.map((s) => + chosen.has(s.id) ? { ...s, assignedJudgeId: chosen.get(s.id)! } : s, + ); + const summary = describeBoardJudging( + { + ...judging, + saved, + plan: planOfSaved(board, judging.timing, saved, judging.roster), + }, + board, + now, + ); + return { + solver: made.solver, + changed: changes.length, + kept: judging.plan.keep.length, + assigned: summary.assigned, + unassigned: summary.unassigned, + warnings: summary.warnings, + }; + }); + + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.schedule.assignJudges", + resourceId: input.hackathonId, + metadata: { + solver: result.solver, + changed: result.changed, + kept: result.kept, + assigned: result.assigned, + unassigned: result.unassigned, + warnings: result.warnings.length, + }, + }); + + return result; + }), + /** Where a presentation with no appointment yet could go, best first. */ assignChoices: isAdmin .input(z.object({ hackathonId: uuid, entryId: uuid, challengeId: uuid })) diff --git a/packages/api/src/routers/points/me.ts b/packages/api/src/routers/points/me.ts index 1a688067..2e665a7a 100644 --- a/packages/api/src/routers/points/me.ts +++ b/packages/api/src/routers/points/me.ts @@ -9,7 +9,9 @@ import { } from "@query/db"; import type { DrizzleDB } from "@query/db"; import { createTRPCRouter, protectedProcedure } from "../../trpc"; +import { notHackathonBanned } from "../../middleware/procedures"; import { rateLimit } from "../../middleware/security"; +import { flaggingRuleBreaks } from "../../services/check-in-flags"; import { isSoldOut, leaderboardName, @@ -213,8 +215,11 @@ export const pointsMeRouter = createTRPCRouter({ }), // Self check-in with the code shown at the event. Every refusal says what - // to do instead, since the hacker is standing in a doorway reading it. - checkIn: protectedProcedure + // to do instead, since the hacker is standing in a doorway reading it. Behind + // the ban gate because it writes: a banned hacker who kept their + // registration could otherwise keep collecting points. The reads above stay + // open, as with the rest of the API. + checkIn: notHackathonBanned .input(scope.extend({ code: z.string().max(32) })) .mutation(async ({ ctx, input }) => { // There are 31^6 codes; this keeps anyone from walking the space. Per @@ -263,6 +268,7 @@ export const pointsMeRouter = createTRPCRouter({ columns: { id: true, name: true, + type: true, points: true, startTime: true, endTime: true, @@ -299,12 +305,20 @@ export const pointsMeRouter = createTRPCRouter({ }); } - const { pointsAwarded } = await recordEventAttendance(db, { - event, - participant: { id: participant.id, name: "You" }, - method: "code", - actorId: ctx.userId, - }); + // A refusal for the check-in rules is recorded for organisers, on the + // pool and outside the transaction it rolled back. The hacker sees only + // the refusal. + const { pointsAwarded } = await flaggingRuleBreaks( + db, + input.hackathonId, + () => + recordEventAttendance(db, { + event, + participant: { id: participant.id, name: "You" }, + method: "code", + actorId: ctx.userId, + }), + ); // The public schedule shows how many people came. ctx.cache.delete(`hackathon:${input.hackathonId}:events`); diff --git a/packages/api/src/services/check-in-flags.test.ts b/packages/api/src/services/check-in-flags.test.ts new file mode 100644 index 00000000..df1afb34 --- /dev/null +++ b/packages/api/src/services/check-in-flags.test.ts @@ -0,0 +1,817 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TRPCError } from "@trpc/server"; +import { PgDialect, getTableConfig } from "drizzle-orm/pg-core"; +import { + hackathonCheckInFlags, + hackathonEvents, + judgingEntries, + judgingEntryMembers, +} from "@query/db"; +import type { DrizzleDB } from "@query/db"; +import { + checkInFlagsForEntries, + entriesForFlags, + flaggedEntryIds, + flaggingRuleBreaks, + openCheckInFlagCount, + recordCheckInFlag, +} from "./check-in-flags"; +import { CheckInRuleBreak } from "./points"; +import type { CheckInRuleBreakDetails } from "./points"; + +const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; +const AT = new Date("2027-02-06T18:00:00Z"); + +const breach: CheckInRuleBreakDetails = { + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + actorId: "user-1", + at: AT, +}; + +const render = (condition: unknown) => + new PgDialect().sqlToQuery(condition as never); + +// --------------------------------------------------------------------------- +// The table + +describe("hackathonCheckInFlags", () => { + const foreignKey = (name: string) => + getTableConfig(hackathonCheckInFlags).foreignKeys.find( + (key) => key.getName() === name, + )!; + + // Deleting an event must not delete the record that someone broke the rules + // at it: prize review reads these long after a session is cancelled. + it.each([ + ["event_id", "check_in_flag_event_fk", hackathonCheckInFlags.eventId], + [ + "conflict_event_id", + "check_in_flag_conflict_event_fk", + hackathonCheckInFlags.conflictEventId, + ], + ])( + "keeps the flag and clears %s when that event is deleted", + (_, name, column) => { + const key = foreignKey(name); + expect(key.reference().foreignTable).toBe(hackathonEvents); + expect(key.onDelete).toBe("set null"); + expect(column.notNull).toBe(false); + }, + ); + + it("still goes with its participant", () => { + const key = foreignKey("check_in_flag_participant_fk"); + expect(key.onDelete).toBe("cascade"); + expect(hackathonCheckInFlags.participantId.notNull).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Writing + +describe("recordCheckInFlag", () => { + const fakeInsertDb = () => { + const writes: { + table: unknown; + values: Record; + conflict: { target: unknown[]; set: Record }; + }[] = []; + const db = { + insert: (table: unknown) => ({ + values: (values: Record) => ({ + onConflictDoUpdate: async (conflict: { + target: unknown[]; + set: Record; + }) => { + writes.push({ table, values, conflict }); + }, + }), + }), + }; + return { db: db as unknown as DrizzleDB, writes }; + }; + + it("inserts one row for the person, the event tried and the event they were in", async () => { + const { db, writes } = fakeInsertDb(); + await recordCheckInFlag(db, HACK, breach); + + expect(writes).toHaveLength(1); + expect(writes[0]!.table).toBe(hackathonCheckInFlags); + // Both timestamps come from the clock the rule was judged on, not from the + // database's now(). + expect(writes[0]!.values).toEqual({ + hackathonId: HACK, + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attemptedById: "user-1", + attempts: 1, + createdAt: AT, + lastAttemptAt: AT, + }); + }); + + it("counts a repeat instead of adding a row, and leaves the dismissal alone", async () => { + const { db, writes } = fakeInsertDb(); + await recordCheckInFlag(db, HACK, breach); + + const { target, set } = writes[0]!.conflict; + expect(target).toEqual([ + hackathonCheckInFlags.participantId, + hackathonCheckInFlags.eventId, + hackathonCheckInFlags.conflictEventId, + ]); + expect(Object.keys(set).sort()).toEqual(["attempts", "lastAttemptAt"]); + expect(set.lastAttemptAt).toBe(AT); + expect(render(set.attempts).sql).toBe( + '"hackathon_check_in_flag"."attempts" + 1', + ); + }); + + it("keeps the staff member as the one who tried when there is no hacker", async () => { + const { db, writes } = fakeInsertDb(); + await recordCheckInFlag(db, HACK, { + ...breach, + method: "scan", + rule: "too_soon", + actorId: null, + }); + expect(writes[0]!.values).toMatchObject({ + method: "scan", + rule: "too_soon", + attemptedById: null, + }); + }); +}); + +describe("flaggingRuleBreaks", () => { + afterEach(() => vi.restoreAllMocks()); + + const refusal = () => + new TRPCError({ + code: "FORBIDDEN", + message: "You're already checked into BNY Workshop.", + cause: new CheckInRuleBreak(breach), + }); + + // A pool whose insert chain ends in `onConflictDoUpdate`, noting each flag. + const pool = (opts: { failWith?: Error } = {}) => { + const flags: unknown[] = []; + const db = { + insert: () => ({ + values: (values: unknown) => ({ + onConflictDoUpdate: async () => { + if (opts.failWith) throw opts.failWith; + flags.push(values); + }, + }), + }), + }; + return { db: db as unknown as DrizzleDB, flags }; + }; + + it("returns what the write returned and records nothing", async () => { + const { db, flags } = pool(); + await expect( + flaggingRuleBreaks(db, HACK, async () => ({ pointsAwarded: 5 })), + ).resolves.toEqual({ pointsAwarded: 5 }); + expect(flags).toEqual([]); + }); + + it("records a refusal for the check-in rules and rethrows the same error", async () => { + const { db, flags } = pool(); + const error = refusal(); + + const caught = await flaggingRuleBreaks(db, HACK, async () => { + throw error; + }).catch((e: unknown) => e); + + expect(caught).toBe(error); + expect(flags).toEqual([ + expect.objectContaining({ + hackathonId: HACK, + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attemptedById: "user-1", + }), + ]); + }); + + it("ignores every other failure", async () => { + for (const error of [ + new Error("connection reset"), + new TRPCError({ code: "CONFLICT", message: "Already checked in." }), + new TRPCError({ + code: "FORBIDDEN", + message: "Not accepted.", + cause: new Error("something else"), + }), + // A cause shaped like a rule break is not one. + new TRPCError({ + code: "FORBIDDEN", + message: "Lookalike.", + cause: Object.assign(new Error("lookalike"), { ...breach }), + }), + ]) { + const { db, flags } = pool(); + const caught = await flaggingRuleBreaks(db, HACK, async () => { + throw error; + }).catch((e: unknown) => e); + expect(caught).toBe(error); + expect(flags).toEqual([]); + } + }); + + it("still throws the original refusal when the flag cannot be written", async () => { + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const { db } = pool({ failWith: new Error("database is down") }); + const error = refusal(); + + const caught = await flaggingRuleBreaks(db, HACK, async () => { + throw error; + }).catch((e: unknown) => e); + + // The person at the door reads the same sentence either way. + expect(caught).toBe(error); + expect((caught as TRPCError).code).toBe("FORBIDDEN"); + expect((caught as TRPCError).message).toBe( + "You're already checked into BNY Workshop.", + ); + expect(consoleError).toHaveBeenCalledTimes(1); + }); + + it("is not thrown off when the pool itself is unusable", async () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const db = { + insert: () => { + throw new Error("pool is closed"); + }, + } as unknown as DrizzleDB; + const error = refusal(); + + await expect( + flaggingRuleBreaks(db, HACK, async () => { + throw error; + }), + ).rejects.toBe(error); + }); +}); + +// --------------------------------------------------------------------------- +// Reading + +describe("entriesForFlags", () => { + const flag = ( + id: string, + over: Partial<{ + participantId: string; + teamId: string | null; + userId: string; + }> = {}, + ) => ({ + id, + participantId: `part-${id}`, + teamId: null as string | null, + userId: `user-${id}`, + ...over, + }); + + it("puts a hacker's flag on their team's portal entry", () => { + const result = entriesForFlags( + [flag("f1", { teamId: "team-1" })], + [ + { entryId: "e-team", teamId: "team-1", submittedById: "part-cap" }, + { entryId: "e-other", teamId: "team-2", submittedById: "part-x" }, + ], + [], + ); + expect(result.get("f1")).toEqual(new Set(["e-team"])); + }); + + it("matches a solo project by who filed it", () => { + const result = entriesForFlags( + [flag("f1")], + [ + { entryId: "e-solo", teamId: null, submittedById: "part-f1" }, + { entryId: "e-not-me", teamId: null, submittedById: "part-other" }, + ], + [], + ); + expect(result.get("f1")).toEqual(new Set(["e-solo"])); + }); + + it("does not treat a former captain as a member of the team's project", () => { + // They filed it, then moved to team-2: only the team link counts for a + // project that has a team. + const result = entriesForFlags( + [flag("f1", { teamId: "team-2" })], + [{ entryId: "e-team-1", teamId: "team-1", submittedById: "part-f1" }], + [], + ); + expect(result.get("f1")).toEqual(new Set()); + }); + + it("does not match teamless hackers to teamless projects by their missing team", () => { + const result = entriesForFlags( + [flag("f1", { teamId: null })], + [{ entryId: "e-solo", teamId: null, submittedById: "part-other" }], + [], + ); + expect(result.get("f1")).toEqual(new Set()); + }); + + it("matches a member by account, which covers Devpost entries and claimed teammates", () => { + const result = entriesForFlags( + [flag("f1")], + [], + [ + { entryId: "e-devpost", userId: "user-f1" }, + { entryId: "e-unclaimed", userId: null }, + { entryId: "e-else", userId: "user-else" }, + ], + ); + expect(result.get("f1")).toEqual(new Set(["e-devpost"])); + }); + + it("counts an entry once however many ways the hacker reaches it", () => { + const result = entriesForFlags( + [flag("f1", { teamId: "team-1" })], + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + [ + { entryId: "e1", userId: "user-f1" }, + { entryId: "e2", userId: "user-f1" }, + ], + ); + expect(result.get("f1")).toEqual(new Set(["e1", "e2"])); + }); + + it("gives every flag an answer, empty when it is on nobody's entry", () => { + const result = entriesForFlags( + [flag("f1"), flag("f2", { teamId: "team-1" })], + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + [], + ); + expect(result.get("f1")).toEqual(new Set()); + expect(result.get("f2")).toEqual(new Set(["e1"])); + }); +}); + +describe("reading flags", () => { + // A db that answers by the table a query reads from, and notes each query so + // a test can say how many there were and what they asked for. It does not + // evaluate any SQL, so the conditions are checked by rendering them. + const fakeReadDb = (rows: Map) => { + const calls: { + table: unknown; + fields: Record; + where?: unknown; + }[] = []; + const db = { + select: (fields: Record) => ({ + from: (table: unknown) => { + const call: (typeof calls)[number] = { table, fields }; + calls.push(call); + // Chainable and awaitable, like a drizzle query. + type Query = { + innerJoin: () => Query; + where: (condition: unknown) => Query; + then: ( + resolve: (value: unknown) => unknown, + reject: (reason: unknown) => unknown, + ) => Promise; + }; + const node: Query = { + innerJoin: () => node, + where: (condition) => { + call.where = condition; + return node; + }, + then: (resolve, reject) => + Promise.resolve(rows.get(table) ?? []).then(resolve, reject), + }; + return node; + }, + }), + }; + return { db: db as unknown as DrizzleDB, calls }; + }; + + const flagRow = (id: string, over: Record = {}) => ({ + id, + participantId: `part-${id}`, + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attempts: 1, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T18:00:00Z"), + dismissedAt: null, + teamId: null, + userId: `user-${id}`, + firstName: "Ada", + lastName: "Lovelace", + accountName: "ada l", + email: "ada@example.com", + ...over, + }); + + const events = [ + { id: "event-1", name: "BNY Workshop" }, + { id: "event-2", name: "Full Sail Fireside" }, + ]; + + const table = (...entries: [unknown, unknown[]][]) => new Map(entries); + + describe("checkInFlagsForEntries", () => { + it("asks nothing for an empty list of entries", async () => { + const { db, calls } = fakeReadDb(table()); + const result = await checkInFlagsForEntries(db, HACK, []); + expect(result.size).toBe(0); + expect(calls).toEqual([]); + }); + + it("stops after one read when the edition has no flags", async () => { + const { db, calls } = fakeReadDb(table()); + const result = await checkInFlagsForEntries(db, HACK, null); + expect(result.size).toBe(0); + expect(calls).toHaveLength(1); + }); + + it("describes each flag with names for organisers", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("f1", { + teamId: "team-1", + rule: "too_soon", + method: "scan", + attempts: 3, + }), + ], + ], + [hackathonEvents, events], + [ + judgingEntries, + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + ], + ), + ); + + const result = await checkInFlagsForEntries(db, HACK, ["e1"]); + + expect(result.get("e1")).toEqual([ + { + id: "f1", + participantName: "Ada Lovelace", + rule: "too_soon", + eventName: "Full Sail Fireside", + conflictEventName: "BNY Workshop", + method: "scan", + attempts: 3, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T18:00:00Z"), + dismissed: false, + }, + ]); + }); + + it("still lists a flag whose event was deleted, under a stand-in name", async () => { + const { db, calls } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [flagRow("f1", { teamId: "team-1", eventId: null })], + ], + [hackathonEvents, events], + [ + judgingEntries, + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + ], + ), + ); + + const [flag] = (await checkInFlagsForEntries(db, HACK, ["e1"])).get( + "e1", + )!; + + expect(flag).toMatchObject({ + id: "f1", + eventName: "A deleted event", + conflictEventName: "BNY Workshop", + }); + // Only the events that still have an id are looked up. + const lookup = render( + calls.find((c) => c.table === hackathonEvents)!.where, + ); + expect(lookup.params).toEqual(["event-1"]); + }); + + it("still lists a flag whose conflicting event was deleted", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [flagRow("f1", { teamId: "team-1", conflictEventId: null })], + ], + [hackathonEvents, events], + [ + judgingEntries, + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + ], + ), + ); + + const [flag] = (await checkInFlagsForEntries(db, HACK, ["e1"])).get( + "e1", + )!; + + expect(flag).toMatchObject({ + eventName: "Full Sail Fireside", + conflictEventName: "A deleted event", + }); + }); + + it("still lists a flag when both events were deleted", async () => { + const { db, calls } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("f1", { + teamId: "team-1", + eventId: null, + conflictEventId: null, + }), + ], + ], + [ + judgingEntries, + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + ], + ), + ); + + const [flag] = (await checkInFlagsForEntries(db, HACK, ["e1"])).get( + "e1", + )!; + + expect(flag).toMatchObject({ + eventName: "A deleted event", + conflictEventName: "A deleted event", + dismissed: false, + }); + // With no event ids left there is nothing to look up. + expect(calls.some((c) => c.table === hackathonEvents)).toBe(false); + }); + + it("falls back to the account name, then the email, when the registration has none", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("f1", { firstName: null, lastName: null }), + flagRow("f2", { + firstName: null, + lastName: null, + accountName: null, + }), + ], + ], + [hackathonEvents, events], + [ + judgingEntryMembers, + [ + { entryId: "e1", userId: "user-f1" }, + { entryId: "e1", userId: "user-f2" }, + ], + ], + ), + ); + + const flags = (await checkInFlagsForEntries(db, HACK, null)).get("e1")!; + expect(flags.map((f) => f.participantName).sort()).toEqual([ + "ada l", + "ada@example.com", + ]); + }); + + it("lists open flags before cleared ones, newest attempt first", async () => { + const t = (minute: number) => new Date(Date.UTC(2027, 1, 6, 18, minute)); + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("old-open", { lastAttemptAt: t(1) }), + flagRow("cleared", { + lastAttemptAt: t(30), + dismissedAt: t(40), + }), + flagRow("new-open", { lastAttemptAt: t(20) }), + ], + ], + [hackathonEvents, events], + [ + judgingEntryMembers, + ["old-open", "cleared", "new-open"].map((id) => ({ + entryId: "e1", + userId: `user-${id}`, + })), + ], + ), + ); + + const flags = (await checkInFlagsForEntries(db, HACK, ["e1"])).get("e1")!; + expect(flags.map((f) => [f.id, f.dismissed])).toEqual([ + ["new-open", false], + ["old-open", false], + ["cleared", true], + ]); + }); + + it("keeps to the entries asked for, and to every entry when asked for null", async () => { + const rows = table( + [hackathonCheckInFlags, [flagRow("f1", { teamId: "team-1" })]], + [hackathonEvents, events], + [ + judgingEntries, + [ + { entryId: "e1", teamId: "team-1", submittedById: null }, + { entryId: "e2", teamId: "team-1", submittedById: null }, + ], + ], + ); + + const some = await checkInFlagsForEntries(fakeReadDb(rows).db, HACK, [ + "e2", + "e-unflagged", + ]); + expect([...some.keys()]).toEqual(["e2"]); + + const all = await checkInFlagsForEntries(fakeReadDb(rows).db, HACK, null); + expect([...all.keys()].sort()).toEqual(["e1", "e2"]); + // One flag, one description, wherever it shows. + expect(all.get("e1")![0]).toBe(all.get("e2")![0]); + }); + + it("leaves out a flag whose hacker is on no entry", async () => { + const { db } = fakeReadDb( + table( + [hackathonCheckInFlags, [flagRow("f1")]], + [hackathonEvents, events], + ), + ); + const result = await checkInFlagsForEntries(db, HACK, null); + expect(result.size).toBe(0); + }); + + it("reads in four batched queries however many flags there are", async () => { + const many = Array.from({ length: 40 }, (_, i) => + flagRow(`f${i}`, { teamId: `team-${i}` }), + ); + const { db, calls } = fakeReadDb( + table([hackathonCheckInFlags, many], [hackathonEvents, events]), + ); + + await checkInFlagsForEntries(db, HACK, null); + + expect(calls.map((c) => c.table)).toEqual([ + hackathonCheckInFlags, + hackathonEvents, + judgingEntries, + judgingEntryMembers, + ]); + }); + + it("looks for live entries of this edition only", async () => { + const { db, calls } = fakeReadDb( + table( + [hackathonCheckInFlags, [flagRow("f1", { teamId: "team-1" })]], + [hackathonEvents, events], + ), + ); + await checkInFlagsForEntries(db, HACK, null); + + const portal = render( + calls.find((c) => c.table === judgingEntries)!.where, + ); + const members = render( + calls.find((c) => c.table === judgingEntryMembers)!.where, + ); + for (const query of [portal, members]) { + expect(query.sql).toContain('"deleted_at" is null'); + expect(query.params).toContain(HACK); + } + expect(portal.params).toContain("portal"); + expect(portal.params).toContain("team-1"); + expect(members.params).toContain("user-f1"); + }); + }); + + describe("flaggedEntryIds", () => { + const target = (id: string, over: Record = {}) => ({ + id, + participantId: `part-${id}`, + teamId: null, + userId: `user-${id}`, + ...over, + }); + + it("asks nothing for an empty list of entries", async () => { + const { db, calls } = fakeReadDb(table()); + expect(await flaggedEntryIds(db, HACK, [])).toEqual(new Set()); + expect(calls).toEqual([]); + }); + + it("returns only the asked-for entries that carry a flag", async () => { + const { db } = fakeReadDb( + table( + [hackathonCheckInFlags, [target("f1", { teamId: "team-1" })]], + [ + judgingEntries, + [ + { entryId: "e1", teamId: "team-1", submittedById: null }, + { entryId: "e-not-asked", teamId: "team-1", submittedById: null }, + ], + ], + ), + ); + const flagged = await flaggedEntryIds(db, HACK, ["e1", "e2"]); + expect(flagged).toEqual(new Set(["e1"])); + }); + + it("reads open flags only, and never who or why", async () => { + const { db, calls } = fakeReadDb(table()); + await flaggedEntryIds(db, HACK, ["e1"]); + + const flags = calls.find((c) => c.table === hackathonCheckInFlags)!; + const where = render(flags.where); + expect(where.sql).toContain('"dismissed_at" is null'); + expect(where.params).toContain(HACK); + // Ids to find the team by, and nothing a judge could read. + expect(Object.keys(flags.fields).sort()).toEqual([ + "id", + "participantId", + "teamId", + "userId", + ]); + }); + }); + + describe("openCheckInFlagCount", () => { + it("is zero without reading entries when nothing is open", async () => { + const { db, calls } = fakeReadDb(table()); + expect(await openCheckInFlagCount(db, HACK)).toBe(0); + expect(calls).toHaveLength(1); + }); + + it("counts each open flag that sits on an entry once, and skips the rest", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + { id: "f1", participantId: "p1", teamId: "team-1", userId: "u1" }, + { id: "f2", participantId: "p2", teamId: null, userId: "u2" }, + { id: "f3", participantId: "p3", teamId: null, userId: "u3" }, + ], + ], + [ + judgingEntries, + [ + { entryId: "e1", teamId: "team-1", submittedById: null }, + { entryId: "e2", teamId: "team-1", submittedById: null }, + ], + ], + [judgingEntryMembers, [{ entryId: "e3", userId: "u2" }]], + ), + ); + // f1 is on two entries and counts once, f2 reaches one through its + // account, f3 is on nobody's. + expect(await openCheckInFlagCount(db, HACK)).toBe(2); + }); + + it("reads open flags of this edition only", async () => { + const { db, calls } = fakeReadDb(table()); + await openCheckInFlagCount(db, HACK); + const where = render(calls[0]!.where); + expect(where.sql).toContain('"dismissed_at" is null'); + expect(where.params).toContain(HACK); + }); + }); +}); diff --git a/packages/api/src/services/check-in-flags.ts b/packages/api/src/services/check-in-flags.ts new file mode 100644 index 00000000..32ee02a2 --- /dev/null +++ b/packages/api/src/services/check-in-flags.ts @@ -0,0 +1,413 @@ +import { TRPCError } from "@trpc/server"; +import { and, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { + hackathonCheckInFlags, + hackathonEvents, + hackathonParticipants, + hackathonProjects, + judgingEntries, + judgingEntryMembers, + users, +} from "@query/db"; +import type { DrizzleDB } from "@query/db"; +import type { JudgingDb } from "./judging/config"; +import { CheckInRuleBreak, fullName, personName } from "./points"; +import type { CheckInRuleBreakDetails } from "./points"; + +// Check-in rule breaks, as organisers and judges meet them. recordEventAttendance +// refuses an overlapping or too-soon check-in and the refusal rolls its +// transaction back, so the attempt is written here, on the pool, afterwards. +// A flag belongs to a person; judging shows it on the team they are on, found +// when it is read (see entriesForFlags) because people change teams and +// projects are promoted late. Hackers never read any of this, and nothing here +// may reach a cached or public payload. + +/** One refused attempt, as organisers read it. Judges get only an entry id. */ +export type CheckInFlagView = { + id: string; + participantName: string; + rule: "overlap" | "too_soon"; + /** The event they tried to enter, or "A deleted event" once it is gone. */ + eventName: string; + /** The event they were already in, worded the same way if deleted. */ + conflictEventName: string; + method: "scan" | "code" | "manual"; + attempts: number; + createdAt: Date; + lastAttemptAt: Date; + /** An organiser reviewed it and cleared it. */ + dismissed: boolean; +}; + +/** + * Stores a refused attempt. The same person trying the same pair of events + * again adds to `attempts` instead of a second row; a dismissal stays as it + * is, so a cleared flag does not reopen itself but still shows the retry count. + */ +export async function recordCheckInFlag( + db: DrizzleDB, + hackathonId: string, + breach: CheckInRuleBreakDetails, +): Promise { + await db + .insert(hackathonCheckInFlags) + .values({ + hackathonId, + participantId: breach.participantId, + eventId: breach.eventId, + conflictEventId: breach.conflictEventId, + rule: breach.rule, + method: breach.method, + attemptedById: breach.actorId, + attempts: 1, + createdAt: breach.at, + lastAttemptAt: breach.at, + }) + .onConflictDoUpdate({ + target: [ + hackathonCheckInFlags.participantId, + hackathonCheckInFlags.eventId, + hackathonCheckInFlags.conflictEventId, + ], + set: { + attempts: sql`${hackathonCheckInFlags.attempts} + 1`, + lastAttemptAt: breach.at, + }, + }); +} + +/** + * Runs an attendance write and, if the check-in rules refuse it, records the + * attempt on the pool before rethrowing. `db` has to be the pool and not the + * transaction the refusal rolled back: a row written inside it would go with + * it. Writing the flag is best effort. The person at the door must see the same + * refusal whether or not the flag landed, so a failure here is logged and the + * original error still goes out. + */ +export async function flaggingRuleBreaks( + db: DrizzleDB, + hackathonId: string, + run: () => Promise, +): Promise { + try { + return await run(); + } catch (error) { + if (error instanceof TRPCError && error.cause instanceof CheckInRuleBreak) { + try { + await recordCheckInFlag(db, hackathonId, error.cause); + } catch (flagError) { + // Deliberate server-side logging: the refusal is the answer the person + // gets, so the console is the only place a lost flag can show up. + // eslint-disable-next-line no-console + console.error( + "[CheckInFlags] Failed to record a rule break:", + flagError, + ); + } + } + throw error; + } +} + +// --------------------------------------------------------------------------- +// Reading flags against judging entries + +/** What finding a flag's entries needs from the flag and its participant. */ +export type FlagTarget = { + id: string; + participantId: string; + teamId: string | null; + userId: string; +}; + +/** A portal entry and the project it was promoted from. */ +export type PortalEntryLink = { + entryId: string; + teamId: string | null; + submittedById: string | null; +}; + +/** A person on an entry, claimed or not. */ +export type MemberEntryLink = { entryId: string; userId: string | null }; + +const pushTo = (map: Map, key: string, value: string) => { + const list = map.get(key); + if (list) list.push(value); + else map.set(key, [value]); +}; + +/** + * The entries each flag belongs to, by flag id. A participant is on an entry + * when it is a portal project filed by their team, or by them alone when it + * has no team, or when they are one of its members (which covers Devpost + * entries and claimed teammates). Reaching the same entry by more than one + * route counts once. A flag on nobody's entry maps to an empty set. Pure, so + * the matching is testable without a database. + */ +export function entriesForFlags( + flags: readonly FlagTarget[], + portal: readonly PortalEntryLink[], + members: readonly MemberEntryLink[], +): Map> { + const byTeam = new Map(); + const bySoloSubmitter = new Map(); + for (const link of portal) { + if (link.teamId) pushTo(byTeam, link.teamId, link.entryId); + else if (link.submittedById) { + pushTo(bySoloSubmitter, link.submittedById, link.entryId); + } + } + const byUser = new Map(); + for (const link of members) { + if (link.userId) pushTo(byUser, link.userId, link.entryId); + } + + const result = new Map>(); + for (const flag of flags) { + const entries = new Set(); + const sources = [ + flag.teamId ? byTeam.get(flag.teamId) : undefined, + bySoloSubmitter.get(flag.participantId), + byUser.get(flag.userId), + ]; + for (const ids of sources) for (const id of ids ?? []) entries.add(id); + result.set(flag.id, entries); + } + return result; +} + +/** Resolves flags to live entries of the edition in two batched reads. */ +async function entriesFor( + db: JudgingDb, + hackathonId: string, + flags: readonly FlagTarget[], +): Promise>> { + if (flags.length === 0) return new Map(); + + const teamIds = [ + ...new Set(flags.flatMap((flag) => (flag.teamId ? [flag.teamId] : []))), + ]; + const participantIds = [...new Set(flags.map((flag) => flag.participantId))]; + const userIds = [...new Set(flags.map((flag) => flag.userId))]; + + const [portal, members] = await Promise.all([ + db + .select({ + entryId: judgingEntries.id, + teamId: hackathonProjects.teamId, + submittedById: hackathonProjects.submittedById, + }) + .from(judgingEntries) + .innerJoin( + hackathonProjects, + eq(hackathonProjects.id, judgingEntries.sourceProjectId), + ) + .where( + and( + eq(judgingEntries.hackathonId, hackathonId), + eq(judgingEntries.source, "portal"), + isNull(judgingEntries.deletedAt), + or( + teamIds.length > 0 + ? inArray(hackathonProjects.teamId, teamIds) + : undefined, + and( + isNull(hackathonProjects.teamId), + inArray(hackathonProjects.submittedById, participantIds), + ), + ), + ), + ), + db + .select({ + entryId: judgingEntryMembers.entryId, + userId: judgingEntryMembers.userId, + }) + .from(judgingEntryMembers) + .innerJoin( + judgingEntries, + eq(judgingEntries.id, judgingEntryMembers.entryId), + ) + .where( + and( + eq(judgingEntryMembers.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + inArray(judgingEntryMembers.userId, userIds), + ), + ), + ]); + + return entriesForFlags(flags, portal, members); +} + +/** The open flags of an edition with just enough to find their entries. */ +async function openFlagTargets( + db: JudgingDb, + hackathonId: string, +): Promise { + return db + .select({ + id: hackathonCheckInFlags.id, + participantId: hackathonCheckInFlags.participantId, + teamId: hackathonParticipants.teamId, + userId: hackathonParticipants.userId, + }) + .from(hackathonCheckInFlags) + .innerJoin( + hackathonParticipants, + eq(hackathonParticipants.id, hackathonCheckInFlags.participantId), + ) + .where( + and( + eq(hackathonCheckInFlags.hackathonId, hackathonId), + isNull(hackathonCheckInFlags.dismissedAt), + ), + ); +} + +/** + * Every flag on the given entries, grouped by entry, for organisers: names and + * events included. Open flags come first, then the most recent attempt. Null + * entry ids means every live entry of the edition (the "Flagged" filter). An + * entry with no flags is not in the map. Never pass the result to a judge or + * put it in a cached payload. + */ +export async function checkInFlagsForEntries( + db: JudgingDb, + hackathonId: string, + entryIds: string[] | null, +): Promise> { + const result = new Map(); + if (entryIds && entryIds.length === 0) return result; + + const rows = await db + .select({ + id: hackathonCheckInFlags.id, + participantId: hackathonCheckInFlags.participantId, + eventId: hackathonCheckInFlags.eventId, + conflictEventId: hackathonCheckInFlags.conflictEventId, + rule: hackathonCheckInFlags.rule, + method: hackathonCheckInFlags.method, + attempts: hackathonCheckInFlags.attempts, + createdAt: hackathonCheckInFlags.createdAt, + lastAttemptAt: hackathonCheckInFlags.lastAttemptAt, + dismissedAt: hackathonCheckInFlags.dismissedAt, + teamId: hackathonParticipants.teamId, + userId: hackathonParticipants.userId, + firstName: hackathonParticipants.firstName, + lastName: hackathonParticipants.lastName, + accountName: users.name, + email: users.email, + }) + .from(hackathonCheckInFlags) + .innerJoin( + hackathonParticipants, + eq(hackathonParticipants.id, hackathonCheckInFlags.participantId), + ) + .innerJoin(users, eq(users.id, hackathonParticipants.userId)) + .where(eq(hackathonCheckInFlags.hackathonId, hackathonId)); + if (rows.length === 0) return result; + + // A deleted event leaves its id null on the flag; there is nothing to look up. + const eventIds = [ + ...new Set( + rows + .flatMap((row) => [row.eventId, row.conflictEventId]) + .filter((id) => id !== null), + ), + ]; + const [events, entriesByFlag] = await Promise.all([ + eventIds.length === 0 + ? ([] as { id: string; name: string }[]) + : db + .select({ id: hackathonEvents.id, name: hackathonEvents.name }) + .from(hackathonEvents) + .where(inArray(hackathonEvents.id, eventIds)), + entriesFor(db, hackathonId, rows), + ]); + const eventName = new Map(events.map((event) => [event.id, event.name])); + const nameOf = (id: string | null) => + (id ? eventName.get(id) : undefined) ?? "A deleted event"; + + const wanted = entryIds ? new Set(entryIds) : null; + const views: [CheckInFlagView, string[]][] = []; + for (const row of rows) { + const entries = [...(entriesByFlag.get(row.id) ?? [])].filter( + (id) => !wanted || wanted.has(id), + ); + if (entries.length === 0) continue; + views.push([ + { + id: row.id, + participantName: fullName(personName(row), row.email), + rule: row.rule, + eventName: nameOf(row.eventId), + conflictEventName: nameOf(row.conflictEventId), + method: row.method, + attempts: row.attempts, + createdAt: row.createdAt, + lastAttemptAt: row.lastAttemptAt, + dismissed: row.dismissedAt !== null, + }, + entries, + ]); + } + + // One order for every entry: open before cleared, newest attempt first, and + // the id last so equal timestamps never shuffle between polls. + views.sort( + ([a], [b]) => + Number(a.dismissed) - Number(b.dismissed) || + b.lastAttemptAt.getTime() - a.lastAttemptAt.getTime() || + (a.id < b.id ? -1 : a.id > b.id ? 1 : 0), + ); + for (const [view, entries] of views) { + for (const entryId of entries) { + const list = result.get(entryId); + if (list) list.push(view); + else result.set(entryId, [view]); + } + } + return result; +} + +/** + * Which of the given entries carry at least one flag nobody has dismissed. + * Ids only, which is all a judge is told: that the team was flagged, not who + * or why. + */ +export async function flaggedEntryIds( + db: JudgingDb, + hackathonId: string, + entryIds: string[], +): Promise> { + if (entryIds.length === 0) return new Set(); + + const targets = await openFlagTargets(db, hackathonId); + const entriesByFlag = await entriesFor(db, hackathonId, targets); + + const wanted = new Set(entryIds); + const flagged = new Set(); + for (const entries of entriesByFlag.values()) { + for (const id of entries) if (wanted.has(id)) flagged.add(id); + } + return flagged; +} + +/** + * Open flags in the edition that sit on a live entry, for the organiser + * header. A flag whose hacker is on no entry is left out: there is nowhere in + * judging to review it, and a count of things nobody can open is noise. + */ +export async function openCheckInFlagCount( + db: JudgingDb, + hackathonId: string, +): Promise { + const targets = await openFlagTargets(db, hackathonId); + const entriesByFlag = await entriesFor(db, hackathonId, targets); + + let count = 0; + for (const entries of entriesByFlag.values()) + if (entries.size > 0) count += 1; + return count; +} diff --git a/packages/api/src/services/judging/appointment-status.test.ts b/packages/api/src/services/judging/appointment-status.test.ts index 8ae78bb6..29f8415a 100644 --- a/packages/api/src/services/judging/appointment-status.test.ts +++ b/packages/api/src/services/judging/appointment-status.test.ts @@ -15,13 +15,13 @@ describe("appointmentStatus", () => { expect(appointmentStatus(slot, after)).toBe("missed"); }); - it("is complete when every submitted evaluation is complete", () => { + it("is complete when every submitted evaluation holds every rating", () => { const slot = { startsAt, endsAt, evaluations: [ - { judgeId: "a", isComplete: true }, - { judgeId: "b", isComplete: true }, + { judgeId: "a", ratingsComplete: true }, + { judgeId: "b", ratingsComplete: true }, ], presentJudgeIds: ["a", "b"], }; @@ -29,13 +29,13 @@ describe("appointmentStatus", () => { expect(appointmentStatus(slot, after)).toBe("complete"); }); - it("is incomplete when an evaluation was auto-submitted unfinished", () => { + it("is incomplete when an evaluation was auto-submitted short of a rating", () => { const slot = { startsAt, endsAt, evaluations: [ - { judgeId: "a", isComplete: true }, - { judgeId: "b", isComplete: false }, + { judgeId: "a", ratingsComplete: true }, + { judgeId: "b", ratingsComplete: false }, ], }; expect(appointmentStatus(slot, after)).toBe("incomplete"); @@ -45,7 +45,7 @@ describe("appointmentStatus", () => { const slot = { startsAt, endsAt, - evaluations: [{ judgeId: "a", isComplete: true }], + evaluations: [{ judgeId: "a", ratingsComplete: true }], presentJudgeIds: ["a", "b"], }; expect(appointmentStatus(slot, during)).toBe("incomplete"); @@ -55,7 +55,7 @@ describe("appointmentStatus", () => { const slot = { startsAt, endsAt, - evaluations: [{ judgeId: "a", isComplete: true }], + evaluations: [{ judgeId: "a", ratingsComplete: true }], }; expect(appointmentStatus(slot, before)).toBe("complete"); }); diff --git a/packages/api/src/services/judging/appointment-status.ts b/packages/api/src/services/judging/appointment-status.ts index 46cffa53..b63f02c3 100644 --- a/packages/api/src/services/judging/appointment-status.ts +++ b/packages/api/src/services/judging/appointment-status.ts @@ -2,7 +2,8 @@ // hacker itinerary alike. Evaluations decide it once any exist; until then the // clock does. // -// complete every judge who was in the room has a complete evaluation +// complete every judge who was in the room has an evaluation that holds +// every rating, the same rule placings count by // incomplete some evaluation exists, but not all of the above // missed the slot is over and nobody evaluated it // pending the slot has started, nothing submitted yet @@ -19,7 +20,7 @@ export type AppointmentStatusInput = { startsAt: Date; endsAt: Date; /** Evaluations of this appointment's project for its challenge. */ - evaluations: readonly { judgeId: string; isComplete: boolean }[]; + evaluations: readonly { judgeId: string; ratingsComplete: boolean }[]; /** Judges whose presence in the room overlapped the slot. */ presentJudgeIds?: readonly string[]; }; @@ -30,9 +31,11 @@ export function appointmentStatus( ): AppointmentStatus { if (input.evaluations.length > 0) { const completeBy = new Set( - input.evaluations.filter((e) => e.isComplete).map((e) => e.judgeId), + input.evaluations.filter((e) => e.ratingsComplete).map((e) => e.judgeId), + ); + const allSubmittedComplete = input.evaluations.every( + (e) => e.ratingsComplete, ); - const allSubmittedComplete = input.evaluations.every((e) => e.isComplete); const everyPresentJudged = (input.presentJudgeIds ?? []).every((id) => completeBy.has(id), ); diff --git a/packages/api/src/services/judging/calibration.test.ts b/packages/api/src/services/judging/calibration.test.ts new file mode 100644 index 00000000..9314bff3 --- /dev/null +++ b/packages/api/src/services/judging/calibration.test.ts @@ -0,0 +1,757 @@ +import { describe, expect, it } from "vitest"; +import { K_A, K_B, K_D, MIN_DRIFT_GRADES, calibrate } from "./calibration"; +import type { CalibrationEvaluation } from "./calibration"; + +// A small seeded generator (mulberry32), so every dataset below is the same on +// every run. No Math.random. +function seeded(seed: number) { + let state = seed >>> 0; + const uniform = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + // Roughly normal: the sum of four uniforms, centred and scaled to sd 1. + const normal = () => + (uniform() + uniform() + uniform() + uniform() - 2) * Math.sqrt(3); + const shuffle = (items: readonly T[]) => { + const copy = [...items]; + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(uniform() * (i + 1)); + [copy[i], copy[j]] = [copy[j] as T, copy[i] as T]; + } + return copy; + }; + return { uniform, normal, shuffle }; +} + +const DAY = Date.UTC(2027, 1, 6, 10, 0, 0); + +const evaluation = ( + id: string, + judgeId: string, + score: number, + over: Partial = {}, +): CalibrationEvaluation => ({ + id, + judgeId, + challengeId: "general", + entryId: `entry-${id}`, + submittedAt: new Date(DAY), + ratingsComplete: true, + ratings: [score], + ...over, +}); + +const clampRating = (value: number) => Math.min(5, Math.max(1, value)); + +type Habit = { + judgeId: string; + /** Added to every score. */ + offset?: number; + /** Multiplies the distance from 3. */ + spread?: number; + /** Points added per grade, in grading order. */ + drift?: number; + noise?: number; +}; + +/** + * A pool of projects with a known quality, dealt out at random to judges with + * known habits. Returns the evaluations and each project's true quality. + */ +function pool(options: { + projects: number; + judges: Habit[]; + seed: number; + challengeId?: string; +}) { + const random = seeded(options.seed); + const quality = Array.from( + { length: options.projects }, + () => 1.8 + 2.4 * random.uniform(), + ); + const order = random.shuffle(quality.map((_, index) => index)); + const evaluations: CalibrationEvaluation[] = []; + const truth = new Map(); + + order.forEach((project, position) => { + const habit = options.judges[position % options.judges.length]!; + const turn = Math.floor(position / options.judges.length); + const raw = + 3 + + (habit.spread ?? 1) * ((quality[project] as number) - 3) + + (habit.offset ?? 0) + + (habit.drift ?? 0) * turn + + (habit.noise ?? 0.2) * random.normal(); + const id = `${options.challengeId ?? "general"}-${habit.judgeId}-${turn}`; + truth.set(id, quality[project] as number); + evaluations.push( + evaluation(id, habit.judgeId, clampRating(raw), { + challengeId: options.challengeId ?? "general", + entryId: `project-${project}`, + submittedAt: new Date(DAY + turn * 60_000), + }), + ); + }); + return { evaluations, truth }; +} + +function ranks(values: readonly number[]) { + const sorted = values + .map((value, index) => ({ value, index })) + .sort((a, b) => a.value - b.value); + const result = new Array(values.length); + sorted.forEach(({ index }, position) => { + result[index] = position; + }); + return result; +} + +/** Spearman rank correlation. */ +function spearman(a: readonly number[], b: readonly number[]) { + const ra = ranks(a); + const rb = ranks(b); + const mean = (ra.length - 1) / 2; + let top = 0; + let sa = 0; + let sb = 0; + for (let i = 0; i < ra.length; i++) { + top += (ra[i]! - mean) * (rb[i]! - mean); + sa += (ra[i]! - mean) ** 2; + sb += (rb[i]! - mean) ** 2; + } + return top / Math.sqrt(sa * sb); +} + +const scoreOf = (e: CalibrationEvaluation) => + e.ratings.reduce((sum, rating) => sum + rating, 0) / e.ratings.length; + +function sdOf(values: readonly number[]) { + const mean = values.reduce((sum, value) => sum + value, 0) / values.length; + return Math.sqrt( + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / values.length, + ); +} + +/** Least-squares slope of the values on their position. */ +function slopeOf(values: readonly number[]) { + const centre = (values.length - 1) / 2; + const mean = values.reduce((sum, value) => sum + value, 0) / values.length; + let along = 0; + let across = 0; + values.forEach((value, index) => { + along += (index - centre) * (value - mean); + across += (index - centre) ** 2; + }); + return along / across; +} + +describe("calibrate", () => { + it("uses the documented pseudo-evaluation counts", () => { + expect([K_A, K_B, K_D]).toEqual([3, 6, 150]); + expect(MIN_DRIFT_GRADES).toBe(20); + }); + + it("returns nothing for no evaluations", () => { + const result = calibrate([]); + expect(result.adjusted.size).toBe(0); + expect(result.judges).toEqual([]); + }); + + it("leaves one evaluation as it is", () => { + const result = calibrate([evaluation("only", "judge", 4.25)]); + expect(result.adjusted.get("only")).toBeCloseTo(4.25, 10); + expect(result.judges).toEqual([ + { + judgeId: "judge", + challengeId: "general", + evaluations: 1, + offset: 0, + spread: 1, + drift: 0, + }, + ]); + }); + + it("scores an evaluation as the mean of its ratings", () => { + const result = calibrate([ + evaluation("a", "judge", 0, { ratings: [4, 5, 3, 4] }), + ]); + expect(result.adjusted.get("a")).toBeCloseTo(4, 10); + }); + + it("ignores evaluations short of a rating and ones with no ratings", () => { + const result = calibrate([ + evaluation("done", "judge", 4), + evaluation("short", "judge", 1, { ratingsComplete: false }), + evaluation("blank", "judge", 0, { ratings: [] }), + ]); + expect([...result.adjusted.keys()]).toEqual(["done"]); + expect(result.judges).toHaveLength(1); + expect(result.judges[0]?.evaluations).toBe(1); + }); + + it("does not correct the offset of the only judge in a pool", () => { + const { evaluations } = pool({ + projects: 30, + judges: [{ judgeId: "alone", offset: 0.9 }], + seed: 5, + }); + const result = calibrate(evaluations); + // Nobody to compare them with: no offset and no spread. + expect(result.judges[0]?.offset).toBeCloseTo(0, 10); + expect(result.judges[0]?.spread).toBeCloseTo(1, 10); + for (const e of evaluations) { + expect(result.adjusted.get(e.id)).toBeCloseTo(scoreOf(e), 0); + } + }); + + it("matches the model on a small pool worked by hand", () => { + // Pool scores 2, 4, 4, 5: mu = 3.75. Taking each judge's mean out leaves + // -1, 1 and -0.5, 0.5, so the sd a judge is measured against is + // sqrt(2.5 / 4), not the plain pool sd of sqrt(1.1875). + const mu = 3.75; + const sd = Math.sqrt(2.5 / 4); + // Judge a has 2 and 4: n = 2, so no drift. Mean 3, population sd 1. + const offset = (2 / (2 + K_A)) * (3 - mu); + const spread = 1 + (2 / (2 + K_B)) * (1 / sd - 1); + const result = calibrate([ + evaluation("a1", "a", 2, { submittedAt: new Date(DAY) }), + evaluation("a2", "a", 4, { submittedAt: new Date(DAY + 60_000) }), + evaluation("b1", "b", 4, { submittedAt: new Date(DAY) }), + evaluation("b2", "b", 5, { submittedAt: new Date(DAY + 60_000) }), + ]); + expect(result.adjusted.get("a1")).toBeCloseTo( + mu + (2 - mu - offset) / spread, + 10, + ); + expect(result.adjusted.get("a2")).toBeCloseTo( + mu + (4 - mu - offset) / spread, + 10, + ); + const a = result.judges.find((j) => j.judgeId === "a"); + expect(a?.offset).toBeCloseTo(offset, 10); + expect(a?.spread).toBeCloseTo(spread, 10); + expect(a?.drift).toBe(0); + }); + + describe("strict and lenient judges", () => { + const { evaluations, truth } = pool({ + projects: 80, + judges: [ + { judgeId: "strict", offset: -0.8 }, + { judgeId: "lenient", offset: 0.8 }, + ], + seed: 11, + }); + const result = calibrate(evaluations); + + it("rank projects far closer to their true quality than raw means do", () => { + const quality = evaluations.map((e) => truth.get(e.id) as number); + const raw = spearman(quality, evaluations.map(scoreOf)); + const adjusted = spearman( + quality, + evaluations.map((e) => result.adjusted.get(e.id) as number), + ); + expect(raw).toBeLessThan(0.65); + expect(adjusted).toBeGreaterThan(0.9); + expect(adjusted - raw).toBeGreaterThan(0.25); + }); + + it("moves their scores by roughly the size of their bias", () => { + const moves = evaluations.map((e) => + Math.abs((result.adjusted.get(e.id) as number) - scoreOf(e)), + ); + const average = moves.reduce((sum, move) => sum + move, 0) / moves.length; + expect(average).toBeGreaterThan(0.6); + expect(average).toBeLessThan(1); + }); + + it("reports which one is strict and which lenient", () => { + const strict = result.judges.find((j) => j.judgeId === "strict"); + const lenient = result.judges.find((j) => j.judgeId === "lenient"); + expect(strict?.offset).toBeLessThan(-0.5); + expect(lenient?.offset).toBeGreaterThan(0.5); + expect(strict?.evaluations).toBe(40); + }); + + it("does not mistake the gap between them for bunched scores", () => { + // Both mark with an ordinary spread. The pool's plain spread also holds + // the gap between their means; measured against that, both would look + // bunched and every score would be stretched. + for (const judge of result.judges) { + expect(judge.spread).toBeGreaterThan(0.9); + expect(judge.spread).toBeLessThan(1.1); + } + const quality = evaluations.map((e) => truth.get(e.id) as number); + const adjusted = evaluations.map( + (e) => result.adjusted.get(e.id) as number, + ); + expect(sdOf(adjusted)).toBeLessThan(sdOf(quality) * 1.15); + }); + + it("brings the two judges' average scores together", () => { + const meanOf = ( + judgeId: string, + score: (e: CalibrationEvaluation) => number, + ) => { + const mine = evaluations.filter((e) => e.judgeId === judgeId); + return mine.reduce((sum, e) => sum + score(e), 0) / mine.length; + }; + const adjusted = (e: CalibrationEvaluation) => + result.adjusted.get(e.id) as number; + expect( + meanOf("lenient", scoreOf) - meanOf("strict", scoreOf), + ).toBeGreaterThan(1.3); + expect( + Math.abs(meanOf("lenient", adjusted) - meanOf("strict", adjusted)), + ).toBeLessThan(0.2); + }); + }); + + describe("a bunched judge", () => { + const { evaluations } = pool({ + projects: 80, + judges: [{ judgeId: "bunched", spread: 0.3 }, { judgeId: "normal" }], + seed: 23, + }); + const result = calibrate(evaluations); + const of = (judgeId: string) => + evaluations.filter((e) => e.judgeId === judgeId); + + it("is stretched toward the pool's spread", () => { + const bunched = of("bunched"); + const before = sdOf(bunched.map(scoreOf)); + const after = sdOf( + bunched.map((e) => result.adjusted.get(e.id) as number), + ); + expect( + result.judges.find((j) => j.judgeId === "bunched")?.spread, + ).toBeLessThan(0.7); + expect(after).toBeGreaterThan(before * 1.4); + }); + + it("leaves the normal judge's spread close to where it was", () => { + const normal = of("normal"); + const before = sdOf(normal.map(scoreOf)); + const after = sdOf( + normal.map((e) => result.adjusted.get(e.id) as number), + ); + expect(Math.abs(after - before)).toBeLessThan(before * 0.3); + }); + }); + + describe("a drifting judge", () => { + const { evaluations } = pool({ + projects: 90, + judges: [{ judgeId: "tiring", drift: -0.05 }, { judgeId: "steady" }], + seed: 31, + }); + const result = calibrate(evaluations); + + const mine = evaluations + .filter((e) => e.judgeId === "tiring") + .sort((a, b) => a.submittedAt.getTime() - b.submittedAt.getTime()); + const grades = mine.length; + // What shrink leaves of a fitted trend with this many grades. + const trusted = grades / (grades + K_D); + + it("is detrended in grading order, by the shrunk trend", () => { + const before = slopeOf(mine.map(scoreOf)); + const after = slopeOf( + mine.map((e) => result.adjusted.get(e.id) as number), + ); + expect(before).toBeLessThan(-0.04); + expect(Math.abs(after)).toBeLessThan(Math.abs(before)); + expect(after * before).toBeGreaterThan(0); + // Only the trusted share of the trend is taken out (spread and offset + // rescale what is left, so this is not exact). + expect(Math.abs(after)).toBeGreaterThan( + Math.abs(before) * (1 - trusted) * 0.7, + ); + }); + + it("reports a drift that is negative for a judge who got harsher", () => { + const tiring = result.judges.find((j) => j.judgeId === "tiring"); + const steady = result.judges.find((j) => j.judgeId === "steady"); + // It is the fitted drift over the whole day, times the share trusted. + expect(tiring?.drift).toBeCloseTo( + slopeOf(mine.map(scoreOf)) * (grades - 1) * trusted, + 9, + ); + expect(tiring?.drift).toBeLessThan(-0.3); + expect(Math.abs(steady?.drift ?? 1)).toBeLessThan(0.3); + }); + }); + + it("breaks a tie in submission time by id when ordering a judge's grades", () => { + // Twenty grades at one instant, so the ids alone say which came first. + const day = (idOf: (position: number) => string) => + calibrate( + Array.from({ length: MIN_DRIFT_GRADES }, (_, position) => + evaluation( + idOf(position), + "judge", + 1 + (4 * position) / (MIN_DRIFT_GRADES - 1), + ), + ), + ); + const id = (n: number) => `g${String(n).padStart(2, "0")}`; + const climbing = day((position) => id(position)); + const falling = day((position) => id(MIN_DRIFT_GRADES - 1 - position)); + expect(climbing.judges[0]?.drift).toBeGreaterThan(0); + expect(falling.judges[0]?.drift).toBeLessThan(0); + }); + + describe("drift tuning", () => { + // Simulated days: pools of 80 judges who each grade `grades` projects in + // turn, three pools per size, so every number below is over 240 judges and + // the same on every run. Project quality varies by about 0.7 of a point + // and a judge adds 0.2 of noise (pool above). `reported` is the drift + // calibrate removes from each judge; `raw` is the unshrunk least-squares + // drift from first grade to last, taken from the same scores. + const day = (grades: number, drift: number) => { + const reported: number[] = []; + const raw: number[] = []; + for (const seed of [1, 2, 3]) { + const { evaluations } = pool({ + projects: 80 * grades, + judges: Array.from({ length: 80 }, (_, i) => ({ + judgeId: `j${i}`, + drift: drift / (grades - 1), + })), + seed: 1000 * grades + seed, + }); + const byJudge = new Map(); + for (const e of [...evaluations].sort( + (a, b) => a.submittedAt.getTime() - b.submittedAt.getTime(), + )) { + byJudge.set(e.judgeId, [ + ...(byJudge.get(e.judgeId) ?? []), + scoreOf(e), + ]); + } + for (const judge of calibrate(evaluations).judges) { + reported.push(judge.drift); + raw.push( + slopeOf(byJudge.get(judge.judgeId) as number[]) * (grades - 1), + ); + } + } + return { reported, raw }; + }; + const meanOf = (values: readonly number[]) => + values.reduce((sum, value) => sum + value, 0) / values.length; + + // From the shortest day that is fitted to the longest there is. + const sizes = [MIN_DRIFT_GRADES, 30, 45, 60, 100]; + + it.each(sizes)( + "invents under 0.1 of a point of drift for a judge with none, at %i grades", + (grades) => { + const { reported, raw } = day(grades, 0); + expect(sdOf(reported)).toBeLessThan(0.1); + // The same judges under the old K_D = 10, which the review measured at + // about 0.33. + const old = raw.map((d) => (d * grades) / (grades + 10)); + expect(sdOf(old)).toBeGreaterThan(0.2); + }, + ); + + it.each(sizes)( + "takes out the share n / (n + K_D) of a real drift, at %i grades", + (grades) => { + for (const drift of [0.5, 1]) { + const { reported, raw } = day(grades, drift); + // The fit itself is unbiased: the raw drifts average the real one. + expect(Math.abs(meanOf(raw) - drift)).toBeLessThan(0.1); + const removed = meanOf(reported) / drift; + expect(Math.abs(removed - grades / (grades + K_D))).toBeLessThan( + 0.07, + ); + } + }, + ); + + it("does not fit a drift below the fewest grades", () => { + const grades = MIN_DRIFT_GRADES - 1; + const { evaluations } = pool({ + projects: 20 * grades, + judges: Array.from({ length: 20 }, (_, i) => ({ + judgeId: `j${i}`, + drift: -3 / (grades - 1), + })), + seed: 7, + }); + const result = calibrate(evaluations); + expect(result.judges.every((judge) => judge.drift === 0)).toBe(true); + // Nothing detrended: with one habit shared by every judge, the offset and + // spread are the only other corrections, and they are about 0 here. + const moved = evaluations.map((e) => + Math.abs((result.adjusted.get(e.id) as number) - scoreOf(e)), + ); + expect(meanOf(moved)).toBeLessThan(0.25); + }); + }); + + it("barely corrects a judge with a single grade", () => { + const { evaluations } = pool({ + projects: 40, + judges: [{ judgeId: "a" }, { judgeId: "b", offset: 0.3 }], + seed: 41, + }); + const once = evaluation("once", "visitor", 4.9); + const all = [...evaluations, once]; + const result = calibrate(all); + const visitor = result.judges.find((j) => j.judgeId === "visitor"); + expect(visitor?.evaluations).toBe(1); + expect(visitor?.spread).toBe(1); + expect(visitor?.drift).toBe(0); + + // One grade is mostly the project, not the judge: the pull toward the pool + // is 1 / (1 + K_A) of the distance, a quarter. + const mu = all.reduce((sum, e) => sum + scoreOf(e), 0) / all.length; + expect(visitor?.offset).toBeCloseTo((4.9 - mu) / (1 + K_A), 10); + const moved = Math.abs(4.9 - (result.adjusted.get("once") as number)); + expect(moved).toBeLessThan(Math.abs(4.9 - mu) * 0.3); + }); + + it("leaves unbiased judges almost where they were", () => { + const { evaluations } = pool({ + projects: 90, + judges: [{ judgeId: "a" }, { judgeId: "b" }, { judgeId: "c" }], + seed: 53, + }); + const result = calibrate(evaluations); + const moves = evaluations.map((e) => + Math.abs((result.adjusted.get(e.id) as number) - scoreOf(e)), + ); + const average = moves.reduce((sum, move) => sum + move, 0) / moves.length; + // Chance alone gives each judge's thirty projects a slightly different + // mean, so some movement is expected; a biased judge moves several times as far. + expect(average).toBeLessThan(0.2); + expect(Math.max(...moves)).toBeLessThan(0.6); + expect( + spearman( + evaluations.map(scoreOf), + evaluations.map((e) => result.adjusted.get(e.id) as number), + ), + ).toBeGreaterThan(0.95); + for (const judge of result.judges) { + expect(Math.abs(judge.offset)).toBeLessThan(0.2); + expect(judge.spread).toBeGreaterThan(0.8); + expect(judge.spread).toBeLessThan(1.25); + } + }); + + describe("pools", () => { + const table = pool({ + projects: 40, + judges: [{ judgeId: "t1" }, { judgeId: "t2", offset: 0.5 }], + seed: 61, + challengeId: "general", + }).evaluations; + // A sponsor whose guests are harsh, and one guest who also judges at the tables. + const sponsor = pool({ + projects: 20, + judges: [ + { judgeId: "g1", offset: -1 }, + { judgeId: "t1", offset: -1 }, + ], + seed: 67, + challengeId: "sponsor", + }).evaluations; + + it("never mix: a pool calibrates the same alone or beside another", () => { + const together = calibrate([...table, ...sponsor]); + const alone = calibrate(table); + for (const e of table) { + expect(together.adjusted.get(e.id)).toBe(alone.adjusted.get(e.id)); + } + const sponsorAlone = calibrate(sponsor); + for (const e of sponsor) { + expect(together.adjusted.get(e.id)).toBe( + sponsorAlone.adjusted.get(e.id), + ); + } + }); + + it("give a judge in two challenges separate parameters in each", () => { + const together = calibrate([...table, ...sponsor]); + const both = together.judges.filter((j) => j.judgeId === "t1"); + expect(both.map((j) => j.challengeId).sort()).toEqual([ + "general", + "sponsor", + ]); + const [general, other] = [ + both.find((j) => j.challengeId === "general")!, + both.find((j) => j.challengeId === "sponsor")!, + ]; + expect(general.evaluations).toBe(20); + expect(other.evaluations).toBe(10); + // Stricter than the lenient t2 at the tables; as harsh as the sponsor's + // own guest, so near level there (ten grades each leave some chance in it). + expect(general.offset).toBeLessThan(-0.1); + expect(Math.abs(other.offset)).toBeLessThan(0.25); + expect(other.offset - general.offset).toBeGreaterThan(0.2); + }); + }); + + it("keeps every output between 1 and 5", () => { + const random = seeded(71); + const extreme: CalibrationEvaluation[] = []; + for (let i = 0; i < 60; i++) { + // A judge who only ever gives 5s next to one who only gives 1s and 2s, + // and one who swings between the ends. + const judgeId = ["high", "low", "swing"][i % 3]!; + const score = + judgeId === "high" + ? 5 + : judgeId === "low" + ? 1 + Math.floor(random.uniform() * 2) + : random.uniform() < 0.5 + ? 1 + : 5; + extreme.push( + evaluation(`x-${i}`, judgeId, score, { + submittedAt: new Date(DAY + i * 60_000), + }), + ); + } + const result = calibrate(extreme); + expect(result.adjusted.size).toBe(60); + for (const value of result.adjusted.values()) { + expect(value).toBeGreaterThanOrEqual(1); + expect(value).toBeLessThanOrEqual(5); + } + for (const judge of result.judges) { + expect(judge.spread).toBeGreaterThanOrEqual(0.5); + expect(judge.spread).toBeLessThanOrEqual(2); + } + }); + + describe("a judge who gives every project the same score", () => { + const { evaluations } = pool({ + projects: 40, + judges: [{ judgeId: "varied" }, { judgeId: "flat", spread: 0, noise: 0 }], + seed: 83, + }); + const flat = evaluations.filter((e) => e.judgeId === "flat"); + const result = calibrate(evaluations); + + it("is clamped at a spread of 0.5 and given one sane score throughout", () => { + expect(new Set(flat.map(scoreOf)).size).toBe(1); + const judge = result.judges.find((j) => j.judgeId === "flat"); + expect(judge?.spread).toBe(0.5); + expect(judge?.drift).toBeCloseTo(0, 10); + + const values = flat.map((e) => result.adjusted.get(e.id) as number); + for (const value of values) { + expect(Number.isFinite(value)).toBe(true); + expect(value).toBeGreaterThanOrEqual(1); + expect(value).toBeLessThanOrEqual(5); + expect(value).toBeCloseTo(values[0] as number, 10); + } + }); + + it("does not disturb the other judge", () => { + const varied = evaluations.filter((e) => e.judgeId === "varied"); + for (const e of varied) { + const adjusted = result.adjusted.get(e.id) as number; + expect(Number.isFinite(adjusted)).toBe(true); + expect(Math.abs(adjusted - scoreOf(e))).toBeLessThan(0.5); + } + }); + }); + + it("copes with a pool where every score is the same", () => { + const result = calibrate( + ["a", "b", "c", "d"].map((id, i) => + evaluation(id, i % 2 ? "x" : "y", 3.5, { + submittedAt: new Date(DAY + i * 60_000), + }), + ), + ); + for (const value of result.adjusted.values()) + expect(value).toBeCloseTo(3.5, 10); + for (const judge of result.judges) { + expect(judge.spread).toBe(1); + expect(judge.offset).toBeCloseTo(0, 10); + } + }); + + it("handles projects with several evaluations, as older data has", () => { + const { evaluations } = pool({ + projects: 30, + judges: [ + { judgeId: "a", offset: -0.6 }, + { judgeId: "b", offset: 0.6 }, + { judgeId: "c" }, + ], + seed: 97, + }); + // Every judge also graded the first ten projects. + const repeats = evaluations.slice(0, 10).flatMap((first) => + ["a", "b", "c"] + .filter((judgeId) => judgeId !== first.judgeId) + .map((judgeId) => + evaluation(`again-${first.id}-${judgeId}`, judgeId, scoreOf(first), { + entryId: first.entryId, + submittedAt: new Date(DAY + 3_600_000), + }), + ), + ); + const all = [...evaluations, ...repeats]; + const result = calibrate(all); + expect(result.adjusted.size).toBe(all.length); + expect(new Set(all.map((e) => e.entryId)).size).toBe(30); + for (const value of result.adjusted.values()) { + expect(value).toBeGreaterThanOrEqual(1); + expect(value).toBeLessThanOrEqual(5); + } + }); + + it("gives the same answer whatever order the evaluations come in", () => { + const { evaluations } = pool({ + projects: 60, + judges: [ + { judgeId: "strict", offset: -0.7, drift: -0.03 }, + { judgeId: "lenient", offset: 0.7, spread: 0.6 }, + { judgeId: "plain" }, + ], + seed: 101, + }); + const other = pool({ + projects: 20, + judges: [{ judgeId: "plain", offset: 0.4 }, { judgeId: "guest" }], + seed: 103, + challengeId: "sponsor", + }).evaluations; + const all = [...evaluations, ...other]; + + const forward = calibrate(all); + const shuffled = calibrate(seeded(7).shuffle(all)); + const reversed = calibrate([...all].reverse()); + + for (const result of [shuffled, reversed]) { + expect(result.judges).toEqual(forward.judges); + expect([...result.adjusted.entries()].sort()).toEqual( + [...forward.adjusted.entries()].sort(), + ); + } + }); + + it("does not change its input", () => { + const { evaluations } = pool({ + projects: 12, + judges: [{ judgeId: "a" }, { judgeId: "b" }], + seed: 107, + }); + const before = JSON.stringify(evaluations); + calibrate(evaluations); + expect(JSON.stringify(evaluations)).toBe(before); + }); +}); diff --git a/packages/api/src/services/judging/calibration.ts b/packages/api/src/services/judging/calibration.ts new file mode 100644 index 00000000..4ac284dc --- /dev/null +++ b/packages/api/src/services/judging/calibration.ts @@ -0,0 +1,257 @@ +// Judge calibration. Each project is graded once per challenge, by one judge, +// so a strict, lenient, bunched or drifting judge decides the score of every +// project they happened to grade. This takes that judge out of the score. +// +// It is valid only because the judge queue (queue.ts) hands each judge a random +// sample of the projects: a random sample is as good as the pool, so a judge +// whose mean, spread or trend differs from the pool's is showing their own +// habits, not an easier or harder set of projects. Hand a judge picked projects +// and the correction below would punish them for it. +// +// A pool is one root challenge: sponsor judges are compared only with the other +// judges of that sponsor's challenge, table judges with table judges, and a +// judge who grades in two challenges gets separate parameters in each. Within a +// pool, over evaluations that hold every rating, with a score s (the mean of +// its ratings): +// +// mu the pool's mean +// drift a judge's grades in order (submittedAt, then id), from +// MIN_DRIFT_GRADES of them: the slope of s on the centred order +// index, shrunk, removed: s' = s - slope * (i - mean i) +// sd the pool's population standard deviation of s' with each judge's +// own mean taken out (judges with two or more grades) +// offset a = shrink * (mean(s') - mu): stricter (below 0) or more lenient +// spread b = 1 + shrink * (sd_j(s') / sd - 1), in [0.5, 2]: bunched or wide +// +// adjusted = clamp(mu + (s' - mu - a) / b, 1, 5) +// +// where shrink = n / (n + K) for a judge with n grades (K_A, K_B, K_D below). + +import { evaluationScore } from "./scoring"; + +// Pseudo-evaluations: a judge with n grades is trusted n / (n + K) of the way, +// so one with few grades is pulled toward no correction. K_A is for the offset, +// K_B for the spread, K_D for the drift. +// +// K_D is far the largest because a trend is the hardest habit to see. Each +// grade is also the quality of the project it happened to be, which varies by +// about 0.7 of a point, so a judge with no drift at all still has a fitted one +// from first grade to last, off by an sd of 0.5 of a point over 20 grades, 0.4 +// over 30 and 0.3 over 60. K_D = 10 passed most of that on as correction: the +// sd of the drift taken off judges with none was 0.32 of a point at 30 grades. +// K_D = 150 holds it under 0.1 at every size from 20 to 100 grades (0.07 at 30, +// 0.09 at 60). Both are measured on simulated judges in calibration.test.ts, +// "drift tuning". +// +// The price is that a real drift is taken out by the same share, n / (n + K_D): +// 17% of it at 30 grades, 29% at 60. Half a point of real drift across a day is +// mostly left in. Taking most of it out as well was the aim, and no weight does +// both: at that size a real drift and the one chance invents look alike, so a +// smaller K_D gives the invented correction back as fast as the real one (0.15 +// of a point at K_D = 60). Both would take some 150 grades from each judge. +export const K_A = 3; +export const K_B = 6; +export const K_D = 150; +// Fewest grades before a drift is fitted. Below this chance alone moves the +// fitted drift by half a point or more (0.9 at 6 grades), as much as a real +// one worth finding, so none is fitted. +export const MIN_DRIFT_GRADES = 20; + +const MIN_SPREAD = 0.5; +const MAX_SPREAD = 2; +const MIN_RATING = 1; +const MAX_RATING = 5; +// Below this a standard deviation is float noise, not a spread. +const FLAT = 1e-9; + +export type CalibrationEvaluation = { + id: string; + judgeId: string; + /** The root challenge. It is the pool: nothing is compared across it. */ + challengeId: string; + entryId: string; + submittedAt: Date; + /** Holds a rating for every rating item of the rubric; only these count. */ + ratingsComplete: boolean; + /** The 1-5 rating values it holds. */ + ratings: readonly number[]; +}; + +export type JudgeCalibration = { + judgeId: string; + challengeId: string; + /** Counted, scored evaluations the judge has in this challenge. */ + evaluations: number; + /** Rating points taken off the judge's scores. Positive: more lenient than the pool. */ + offset: number; + /** What the judge's spread was divided by. Below 1: bunched. Above 1: wider than the pool. */ + spread: number; + /** Points gained (positive) or lost (negative) from the judge's first grade to their last. */ + drift: number; +}; + +export type Calibration = { + /** The calibrated score of each counted, scored evaluation, by id. */ + adjusted: Map; + judges: JudgeCalibration[]; +}; + +type Graded = { + id: string; + judgeId: string; + challengeId: string; + at: number; + score: number; +}; + +const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + +const mean = (values: readonly number[]) => { + let sum = 0; + for (const value of values) sum += value; + return sum / values.length; +}; + +const populationSd = (values: readonly number[]) => { + const centre = mean(values); + let sum = 0; + for (const value of values) sum += (value - centre) ** 2; + return Math.sqrt(sum / values.length); +}; + +const clamp = (value: number, low: number, high: number) => + Math.min(high, Math.max(low, value)); + +const shrink = (n: number, k: number) => n / (n + k); + +function groupBy(items: readonly T[], key: (item: T) => string) { + const groups = new Map(); + for (const item of items) { + const list = groups.get(key(item)) ?? []; + list.push(item); + groups.set(key(item), list); + } + return groups; +} + +/** + * Calibrated scores for every evaluation that holds every rating, and what was + * learned about each judge. Evaluations short of a rating, and ones with no + * ratings at all, are left out of both. The result does not depend on the order + * of the input. + */ +export function calibrate( + evaluations: readonly CalibrationEvaluation[], +): Calibration { + const graded: Graded[] = []; + for (const evaluation of evaluations) { + if (!evaluation.ratingsComplete) continue; + const score = evaluationScore(evaluation.ratings); + if (score === null) continue; + graded.push({ + id: evaluation.id, + judgeId: evaluation.judgeId, + challengeId: evaluation.challengeId, + at: evaluation.submittedAt.getTime(), + score, + }); + } + + // One canonical order, so the sums below add up the same way whatever order + // the caller listed the evaluations in. + graded.sort( + (a, b) => + compareText(a.challengeId, b.challengeId) || + compareText(a.judgeId, b.judgeId) || + a.at - b.at || + compareText(a.id, b.id), + ); + + const adjusted = new Map(); + const judges: JudgeCalibration[] = []; + + for (const pool of groupBy(graded, (g) => g.challengeId).values()) { + const mu = mean(pool.map((g) => g.score)); + + const byJudge = [...groupBy(pool, (g) => g.judgeId).values()].map( + (grades) => { + const n = grades.length; + const scores = grades.map((g) => g.score); + const centre = (n - 1) / 2; + + // Least-squares slope of score on the centred order index. + let slope = 0; + if (n >= MIN_DRIFT_GRADES) { + const average = mean(scores); + let along = 0; + let across = 0; + scores.forEach((score, index) => { + along += (index - centre) * (score - average); + across += (index - centre) ** 2; + }); + slope = (along / across) * shrink(n, K_D); + } + const detrended = scores.map( + (score, index) => score - slope * (index - centre), + ); + return { grades, slope, detrended }; + }, + ); + + // What a judge's spread is measured against: the pool's spread once each + // judge's own mean is taken out. The plain spread of the pool also holds + // the gaps between judges' means, so beside a lenient judge a strict one + // with an ordinary spread would look bunched, and both would be stretched. + // A judge with one grade has no spread of their own and is left out. + let squares = 0; + let counted = 0; + for (const { detrended } of byJudge) { + if (detrended.length < 2) continue; + const centre = mean(detrended); + for (const value of detrended) squares += (value - centre) ** 2; + counted += detrended.length; + } + const sd = counted > 0 ? Math.sqrt(squares / counted) : 0; + + for (const { grades, slope, detrended } of byJudge) { + const n = grades.length; + const offset = shrink(n, K_A) * (mean(detrended) - mu); + + // A judge who gave every project the same score has a spread of 0, which + // would be a divisor near 0 below. The clamp stops the divisor at 0.5 + // (reached once they have 6 or more grades; with fewer, the shrink keeps + // it higher), so what is left of their score after the offset is at most + // doubled. Every project they graded lands on the same value, inside + // 1 to 5: they said nothing about which project was better. + let spread = 1; + if (n >= 2 && sd > FLAT) { + spread = clamp( + 1 + shrink(n, K_B) * (populationSd(detrended) / sd - 1), + MIN_SPREAD, + MAX_SPREAD, + ); + } + + grades.forEach((grade, index) => { + adjusted.set( + grade.id, + clamp( + mu + ((detrended[index] as number) - mu - offset) / spread, + MIN_RATING, + MAX_RATING, + ), + ); + }); + judges.push({ + judgeId: grades[0]!.judgeId, + challengeId: grades[0]!.challengeId, + evaluations: n, + offset, + spread, + drift: slope * (n - 1), + }); + } + } + + return { adjusted, judges }; +} diff --git a/packages/api/src/services/judging/checklist.ts b/packages/api/src/services/judging/checklist.ts index 06ec5661..45882e78 100644 --- a/packages/api/src/services/judging/checklist.ts +++ b/packages/api/src/services/judging/checklist.ts @@ -11,6 +11,7 @@ import { import type { JudgingRoomKind } from "@query/db"; import { ensureJudgingConfig } from "./config"; import type { JudgingDb } from "./config"; +import { outerColumn } from "./outer-column"; // The organiser's launch checklist: what has to be true before judges take // their rooms and tables and hackers look up their times. Reads every area's @@ -282,7 +283,7 @@ export async function readChecklistFacts( kind: judgingRooms.kind, challengeId: judgingRooms.challengeId, hasBuilding: sql`${judgingRooms.buildingId} is not null`, - hasLink: sql`exists (select 1 from ${judgingRoomLinks} where ${judgingRoomLinks.roomId} = ${judgingRooms.id} and ${judgingRoomLinks.revokedAt} is null)`, + hasLink: sql`exists (select 1 from ${judgingRoomLinks} where ${judgingRoomLinks.roomId} = ${outerColumn(judgingRooms, judgingRooms.id)} and ${judgingRoomLinks.revokedAt} is null)`, }) .from(judgingRooms) .where( diff --git a/packages/api/src/services/judging/evaluation-store.test.ts b/packages/api/src/services/judging/evaluation-store.test.ts index b7474e6f..d41bf2bf 100644 --- a/packages/api/src/services/judging/evaluation-store.test.ts +++ b/packages/api/src/services/judging/evaluation-store.test.ts @@ -1,11 +1,27 @@ import { describe, expect, it } from "vitest"; +import { getTableName } from "drizzle-orm"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import { + judgingConfigs, + judgingEntries, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingJudges, + judgingRubricItems, +} from "@query/db"; +import { + claimGrading, evaluatedChallenge, guestScope, inJudgeScope, + memberScope, reconcileDueDrafts, } from "./evaluation-store"; import type { ChallengeRow, Seat } from "./evaluation-store"; +import type { Answers } from "./evaluation"; +import { GRADING_HOLD_MS } from "./grading-hold"; import type { Judge } from "./principal"; import type { DrizzleDB } from "@query/db"; @@ -71,11 +87,34 @@ describe("guest scope", () => { }); }); +describe("member scope", () => { + it("is General's challenges alone for a judge with no seat", () => { + expect([...memberScope(challenges, null)]).toEqual(["general"]); + }); + + it("adds a sponsor's challenge, and what is below it, from its room", () => { + expect([...memberScope(challenges, seat("sponsor"))].sort()).toEqual([ + "general", + "sponsor", + ]); + expect([...memberScope(challenges, seat("mlh"))].sort()).toEqual([ + "domain", + "general", + "mlh", + "mongo", + ]); + }); + + it("does not let a seat at General open a sponsor's challenge", () => { + expect([...memberScope(challenges, seat("general"))]).toEqual(["general"]); + }); +}); + describe("evaluatedChallenge", () => { it("evaluates a child challenge under its root", () => { - expect(evaluatedChallenge(member, challenges, "mongo", null).id).toBe( - "mlh", - ); + expect( + evaluatedChallenge(member, challenges, "mongo", seat("mlh")).id, + ).toBe("mlh"); }); it("defaults a member to their room's challenge, then General", () => { @@ -87,11 +126,64 @@ describe("evaluatedChallenge", () => { ); }); - it("falls back to the first root when there is no General", () => { + it("has nothing for a member to default to when there is only sponsors'", () => { const noGeneral = challenges.filter((c) => c.id !== "general"); - expect(evaluatedChallenge(member, noGeneral, undefined, null).id).toBe( - "sponsor", - ); + expect(() => + evaluatedChallenge(member, noGeneral, undefined, null), + ).toThrowError(expect.objectContaining({ code: "FORBIDDEN" })); + expect( + evaluatedChallenge(member, noGeneral, undefined, seat("sponsor")).id, + ).toBe("sponsor"); + }); + + describe("sponsor challenges", () => { + const refused = (label: string) => + expect.objectContaining({ + code: "FORBIDDEN", + message: `Sponsor judges grade ${label}. Sit in its room to grade it.`, + }); + + it("refuses a table judge with no seat", () => { + expect(() => + evaluatedChallenge(member, challenges, "sponsor", null), + ).toThrowError(refused("sponsor")); + }); + + it("refuses a table judge seated at another challenge's room", () => { + expect(() => + evaluatedChallenge(member, challenges, "sponsor", seat("general")), + ).toThrowError(refused("sponsor")); + expect(() => + evaluatedChallenge(member, challenges, "sponsor", seat("mlh")), + ).toThrowError(refused("sponsor")); + }); + + it("names the root when the judge asked for a prize inside a group", () => { + expect(() => + evaluatedChallenge(member, challenges, "mongo", null), + ).toThrowError(refused("mlh")); + }); + + it("lets a table judge grade it from a seat in its room", () => { + expect( + evaluatedChallenge(member, challenges, "sponsor", seat("sponsor")).id, + ).toBe("sponsor"); + }); + + it("keeps General open whatever the seat", () => { + expect(evaluatedChallenge(member, challenges, "general", null).id).toBe( + "general", + ); + expect( + evaluatedChallenge(member, challenges, "general", seat("sponsor")).id, + ).toBe("general"); + }); + + it("leaves a guest on their room's challenge, seat or none", () => { + expect(evaluatedChallenge(guest, challenges, "mongo", null).id).toBe( + "mlh", + ); + }); }); it("keeps a guest on their room's challenge", () => { @@ -142,3 +234,356 @@ describe("reconcileDueDrafts", () => { expect(transactions).toBe(0); }); }); + +type Rows = unknown[] | ((call: number) => unknown[]); +type Event = { + kind: "select" | "execute" | "insert" | "update" | "delete"; + table?: string; + where?: SQL; + query?: SQL; +}; + +/** + * A database whose selects answer by the tables they read (and join), that + * keeps every statement in order. `answers` may be a function of how many times + * that key was read, for a table read more than once in a run. + */ +function fakeDb(answers: Record) { + const events: Event[] = []; + const reads = new Map(); + const via = (...tables: unknown[]) => + tables.map((table) => getTableName(table as never)).join(">"); + const rowsFor = (key: string) => { + const call = reads.get(key) ?? 0; + reads.set(key, call + 1); + const answer = answers[key] ?? []; + return typeof answer === "function" ? answer(call) : answer; + }; + + const select = () => { + const joined: unknown[] = []; + let from: unknown; + let where: SQL | undefined; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + where: (clause: SQL) => { + where = clause; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (e: unknown) => unknown, + ) => { + const key = via(from, ...joined); + events.push({ kind: "select", table: key, where }); + return Promise.resolve(rowsFor(key)).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown) => { + joined.push(table); + return chain; + }; + } + for (const method of ["orderBy", "limit", "for"]) + chain[method] = () => chain; + return chain; + }; + + const db = { + select, + execute: (query: SQL) => { + events.push({ kind: "execute", query }); + return Promise.resolve(); + }, + insert: (table: unknown) => ({ + values: (values: unknown) => { + events.push({ kind: "insert", table: via(table) }); + const done = Promise.resolve([{ id: "new", values }]); + return Object.assign(done, { returning: () => done }); + }, + }), + update: (table: unknown) => ({ + set: () => ({ + where: (where: SQL) => { + events.push({ kind: "update", table: via(table), where }); + return Promise.resolve(); + }, + }), + }), + delete: (table: unknown) => ({ + where: (where: SQL) => { + events.push({ kind: "delete", table: via(table), where }); + return Promise.resolve(); + }, + }), + transaction: (run: (tx: unknown) => Promise) => run(db), + }; + return { db: db as never, events, via }; +} + +const dialect = new PgDialect(); +const textOf = (clause: SQL) => dialect.sqlToQuery(clause).sql; +const lockKeys = (events: Event[]) => + events + .filter((event) => event.kind === "execute") + .map((event) => dialect.sqlToQuery(event.query as SQL).params[0]); + +const RATING = "10000000-0000-4000-8000-000000000001"; +const RESPONSE = "10000000-0000-4000-8000-000000000002"; +const due = ( + id: string, + judgeId: string, + entryId: string, + challengeId = "c1", + answers: Answers = { ratings: { [RATING]: 4 }, responses: {} }, +) => ({ + id, + judgeId, + entryId, + challengeId, + appointmentId: "a1", + ...answers, + deadlineAt: new Date("2026-04-18T15:00:00Z"), +}); + +describe("reconcileDueDrafts and graded once", () => { + const dueKey = `${getTableName(judgingEvaluationDrafts)}>${getTableName(judgingConfigs)}`; + const answersFor = ( + drafts: ReturnType[], + evaluations: Rows = [], + ): Record => { + const name = (table: unknown) => getTableName(table as never); + return { + [dueKey]: drafts, + [name(judgingRubricItems)]: [ + { + id: RATING, + kind: "rating", + label: "Impact", + description: "", + required: true, + guestVisibility: null, + }, + { + id: RESPONSE, + kind: "response", + label: "Feedback", + description: "", + required: false, + guestVisibility: null, + }, + ], + [name(judgingJudges)]: [ + ...new Set(drafts.map((draft) => draft.judgeId)), + ].map((id) => ({ id, kind: "member" })), + [`${name(judgingEntries)}>${name(judgingEntryChallenges)}`]: [ + { id: "entry" }, + ], + [name(judgingEvaluations)]: evaluations, + }; + }; + const evaluationReads = (events: Event[]) => + events.filter( + (e) => + e.kind === "select" && e.table === getTableName(judgingEvaluations), + ); + const written = (events: Event[], table: unknown) => + events.filter( + (e) => e.kind === "insert" && e.table === getTableName(table as never), + ); + + it("submits a due draft for a project nobody has graded", async () => { + const { db, events } = fakeDb(answersFor([due("d1", "j1", "e1")])); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(1); + expect(events.some((e) => e.kind === "delete")).toBe(true); + }); + + it("keeps a draft when another judge already graded the project", async () => { + const { db, events } = fakeDb( + answersFor([due("d1", "j1", "e1")], [{ id: "theirs" }]), + ); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(0); + // The answers stay; only the deadline goes. + expect(events.some((e) => e.kind === "delete")).toBe(false); + expect(events.some((e) => e.kind === "update")).toBe(true); + }); + + it("keeps a due draft with nothing in it, from a judge who only opened the project", async () => { + const opened = due("d1", "j1", "e1", "c1", { ratings: {}, responses: {} }); + const { db, events } = fakeDb(answersFor([opened])); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(0); + expect(events.some((e) => e.kind === "delete")).toBe(false); + expect(events.some((e) => e.kind === "update")).toBe(true); + }); + + it("still submits a due draft with only a written response in it", async () => { + const answered = due("d1", "j1", "e1", "c1", { + ratings: {}, + responses: { [RESPONSE]: { value: "Clear demo.", isPublic: false } }, + }); + const { db, events } = fakeDb(answersFor([answered])); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(1); + }); + + it("looks for a grade from any judge, not just the draft's", async () => { + const { db, events } = fakeDb(answersFor([due("d1", "j1", "e1")])); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + const [read] = evaluationReads(events); + const where = textOf(read.where as SQL); + expect(where).toContain('"entry_id"'); + expect(where).toContain('"challenge_id"'); + expect(where).not.toContain('"judge_id"'); + }); + + it("submits only the first of two drafts for one project", async () => { + // The second draft sees the evaluation the first just wrote. + const { db, events } = fakeDb( + answersFor([due("d1", "j1", "e1"), due("d2", "j2", "e1")], (call) => + call === 0 ? [] : [{ id: "new" }], + ), + ); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(1); + expect(events.filter((e) => e.kind === "delete")).toHaveLength(1); + }); + + it("locks each project once, in key order, before it writes anything", async () => { + const { db, events } = fakeDb( + answersFor([ + due("d1", "j1", "e9"), + due("d2", "j2", "e1"), + due("d3", "j3", "e9"), + ]), + ); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(lockKeys(events)).toEqual(["grading:e1:c1", "grading:e9:c1"]); + const firstWrite = events.findIndex((e) => + ["insert", "update", "delete"].includes(e.kind), + ); + const lastLock = events.map((e) => e.kind).lastIndexOf("execute"); + expect(lastLock).toBeLessThan(firstWrite); + }); +}); + +describe("claimGrading", () => { + const judge = "me"; + const challenge: ChallengeRow = row("general", { + label: "General", + isGeneral: true, + }); + const NOW = new Date("2026-04-18T15:00:00Z"); + const evaluationsKey = getTableName(judgingEvaluations); + const draftsKey = getTableName(judgingEvaluationDrafts); + const claim = (answers: Record) => { + const fake = fakeDb(answers); + return { + ...fake, + run: () => + claimGrading( + fake.db, + { judgeId: judge, entryId: "e1", challenge }, + NOW, + ), + }; + }; + + it("lets a judge take a project nobody has", async () => { + const { run, events } = claim({}); + await expect(run()).resolves.toBeUndefined(); + expect(lockKeys(events)).toEqual(["grading:e1:general"]); + }); + + it("takes the lock before it reads", async () => { + const { run, events } = claim({}); + await run(); + expect(events[0].kind).toBe("execute"); + expect(events.slice(1).every((e) => e.kind === "select")).toBe(true); + }); + + it("refuses a project another judge already graded", async () => { + const { run } = claim({ [evaluationsKey]: [{ judgeId: "them" }] }); + await expect(run()).rejects.toMatchObject({ + code: "CONFLICT", + message: + "Another judge has already graded this project for General. Each project is graded once.", + }); + }); + + it("refuses a project another judge is grading", async () => { + const { run } = claim({ + [draftsKey]: [ + { + judgeId: "them", + updatedAt: new Date(NOW.getTime() - 60_000), + deadlineAt: null, + }, + ], + }); + await expect(run()).rejects.toMatchObject({ + code: "CONFLICT", + message: + "Another judge is grading this project right now. Pick another one.", + }); + }); + + it("refuses while a presentation's draft is before its deadline", async () => { + const { run } = claim({ + [draftsKey]: [ + { + judgeId: "them", + updatedAt: new Date(NOW.getTime() - GRADING_HOLD_MS * 3), + deadlineAt: new Date(NOW.getTime() + 60_000), + }, + ], + }); + await expect(run()).rejects.toMatchObject({ code: "CONFLICT" }); + }); + + it("lets a judge take over a draft left alone past the hold", async () => { + const { run } = claim({ + [draftsKey]: [ + { + judgeId: "them", + updatedAt: new Date(NOW.getTime() - GRADING_HOLD_MS - 1), + deadlineAt: null, + }, + ], + }); + await expect(run()).resolves.toBeUndefined(); + }); + + it("only looks at other judges' rows", async () => { + const { run, events } = claim({}); + await run(); + const reads = events.filter((e) => e.kind === "select"); + expect(reads).toHaveLength(2); + for (const read of reads) { + const where = textOf(read.where as SQL); + expect(where).toContain('"judge_id" <> $'); + expect(where).toContain('"entry_id" = $'); + expect(where).toContain('"challenge_id" = $'); + } + }); +}); diff --git a/packages/api/src/services/judging/evaluation-store.ts b/packages/api/src/services/judging/evaluation-store.ts index 591173e8..d1ce21ec 100644 --- a/packages/api/src/services/judging/evaluation-store.ts +++ b/packages/api/src/services/judging/evaluation-store.ts @@ -1,5 +1,16 @@ import { TRPCError } from "@trpc/server"; -import { and, asc, eq, gt, inArray, isNull, lte, or } from "drizzle-orm"; +import { + and, + asc, + eq, + gt, + inArray, + isNull, + lte, + ne, + or, + sql, +} from "drizzle-orm"; import { judgingAppointments, judgingChallenges, @@ -23,11 +34,21 @@ import type { DrizzleDB } from "@query/db"; import type { JudgingDb, JudgingTx } from "./config"; import { checkAnswers, snapshotOf } from "./evaluation"; import type { CheckedAnswers } from "./evaluation"; +import { gradingLockKey, gradingRefusal, gradingStateOf } from "./grading-hold"; import { GUEST_SESSION_ENDED } from "./principal"; import type { Judge } from "./principal"; -import { descendantsOf, effectiveMode, rootOf } from "./scope"; +import { + descendantsOf, + effectiveMode, + isSponsorChallenge, + rootOf, +} from "./scope"; import type { ChallengeMode } from "./scope"; -import { evaluationTiming } from "./timing"; +import { + DEADLINE_GRACE_MS, + evaluationTiming, + reservedUntil as reservedUntilOf, +} from "./timing"; import type { EvaluationTiming } from "./timing"; // The judge desk's reads and writes around one evaluation: which challenge a @@ -126,10 +147,37 @@ export async function seatOf( }; } +/** + * Sponsor challenges belong to the sponsor's own judges: a member (table) + * judge grades one only while seated in a room that serves it. + */ +function memberMayGrade(root: ChallengeRow, seat: Seat | null): boolean { + return !isSponsorChallenge(root) || seat?.challengeId === root.id; +} + +/** + * The challenges a member judge may pick right now: the general ones, and a + * sponsor's (with everything below it) only from a seat in its room. + */ +export function memberScope( + challenges: readonly ChallengeRow[], + seat: Seat | null, +): Set { + return new Set( + challenges + .filter((c) => { + const root = rootOf(challenges, c.id); + return !!root && memberMayGrade(root, seat); + }) + .map((c) => c.id), + ); +} + /** * The root challenge a judge is evaluating. A guest always evaluates their * room's; a filter they pass has to sit inside it. A member evaluates the root - * of what they picked, or else the room they sit in, or else General. + * of what they picked, or else the room they sit in, or else General, and + * only a sponsor's challenge from a seat in the room that serves it. */ export function evaluatedChallenge( judge: Judge, @@ -162,6 +210,12 @@ export function evaluatedChallenge( "This hackathon has no judging challenges yet. Ask an organiser to import projects first.", }); } + if (judge.kind === "member" && !memberMayGrade(root, seat)) { + throw new TRPCError({ + code: "FORBIDDEN", + message: `Sponsor judges grade ${root.label}. Sit in its room to grade it.`, + }); + } return root; } @@ -306,12 +360,21 @@ export type Presentation = { startsAt: Date; deadlineAt: Date; endsAt: Date; + /** For the timing rules only: never send it to a judge. */ + assignedJudgeId: string | null; }; /** * Whether `judge` may write their evaluation of an entry for a challenge now, * with the presentation it is tied to. Reads the saved schedule and where the - * judge sits; the rule itself is services/judging/timing.ts. + * judge sits; the rule itself is services/judging/timing.ts. A presentation + * assigned to another judge is refused until its takeover time, which is how a + * claim honours the assignment: every claim runs this inside the transaction + * that holds the grading lock (claimGrading), and the assigned judge's draft + * keeps others off once they have started. + * + * `reservedUntil` is when the project opens to this judge if it is held back + * for another judge until then, for the grading state a read reports. */ export async function evaluationAccess( db: JudgingDb, @@ -322,9 +385,15 @@ export async function evaluationAccess( challenge: ChallengeRow; challenges: readonly ChallengeRow[]; evaluation: { appointmentId: string | null } | null; + /** DEADLINE_GRACE_MS when the caller is writing. */ + graceMs?: number; }, now = new Date(), -): Promise<{ timing: EvaluationTiming; presentation: Presentation | null }> { +): Promise<{ + timing: EvaluationTiming; + presentation: Presentation | null; + reservedUntil: Date | null; +}> { const [schedule] = await db .select({ id: judgingSchedules.id }) .from(judgingSchedules) @@ -339,6 +408,7 @@ export async function evaluationAccess( startsAt: judgingAppointments.startsAt, deadlineAt: judgingAppointments.deadlineAt, endsAt: judgingAppointments.endsAt, + assignedJudgeId: judgingAppointments.assignedJudgeId, }) .from(judgingAppointments) .where( @@ -352,18 +422,95 @@ export async function evaluationAccess( : []; const seat = await seatOf(db, input.judgeId, input.challenges); + const mode = + effectiveMode(input.challenges, input.challenge.id) ?? "scheduled"; const timing = evaluationTiming( { + judgeId: input.judgeId, hasSchedule: !!schedule, - mode: effectiveMode(input.challenges, input.challenge.id) ?? "scheduled", + mode, challengeId: input.challenge.id, evaluation: input.evaluation, appointment: presentation ?? null, presenceChallengeId: seat?.challengeId ?? null, + graceMs: input.graceMs, }, now, ); - return { timing, presentation: presentation ?? null }; + // A judge's own evaluation is theirs whoever the presentation was assigned to. + const reservedUntil = + schedule && mode === "scheduled" && !input.evaluation + ? reservedUntilOf(presentation ?? null, input.judgeId, now) + : null; + return { timing, presentation: presentation ?? null, reservedUntil }; +} + +/** + * Serialises the writers of one project's grade for one challenge until the + * transaction ends: a judge's submit, another judge's first save, and the + * deadline submission each check that nobody else has the project and then + * write, which two of them cannot do at once. Transaction-level, so it is safe + * behind the connection pooler. Take a draft's row lock before this one, and + * take several in key order (gradingLockKey), so the locks cannot deadlock. + */ +export async function lockGrading( + tx: JudgingTx, + entryId: string, + challengeId: string, +) { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${gradingLockKey(entryId, challengeId)}::text, 0))`, + ); +} + +/** + * For a judge who has not graded a project yet and is about to write to it: + * takes the project's grading lock, then refuses if another judge has graded + * it or is grading it now (see grading-hold.ts). Run it after the judge's own + * rows are locked, and skip it for a judge editing their own evaluation. + * + * This is also what keeps a late takeover off a presentation whose assigned + * judge has started: their draft holds it. Whether the judge may claim a + * presentation assigned to someone else yet is evaluationAccess's call, run + * next under this lock. + */ +export async function claimGrading( + tx: JudgingTx, + input: { judgeId: string; entryId: string; challenge: ChallengeRow }, + now = new Date(), +) { + await lockGrading(tx, input.entryId, input.challenge.id); + + const evaluations = await tx + .select({ judgeId: judgingEvaluations.judgeId }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.entryId, input.entryId), + eq(judgingEvaluations.challengeId, input.challenge.id), + ne(judgingEvaluations.judgeId, input.judgeId), + ), + ); + const drafts = await tx + .select({ + judgeId: judgingEvaluationDrafts.judgeId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + }) + .from(judgingEvaluationDrafts) + .where( + and( + eq(judgingEvaluationDrafts.entryId, input.entryId), + eq(judgingEvaluationDrafts.challengeId, input.challenge.id), + ne(judgingEvaluationDrafts.judgeId, input.judgeId), + ), + ); + + const refusal = gradingRefusal( + gradingStateOf(input.judgeId, evaluations, drafts, now), + input.challenge.label, + ); + if (refusal) throw new TRPCError({ code: "CONFLICT", message: refusal }); } /** @@ -483,16 +630,26 @@ const RECONCILE_BATCH = 50; * draft is submitted once, and a judge's own submit (which locks the draft * first) wins over this. A draft that can no longer be submitted (the guest's * session was revoked before the deadline, the project was withdrawn, the - * schedule was dropped) keeps its answers and loses its deadline. + * schedule was dropped, another judge already graded the project) keeps its + * answers and loses its deadline. So does a draft with no answers in it: the + * judge only opened the project (desk.startGrading), and that is not a grade. + * + * Each project is graded once, so the batch takes the grading lock of every + * project it touches before it writes anything: in key order, and ahead of any + * write, so it cannot deadlock with another batch or with a judge's submit. */ export async function reconcileDueDrafts( db: DrizzleDB, hackathonId: string, now = new Date(), ) { + // Waits out the grace a judge's own submit gets, so the two never race. const due = and( eq(judgingEvaluationDrafts.hackathonId, hackathonId), - lte(judgingEvaluationDrafts.deadlineAt, now), + lte( + judgingEvaluationDrafts.deadlineAt, + new Date(now.getTime() - DEADLINE_GRACE_MS), + ), or( eq(judgingConfigs.state, "open"), and( @@ -549,6 +706,17 @@ export async function reconcileDueDrafts( ); const kindOf = new Map(kinds.map((judge) => [judge.id, judge.kind])); + const locks = new Map( + drafts.map((draft) => [ + gradingLockKey(draft.entryId, draft.challengeId), + draft, + ]), + ); + for (const key of [...locks.keys()].sort()) { + const draft = locks.get(key)!; + await lockGrading(tx, draft.entryId, draft.challengeId); + } + const retain = (draftId: string) => tx .update(judgingEvaluationDrafts) @@ -575,20 +743,24 @@ export async function reconcileDueDrafts( .from(judgingEvaluations) .where( and( - eq(judgingEvaluations.judgeId, draft.judgeId), eq(judgingEvaluations.entryId, draft.entryId), eq(judgingEvaluations.challengeId, draft.challengeId), ), ) .limit(1); if (existing) { - // Never overwrite a result the judge submitted themselves. + // Never overwrite a result the judge submitted themselves, and a + // project another judge graded is graded. await retain(draft.id); continue; } const check = checkAnswers(rubric, draft, kind, "lenient"); - if (!check.ok) { + if ( + !check.ok || + (check.answers.ratings.length === 0 && + check.answers.responses.length === 0) + ) { await retain(draft.id); continue; } diff --git a/packages/api/src/services/judging/grading-hold.test.ts b/packages/api/src/services/judging/grading-hold.test.ts new file mode 100644 index 00000000..0fdf9ef7 --- /dev/null +++ b/packages/api/src/services/judging/grading-hold.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from "vitest"; +import { + GRADING_HOLD_MS, + gradingLockKey, + gradingRefusal, + gradingStateOf, + gradingStates, + holdsGrade, +} from "./grading-hold"; + +const NOW = new Date("2026-04-18T15:00:00Z"); +const ago = (ms: number) => new Date(NOW.getTime() - ms); +const ahead = (ms: number) => new Date(NOW.getTime() + ms); + +const draft = ( + judgeId: string, + over: { updatedAt?: Date; deadlineAt?: Date | null } = {}, +) => ({ + judgeId, + updatedAt: over.updatedAt ?? NOW, + deadlineAt: over.deadlineAt ?? null, +}); + +describe("GRADING_HOLD_MS", () => { + it("is ten minutes", () => { + expect(GRADING_HOLD_MS).toBe(10 * 60 * 1000); + }); +}); + +describe("holdsGrade", () => { + it("holds a project while its draft was saved inside the hold", () => { + expect(holdsGrade(draft("a"), NOW)).toBe(true); + expect( + holdsGrade( + { updatedAt: ago(GRADING_HOLD_MS - 1), deadlineAt: null }, + NOW, + ), + ).toBe(true); + }); + + it("lets go once the draft has sat for the whole hold", () => { + expect( + holdsGrade({ updatedAt: ago(GRADING_HOLD_MS), deadlineAt: null }, NOW), + ).toBe(false); + expect( + holdsGrade( + { updatedAt: ago(GRADING_HOLD_MS * 3), deadlineAt: null }, + NOW, + ), + ).toBe(false); + }); + + it("holds a stale draft until its presentation's deadline", () => { + const stale = ago(GRADING_HOLD_MS * 3); + expect(holdsGrade({ updatedAt: stale, deadlineAt: ahead(1000) }, NOW)).toBe( + true, + ); + expect(holdsGrade({ updatedAt: stale, deadlineAt: ago(1) }, NOW)).toBe( + false, + ); + expect(holdsGrade({ updatedAt: stale, deadlineAt: NOW }, NOW)).toBe(false); + }); +}); + +describe("gradingStateOf", () => { + it("is open when nobody has the project", () => { + expect(gradingStateOf("me", [], [], NOW)).toBe("open"); + }); + + it("is mine for a judge's own evaluation", () => { + expect(gradingStateOf("me", [{ judgeId: "me" }], [], NOW)).toBe("mine"); + }); + + it("is graded once another judge has submitted", () => { + expect(gradingStateOf("me", [{ judgeId: "them" }], [], NOW)).toBe("graded"); + }); + + it("keeps a project mine when old duplicates exist", () => { + expect( + gradingStateOf("me", [{ judgeId: "them" }, { judgeId: "me" }], [], NOW), + ).toBe("mine"); + }); + + it("is graded even if the judge has a draft of their own", () => { + expect( + gradingStateOf("me", [{ judgeId: "them" }], [draft("me")], NOW), + ).toBe("graded"); + }); + + it("is mine while the judge holds it with a draft", () => { + expect(gradingStateOf("me", [], [draft("me")], NOW)).toBe("mine"); + }); + + it("is in progress while another judge holds it", () => { + expect(gradingStateOf("me", [], [draft("them")], NOW)).toBe("in_progress"); + expect( + gradingStateOf( + "me", + [], + [ + draft("them", { + updatedAt: ago(GRADING_HOLD_MS * 2), + deadlineAt: ahead(60_000), + }), + ], + NOW, + ), + ).toBe("in_progress"); + }); + + it("is open past another judge's stale draft", () => { + expect( + gradingStateOf( + "me", + [], + [draft("them", { updatedAt: ago(GRADING_HOLD_MS + 1) })], + NOW, + ), + ).toBe("open"); + }); + + it("is open past the judge's own stale draft", () => { + expect( + gradingStateOf( + "me", + [], + [draft("me", { updatedAt: ago(GRADING_HOLD_MS + 1) })], + NOW, + ), + ).toBe("open"); + }); + + it("hands the hold to whoever started last", () => { + const old = draft("them", { updatedAt: ago(GRADING_HOLD_MS + 1) }); + const fresh = draft("me"); + expect(gradingStateOf("me", [], [old, fresh], NOW)).toBe("mine"); + expect(gradingStateOf("them", [], [old, fresh], NOW)).toBe("in_progress"); + }); + + describe("reserved for the judge assigned to it", () => { + const until = ahead(3 * 60_000); + + it("is reserved while a free project is kept back", () => { + expect(gradingStateOf("me", [], [], NOW, until)).toBe("reserved"); + }); + + it("is open once nothing is kept back", () => { + expect(gradingStateOf("me", [], [], NOW, null)).toBe("open"); + }); + + it("is reserved past another judge's stale draft", () => { + expect( + gradingStateOf( + "me", + [], + [draft("them", { updatedAt: ago(GRADING_HOLD_MS + 1) })], + NOW, + until, + ), + ).toBe("reserved"); + }); + + it("gives way to the assigned judge's draft: in progress, not reserved", () => { + expect(gradingStateOf("me", [], [draft("them")], NOW, until)).toBe( + "in_progress", + ); + }); + + it("gives way to a grade, and to the judge's own work", () => { + expect(gradingStateOf("me", [{ judgeId: "them" }], [], NOW, until)).toBe( + "graded", + ); + expect(gradingStateOf("me", [{ judgeId: "me" }], [], NOW, until)).toBe( + "mine", + ); + expect(gradingStateOf("me", [], [draft("me")], NOW, until)).toBe("mine"); + }); + }); +}); + +describe("gradingStates", () => { + it("reads each project from its own rows", () => { + const states = gradingStates( + "me", + ["mine", "theirs", "held", "free"], + [ + { entryId: "mine", judgeId: "me" }, + { entryId: "theirs", judgeId: "them" }, + ], + [{ entryId: "held", ...draft("them") }], + NOW, + ); + expect(Object.fromEntries(states)).toEqual({ + mine: "mine", + theirs: "graded", + held: "in_progress", + free: "open", + }); + }); + + it("marks the projects kept back for another judge", () => { + const until = ahead(60_000); + const states = gradingStates( + "me", + ["kept", "kept-but-held", "free"], + [], + [{ entryId: "kept-but-held", ...draft("them") }], + NOW, + new Map([ + ["kept", until], + ["kept-but-held", until], + ]), + ); + expect(Object.fromEntries(states)).toEqual({ + kept: "reserved", + "kept-but-held": "in_progress", + free: "open", + }); + }); + + it("ignores rows for projects it was not asked about", () => { + const states = gradingStates( + "me", + ["a"], + [{ entryId: "b", judgeId: "them" }], + [], + NOW, + ); + expect([...states.keys()]).toEqual(["a"]); + expect(states.get("a")).toBe("open"); + }); +}); + +describe("gradingRefusal", () => { + it("names the challenge when another judge graded the project", () => { + expect(gradingRefusal("graded", "Best Use of AI")).toBe( + "Another judge has already graded this project for Best Use of AI. Each project is graded once.", + ); + }); + + it("asks the judge to pick another while one is grading", () => { + expect(gradingRefusal("in_progress", "General")).toBe( + "Another judge is grading this project right now. Pick another one.", + ); + }); + + it("lets a judge write to a project that is theirs or open", () => { + expect(gradingRefusal("mine", "General")).toBeNull(); + expect(gradingRefusal("open", "General")).toBeNull(); + }); + + it("leaves a reserved project to the timing rules, which say when it opens", () => { + expect(gradingRefusal("reserved", "General")).toBeNull(); + }); +}); + +describe("gradingLockKey", () => { + it("is the same for the same pair and differs for another", () => { + expect(gradingLockKey("e1", "c1")).toBe(gradingLockKey("e1", "c1")); + expect(gradingLockKey("e1", "c1")).not.toBe(gradingLockKey("e1", "c2")); + expect(gradingLockKey("e1", "c1")).not.toBe(gradingLockKey("e2", "c1")); + expect(gradingLockKey("e1", "c2")).not.toBe(gradingLockKey("e2", "c1")); + }); +}); diff --git a/packages/api/src/services/judging/grading-hold.ts b/packages/api/src/services/judging/grading-hold.ts new file mode 100644 index 00000000..431f9049 --- /dev/null +++ b/packages/api/src/services/judging/grading-hold.ts @@ -0,0 +1,127 @@ +// Each project is graded once per (root) challenge, by one judge. Pure: the +// desk loads the rows, this decides. +// +// A project is closed to other judges once someone has submitted an evaluation +// of it. Until then a judge who is writing one holds it: their draft keeps +// other judges off for GRADING_HOLD_MS after its last save, and a scheduled +// presentation's draft holds until its deadline. A draft left alone past that +// holds nothing, and a judge who starts afterwards takes the hold over. +// Evaluations that already exist twice, from before this rule, are left as they +// are. +// +// A presentation assigned to another judge is reserved for them until its +// takeover time (timing.ts). An open project in that window reads "reserved". +// Once the assigned judge has a draft it reads "in_progress" like any other +// held project, so a judge who takes over after the window cannot take one the +// assigned judge started. + +/** How long after its last save a draft keeps other judges off the project. */ +export const GRADING_HOLD_MS = 10 * 60_000; + +/** + * Where a project stands for one judge: "mine" (they graded it, or hold it), + * "graded" (another judge did), "in_progress" (another judge holds it), + * "reserved" (free, but kept for the judge assigned to it until a time), or + * "open". + */ +export type GradingState = + "mine" | "graded" | "in_progress" | "reserved" | "open"; + +type HeldDraft = { updatedAt: Date; deadlineAt: Date | null }; + +/** Whether a draft still keeps other judges off its project. */ +export function holdsGrade(draft: HeldDraft, now: Date): boolean { + return ( + now.getTime() - draft.updatedAt.getTime() < GRADING_HOLD_MS || + (draft.deadlineAt !== null && draft.deadlineAt.getTime() > now.getTime()) + ); +} + +/** + * One project's state for `judgeId`, from its evaluations and drafts for the + * challenge (every judge's, theirs included). A judge's own evaluation wins + * over another's, so a project both graded stays theirs to edit. + * `reservedUntil` is when the project opens to this judge, if it is held back + * for another judge until then (timing.ts reservedUntil); it only shows on a + * project nobody has graded or holds. + */ +export function gradingStateOf( + judgeId: string, + evaluations: readonly { judgeId: string }[], + drafts: readonly (HeldDraft & { judgeId: string })[], + now: Date, + reservedUntil: Date | null = null, +): GradingState { + if (evaluations.some((evaluation) => evaluation.judgeId === judgeId)) { + return "mine"; + } + if (evaluations.length > 0) return "graded"; + + const holding = drafts.filter((draft) => holdsGrade(draft, now)); + if (holding.some((draft) => draft.judgeId === judgeId)) return "mine"; + if (holding.length > 0) return "in_progress"; + return reservedUntil ? "reserved" : "open"; +} + +/** + * The state of each of `entryIds`, from the rows of one challenge. + * `reservedUntil` holds the entries kept back for another judge, with when + * each opens to this one. + */ +export function gradingStates( + judgeId: string, + entryIds: readonly string[], + evaluations: readonly { entryId: string; judgeId: string }[], + drafts: readonly (HeldDraft & { entryId: string; judgeId: string })[], + now: Date, + reservedUntil: ReadonlyMap = new Map(), +): Map { + const evaluationsOf = new Map(); + for (const evaluation of evaluations) { + const list = evaluationsOf.get(evaluation.entryId) ?? []; + list.push(evaluation); + evaluationsOf.set(evaluation.entryId, list); + } + const draftsOf = new Map(); + for (const draft of drafts) { + const list = draftsOf.get(draft.entryId) ?? []; + list.push(draft); + draftsOf.set(draft.entryId, list); + } + + return new Map( + entryIds.map((entryId) => [ + entryId, + gradingStateOf( + judgeId, + evaluationsOf.get(entryId) ?? [], + draftsOf.get(entryId) ?? [], + now, + reservedUntil.get(entryId) ?? null, + ), + ]), + ); +} + +/** + * Why a judge cannot write to a project in this state, or null if they can. + * A reserved project is refused by the timing rules instead, which say when it + * opens (timing.ts). + */ +export function gradingRefusal( + state: GradingState, + challengeLabel: string, +): string | null { + if (state === "graded") { + return `Another judge has already graded this project for ${challengeLabel}. Each project is graded once.`; + } + if (state === "in_progress") { + return "Another judge is grading this project right now. Pick another one."; + } + return null; +} + +/** What writers of one project's grade for one challenge lock on. */ +export function gradingLockKey(entryId: string, challengeId: string): string { + return `grading:${entryId}:${challengeId}`; +} diff --git a/packages/api/src/services/judging/itinerary.test.ts b/packages/api/src/services/judging/itinerary.test.ts index 6fb6b531..48b05f22 100644 --- a/packages/api/src/services/judging/itinerary.test.ts +++ b/packages/api/src/services/judging/itinerary.test.ts @@ -118,25 +118,25 @@ describe("evaluationsForAppointment", () => { judgeId: "a", challengeId: "general", appointmentId: "s1", - isComplete: true, + ratingsComplete: true, }, { judgeId: "b", challengeId: "general", appointmentId: null, - isComplete: true, + ratingsComplete: true, }, { judgeId: "c", challengeId: "general", appointmentId: "s9", - isComplete: true, + ratingsComplete: true, }, { judgeId: "d", challengeId: "sponsor", appointmentId: null, - isComplete: true, + ratingsComplete: true, }, ]; expect( @@ -269,7 +269,7 @@ describe("buildItinerary", () => { judgeId: "j1", challengeId: "general", appointmentId: "s1", - isComplete: true, + ratingsComplete: true, }, ], presence: [ @@ -295,7 +295,7 @@ describe("buildItinerary", () => { judgeId: "j1", challengeId: "general", appointmentId: "s1", - isComplete: true, + ratingsComplete: true, }, ], presence: [ @@ -319,6 +319,22 @@ describe("buildItinerary", () => { expect(itinerary.appointments[0]!.status).toBe("incomplete"); }); + it("keeps a slot incomplete while an evaluation lacks a rating", () => { + const itinerary = buildItinerary({ + ...base, + evaluations: [ + { + judgeId: "j1", + challengeId: "general", + appointmentId: "s1", + ratingsComplete: false, + }, + ], + now: at(12), + }); + expect(itinerary.appointments[0]!.status).toBe("incomplete"); + }); + it("lists unscheduled challenges with their live rooms and whether judged", () => { const pending = buildItinerary({ ...base, now: at(5) }); expect(pending.unscheduled).toEqual([ @@ -341,12 +357,26 @@ describe("buildItinerary", () => { judgeId: "j1", challengeId: "mlh", appointmentId: null, - isComplete: true, + ratingsComplete: true, }, ], now: at(5), }); expect(judged.unscheduled[0]!.judged).toBe(true); + + const unrated = buildItinerary({ + ...base, + evaluations: [ + { + judgeId: "j1", + challengeId: "mlh", + appointmentId: null, + ratingsComplete: false, + }, + ], + now: at(5), + }); + expect(unrated.unscheduled[0]!.judged).toBe(false); }); }); diff --git a/packages/api/src/services/judging/itinerary.ts b/packages/api/src/services/judging/itinerary.ts index 0899078a..02f8c87b 100644 --- a/packages/api/src/services/judging/itinerary.ts +++ b/packages/api/src/services/judging/itinerary.ts @@ -39,7 +39,8 @@ export type ItineraryEvaluation = { judgeId: string; challengeId: string; appointmentId: string | null; - isComplete: boolean; + /** Holds every rating (scoring.hasEveryRating), so placings count it. */ + ratingsComplete: boolean; }; export type ItineraryPresence = { @@ -159,7 +160,7 @@ export type UnscheduledChallengeView = { challenge: string; children: string[]; rooms: { name: string; kind: JudgingRoomKind; building: string | null }[]; - /** A judge has finished an evaluation of the project in this challenge. */ + /** A judge has rated the project in this challenge, every rating in. */ judged: boolean; }; @@ -243,7 +244,7 @@ export function buildItinerary(input: { })), judged: evaluations.some( (evaluation) => - evaluation.isComplete && + evaluation.ratingsComplete && rootId(evaluation.challengeId) === challenge.id, ), })); @@ -290,8 +291,8 @@ export type FeedbackItem = { /** * Written feedback, one item per evaluation that has any to share, in * challenge order then the order it arrived. Carries no judge and no score: - * the router passes only complete member evaluations and their public - * responses. + * the router passes only member evaluations that hold every rating, and their + * public responses. */ export function groupFeedback( evaluations: readonly FeedbackEvaluation[], diff --git a/packages/api/src/services/judging/outer-column.test.ts b/packages/api/src/services/judging/outer-column.test.ts new file mode 100644 index 00000000..77c516e3 --- /dev/null +++ b/packages/api/src/services/judging/outer-column.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { + judgingAppointments, + judgingRoomLinks, + judgingRooms, + judgingSchedules, +} from "@query/db"; +import { outerColumn } from "./outer-column"; + +const db = drizzle.mock(); + +describe("outerColumn", () => { + it("keeps the outer table's name in a one-table select", () => { + const query = db + .select({ + appointments: sql`(select count(*)::int from ${judgingAppointments} where ${judgingAppointments.scheduleId} = ${outerColumn(judgingSchedules, judgingSchedules.id)})`, + }) + .from(judgingSchedules) + .toSQL(); + + expect(query.sql).toContain( + `where "schedule_id" = "judging_schedule"."id"`, + ); + }); + + it("is needed: a plain column loses its table name there", () => { + const query = db + .select({ + hasLink: sql`exists (select 1 from ${judgingRoomLinks} where ${judgingRoomLinks.roomId} = ${judgingRooms.id})`, + }) + .from(judgingRooms) + .toSQL(); + + expect(query.sql).toContain(`where "room_id" = "id"`); + }); +}); diff --git a/packages/api/src/services/judging/outer-column.ts b/packages/api/src/services/judging/outer-column.ts new file mode 100644 index 00000000..f82a5dd0 --- /dev/null +++ b/packages/api/src/services/judging/outer-column.ts @@ -0,0 +1,10 @@ +import { getTableName, sql } from "drizzle-orm"; +import type { Column, Table } from "drizzle-orm"; + +// A select from one table with no joins drops the table name from every +// column in a sql expression. In a correlated subquery the outer row's column +// then resolves to the subquery's own column of the same name, so the +// subquery compares its rows with themselves. Name the outer column with this +// instead; the query must read the table under its own name. +export const outerColumn = (table: Table, column: Column) => + sql`${sql.identifier(getTableName(table))}.${sql.identifier(column.name)}`; diff --git a/packages/api/src/services/judging/placements.test.ts b/packages/api/src/services/judging/placements.test.ts index f8161b6a..bc8f401c 100644 --- a/packages/api/src/services/judging/placements.test.ts +++ b/packages/api/src/services/judging/placements.test.ts @@ -1,13 +1,51 @@ -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { calibrate } from "./calibration"; +import type * as CalibrationModule from "./calibration"; import { computePlacements, formatScore } from "./placements"; import type { PlacementEvaluation } from "./placements"; +// The real calibrate, but spied on so a test can swap it out. The first group +// below is about ranking, scopes and ties, so it runs on scores that are left +// as the raw means; the second runs the real thing. +vi.mock("./calibration", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, calibrate: vi.fn(actual.calibrate) }; +}); +const actualCalibrate = async () => + (await vi.importActual("./calibration")).calibrate; + +const leaveAsRaw: typeof calibrate = (evaluations) => ({ + adjusted: new Map( + evaluations + .filter((e) => e.ratingsComplete && e.ratings.length > 0) + .map((e) => [ + e.id, + e.ratings.reduce((sum, value) => sum + value, 0) / e.ratings.length, + ]), + ), + judges: [], +}); + +let count = 0; const evaluation = ( entryId: string, challengeId: string, ratings: number[], - isComplete = true, -): PlacementEvaluation => ({ entryId, challengeId, ratings, isComplete }); + ratingsComplete = true, + over: Partial = {}, +): PlacementEvaluation => { + count += 1; + return { + id: `evaluation-${count}`, + judgeId: "judge", + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, 0, count)), + entryId, + challengeId, + ratings, + ratingsComplete, + ...over, + }; +}; const challenges = [ { id: "general", label: "General" }, @@ -15,6 +53,12 @@ const challenges = [ ]; describe("computePlacements", () => { + beforeEach(() => { + count = 0; + vi.mocked(calibrate).mockReset(); + vi.mocked(calibrate).mockImplementation(leaveAsRaw); + }); + it("ranks overall and per challenge, ties sharing a placing", () => { const placements = computePlacements({ entryIds: ["a", "b", "c"], @@ -68,7 +112,7 @@ describe("computePlacements", () => { ); }); - it("counts complete evaluations only and leaves unscored projects out", () => { + it("counts evaluations with every rating only and leaves unscored projects out", () => { const placements = computePlacements({ entryIds: ["a", "b"], memberships: [ @@ -119,6 +163,164 @@ describe("computePlacements", () => { }); }); +describe("computePlacements with judge calibration", () => { + beforeEach(async () => { + count = 0; + vi.mocked(calibrate).mockReset(); + vi.mocked(calibrate).mockImplementation(await actualCalibrate()); + }); + + // A strict judge and a lenient one each graded six projects, dealt out at + // random, so the two sets are alike in quality and the lenient judge's higher + // marks are the judge's, not the projects'. + const strictScores = [3.0, 3.2, 3.4, 3.6, 3.8, 4.0]; + const lenientScores = [3.8, 4.0, 4.2, 4.4, 4.6, 4.8]; + // Marked in a scrambled order, so neither judge looks to be drifting. + const gradingOrder = [2, 5, 0, 4, 1, 3]; + const twoJudges = () => { + const graded = (prefix: string, judgeId: string, scores: number[]) => + gradingOrder.map((i) => + evaluation(`${prefix}${i + 1}`, "general", [scores[i]!], true, { + judgeId, + }), + ); + const evaluations = [ + ...graded("s", "strict", strictScores), + ...graded("l", "lenient", lenientScores), + ]; + const entryIds = evaluations.map((e) => e.entryId); + return computePlacements({ + entryIds, + memberships: entryIds.map((entryId) => ({ + entryId, + challengeId: "general", + })), + challenges, + evaluations, + }); + }; + + it("ranks on the calibrated score, so a strict judge's best is not buried", () => { + const overall = new Map( + twoJudges() + .filter((p) => p.scope === "overall") + .map((p) => [p.entryId, p]), + ); + const place = (entryId: string) => overall.get(entryId)!.placement; + const raw = (entryId: string) => overall.get(entryId)!.rawScore; + + // On raw marks the strict judge's best (4.0) is below the lenient judge's + // third best (4.4). + expect(raw("s6")).toBeLessThan(raw("l4")); + // Calibrated, each of the strict judge's projects beats the lenient + // judge's project whose raw mark is 0.4 above it. + expect(place("s6")).toBeLessThan(place("l4")); + expect(place("s5")).toBeLessThan(place("l3")); + expect(place("s4")).toBeLessThan(place("l2")); + expect(place("s3")).toBeLessThan(place("l1")); + // Within a judge, the order is the judge's own. + expect(place("l6")).toBeLessThan(place("l5")); + expect(place("s2")).toBeLessThan(place("s1")); + }); + + it("keeps the plain mean beside the calibrated score", () => { + const placements = twoJudges(); + const byEntry = new Map( + placements + .filter((p) => p.scope === "overall") + .map((p) => [p.entryId, p]), + ); + expect(byEntry.get("s6")?.rawScore).toBeCloseTo(4.0, 10); + expect(byEntry.get("l1")?.rawScore).toBeCloseTo(3.8, 10); + // The strict judge's score moves up, the lenient judge's down. + expect(byEntry.get("s6")!.score).toBeGreaterThan(4.0); + expect(byEntry.get("l6")!.score).toBeLessThan(4.8); + }); + + it("scores a project the same in a challenge as overall when it has one evaluation", () => { + const placements = twoJudges(); + const pick = (scope: string, entryId: string) => + placements.find((p) => p.scope === scope && p.entryId === entryId); + expect(pick("general", "s3")?.score).toBe(pick("overall", "s3")?.score); + expect(pick("general", "s3")?.rawScore).toBe( + pick("overall", "s3")?.rawScore, + ); + }); + + it("calibrates once, over the live entries' evaluations only", () => { + const evaluations = [ + evaluation("a", "general", [4], true, { judgeId: "one" }), + evaluation("b", "sponsor", [3], true, { judgeId: "two" }), + evaluation("a", "general", [2], false, { judgeId: "one" }), + // Pulled from prizes: judges scored it, it must not shift anyone else. + evaluation("pulled", "general", [5], true, { judgeId: "two" }), + ]; + computePlacements({ + entryIds: ["a", "b"], + memberships: [ + { entryId: "a", challengeId: "general" }, + { entryId: "b", challengeId: "sponsor" }, + ], + challenges, + evaluations, + }); + + expect(calibrate).toHaveBeenCalledTimes(1); + const given = vi.mocked(calibrate).mock.calls[0]![0]; + expect(given.map((e) => e.entryId).sort()).toEqual(["a", "a", "b"]); + }); + + it("does not let a pulled project's grades move anyone else's score", () => { + const base = [ + evaluation("a", "general", [4], true, { judgeId: "one" }), + evaluation("b", "general", [3], true, { judgeId: "one" }), + evaluation("c", "general", [3.5], true, { judgeId: "two" }), + ]; + const run = (extra: PlacementEvaluation[]) => + computePlacements({ + entryIds: ["a", "b", "c"], + memberships: ["a", "b", "c"].map((entryId) => ({ + entryId, + challengeId: "general", + })), + challenges, + evaluations: [...base, ...extra], + }).map((p) => [p.scope, p.entryId, p.score]); + + expect( + run([evaluation("pulled", "general", [5], true, { judgeId: "two" })]), + ).toEqual(run([])); + }); + + it("does not compare a sponsor's judges with the table judges", () => { + // The sponsor's two guests both mark high; that must not drag the tables. + const tables = [ + evaluation("a", "general", [3], true, { judgeId: "t1" }), + evaluation("b", "general", [4], true, { judgeId: "t2" }), + ]; + const sponsor = [ + evaluation("c", "sponsor", [5], true, { judgeId: "g1" }), + evaluation("d", "sponsor", [5], true, { judgeId: "g2" }), + ]; + const input = { + entryIds: ["a", "b", "c", "d"], + memberships: [ + { entryId: "a", challengeId: "general" }, + { entryId: "b", challengeId: "general" }, + { entryId: "c", challengeId: "sponsor" }, + { entryId: "d", challengeId: "sponsor" }, + ], + challenges, + }; + const general = (evaluations: PlacementEvaluation[]) => + computePlacements({ ...input, evaluations }) + .filter((p) => p.scope === "general") + .map((p) => [p.entryId, p.score]); + + expect(general([...tables, ...sponsor])).toEqual(general(tables)); + }); +}); + describe("formatScore", () => { it("writes two decimal places for numeric(4,2)", () => { expect(formatScore(4)).toBe("4.00"); diff --git a/packages/api/src/services/judging/placements.ts b/packages/api/src/services/judging/placements.ts index 16fdc4de..ec26ffac 100644 --- a/packages/api/src/services/judging/placements.ts +++ b/packages/api/src/services/judging/placements.ts @@ -1,9 +1,11 @@ // Turns evaluations into placings: one ranking per challenge and one overall. -// A project's score is the mean of its complete evaluations (scoring.ts); -// equal scores share a placing. Projects nobody scored are left out rather -// than listed last, since a placing is published and "8th of 8" for a project -// no judge reached would be a claim nobody made. +// A project's score is the mean of its counted evaluations (scoring.ts), each +// calibrated for its judge's habits first (calibration.ts); equal scores share +// a placing. The raw mean is kept beside it. Projects nobody scored are left +// out rather than listed last, since a placing is published and "8th of 8" for +// a project no judge reached would be a claim nobody made. +import { calibrate } from "./calibration"; import { overallScore, rankByScore, scopedScore } from "./scoring"; import type { ScoredEvaluation } from "./scoring"; @@ -11,7 +13,14 @@ import type { ScoredEvaluation } from "./scoring"; export const OVERALL_SCOPE = "overall"; export const OVERALL_LABEL = "Overall"; -export type PlacementEvaluation = ScoredEvaluation & { entryId: string }; +export type PlacementEvaluation = ScoredEvaluation & { + /** Calibration counts evaluations by this alone. */ + ratingsComplete: boolean; + id: string; + entryId: string; + judgeId: string; + submittedAt: Date; +}; export type ComputedPlacement = { /** "overall" or a challenge id. */ @@ -19,7 +28,10 @@ export type ComputedPlacement = { scopeLabel: string; entryId: string; placement: number; + /** Calibrated for judge bias. This is what ranks. */ score: number; + /** The plain mean of the same evaluations. */ + rawScore: number; evaluationCount: number; }; @@ -36,11 +48,18 @@ export function computePlacements(input: { const entryIds = [...new Set(input.entryIds)].sort(); const live = new Set(entryIds); + // Calibrated once, over every counted evaluation of a live entry, so a + // judge's habits are learned from all of their grades and not from the + // slice that fell in one scope. + const liveEvaluations = input.evaluations.filter((evaluation) => + live.has(evaluation.entryId), + ); + const { adjusted } = calibrate(liveEvaluations); + const evaluationsOf = new Map(); - for (const evaluation of input.evaluations) { - if (!live.has(evaluation.entryId)) continue; + for (const evaluation of liveEvaluations) { const list = evaluationsOf.get(evaluation.entryId) ?? []; - list.push(evaluation); + list.push({ ...evaluation, adjustedScore: adjusted.get(evaluation.id) }); evaluationsOf.set(evaluation.entryId, list); } @@ -54,13 +73,29 @@ export function computePlacements(input: { }, ): ComputedPlacement[] => { const scored = candidates - .map((entryId) => ({ - entryId, - ...summarize(evaluationsOf.get(entryId) ?? []), - })) + .map((entryId) => { + const evals = evaluationsOf.get(entryId) ?? []; + return { + entryId, + ...summarize(evals), + // The same evaluations on their plain ratings. + rawScore: summarize( + evals.map((evaluation) => ({ + ...evaluation, + adjustedScore: undefined, + })), + ).score, + }; + }) .filter( - (row): row is { entryId: string; score: number; count: number } => - row.score !== null, + ( + row, + ): row is { + entryId: string; + score: number; + count: number; + rawScore: number; + } => row.score !== null && row.rawScore !== null, ); return rankByScore(scored).map((row) => ({ scope, @@ -68,6 +103,7 @@ export function computePlacements(input: { entryId: row.entryId, placement: row.rank, score: row.score, + rawScore: row.rawScore, evaluationCount: row.count, })); }; diff --git a/packages/api/src/services/judging/queue-store.test.ts b/packages/api/src/services/judging/queue-store.test.ts new file mode 100644 index 00000000..8d03f681 --- /dev/null +++ b/packages/api/src/services/judging/queue-store.test.ts @@ -0,0 +1,543 @@ +import { describe, expect, it } from "vitest"; +import { getTableName } from "drizzle-orm"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + judgingAppointments, + judgingEntries, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingSchedules, +} from "@query/db"; +import { QUEUE_REFUSAL, assertQueueOffers, loadQueue } from "./queue-store"; +import { QUEUE_WINDOW, queueOrder, queueWindow } from "./queue"; +import type { ChallengeRow, Seat } from "./evaluation-store"; +import { GRADING_HOLD_MS } from "./grading-hold"; + +const HACK = "hackathon"; +const JUDGE = "judge-1"; + +const row = (id: string, over: Partial = {}): ChallengeRow => ({ + id, + parentId: null, + mode: "unscheduled", + label: id, + isGeneral: false, + isGroup: false, + ...over, +}); +const general = row("general", { isGeneral: true }); +const sponsor = row("sponsor", { mode: "scheduled" }); +const mlh = row("mlh", { isGroup: true }); +const mongo = row("mongo", { parentId: "mlh" }); +const challenges = [general, sponsor, mlh, mongo]; + +const seat = (challengeId: string): Seat => ({ + roomId: "room-1", + roomName: "Table 4", + challengeId, +}); + +const entryIds = Array.from({ length: 8 }, (_, i) => `entry-${i}`); + +const name = (table: unknown) => getTableName(table as never); +const via = (...tables: unknown[]) => tables.map(name).join(">"); +const entries = via(judgingEntries, judgingEntryChallenges); +const evaluations = via(judgingEvaluations); +const drafts = via(judgingEvaluationDrafts); +const schedules = via(judgingSchedules); +const appointments = via(judgingAppointments); +const memberships = via(judgingEntryChallenges); + +type Rows = unknown[] | ((call: number) => unknown[]); +type Read = { table: string; where?: SQL; on?: SQL }; + +/** A database whose selects answer by the tables they read, in order. */ +function fakeDb(answers: Record) { + const reads: Read[] = []; + const counts = new Map(); + const select = () => { + const joined: unknown[] = []; + let from: unknown; + let where: SQL | undefined; + let on: SQL | undefined; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + where: (clause: SQL) => { + where = clause; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (e: unknown) => unknown, + ) => { + const key = via(from, ...joined); + reads.push({ table: key, where, on }); + const call = counts.get(key) ?? 0; + counts.set(key, call + 1); + const answer = answers[key] ?? []; + return Promise.resolve( + typeof answer === "function" ? answer(call) : answer, + ).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown, clause: SQL) => { + joined.push(table); + on = clause; + return chain; + }; + } + for (const method of ["orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }; + return { db: { select } as never, reads }; +} + +const ids = (list: readonly string[]) => list.map((id) => ({ id })); +const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000); +const minutesAhead = (minutes: number) => + new Date(Date.now() + minutes * 60_000); + +const input = ( + over: Partial[1]> = {}, +): Parameters[1] => ({ + hackathonId: HACK, + judgeId: JUDGE, + challenge: general, + seat: null, + ...over, +}); + +describe("loadQueue", () => { + it("says where each project stands for the judge", async () => { + const { db } = fakeDb({ + [entries]: ids(entryIds.slice(0, 5)), + [evaluations]: [ + { entryId: "entry-0", judgeId: JUDGE }, + { entryId: "entry-1", judgeId: "other" }, + ], + [drafts]: [ + { + entryId: "entry-2", + judgeId: "other", + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: "entry-3", + judgeId: "other", + updatedAt: minutesAgo(60), + deadlineAt: null, + }, + ], + }); + + const queue = await loadQueue(db, input()); + + expect(queue.timed).toBe(false); + expect(queue.needsRoom).toBe(false); + expect( + Object.fromEntries(queue.candidates.map((c) => [c.entryId, c.grading])), + ).toEqual({ + "entry-0": "mine", + "entry-1": "graded", + "entry-2": "in_progress", + "entry-3": "open", + "entry-4": "open", + }); + }); + + it("keeps a project the judge only opened on offer to them", async () => { + const mine = (entryId: string, revision: number) => ({ + entryId, + judgeId: JUDGE, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision, + }); + const { db } = fakeDb({ + [entries]: ids(entryIds.slice(0, 4)), + [evaluations]: [ + { entryId: "entry-2", judgeId: "other" }, + { entryId: "entry-3", judgeId: JUDGE }, + ], + [drafts]: [ + mine("entry-0", 0), + mine("entry-1", 2), + mine("entry-2", 0), + mine("entry-3", 0), + ], + }); + + const queue = await loadQueue(db, input()); + + expect( + Object.fromEntries(queue.candidates.map((c) => [c.entryId, c.grading])), + ).toEqual({ + // Opened, nothing saved: still on offer. + "entry-0": "open", + // Saved to: the judge is grading it. + "entry-1": "mine", + // Another judge's grade, or the judge's own, wins over an empty draft. + "entry-2": "graded", + "entry-3": "mine", + }); + }); + + it("has no candidates, and reads no grades, for a challenge without projects", async () => { + const { db, reads } = fakeDb({}); + + const queue = await loadQueue(db, input()); + + expect(queue.candidates).toEqual([]); + expect(reads.map((r) => r.table)).toEqual([schedules, entries]); + }); + + it("narrows the projects to a prize when asked", async () => { + const { db, reads } = fakeDb({ [entries]: ids(["entry-0"]) }); + + await loadQueue(db, input({ challenge: mlh, filterId: "mongo" })); + + const params = new PgDialect().sqlToQuery( + reads.find((r) => r.table === entries)?.on as SQL, + ).params; + expect(params).toContain("mongo"); + }); + + describe("at a scheduled challenge under a saved schedule", () => { + const world = (over: Record = {}) => ({ + [schedules]: [{ id: "schedule" }], + [entries]: ids(["entry-0", "entry-1", "entry-2"]), + ...over, + }); + + it("is timed, and asks a judge seated elsewhere to join the room", async () => { + const { db, reads } = fakeDb(world()); + + const queue = await loadQueue(db, input({ challenge: sponsor })); + expect(queue).toEqual({ timed: true, needsRoom: true, candidates: [] }); + + const elsewhere = await loadQueue( + db, + input({ challenge: sponsor, seat: seat("general") }), + ); + expect(elsewhere.needsRoom).toBe(true); + expect(reads.some((r) => r.table === entries)).toBe(false); + }); + + it("offers only the presentations running now in the judge's room", async () => { + const slot = (entryId: string, startsAt: Date, deadlineAt: Date) => ({ + id: `slot-${entryId}`, + entryId, + startsAt, + deadlineAt, + }); + const { db } = fakeDb( + world({ + [appointments]: [ + slot("entry-0", minutesAgo(30), minutesAgo(20)), + slot("entry-1", minutesAgo(5), minutesAhead(5)), + slot("entry-2", minutesAhead(20), minutesAhead(30)), + ], + }), + ); + + const queue = await loadQueue( + db, + input({ challenge: sponsor, seat: seat("sponsor") }), + ); + + expect(queue.timed).toBe(true); + expect(queue.needsRoom).toBe(false); + expect(queue.candidates.map((c) => c.entryId)).toEqual(["entry-1"]); + }); + + describe("with a judge assigned to each presentation", () => { + const running = (entryId: string, assignedJudgeId: string | null) => ({ + id: `slot-${entryId}`, + entryId, + startsAt: minutesAgo(2), + deadlineAt: minutesAhead(8), + assignedJudgeId, + }); + const here = input({ challenge: sponsor, seat: seat("sponsor") }); + + it("leaves out a presentation still reserved for another judge", async () => { + const { db } = fakeDb( + world({ + [appointments]: [ + running("entry-0", "other"), + running("entry-1", JUDGE), + running("entry-2", null), + ], + }), + ); + + const queue = await loadQueue(db, here); + + expect(queue.candidates.map((c) => c.entryId)).toEqual([ + "entry-1", + "entry-2", + ]); + }); + + it("says which presentations are the judge's own", async () => { + const { db } = fakeDb( + world({ + [appointments]: [ + running("entry-0", "other"), + running("entry-1", JUDGE), + running("entry-2", null), + ], + }), + ); + + const queue = await loadQueue(db, here); + + expect( + Object.fromEntries( + queue.candidates.map((c) => [c.entryId, c.assigned ?? false]), + ), + ).toEqual({ "entry-1": true, "entry-2": false }); + }); + + it("opens a presentation to the judge once its takeover time has passed", async () => { + const late = (entryId: string) => ({ + ...running(entryId, "other"), + startsAt: minutesAgo(6), + deadlineAt: minutesAhead(4), + }); + const { db } = fakeDb( + world({ + [appointments]: [late("entry-0"), running("entry-1", "other")], + }), + ); + + const queue = await loadQueue(db, here); + + expect(queue.candidates.map((c) => c.entryId)).toEqual(["entry-0"]); + // Open for takeover, not the judge's own. + expect(queue.candidates[0].assigned).toBeUndefined(); + }); + + it("offers the judge's own presentation before the ones open to everyone", async () => { + const { db } = fakeDb( + world({ + [appointments]: [ + running("entry-0", null), + running("entry-1", null), + running("entry-2", JUDGE), + ], + }), + ); + + const queue = await loadQueue(db, here); + + expect( + queueWindow(JUDGE, "sponsor", queue.candidates, Infinity)[0], + ).toBe("entry-2"); + }); + + it("still keeps a presentation another judge holds off the judge", async () => { + const { db } = fakeDb( + world({ + [appointments]: [ + { ...running("entry-0", "other"), startsAt: minutesAgo(6) }, + ], + [drafts]: [ + { + entryId: "entry-0", + judgeId: "other", + updatedAt: minutesAgo(1), + deadlineAt: minutesAhead(4), + revision: 1, + }, + ], + }), + ); + + const queue = await loadQueue(db, here); + + expect(queue.candidates).toEqual([ + { entryId: "entry-0", grading: "in_progress" }, + ]); + expect( + queueWindow(JUDGE, "sponsor", queue.candidates, Infinity), + ).toEqual([]); + }); + }); + }); + + it("is untimed for a scheduled challenge nobody has scheduled yet", async () => { + const { db } = fakeDb({ [entries]: ids(["entry-0"]) }); + + const queue = await loadQueue(db, input({ challenge: sponsor })); + + expect(queue.timed).toBe(false); + expect(queue.candidates).toHaveLength(1); + }); +}); + +describe("assertQueueOffers", () => { + const order = queueOrder(JUDGE, "general", entryIds); + const claim = ( + entryId: string, + answers: Record, + over: Partial[1]> = {}, + ) => { + const fake = fakeDb({ [entries]: ids(entryIds), ...answers }); + return { + ...fake, + claim: () => + assertQueueOffers(fake.db, { + hackathonId: HACK, + judgeId: JUDGE, + entryId, + challenge: general, + challenges, + ...over, + }), + }; + }; + + it("lets a judge start any of the first projects their queue offers", async () => { + for (const entryId of order.slice(0, QUEUE_WINDOW)) { + await expect(claim(entryId, {}).claim()).resolves.toBeUndefined(); + } + }); + + it("refuses a project further down their queue", async () => { + for (const entryId of order.slice(QUEUE_WINDOW)) { + await expect(claim(entryId, {}).claim()).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: QUEUE_REFUSAL, + }); + } + }); + + it("says what to do instead", () => { + expect(QUEUE_REFUSAL).toBe( + "Grade the projects your queue offers so every judge sees a random mix. Use Grade next project.", + ); + }); + + it("slides the window past projects that are graded or held", async () => { + const fourth = order[QUEUE_WINDOW]; + const { claim: go } = claim(fourth, { + [evaluations]: [{ entryId: order[0], judgeId: "other" }], + [drafts]: [ + { + entryId: order[1], + judgeId: "other", + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: order[2], + judgeId: "other", + updatedAt: minutesAgo(2), + deadlineAt: null, + }, + ], + }); + + await expect(go()).resolves.toBeUndefined(); + }); + + it("does not slide the window past projects the judge only opened", async () => { + const held = (entryId: string, revision: number) => ({ + entryId, + judgeId: JUDGE, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision, + }); + const opened = order.slice(0, QUEUE_WINDOW).map((id) => held(id, 0)); + await expect( + claim(order[QUEUE_WINDOW], { [drafts]: opened }).claim(), + ).rejects.toMatchObject({ message: QUEUE_REFUSAL }); + + // Once they save to one, they are grading it, and the window moves on. + const started = [held(order[0], 1), ...opened.slice(1)]; + await expect( + claim(order[QUEUE_WINDOW], { [drafts]: started }).claim(), + ).resolves.toBeUndefined(); + }); + + it("does not count a draft left alone past the hold", async () => { + const { claim: go } = claim(order[0], { + [drafts]: [ + { + entryId: order[0], + judgeId: "other", + updatedAt: new Date(Date.now() - GRADING_HOLD_MS - 60_000), + deadlineAt: null, + }, + ], + }); + + await expect(go()).resolves.toBeUndefined(); + }); + + it("asks nothing of a scheduled challenge: the schedule decides", async () => { + const { claim: go, reads } = claim( + order[7], + { [schedules]: [{ id: "schedule" }] }, + { challenge: sponsor }, + ); + + await expect(go()).resolves.toBeUndefined(); + expect(reads.some((r) => r.table === entries)).toBe(false); + }); + + it("still asks it of a scheduled challenge nobody has scheduled yet", async () => { + const worst = queueOrder(JUDGE, "sponsor", entryIds)[7]; + const { claim: go } = claim(worst, {}, { challenge: sponsor }); + + await expect(go()).rejects.toMatchObject({ message: QUEUE_REFUSAL }); + }); + + describe("under a group, with the list narrowed to one prize", () => { + const wholeOrder = queueOrder(JUDGE, "mlh", entryIds); + const prize = (entryId: string, prizeEntries: string[]) => + claim( + entryId, + { + [entries]: (call) => ids(call === 0 ? entryIds : prizeEntries), + [memberships]: [{ challengeId: "mongo" }], + }, + { challenge: mlh }, + ); + + it("offers a project that tops its prize's queue though it is far down the group's", async () => { + const far = wholeOrder[7]; + + await expect(prize(far, [far]).claim()).resolves.toBeUndefined(); + }); + + it("refuses a project that is outside the window of the group and of its prize", async () => { + const prizeEntries = wholeOrder.slice(3, 8); + const prizeOrder = queueOrder(JUDGE, "mlh", prizeEntries); + + await expect( + prize(prizeOrder[QUEUE_WINDOW], prizeEntries).claim(), + ).rejects.toMatchObject({ message: QUEUE_REFUSAL }); + }); + + it("checks the group first and stops there when it offers the project", async () => { + const { claim: go, reads } = prize(wholeOrder[0], []); + + await go(); + + expect(reads.filter((r) => r.table === entries)).toHaveLength(1); + expect(reads.some((r) => r.table === memberships)).toBe(false); + }); + }); +}); diff --git a/packages/api/src/services/judging/queue-store.ts b/packages/api/src/services/judging/queue-store.ts new file mode 100644 index 00000000..e6d85780 --- /dev/null +++ b/packages/api/src/services/judging/queue-store.ts @@ -0,0 +1,255 @@ +import { TRPCError } from "@trpc/server"; +import { and, eq, inArray, isNull } from "drizzle-orm"; +import { + judgingAppointments, + judgingEntries, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingSchedules, +} from "@query/db"; +import type { JudgingDb } from "./config"; +import type { ChallengeRow, Seat } from "./evaluation-store"; +import { gradingStates } from "./grading-hold"; +import type { GradingState } from "./grading-hold"; +import { queueWindow } from "./queue"; +import { descendantsOf } from "./scope"; +import { evaluationTiming } from "./timing"; + +// The projects a judge is offered at a challenge, read from the database. The +// desk's next-project pick and the check on a judge's first claim of a project +// both go through loadQueue, so they never disagree about what is on offer. + +export type Queue = { + /** A scheduled challenge under a saved schedule: the schedule decides who sees what. */ + timed: boolean; + /** Timed, and the judge is not seated in a room that serves the challenge. */ + needsRoom: boolean; + /** + * The challenge's projects and where each stands for the judge. A project + * the judge has only opened counts as open (see loadQueue). For a timed + * challenge, only the presentations running now in the judge's room that + * the judge may start: not one still reserved for another judge (timing.ts). + * Those assigned to the judge say so. + */ + candidates: { entryId: string; grading: GradingState; assigned?: boolean }[]; +}; + +/** + * `challenge` is the root being graded. `filterId` narrows the projects to one + * prize inside it, as the desk's challenge filter does. + */ +export async function loadQueue( + db: JudgingDb, + input: { + hackathonId: string; + judgeId: string; + challenge: ChallengeRow; + seat: Seat | null; + filterId?: string; + }, + now = new Date(), +): Promise { + const { challenge, seat } = input; + const [schedule] = await db + .select({ id: judgingSchedules.id }) + .from(judgingSchedules) + .where(eq(judgingSchedules.hackathonId, input.hackathonId)) + .limit(1); + // A scheduled challenge is judged where it presents. + const timed = schedule !== undefined && challenge.mode === "scheduled"; + if (timed && seat?.challengeId !== challenge.id) { + return { timed, needsRoom: true, candidates: [] }; + } + const queue = (candidates: Queue["candidates"]): Queue => ({ + timed, + needsRoom: false, + candidates, + }); + + const entries = await db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .innerJoin( + judgingEntryChallenges, + and( + eq(judgingEntryChallenges.entryId, judgingEntries.id), + eq(judgingEntryChallenges.challengeId, input.filterId ?? challenge.id), + ), + ) + .where( + and( + eq(judgingEntries.hackathonId, input.hackathonId), + isNull(judgingEntries.deletedAt), + ), + ); + if (entries.length === 0) return queue([]); + const entryIds = entries.map((entry) => entry.id); + + const evaluations = await db + .select({ + entryId: judgingEvaluations.entryId, + judgeId: judgingEvaluations.judgeId, + }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.challengeId, challenge.id), + inArray(judgingEvaluations.entryId, entryIds), + ), + ); + const drafts = await db + .select({ + entryId: judgingEvaluationDrafts.entryId, + judgeId: judgingEvaluationDrafts.judgeId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + revision: judgingEvaluationDrafts.revision, + }) + .from(judgingEvaluationDrafts) + .where( + and( + eq(judgingEvaluationDrafts.challengeId, challenge.id), + inArray(judgingEvaluationDrafts.entryId, entryIds), + ), + ); + const grading = gradingStates( + input.judgeId, + entryIds, + evaluations, + drafts, + now, + ); + // A project the judge has only opened (desk.startGrading's draft, never + // saved to) stays on offer to them. Otherwise opening projects and backing + // out of them, or skipping them, would walk the judge down their queue. + const graded = new Set( + evaluations + .filter((evaluation) => evaluation.judgeId === input.judgeId) + .map((evaluation) => evaluation.entryId), + ); + const opened = new Set( + drafts + .filter( + (draft) => + draft.judgeId === input.judgeId && + draft.revision === 0 && + !graded.has(draft.entryId), + ) + .map((draft) => draft.entryId), + ); + let candidates = entryIds.map((entryId) => { + const state = grading.get(entryId) ?? ("open" as const); + return { + entryId, + grading: + state === "mine" && opened.has(entryId) ? ("open" as const) : state, + }; + }); + + if (timed && seat) { + const slots = await db + .select({ + id: judgingAppointments.id, + entryId: judgingAppointments.entryId, + startsAt: judgingAppointments.startsAt, + deadlineAt: judgingAppointments.deadlineAt, + assignedJudgeId: judgingAppointments.assignedJudgeId, + }) + .from(judgingAppointments) + .where( + and( + eq(judgingAppointments.scheduleId, schedule.id), + eq(judgingAppointments.challengeId, challenge.id), + eq(judgingAppointments.roomId, seat.roomId), + inArray(judgingAppointments.entryId, entryIds), + ), + ); + const running = new Set( + slots + .filter( + (slot) => + evaluationTiming( + { + judgeId: input.judgeId, + hasSchedule: true, + mode: challenge.mode, + challengeId: challenge.id, + evaluation: null, + appointment: slot, + presenceChallengeId: seat.challengeId, + }, + now, + ).canEdit, + ) + .map((slot) => slot.entryId), + ); + const assigned = new Set( + slots + .filter((slot) => slot.assignedJudgeId === input.judgeId) + .map((slot) => slot.entryId), + ); + candidates = candidates + .filter((c) => running.has(c.entryId)) + .map((c) => (assigned.has(c.entryId) ? { ...c, assigned: true } : c)); + } + + return queue(candidates); +} + +export const QUEUE_REFUSAL = + "Grade the projects your queue offers so every judge sees a random mix. Use Grade next project."; + +/** + * For a judge's first claim of a project (they hold no draft of it and have + * not graded it): at an untimed challenge it has to be one of the projects + * their queue offers (see QUEUE_WINDOW). A judge who narrowed the list to one + * prize is offered that prize's own window, so the project counts if it sits + * in the window of any prize it is entered in. Run it after claimGrading, in + * the same transaction. + */ +export async function assertQueueOffers( + tx: JudgingDb, + input: { + hackathonId: string; + judgeId: string; + entryId: string; + challenge: ChallengeRow; + challenges: readonly ChallengeRow[]; + }, + now = new Date(), +) { + const { challenge } = input; + const offers = (queue: Queue) => + queueWindow(input.judgeId, challenge.id, queue.candidates).includes( + input.entryId, + ); + // A timed challenge stops at the schedule check, so the seat is never read. + const base = { hackathonId: input.hackathonId, judgeId: input.judgeId }; + + const whole = await loadQueue(tx, { ...base, challenge, seat: null }, now); + if (whole.timed || offers(whole)) return; + + const below = descendantsOf(input.challenges, challenge.id).map((c) => c.id); + if (below.length > 0) { + const entered = await tx + .select({ challengeId: judgingEntryChallenges.challengeId }) + .from(judgingEntryChallenges) + .where( + and( + eq(judgingEntryChallenges.entryId, input.entryId), + inArray(judgingEntryChallenges.challengeId, below), + ), + ); + for (const { challengeId } of entered) { + const prize = await loadQueue( + tx, + { ...base, challenge, seat: null, filterId: challengeId }, + now, + ); + if (offers(prize)) return; + } + } + + throw new TRPCError({ code: "PRECONDITION_FAILED", message: QUEUE_REFUSAL }); +} diff --git a/packages/api/src/services/judging/queue.test.ts b/packages/api/src/services/judging/queue.test.ts new file mode 100644 index 00000000..474f8613 --- /dev/null +++ b/packages/api/src/services/judging/queue.test.ts @@ -0,0 +1,269 @@ +import { describe, expect, it } from "vitest"; +import { QUEUE_WINDOW, nextToGrade, queueOrder, queueWindow } from "./queue"; +import type { GradingState } from "./grading-hold"; + +const CHALLENGE = "c0000000-0000-4000-8000-000000000001"; +const entries = Array.from( + { length: 30 }, + (_, i) => `e${String(i).padStart(7, "0")}-0000-4000-8000-000000000000`, +); +const judges = Array.from( + { length: 40 }, + (_, i) => `j${String(i).padStart(7, "0")}-0000-4000-8000-000000000000`, +); + +const open = (entryId: string) => ({ entryId, grading: "open" as const }); +const state = (entryId: string, grading: GradingState) => ({ + entryId, + grading, +}); + +describe("queueOrder", () => { + it("is the same every time for a judge", () => { + expect(queueOrder(judges[0], CHALLENGE, entries)).toEqual( + queueOrder(judges[0], CHALLENGE, entries), + ); + }); + + it("does not depend on the order the projects arrive in", () => { + expect(queueOrder(judges[0], CHALLENGE, [...entries].reverse())).toEqual( + queueOrder(judges[0], CHALLENGE, entries), + ); + }); + + it("holds every project exactly once", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + expect(order).toHaveLength(entries.length); + expect(new Set(order)).toEqual(new Set(entries)); + }); + + it("gives a different judge a different order", () => { + expect(queueOrder(judges[0], CHALLENGE, entries)).not.toEqual( + queueOrder(judges[1], CHALLENGE, entries), + ); + }); + + it("gives the same judge a different order in another challenge", () => { + expect(queueOrder(judges[0], CHALLENGE, entries)).not.toEqual( + queueOrder(judges[0], "c0000000-0000-4000-8000-000000000002", entries), + ); + }); + + it("spreads the first pick across judges", () => { + const firsts = new Map(); + for (const judge of judges) { + const [first] = queueOrder(judge, CHALLENGE, entries); + firsts.set(first, (firsts.get(first) ?? 0) + 1); + } + // 40 judges over 30 projects: no one project is everyone's first. + expect(firsts.size).toBeGreaterThanOrEqual(15); + expect(Math.max(...firsts.values())).toBeLessThanOrEqual(6); + }); + + it("rarely sends two judges to the same project first", () => { + const firsts = judges.map( + (judge) => queueOrder(judge, CHALLENGE, entries)[0], + ); + let pairs = 0; + let collisions = 0; + for (let a = 0; a < firsts.length; a++) { + for (let b = a + 1; b < firsts.length; b++) { + pairs += 1; + if (firsts[a] === firsts[b]) collisions += 1; + } + } + // One in 30 if the order is random. + expect(collisions / pairs).toBeLessThan(0.1); + }); + + it("puts a project in no fixed place from one judge to the next", () => { + const places = judges.map((judge) => + queueOrder(judge, CHALLENGE, entries).indexOf(entries[0]), + ); + expect(new Set(places).size).toBeGreaterThanOrEqual(10); + }); + + it("is empty for no projects", () => { + expect(queueOrder(judges[0], CHALLENGE, [])).toEqual([]); + }); +}); + +describe("nextToGrade", () => { + it("is the judge's first project in their own order", () => { + expect(nextToGrade(judges[0], CHALLENGE, entries.map(open))).toBe( + queueOrder(judges[0], CHALLENGE, entries)[0], + ); + }); + + it("is the same on every call", () => { + const candidates = entries.map(open); + expect(nextToGrade(judges[3], CHALLENGE, candidates)).toBe( + nextToGrade(judges[3], CHALLENGE, candidates), + ); + }); + + it("passes over projects that are not open", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + const candidates = entries.map((entryId) => { + if (entryId === order[0]) return state(entryId, "graded"); + if (entryId === order[1]) return state(entryId, "in_progress"); + if (entryId === order[2]) return state(entryId, "mine"); + return open(entryId); + }); + expect(nextToGrade(judges[0], CHALLENGE, candidates)).toBe(order[3]); + }); + + it("never returns a project that is not open", () => { + const candidates = entries.map((entryId, i) => + state(entryId, (["graded", "in_progress", "mine"] as const)[i % 3]), + ); + expect(nextToGrade(judges[0], CHALLENGE, candidates)).toBeNull(); + }); + + it("is null when there is nothing to grade", () => { + expect(nextToGrade(judges[0], CHALLENGE, [])).toBeNull(); + }); + + it("only picks from the projects it is given", () => { + const scope = entries.slice(0, 3); + const picked = judges.map((judge) => + nextToGrade(judge, CHALLENGE, scope.map(open)), + ); + for (const entryId of picked) expect(scope).toContain(entryId); + expect(new Set(picked).size).toBeGreaterThan(1); + }); +}); + +describe("queueWindow", () => { + it("is the first few open projects in the judge's order", () => { + expect(QUEUE_WINDOW).toBe(3); + expect(queueWindow(judges[0], CHALLENGE, entries.map(open))).toEqual( + queueOrder(judges[0], CHALLENGE, entries).slice(0, QUEUE_WINDOW), + ); + }); + + it("counts only open projects, so held and graded ones do not use it up", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + const candidates = entries.map((entryId) => { + if (entryId === order[0]) return state(entryId, "graded"); + if (entryId === order[1]) return state(entryId, "in_progress"); + if (entryId === order[2]) return state(entryId, "mine"); + return open(entryId); + }); + expect(queueWindow(judges[0], CHALLENGE, candidates)).toEqual( + order.slice(3, 3 + QUEUE_WINDOW), + ); + }); + + it("is shorter when fewer projects are open", () => { + expect( + queueWindow(judges[0], CHALLENGE, entries.slice(0, 2).map(open)), + ).toHaveLength(2); + expect(queueWindow(judges[0], CHALLENGE, [])).toEqual([]); + }); + + it("is every open project when the window is unlimited", () => { + expect( + queueWindow(judges[0], CHALLENGE, entries.map(open), Infinity), + ).toEqual(queueOrder(judges[0], CHALLENGE, entries)); + }); + + it("differs from judge to judge, so each draws their own sample", () => { + const windows = new Set( + judges.map((judge) => + queueWindow(judge, CHALLENGE, entries.map(open)).join(), + ), + ); + expect(windows.size).toBeGreaterThan(30); + }); +}); + +describe("presentations assigned to the judge", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + const assigned = (entryId: string) => ({ + entryId, + grading: "open" as const, + assigned: true, + }); + // The judge's own come first, whatever place their queue gives them. + const mine = [order[7], order[3]]; + const candidates = entries.map((entryId) => + mine.includes(entryId) ? assigned(entryId) : open(entryId), + ); + + it("leads the window, each group in the judge's own order", () => { + expect( + queueWindow(judges[0], CHALLENGE, candidates, Infinity).slice(0, 4), + ).toEqual([order[3], order[7], order[0], order[1]]); + }); + + it("is what the judge is sent to next, ahead of the rest of the queue", () => { + expect( + nextToGrade(judges[0], CHALLENGE, candidates, { window: Infinity }), + ).toBe(order[3]); + }); + + it("passes over a skipped one, then the other, then the rest", () => { + const next = (skip: string[]) => + nextToGrade(judges[0], CHALLENGE, candidates, { skip, window: Infinity }); + expect(next([order[3]])).toBe(order[7]); + expect(next([order[3], order[7]])).toBe(order[0]); + }); + + it("does not offer one that is not open", () => { + const taken = entries.map((entryId) => + entryId === order[3] + ? { entryId, grading: "graded" as const, assigned: true } + : mine.includes(entryId) + ? assigned(entryId) + : open(entryId), + ); + expect(nextToGrade(judges[0], CHALLENGE, taken, { window: Infinity })).toBe( + order[7], + ); + }); + + it("changes nothing for projects with no assignment", () => { + expect(queueWindow(judges[0], CHALLENGE, entries.map(open))).toEqual( + order.slice(0, QUEUE_WINDOW), + ); + expect( + queueWindow( + judges[0], + CHALLENGE, + entries.map((entryId) => ({ + entryId, + grading: "open" as const, + assigned: false, + })), + ), + ).toEqual(order.slice(0, QUEUE_WINDOW)); + }); +}); + +describe("nextToGrade with skips", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + const next = (skip: string[], window?: number) => + nextToGrade(judges[0], CHALLENGE, entries.map(open), { skip, window }); + + it("passes over a skipped project to the next one in the window", () => { + expect(next([order[0]])).toBe(order[1]); + expect(next([order[0], order[1]])).toBe(order[2]); + }); + + it("is nothing once every project in the window is skipped", () => { + expect(next(order.slice(0, QUEUE_WINDOW))).toBeNull(); + }); + + it("never reaches past the window for a project that was not skipped", () => { + expect(next(order.slice(0, QUEUE_WINDOW))).not.toBe(order[QUEUE_WINDOW]); + }); + + it("ignores a skipped project outside the window", () => { + expect(next([order[10]])).toBe(order[0]); + }); + + it("reaches as far as an unlimited window goes", () => { + expect(next(order.slice(0, 5), Infinity)).toBe(order[5]); + }); +}); diff --git a/packages/api/src/services/judging/queue.ts b/packages/api/src/services/judging/queue.ts new file mode 100644 index 00000000..d0c30a34 --- /dev/null +++ b/packages/api/src/services/judging/queue.ts @@ -0,0 +1,94 @@ +import { createHash } from "node:crypto"; +import type { GradingState } from "./grading-hold"; + +// The order a judge is sent through a challenge's projects. Every judge gets +// their own shuffle of the same projects, so judges at one table draw a random +// sample each and rarely reach for the same project at once. The shuffle is a +// hash, not a random draw: it does not move between polls, and a judge who +// reloads is sent to the same project. + +/** A project's place in one judge's queue for a challenge. */ +function queueKey(judgeId: string, challengeId: string, entryId: string) { + return createHash("sha256") + .update(`${judgeId}:${entryId}:${challengeId}`) + .digest("hex"); +} + +/** `entryIds` in the order `judgeId` is sent through them for the challenge. */ +export function queueOrder( + judgeId: string, + challengeId: string, + entryIds: readonly string[], +): string[] { + return entryIds + .map((entryId) => ({ + entryId, + key: queueKey(judgeId, challengeId, entryId), + })) + .sort((a, b) => + a.key === b.key + ? a.entryId.localeCompare(b.entryId) + : a.key < b.key + ? -1 + : 1, + ) + .map((item) => item.entryId); +} + +type Candidate = { + entryId: string; + grading: GradingState; + /** A presentation assigned to this judge (scheduled challenges only). */ + assigned?: boolean; +}; + +// How many open projects at the front of their queue a judge may start at an +// untimed challenge. Calibration depends on each judge's sample being random. +// A small window lets a judge skip a team that is away from its table without +// letting them pick projects. +export const QUEUE_WINDOW = 3; + +/** + * The open projects a judge is offered: the first `window` of them in their + * queue order, with the presentations assigned to the judge ahead of the rest + * (each group in that order). A scheduled challenge passes Infinity, since the + * schedule already decides who sees what. + */ +export function queueWindow( + judgeId: string, + challengeId: string, + candidates: readonly Candidate[], + window = QUEUE_WINDOW, +): string[] { + const open = candidates.filter((candidate) => candidate.grading === "open"); + const assigned = new Set( + open.filter((c) => c.assigned).map((c) => c.entryId), + ); + const ordered = queueOrder( + judgeId, + challengeId, + open.map((candidate) => candidate.entryId), + ); + return [ + ...ordered.filter((entryId) => assigned.has(entryId)), + ...ordered.filter((entryId) => !assigned.has(entryId)), + ].slice(0, window); +} + +/** + * The next project for the judge to grade: the first of the projects their + * queue offers that they have not skipped. + */ +export function nextToGrade( + judgeId: string, + challengeId: string, + candidates: readonly Candidate[], + options: { skip?: readonly string[]; window?: number } = {}, +): string | null { + const skip = new Set(options.skip); + return ( + queueWindow(judgeId, challengeId, candidates, options.window).find( + (entryId) => !skip.has(entryId), + ) ?? null + ); +} diff --git a/packages/api/src/services/judging/schedule/board.test.ts b/packages/api/src/services/judging/schedule/board.test.ts index cdb1c1eb..4020ee2e 100644 --- a/packages/api/src/services/judging/schedule/board.test.ts +++ b/packages/api/src/services/judging/schedule/board.test.ts @@ -27,7 +27,7 @@ const evaluation = (over: Partial): BoardEvaluation => ({ entryId: "a", challengeId: "mlh", appointmentId: null, - isComplete: true, + ratingsComplete: true, ...over, }); diff --git a/packages/api/src/services/judging/schedule/board.ts b/packages/api/src/services/judging/schedule/board.ts index 1f756314..dee5cd8b 100644 --- a/packages/api/src/services/judging/schedule/board.ts +++ b/packages/api/src/services/judging/schedule/board.ts @@ -12,7 +12,8 @@ export type BoardEvaluation = { entryId: string; challengeId: string; appointmentId: string | null; - isComplete: boolean; + /** Holds every rating (scoring.hasEveryRating), so placings count it. */ + ratingsComplete: boolean; }; export type BoardPresence = { @@ -32,6 +33,21 @@ type Booked = { endsAt: Date; }; +/** + * Whether an evaluation or draft of the appointment's project belongs to the + * appointment: it names it, or, when it names none, its challenge sits under + * the appointment's. The caller has matched the project. + */ +export function belongsTo( + appointment: { id: string; challengeId: string }, + row: { challengeId: string; appointmentId: string | null }, + rootIdOf: (challengeId: string) => string | null, +): boolean { + return row.appointmentId + ? row.appointmentId === appointment.id + : rootIdOf(row.challengeId) === appointment.challengeId; +} + /** Rooms with a judge still in them and heard from in the last 15 minutes. */ export function staffedRoomIds( presence: readonly BoardPresence[], @@ -78,9 +94,7 @@ export function withStatus( return appointments.map((appointment) => { const mine = (byEntry.get(appointment.entryId) ?? []).filter((e) => - e.appointmentId - ? e.appointmentId === appointment.id - : rootIdOf(e.challengeId) === appointment.challengeId, + belongsTo(appointment, e, rootIdOf), ); // Seen in the room at some point during the slot. An open row's last // heartbeat stands in for when the judge left. diff --git a/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts b/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts index 620fae94..8e28cf3a 100644 --- a/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts +++ b/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts @@ -1,7 +1,13 @@ import { describe, expect, it } from "vitest"; import { aamoghCPSolver } from "./aamogh-cp-solver"; -import { cpsatProblem, readCpsatSolution } from "./cpsat-bridge"; -import type { PlanInputs } from "./inputs"; +import { + cpsatAssignmentProblem, + cpsatProblem, + readCpsatAssignments, + readCpsatSolution, +} from "./cpsat-bridge"; +import type { Placement, PlanInputs } from "./inputs"; +import type { JudgeAssignment, JudgeInput } from "./judges"; const inputs: PlanInputs = { rooms: [ @@ -58,6 +64,9 @@ describe("cpsatProblem", () => { expect(problem.rooms.map((r) => r.sponsor)).toEqual([false, false, true]); expect(problem.tasks).toHaveLength(5); expect("hint" in problem).toBe(false); + // A file without judges is still the version-1 file it always was. + expect("judges" in problem).toBe(false); + expect("assignmentHint" in problem).toBe(false); }); it("passes a starting schedule along as the hint", () => { @@ -126,4 +135,236 @@ describe("readCpsatSolution", () => { expect(result).toMatchObject({ ok: true, relaxed: true }); if (result.ok) expect(result.reducedBreaks).toHaveLength(1); }); + + it("still reads a version-1 file written before judges", () => { + expect(read(solution)).toMatchObject({ + ok: true, + judging: { source: "in-app", assignments: [] }, + }); + }); +}); + +const judges: JudgeInput[] = [ + { id: "m1", kind: "member", challengeIds: ["general", "sp1"], roomId: null }, + { id: "m2", kind: "member", challengeIds: ["general", "sp1"], roomId: null }, + { id: "gs", kind: "guest", challengeIds: ["sp1"], roomId: "s1" }, +]; + +// What `uv run cpsat.py problem.json --workers 1` wrote for cpsatProblem(inputs, +// "fp-test", undefined, judges), verbatim apart from the timing. +const judged = { + format: "query-judging-schedule-solution", + version: 1, + fingerprint: "fp-test", + status: "optimal", + relaxed: false, + objective: { + judgeShortfall: 0, + sponsorFinish: 2, + sponsorIdle: 0, + finish: 4, + buildingChanges: 0, + breakSlots: 2, + }, + placements: [ + { entryId: "a", challengeId: "general", roomId: "g2", slot: 2 }, + { entryId: "a", challengeId: "sp1", roomId: "s1", slot: 0 }, + { entryId: "b", challengeId: "general", roomId: "g2", slot: 3 }, + { entryId: "b", challengeId: "sp1", roomId: "s1", slot: 1 }, + { entryId: "c", challengeId: "general", roomId: "g2", slot: 1 }, + ], + reasons: [], + solver: { name: "cp-sat", ortools: "9.15.6755", seconds: 0 }, + assignmentStatus: "optimal", + judging: { + unassigned: 0, + memberOnSponsor: 0, + maxLoad: 2, + loadSpread: 1, + repeats: 2, + dayPartsCovered: 5, + avgDistinctRooms: 1.0, + }, + assignments: [ + { entryId: "a", challengeId: "general", judgeId: "m1" }, + { entryId: "a", challengeId: "sp1", judgeId: "gs" }, + { entryId: "b", challengeId: "general", judgeId: "m2" }, + { entryId: "b", challengeId: "sp1", judgeId: "gs" }, + { entryId: "c", challengeId: "general", judgeId: "m2" }, + ], +}; + +const readJudged = (value: unknown, now: JudgeInput[] = judges) => + readCpsatSolution(inputs, "fp-test", JSON.stringify(value), now); + +describe("cpsatProblem with judges", () => { + it("lists the judges with their run limit", () => { + const problem = cpsatProblem(inputs, "fp-test", undefined, judges); + expect(problem.judges).toEqual([ + { id: "m1", kind: "member", challengeIds: ["general", "sp1"], roomId: null, maxConsecutive: 4 }, + { id: "m2", kind: "member", challengeIds: ["general", "sp1"], roomId: null, maxConsecutive: 4 }, + { id: "gs", kind: "guest", challengeIds: ["sp1"], roomId: "s1", maxConsecutive: 4 }, + ]); + expect("assignmentHint" in problem).toBe(false); + expect("judges" in cpsatProblem(inputs, "fp-test", undefined, [])).toBe(false); + }); + + it("sends where the in-app assignment would put the judges for a hint", () => { + const problem = cpsatProblem(inputs, "fp-test", judged.placements, [ + { ...judges[0]!, maxConsecutive: 2 }, + ...judges.slice(1), + ]); + expect(problem.judges![0]!.maxConsecutive).toBe(2); + const hint = (problem as { assignmentHint?: JudgeAssignment[] }).assignmentHint!; + expect(hint).toHaveLength(5); + expect(new Set(hint.map((h) => h.judgeId))).toEqual(new Set(["m1", "m2", "gs"])); + }); +}); + +describe("readCpsatSolution with judges", () => { + it("takes the judges the script chose, once they pass the rules", () => { + const result = readJudged(judged); + expect(result).toMatchObject({ + ok: true, + judging: { + source: "cp-sat", + assignments: judged.assignments, + metrics: { unassigned: 0, maxLoad: 2, loadSpread: 1, repeats: 2 }, + }, + }); + }); + + it("assigns the judges itself when the file has none", () => { + const { assignments: _dropped, ...without } = judged; + const result = readJudged(without); + if (!result.ok) throw new Error("the schedule is fine"); + expect(result.judging.source).toBe("in-app"); + expect(result.judging.metrics.unassigned).toBe(0); + expect(result.judging.assignments).toHaveLength(5); + }); + + it("assigns the judges itself when the file's break the rules for the judges now", () => { + // The guest has since gone: gs is not among the judges any more. + const result = readJudged(judged, judges.slice(0, 2)); + if (!result.ok) throw new Error("the placements are still fine"); + expect(result.judging.source).toBe("in-app"); + expect(result.judging.assignments.every((a) => a.judgeId !== "gs")).toBe(true); + }); + + it("assigns the judges itself when the file puts one judge in two places", () => { + const clashing = { + ...judged, + assignments: judged.assignments.map((a) => + a.entryId === "c" ? { ...a, judgeId: "gs" } : a, + ), + }; + const result = readJudged(clashing); + if (!result.ok) throw new Error("the placements are fine"); + expect(result.judging.source).toBe("in-app"); + }); + + it("leaves presentations without a judge rather than refuse the schedule", () => { + const result = readJudged(judged, []); + if (!result.ok) throw new Error("the placements are fine"); + expect(result.judging.assignments).toEqual([]); + expect(result.judging.metrics.unassigned).toBe(5); + }); +}); + +describe("assigning judges to a schedule already made", () => { + const placements: Placement[] = judged.placements; + const keep: JudgeAssignment[] = [{ entryId: "c", challengeId: "general", judgeId: "m1" }]; + const problem = cpsatAssignmentProblem(inputs, placements, judges, keep); + const answer = (over: Record = {}) => + JSON.stringify({ + ...judged, + fingerprint: problem.fingerprint, + assignments: [ + { entryId: "c", challengeId: "general", judgeId: "m1" }, + { entryId: "a", challengeId: "general", judgeId: "m2" }, + { entryId: "a", challengeId: "sp1", judgeId: "gs" }, + { entryId: "b", challengeId: "general", judgeId: "m2" }, + { entryId: "b", challengeId: "sp1", judgeId: "gs" }, + ], + ...over, + }); + const readAnswer = (text: string, kept = keep) => + readCpsatAssignments(inputs, placements, judges, kept, text); + + it("writes the schedule, the judges and what to keep into the problem", () => { + expect(problem.placements).toEqual(placements); + expect(problem.locked).toEqual(keep); + expect(problem.judges).toHaveLength(3); + expect(problem.assignmentHint).toHaveLength(5); + expect("locked" in cpsatAssignmentProblem(inputs, placements, judges)).toBe(false); + }); + + it("reads the assignments, scored", () => { + expect(readAnswer(answer())).toMatchObject({ + ok: true, + status: "optimal", + metrics: { unassigned: 0, memberOnSponsor: 0 }, + }); + }); + + it("refuses an answer for another schedule", () => { + expect(readAnswer(answer({ fingerprint: "fp-test" }))).toMatchObject({ ok: false }); + }); + + it("refuses a proof that nothing fits and a timeout", () => { + expect(readAnswer(answer({ status: "infeasible" }))).toMatchObject({ ok: false }); + expect(readAnswer(answer({ status: "unknown" }))).toMatchObject({ ok: false }); + }); + + it("refuses an answer with no assignments or no JSON", () => { + const { assignments: _dropped, ...without } = JSON.parse(answer()); + expect(readAnswer(JSON.stringify(without))).toMatchObject({ ok: false }); + expect(readAnswer("nope")).toMatchObject({ ok: false }); + expect(readAnswer("{}")).toMatchObject({ ok: false }); + }); + + it("refuses assignments that break a rule", () => { + // m2 at a:sp1 (slot 0) and c:general (slot 1) in different buildings. + const result = readAnswer( + answer({ + assignments: [ + { entryId: "c", challengeId: "general", judgeId: "m1" }, + { entryId: "a", challengeId: "sp1", judgeId: "m1" }, + ], + }), + ); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) expect(result.reason).toContain("against the rules"); + }); + + it("refuses an answer that moves what it was told to keep", () => { + const result = readAnswer( + answer({ + assignments: [ + { entryId: "c", challengeId: "general", judgeId: "m2" }, + { entryId: "a", challengeId: "general", judgeId: "m1" }, + ], + }), + ); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) expect(result.reason).toContain("keep"); + }); + + it("does not blame the script for a rule the kept assignments already broke", () => { + // m1 kept at b:sp1 (slot 1, HEC) and c:general (slot 1, HEC): two places + // at once, so before CP-SAT ran. + const hasty: JudgeAssignment[] = [ + { entryId: "b", challengeId: "sp1", judgeId: "m1" }, + { entryId: "c", challengeId: "general", judgeId: "m1" }, + ]; + const text = answer({ + assignments: [ + ...hasty, + { entryId: "a", challengeId: "general", judgeId: "m2" }, + { entryId: "a", challengeId: "sp1", judgeId: "gs" }, + { entryId: "b", challengeId: "general", judgeId: "m2" }, + ], + }); + expect(readAnswer(text, hasty)).toMatchObject({ ok: true }); + }); }); diff --git a/packages/api/src/services/judging/schedule/cpsat-bridge.ts b/packages/api/src/services/judging/schedule/cpsat-bridge.ts index 7e8e9f0c..696f35ab 100644 --- a/packages/api/src/services/judging/schedule/cpsat-bridge.ts +++ b/packages/api/src/services/judging/schedule/cpsat-bridge.ts @@ -1,5 +1,13 @@ import { z } from "zod"; +import { candidateKey, taskKey } from "./inputs"; import type { Placement, PlanInputs } from "./inputs"; +import { + MAX_CONSECUTIVE, + assignJudges, + scoreJudgeAssignment, + validateJudgeAssignment, +} from "./judges"; +import type { JudgeAssignment, JudgeInput, JudgeMetrics } from "./judges"; import { scoreSchedule, validateSchedule } from "./rules"; import type { PlanScore, ReducedBreak } from "./rules"; @@ -18,14 +26,33 @@ export const CPSAT_SOLUTION_FORMAT = "query-judging-schedule-solution"; export const CPSAT_COMMAND = "uv run packages/api/scripts/judging-schedule/cpsat.py problem.json -o solution.json"; +const judgesOf = (judges: readonly JudgeInput[]) => + judges.map((judge) => ({ + id: judge.id, + kind: judge.kind, + challengeIds: [...judge.challengeIds], + roomId: judge.roomId, + maxConsecutive: judge.maxConsecutive ?? MAX_CONSECUTIVE, + })); + +const assignmentsOf = (rows: readonly JudgeAssignment[]) => + rows.map((row) => ({ + entryId: row.entryId, + challengeId: row.challengeId, + judgeId: row.judgeId, + })); + /** * The problem file cpsat.py reads. `hint` is a schedule to start from, such - * as the best partial one the in-app solver found. + * as the best partial one the in-app solver found. `judges`, when there are + * any, asks for a judge for every presentation as well, and also keeps each + * challenge's presentations in any slot within its judges. */ export function cpsatProblem( inputs: PlanInputs, fingerprint: string, hint?: readonly Placement[], + judges: readonly JudgeInput[] = [], ) { const sponsorChallenges = new Set( inputs.tasks.filter((t) => t.sponsor).map((t) => t.challengeId), @@ -58,6 +85,51 @@ export function cpsatProblem( })), } : {}), + ...(judges.length > 0 + ? { + judges: judgesOf(judges), + // Where the in-app assignment would put the judges for that hint, + // for CP-SAT to improve on. + ...(hint && hint.length > 0 + ? { + assignmentHint: assignmentsOf( + assignJudges(inputs, hint, judges).assignment, + ), + } + : {}), + } + : {}), + }; +} + +/** Binds an assignment-only run to the placements it assigns judges to. */ +const assignmentFingerprint = (placements: readonly Placement[]) => + candidateKey("judges", false, placements); + +/** + * A problem that only assigns judges: the schedule is given and CP-SAT leaves + * it alone. `keep` stays as it is, such as the presentations already under + * way. `inputs` hold just the presentations in `placements`. + */ +export function cpsatAssignmentProblem( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + keep: readonly JudgeAssignment[] = [], +) { + const base = cpsatProblem(inputs, assignmentFingerprint(placements), undefined, judges); + return { + ...base, + placements: placements.map((p) => ({ + entryId: p.entryId, + challengeId: p.challengeId, + roomId: p.roomId, + slot: p.slot, + })), + ...(keep.length > 0 ? { locked: assignmentsOf(keep) } : {}), + assignmentHint: assignmentsOf( + assignJudges(inputs, placements, judges, { keep }).assignment, + ), }; } @@ -76,8 +148,25 @@ const solutionSchema = z.object({ }), ), reasons: z.array(z.string()).optional(), + assignments: z + .array( + z.object({ + entryId: z.string(), + challengeId: z.string(), + judgeId: z.string(), + }), + ) + .optional(), }); +/** Who grades what for a schedule, and who worked it out. */ +export type JudgeOutcome = { + assignments: JudgeAssignment[]; + metrics: JudgeMetrics; + /** "cp-sat" when the file's own assignments passed the checks. */ + source: "cp-sat" | "in-app"; +}; + export type CpsatImport = | { ok: true; @@ -87,14 +176,44 @@ export type CpsatImport = placements: Placement[]; score: PlanScore; reducedBreaks: ReducedBreak[]; + judging: JudgeOutcome; } | { ok: false; reason: string }; -/** Reads and checks an uploaded solution.json against the current problem. */ +/** + * The file's assignments when it has some and they keep the rules for the + * judges as they are now; otherwise the in-app ones. + */ +function judgeOutcome( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + given: readonly JudgeAssignment[] | undefined, +): JudgeOutcome { + if ( + given && + given.length > 0 && + validateJudgeAssignment(inputs, placements, judges, given).valid + ) { + return { + assignments: assignmentsOf(given), + metrics: scoreJudgeAssignment(inputs, placements, judges, given), + source: "cp-sat", + }; + } + const made = assignJudges(inputs, placements, judges); + return { assignments: made.assignment, metrics: made.metrics, source: "in-app" }; +} + +/** + * Reads and checks an uploaded solution.json against the current problem. + * `judges` are the judges as they are now, whatever the problem listed. + */ export function readCpsatSolution( inputs: PlanInputs, fingerprint: string, text: string, + judges: readonly JudgeInput[] = [], ): CpsatImport { let raw: unknown; try { @@ -154,5 +273,79 @@ export function readCpsatSolution( placements, score: scoreSchedule(inputs, placements), reducedBreaks: check.reducedBreaks, + judging: judgeOutcome(inputs, placements, judges, solution.assignments), + }; +} + +export type CpsatAssignmentImport = + | { + ok: true; + status: "optimal" | "feasible"; + assignments: JudgeAssignment[]; + metrics: JudgeMetrics; + } + | { ok: false; reason: string }; + +/** + * Reads what cpsat.py printed for a cpsatAssignmentProblem. Unlike a new + * schedule, nothing here is optional: the assignments must keep the rules + * for the judges, leave every one in `keep` as it was, and break no rule that + * `keep` did not already break. + */ +export function readCpsatAssignments( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + keep: readonly JudgeAssignment[], + text: string, +): CpsatAssignmentImport { + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { ok: false, reason: "CP-SAT did not print a solution." }; + } + const parsed = solutionSchema.safeParse(raw); + if (!parsed.success) { + return { ok: false, reason: "CP-SAT did not print a solution from cpsat.py." }; + } + const solution = parsed.data; + if (solution.fingerprint !== assignmentFingerprint(placements)) { + return { ok: false, reason: "CP-SAT answered a different schedule." }; + } + if (solution.status === "infeasible" || solution.status === "unknown") { + return { ok: false, reason: "CP-SAT found no assignment of judges in the time it had." }; + } + if (!solution.assignments) { + return { ok: false, reason: "CP-SAT did not assign judges." }; + } + + const assignments = assignmentsOf(solution.assignments); + const before = new Set( + validateJudgeAssignment(inputs, placements, judges, keep).problems, + ); + const broken = validateJudgeAssignment( + inputs, + placements, + judges, + assignments, + ).problems.filter((problem) => !before.has(problem)); + if (broken.length > 0) { + return { + ok: false, + reason: `CP-SAT assigned judges against the rules: ${broken.slice(0, 3).join(" ")}`, + }; + } + const given = new Map( + assignments.map((a) => [taskKey(a.entryId, a.challengeId), a.judgeId]), + ); + if (keep.some((k) => given.get(taskKey(k.entryId, k.challengeId)) !== k.judgeId)) { + return { ok: false, reason: "CP-SAT changed an assignment it was told to keep." }; + } + return { + ok: true, + status: solution.status, + assignments, + metrics: scoreJudgeAssignment(inputs, placements, judges, assignments), }; } diff --git a/packages/api/src/services/judging/schedule/cpsat-runner.test.ts b/packages/api/src/services/judging/schedule/cpsat-runner.test.ts index 46a834c5..9d7c611a 100644 --- a/packages/api/src/services/judging/schedule/cpsat-runner.test.ts +++ b/packages/api/src/services/judging/schedule/cpsat-runner.test.ts @@ -4,7 +4,12 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import type * as ChildProcess from "node:child_process"; import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { cpsatProblem, readCpsatSolution } from "./cpsat-bridge"; +import { + cpsatAssignmentProblem, + cpsatProblem, + readCpsatAssignments, + readCpsatSolution, +} from "./cpsat-bridge"; import { CPSAT_UNAVAILABLE, cpsatAvailable, runCpsat } from "./cpsat-runner"; import type { PlanInputs } from "./inputs"; @@ -212,6 +217,44 @@ process.exitCode = 1; if (run.status === "failed") expect(run.reason).toContain("after 70 seconds"); }); + it("allows one more time limit when the script also assigns judges", async () => { + fakeScript(`setInterval(() => undefined, 1000);`); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const judged = cpsatProblem(inputs, "fp-test", undefined, [ + { id: "m1", kind: "member", challengeIds: ["general", "sp1"], roomId: null }, + ]); + const running = runCpsat(judged, { seconds: 5 }); + while (spawned.length === 0) await new Promise((next) => setImmediate(next)); + // 5 seconds for each of two attempts and for the judges, plus 60. + await vi.advanceTimersByTimeAsync(74_000); + expect(spawned[0]!.child.killed).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + expect(spawned[0]!.child.killed).toBe(true); + + const run = await running; + if (run.status === "failed") expect(run.reason).toContain("after 75 seconds"); + }); + + it("allows a single time limit when the schedule is given and only judges are assigned", async () => { + fakeScript(`setInterval(() => undefined, 1000);`); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const placements = [{ entryId: "a", challengeId: "general", roomId: "g1", slot: 0 }]; + const judged = cpsatAssignmentProblem( + { ...inputs, tasks: [inputs.tasks[0]!] }, + placements, + [{ id: "m1", kind: "member", challengeIds: ["general"], roomId: null }], + ); + const running = runCpsat(judged, { seconds: 5 }); + while (spawned.length === 0) await new Promise((next) => setImmediate(next)); + await vi.advanceTimersByTimeAsync(64_000); + expect(spawned[0]!.child.killed).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + expect(spawned[0]!.child.killed).toBe(true); + + const run = await running; + if (run.status === "failed") expect(run.reason).toContain("after 65 seconds"); + }); + it("kills a run when its signal fires, and removes the problem file", async () => { // Would run for the full 70 seconds if nothing stopped it. fakeScript(`console.error("loading"); setInterval(() => undefined, 1000);`); @@ -288,4 +331,66 @@ describe.runIf(realPython)("runCpsat with cpsat.py", () => { ok: true, }); }); + + it("assigns judges to the bundled example", { timeout: 120_000 }, async () => { + vi.stubEnv("JUDGING_CPSAT_PYTHON", realPython!); + vi.stubEnv("JUDGING_CPSAT_SCRIPT", ""); + const example = JSON.parse( + readFileSync( + fileURLToPath( + new URL("../../../../scripts/judging-schedule/example-problem.json", import.meta.url), + ), + "utf8", + ), + ); + const exampleInputs: PlanInputs = { + rooms: example.rooms.map((r: PlanInputs["rooms"][number]) => ({ + id: r.id, + challengeId: r.challengeId, + buildingId: r.buildingId, + })), + tasks: example.tasks, + slots: example.slotCount, + sameGap: example.sameGap, + crossGap: example.crossGap, + }; + const everything = [...new Set(example.tasks.map((t: { challengeId: string }) => t.challengeId))]; + const judges = Array.from({ length: 8 }, (_, n) => ({ + id: `judge-${n}`, + kind: "member" as const, + challengeIds: everything as string[], + roomId: null, + })); + + const run = await runCpsat( + cpsatProblem(exampleInputs, example.fingerprint, undefined, judges), + { seconds: 5 }, + ); + expect(run).toMatchObject({ status: "solved" }); + if (run.status !== "solved") return; + const read = readCpsatSolution(exampleInputs, example.fingerprint, run.solution, judges); + expect(read).toMatchObject({ ok: true, judging: { source: "cp-sat" } }); + if (!read.ok) return; + + // Then the same schedule again, only the judges: one assignment is kept. + const keep = read.judging.assignments.slice(0, 1); + const again = await runCpsat( + cpsatAssignmentProblem(exampleInputs, read.placements, judges, keep), + { seconds: 5 }, + ); + expect(again).toMatchObject({ status: "solved" }); + if (again.status !== "solved") return; + const assigned = readCpsatAssignments( + exampleInputs, + read.placements, + judges, + keep, + again.solution, + ); + expect(assigned).toMatchObject({ ok: true }); + if (assigned.ok) { + expect(assigned.assignments).toEqual(expect.arrayContaining(keep)); + expect(assigned.assignments).toHaveLength(read.placements.length); + } + }); }); diff --git a/packages/api/src/services/judging/schedule/cpsat-runner.ts b/packages/api/src/services/judging/schedule/cpsat-runner.ts index b8ff9fea..01bbebe4 100644 --- a/packages/api/src/services/judging/schedule/cpsat-runner.ts +++ b/packages/api/src/services/judging/schedule/cpsat-runner.ts @@ -7,7 +7,8 @@ import { dirname, join, resolve } from "node:path"; // Runs the CP-SAT script on this server, for hosts that have a Python with // OR-Tools, so an organiser does not have to download the problem and run it // on a laptop. It is the same script fed the same problem file, and what it -// prints is read exactly like an upload (readCpsatSolution): this file only +// prints is read exactly like an upload (readCpsatSolution, or +// readCpsatAssignments when only the judges are assigned): this file only // starts the process and reports how it ended, it never decides whether a // schedule is good. // @@ -29,6 +30,17 @@ export const CPSAT_SECONDS = { min: 5, max: 60, default: 20 } as const; /** Lines of the script's log kept for error messages. */ const LOG_LINES = 20; +/** + * How many times the script may use its whole time limit on `problem`: twice + * to place the presentations (strict, then relaxed) unless the schedule is + * given, and once more to assign judges when there are any. + */ +function timeLimits(problem: unknown): number { + const shape = problem as { judges?: unknown; placements?: unknown } | null; + const judged = Array.isArray(shape?.judges) && shape.judges.length > 0; + return (Array.isArray(shape?.placements) ? 0 : 2) + (judged ? 1 : 0); +} + type Command = { python: string; script: string }; let probe: { env: string; command: Command | null } | undefined; @@ -108,11 +120,12 @@ export async function runCpsat( Math.max(CPSAT_SECONDS.min, Math.round(options.seconds ?? CPSAT_SECONDS.default)), ); const workers = Math.min(4, availableParallelism()); - // Up to two attempts of `seconds` each. Starting Python, loading OR-Tools - // and building the model for each attempt come on top and do not depend - // on `seconds`: on a busy machine they took close to 30 seconds for a - // 2,000-presentation edition, so the margin leaves room for twice that. - const killAfter = seconds * 2 + 60; + // Up to two attempts of `seconds` each, and one more for the judges. + // Starting Python, loading OR-Tools and building the model for each attempt + // come on top and do not depend on `seconds`: on a busy machine they took + // close to 30 seconds for a 2,000-presentation edition, so the margin leaves + // room for twice that. + const killAfter = seconds * timeLimits(problem) + 60; // A directory of its own, so two runs at once never share a file. const dir = await mkdtemp(join(tmpdir(), "cpsat-")); diff --git a/packages/api/src/services/judging/schedule/inputs.ts b/packages/api/src/services/judging/schedule/inputs.ts index ffb0aae1..b313a523 100644 --- a/packages/api/src/services/judging/schedule/inputs.ts +++ b/packages/api/src/services/judging/schedule/inputs.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { rootOf } from "../scope"; +import { isSponsorChallenge, rootOf } from "../scope"; import type { ChallengeMode } from "../scope"; import { gridOf } from "./slots"; import type { ScheduleTiming } from "./slots"; @@ -86,8 +86,8 @@ export function selectPlan(source: { memberships: readonly SourceMembership[]; }): { rooms: PlanRoom[]; tasks: PlanTask[] } { const scheduled = scheduledChallengeIds(source.challenges); - const general = new Set( - source.challenges.filter((c) => c.isGeneral).map((c) => c.id), + const sponsor = new Set( + source.challenges.filter(isSponsorChallenge).map((c) => c.id), ); const rooms = source.rooms @@ -111,7 +111,7 @@ export function selectPlan(source: { tasks.push({ entryId: row.entryId, challengeId: row.challengeId, - sponsor: !general.has(row.challengeId), + sponsor: sponsor.has(row.challengeId), }); } tasks.sort(byTask); diff --git a/packages/api/src/services/judging/schedule/judge-plan.test.ts b/packages/api/src/services/judging/schedule/judge-plan.test.ts new file mode 100644 index 00000000..74edd1d0 --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-plan.test.ts @@ -0,0 +1,269 @@ +import { describe, expect, it } from "vitest"; +import { + judgeChanges, + judgeChoices, + judgePlanOf, + rosterOf, +} from "./judge-plan"; +import type { JudgeRow, SavedAppointment } from "./judge-plan"; +import type { ChallengeRow } from "../evaluation-store"; + +const challenge = ( + id: string, + over: Partial = {}, +): ChallengeRow => ({ + id, + parentId: null, + mode: "scheduled", + label: id, + isGeneral: false, + isGroup: false, + ...over, +}); + +// General and one sponsor are scheduled; MLH is a group of two unscheduled prizes. +const challenges: ChallengeRow[] = [ + challenge("general", { isGeneral: true }), + challenge("acme"), + challenge("mlh", { isGroup: true, mode: "unscheduled" }), + challenge("mongo", { parentId: "mlh", mode: "unscheduled" }), +]; + +const rooms = [ + { id: "g1", challengeId: "general", buildingId: "ENG" }, + { id: "a1", challengeId: "acme", buildingId: "HEC" }, +]; + +const judgeRows: JudgeRow[] = [ + { id: "m1", kind: "member", userId: "u1", displayName: "Ada" }, + { id: "gs", kind: "guest", userId: null, displayName: "Grace" }, + { id: "lost", kind: "guest", userId: null, displayName: "Nobody" }, +]; + +describe("rosterOf", () => { + const roster = (over: Partial[0]> = {}) => + rosterOf({ + hackathonId: "h", + judges: judgeRows, + seats: [ + { + judgeId: "gs", + roomId: "a1", + createdAt: new Date("2027-02-06T12:00:00Z"), + }, + ], + challenges, + rooms, + ...over, + }); + + it("lets a member grade every scheduled challenge and a guest only their room's", () => { + const { judges } = roster(); + expect(judges).toEqual([ + { id: "gs", kind: "guest", challengeIds: ["acme"], roomId: "a1" }, + { + id: "m1", + kind: "member", + challengeIds: ["acme", "general"], + roomId: null, + }, + ]); + }); + + it("leaves out a guest with no session, or whose room takes no appointments", () => { + expect(roster({ seats: [] }).judges.map((j) => j.id)).toEqual(["m1"]); + expect( + roster({ + seats: [ + { judgeId: "gs", roomId: "unscheduled-room", createdAt: new Date() }, + ], + }).judges.map((j) => j.id), + ).toEqual(["m1"]); + }); + + it("seats a guest in the room of their latest session", () => { + const { judges } = roster({ + seats: [ + { + judgeId: "gs", + roomId: "g1", + createdAt: new Date("2027-02-06T09:00:00Z"), + }, + { + judgeId: "gs", + roomId: "a1", + createdAt: new Date("2027-02-06T13:00:00Z"), + }, + { + judgeId: "gs", + roomId: "g1", + createdAt: new Date("2027-02-06T11:00:00Z"), + }, + ], + }); + expect(judges.find((j) => j.id === "gs")).toMatchObject({ + roomId: "a1", + challengeIds: ["acme"], + }); + }); + + it("names every judge, including those it cannot assign", () => { + expect(roster().names.get("lost")).toBe("Nobody"); + }); +}); + +const appointment = ( + id: string, + over: Partial = {}, +): SavedAppointment => ({ + id, + entryId: `e-${id}`, + challengeId: "general", + roomId: "g1", + slot: 0, + assignedJudgeId: null, + removed: false, + open: true, + ...over, +}); + +describe("judgePlanOf", () => { + const plan = (appointments: SavedAppointment[]) => + judgePlanOf({ + rooms: [ + ...rooms, + { id: "old", challengeId: "general", buildingId: null }, + ], + sponsorChallengeIds: new Set(["acme"]), + grid: { slots: 10, sameGap: 1, crossGap: 2 }, + appointments, + judgeIds: new Set(["m1", "m2"]), + }); + + it("plans the appointments that are open or already have a judge", () => { + const result = plan([ + appointment("open"), + appointment("kept", { open: false, assignedJudgeId: "m1", slot: 3 }), + appointment("under-way", { open: false, slot: 5 }), + appointment("gone", { removed: true, assignedJudgeId: "m1", slot: 6 }), + ]); + expect(result.placements.map((p) => p.entryId)).toEqual([ + "e-open", + "e-kept", + ]); + expect(result.inputs.tasks).toHaveLength(2); + expect(result.keep).toEqual([ + { entryId: "e-kept", challengeId: "general", judgeId: "m1" }, + ]); + expect(result.current).toEqual(result.keep); + }); + + it("marks sponsor challenges and gives a room without a building its own", () => { + const result = plan([ + appointment("s", { challengeId: "acme", roomId: "a1" }), + ]); + expect(result.inputs.tasks[0]).toMatchObject({ sponsor: true }); + expect(result.inputs.rooms.find((r) => r.id === "old")!.buildingId).toBe( + "room:old", + ); + expect(result.inputs).toMatchObject({ slots: 10, sameGap: 1, crossGap: 2 }); + }); + + it("counts a judge on an open appointment as current, not kept", () => { + const result = plan([appointment("o", { assignedJudgeId: "m1" })]); + expect(result.keep).toEqual([]); + expect(result.current).toHaveLength(1); + }); + + it("drops an appointment under way whose judge has left the roster", () => { + // Kept, it would hold a presentation for a judge nobody can plan around, + // and CP-SAT's answer would be refused for not keeping it. + const result = plan([ + appointment("gone-judge", { + open: false, + assignedJudgeId: "g9", + slot: 2, + }), + appointment("open-gone-judge", { assignedJudgeId: "g9", slot: 4 }), + ]); + expect(result.placements.map((p) => p.entryId)).toEqual([ + "e-open-gone-judge", + ]); + expect(result.keep).toEqual([]); + }); +}); + +describe("judgeChanges", () => { + it("lists the open appointments whose judge differs, null where nobody is left", () => { + const appointments = [ + appointment("same", { assignedJudgeId: "m1" }), + appointment("new"), + appointment("moved", { assignedJudgeId: "m1" }), + appointment("dropped", { assignedJudgeId: "m1" }), + appointment("closed", { open: false, assignedJudgeId: "m1" }), + ]; + const changes = judgeChanges(appointments, [ + { entryId: "e-same", challengeId: "general", judgeId: "m1" }, + { entryId: "e-new", challengeId: "general", judgeId: "m2" }, + { entryId: "e-moved", challengeId: "general", judgeId: "m2" }, + { entryId: "e-closed", challengeId: "general", judgeId: "m2" }, + ]); + expect(changes).toEqual([ + { id: "new", assignedJudgeId: "m2" }, + { id: "moved", assignedJudgeId: "m2" }, + { id: "dropped", assignedJudgeId: null }, + ]); + }); +}); + +describe("judgeChoices", () => { + const roster = rosterOf({ + hackathonId: "h", + judges: judgeRows, + seats: [{ judgeId: "gs", roomId: "a1", createdAt: new Date() }], + challenges, + rooms, + }); + + it("offers the judges who may grade the appointment there, and marks the booked", () => { + const target = appointment("t", { slot: 2 }); + const all = [ + target, + appointment("other", { + roomId: "a1", + challengeId: "acme", + slot: 2, + assignedJudgeId: "m1", + }), + ]; + expect(judgeChoices(roster, target, all)).toEqual([ + { judgeId: "m1", bookedWith: "other" }, + ]); + // Free once their other appointment is at another time. + all[1] = { ...all[1]!, slot: 4 }; + expect(judgeChoices(roster, target, all)).toEqual([ + { judgeId: "m1", bookedWith: null }, + ]); + }); + + it("offers a guest only their own room", () => { + const sponsor = appointment("s", { challengeId: "acme", roomId: "a1" }); + expect( + judgeChoices(roster, sponsor, [sponsor]).map((c) => c.judgeId), + ).toEqual(["gs", "m1"]); + }); + + it("ignores a booking for a project no longer entered", () => { + const target = appointment("t", { slot: 2 }); + const gone = appointment("g", { + removed: true, + slot: 2, + assignedJudgeId: "m1", + roomId: "a1", + challengeId: "acme", + }); + expect(judgeChoices(roster, target, [target, gone])[0]).toMatchObject({ + bookedWith: null, + }); + }); +}); diff --git a/packages/api/src/services/judging/schedule/judge-plan.ts b/packages/api/src/services/judging/schedule/judge-plan.ts new file mode 100644 index 00000000..24169e4c --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-plan.ts @@ -0,0 +1,222 @@ +import { inJudgeScope } from "../evaluation-store"; +import type { ChallengeRow } from "../evaluation-store"; +import type { Judge } from "../principal"; +import { scheduledChallengeIds, taskKey } from "./inputs"; +import type { Placement, PlanInputs, PlanRoom } from "./inputs"; +import { mayGrade } from "./judges"; +import type { JudgeAssignment, JudgeInput } from "./judges"; + +// What the router hands the judge assigner and reads back: who can judge, +// which saved appointments are up for assignment, and the numbers the board +// shows. No database in here; the router reads the rows and writes the result. + +export type JudgeRow = { + id: string; + kind: "member" | "guest"; + userId: string | null; + displayName: string; +}; + +/** A guest's session: the room they scanned in at. */ +export type GuestSeat = { judgeId: string; roomId: string; createdAt: Date }; + +export type Roster = { + /** Judges who can be given presentations: members, and guests with a room. */ + judges: JudgeInput[]; + /** Every judge's name, those who cannot be given presentations too. */ + names: Map; +}; + +/** + * The judges an edition can assign. A member may grade whatever the judge + * desk lets them see; a guest only their room's challenge (inJudgeScope), + * and only in that room, which is the room of their latest session. A guest + * with no session, or whose room takes no appointments, has nothing to grade. + */ +export function rosterOf(source: { + hackathonId: string; + judges: readonly JudgeRow[]; + /** Sessions that have not been revoked. */ + seats: readonly GuestSeat[]; + challenges: readonly ChallengeRow[]; + /** The rooms that take appointments. */ + rooms: readonly PlanRoom[]; +}): Roster { + const scheduled = [...scheduledChallengeIds(source.challenges)].sort(); + const rooms = new Map(source.rooms.map((room) => [room.id, room])); + + const seatOf = new Map(); + for (const seat of source.seats) { + const current = seatOf.get(seat.judgeId); + if (!current || seat.createdAt > current.createdAt) + seatOf.set(seat.judgeId, seat); + } + + const judges: JudgeInput[] = []; + for (const row of source.judges) { + if (row.kind === "member") { + const principal: Judge = { + kind: "member", + hackathonId: source.hackathonId, + userId: row.userId ?? "", + judgeId: row.id, + displayName: row.displayName, + }; + judges.push({ + id: row.id, + kind: "member", + challengeIds: scheduled.filter((id) => + inJudgeScope(principal, source.challenges, id), + ), + roomId: null, + }); + continue; + } + const room = rooms.get(seatOf.get(row.id)?.roomId ?? ""); + if (!room) continue; + const principal: Judge = { + kind: "guest", + hackathonId: source.hackathonId, + sessionId: "", + linkId: "", + roomId: room.id, + roomChallengeId: room.challengeId, + judgeId: row.id, + displayName: row.displayName, + }; + judges.push({ + id: row.id, + kind: "guest", + challengeIds: scheduled.filter((id) => + inJudgeScope(principal, source.challenges, id), + ), + roomId: room.id, + }); + } + judges.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + + return { + judges, + names: new Map(source.judges.map((row) => [row.id, row.displayName])), + }; +} + +export type SavedAppointment = { + id: string; + entryId: string; + challengeId: string; + roomId: string; + slot: number; + assignedJudgeId: string | null; + /** Entered no more: needs no judge. */ + removed: boolean; + /** Not started, nothing evaluated or drafted: its judge may be chosen anew. */ + open: boolean; +}; + +/** + * The saved schedule as the judge assigner takes it: the appointments still + * entered that are open or already have a judge. `keep` is the judges of those + * that are not open, which stay; `current` is every judge there is now. + */ +export function judgePlanOf(source: { + /** Every room, archived ones too; a room with no building counts as its own. */ + rooms: readonly { + id: string; + challengeId: string; + buildingId: string | null; + }[]; + sponsorChallengeIds: ReadonlySet; + grid: Pick; + appointments: readonly SavedAppointment[]; + /** The judges on the roster: those who can be given presentations. */ + judgeIds: ReadonlySet; +}) { + // One that is over or under way with nobody assigned has no judge to find, + // and one whose judge has left the roster ties up nobody who is still on it. + const live = source.appointments.filter( + (a) => + !a.removed && + (a.open || + (a.assignedJudgeId !== null && source.judgeIds.has(a.assignedJudgeId))), + ); + const inputs: PlanInputs = { + ...source.grid, + rooms: source.rooms.map((room) => ({ + id: room.id, + challengeId: room.challengeId, + buildingId: room.buildingId ?? `room:${room.id}`, + })), + tasks: live.map((a) => ({ + entryId: a.entryId, + challengeId: a.challengeId, + sponsor: source.sponsorChallengeIds.has(a.challengeId), + })), + }; + const placements: Placement[] = live.map((a) => ({ + entryId: a.entryId, + challengeId: a.challengeId, + roomId: a.roomId, + slot: a.slot, + })); + const given = (a: SavedAppointment): JudgeAssignment => ({ + entryId: a.entryId, + challengeId: a.challengeId, + judgeId: a.assignedJudgeId!, + }); + return { + inputs, + placements, + keep: live.filter((a) => !a.open && a.assignedJudgeId).map(given), + current: live.filter((a) => a.assignedJudgeId).map(given), + }; +} + +/** + * What `assignment` changes on the open appointments: the new judge of every + * one whose judge differs, null where it now has none. + */ +export function judgeChanges( + appointments: readonly SavedAppointment[], + assignment: readonly JudgeAssignment[], +): { id: string; assignedJudgeId: string | null }[] { + const chosen = new Map( + assignment.map((a) => [taskKey(a.entryId, a.challengeId), a.judgeId]), + ); + return appointments + .filter((a) => a.open && !a.removed) + .flatMap((a) => { + const assignedJudgeId = + chosen.get(taskKey(a.entryId, a.challengeId)) ?? null; + return assignedJudgeId === a.assignedJudgeId + ? [] + : [{ id: a.id, assignedJudgeId }]; + }); +} + +export type JudgeChoice = { + judgeId: string; + /** The appointment of theirs in the same slot, if they are already booked. */ + bookedWith: string | null; +}; + +/** Judges who may grade `appointment`, and whether each is free in its slot. */ +export function judgeChoices( + roster: Roster, + appointment: Pick, + all: readonly SavedAppointment[], +): JudgeChoice[] { + return roster.judges + .filter((judge) => mayGrade(judge, appointment)) + .map((judge) => ({ + judgeId: judge.id, + bookedWith: + all.find( + (other) => + other.id !== appointment.id && + !other.removed && + other.assignedJudgeId === judge.id && + other.slot === appointment.slot, + )?.id ?? null, + })); +} diff --git a/packages/api/src/services/judging/schedule/judge-progress.test.ts b/packages/api/src/services/judging/schedule/judge-progress.test.ts new file mode 100644 index 00000000..aa27a690 --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-progress.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest"; +import { DEADLINE_GRACE_MS, takeoverAt } from "../timing"; +import { judgeProgress, judgeProgressById } from "./judge-progress"; + +const at = (time: string) => new Date(`2027-02-08T${time}:00Z`); +const startsAt = at("15:00"); +const deadlineAt = at("15:20"); +const JUDGE = "ada"; +const OTHER = "bea"; + +const input = (over: Partial[0]> = {}) => ({ + startsAt, + deadlineAt, + assignedJudgeId: JUDGE, + evaluations: [], + drafts: [], + ...over, +}); +// The desk stamps a presentation's draft with its deadline. +const draft = ( + judgeId: string, + time = "15:02", + draftDeadline: Date | null = deadlineAt, +) => ({ + judgeId, + updatedAt: at(time), + deadlineAt: draftDeadline, +}); + +describe("judgeProgress", () => { + it("says nothing without an assigned judge", () => { + expect( + judgeProgress( + input({ assignedJudgeId: null, evaluations: [{ judgeId: OTHER }] }), + at("16:00"), + ), + ).toBeNull(); + }); + + it("says nothing before the presentation starts", () => { + expect(judgeProgress(input(), at("14:59"))).toBeNull(); + // A judge cannot have begun, so nothing could show anyway. + expect( + judgeProgress(input({ drafts: [draft(JUDGE)] }), at("14:59")), + ).toBeNull(); + }); + + it("says nothing while the assigned judge still has the presentation to themselves", () => { + expect(judgeProgress(input(), at("15:00"))).toBeNull(); + expect(judgeProgress(input(), at("15:04"))).toBeNull(); + }); + + it("flags an assigned judge who has not begun once the takeover time passes", () => { + // The time the judge desk opens the presentation to everyone else. + const opens = takeoverAt(startsAt); + expect(judgeProgress(input(), new Date(opens.getTime() - 1))).toBeNull(); + expect(judgeProgress(input(), opens)).toBe("not_started"); + expect(judgeProgress(input(), at("15:05"))).toBe("not_started"); + // And stays so once the slot is over. + expect(judgeProgress(input(), at("18:00"))).toBe("not_started"); + }); + + it("is grading while the assigned judge has a draft open", () => { + expect(judgeProgress(input({ drafts: [draft(JUDGE)] }), at("15:01"))).toBe( + "grading", + ); + // Past the takeover time too: they began, so nothing was missed. + expect(judgeProgress(input({ drafts: [draft(JUDGE)] }), at("15:30"))).toBe( + "grading", + ); + }); + + it("is graded once the assigned judge has submitted", () => { + expect( + judgeProgress(input({ evaluations: [{ judgeId: JUDGE }] }), at("15:08")), + ).toBe("graded"); + // An evaluation outranks a draft left beside it. + expect( + judgeProgress( + input({ evaluations: [{ judgeId: JUDGE }], drafts: [draft(JUDGE)] }), + at("15:08"), + ), + ).toBe("graded"); + }); + + it("is taken over when another judge has submitted or has a draft", () => { + expect( + judgeProgress(input({ evaluations: [{ judgeId: OTHER }] }), at("15:09")), + ).toBe("taken_over"); + expect(judgeProgress(input({ drafts: [draft(OTHER)] }), at("15:09"))).toBe( + "taken_over", + ); + }); + + it("reads the judge who saved a draft last as the one who has it", () => { + // Ada's draft was left alone and Bea took over; both rows remain. + expect( + judgeProgress( + input({ drafts: [draft(JUDGE, "15:01"), draft(OTHER, "15:12")] }), + at("15:13"), + ), + ).toBe("taken_over"); + // Or Ada came back and saved after Bea. + expect( + judgeProgress( + input({ drafts: [draft(OTHER, "15:12"), draft(JUDGE, "15:20")] }), + at("15:21"), + ), + ).toBe("grading"); + }); + + it("stops counting a draft that lost its deadline once judging for the presentation is over", () => { + // The deadline and the grace a last write gets. + const over = new Date(deadlineAt.getTime() + DEADLINE_GRACE_MS); + const before = new Date(over.getTime() - 1); + // Opened and left empty: it is kept without its deadline and never + // submitted. + const empty = input({ drafts: [draft(JUDGE, "15:02", null)] }); + expect(judgeProgress(empty, before)).toBe("grading"); + expect(judgeProgress(empty, over)).toBe("not_started"); + // Another judge's, likewise: nobody took it over. + expect( + judgeProgress(input({ drafts: [draft(OTHER, "15:12", null)] }), over), + ).toBe("not_started"); + // One still carrying its deadline is about to be submitted for its judge. + expect(judgeProgress(input({ drafts: [draft(JUDGE)] }), over)).toBe( + "grading", + ); + expect( + judgeProgress(input({ drafts: [draft(OTHER, "15:12")] }), over), + ).toBe("taken_over"); + }); +}); + +describe("judgeProgressById", () => { + const roots: Record = { + general: "general", + "general-ai": "general", + acme: "acme", + }; + const rootIdOf = (id: string) => roots[id] ?? null; + const appointment = (over: Record = {}) => ({ + id: "apt", + entryId: "e1", + challengeId: "general", + startsAt, + deadlineAt, + assignedJudgeId: JUDGE, + ...over, + }); + const row = (over: Record = {}) => ({ + judgeId: JUDGE, + entryId: "e1", + challengeId: "general", + appointmentId: null as string | null, + ...over, + }); + const run = ( + appointments: ReturnType[], + evaluations: ReturnType[], + drafts: (ReturnType & { + updatedAt: Date; + deadlineAt: Date | null; + })[], + now = at("15:30"), + ) => judgeProgressById(appointments, evaluations, drafts, rootIdOf, now); + + it("matches by appointment, or by project and challenge root when none is named", () => { + const progress = run( + [appointment()], + [ + // Another judge's, on a challenge under the appointment's root. + row({ judgeId: OTHER, challengeId: "general-ai" }), + ], + [], + ); + expect(progress.get("apt")).toBe("taken_over"); + + // Written against another appointment, another project, another root. + const unrelated = run( + [appointment()], + [ + row({ appointmentId: "other" }), + row({ entryId: "e2" }), + row({ challengeId: "acme" }), + ], + [], + ); + expect(unrelated.get("apt")).toBe("not_started"); + + const named = run( + [appointment()], + [row({ appointmentId: "apt", challengeId: "acme" })], + [], + ); + expect(named.get("apt")).toBe("graded"); + }); + + it("reads each appointment on its own rows, and leaves out the ones with nothing to say", () => { + const progress = run( + [ + appointment({ id: "a1", entryId: "e1" }), + appointment({ id: "a2", entryId: "e2", assignedJudgeId: OTHER }), + appointment({ id: "a3", entryId: "e3", assignedJudgeId: null }), + appointment({ id: "a4", entryId: "e4", startsAt: at("16:00") }), + ], + [row({ entryId: "e1" })], + [ + { + ...row({ entryId: "e2", judgeId: OTHER }), + updatedAt: at("15:10"), + deadlineAt, + }, + ], + ); + expect(Object.fromEntries(progress)).toEqual({ + a1: "graded", + a2: "grading", + }); + }); +}); diff --git a/packages/api/src/services/judging/schedule/judge-progress.ts b/packages/api/src/services/judging/schedule/judge-progress.ts new file mode 100644 index 00000000..d6533eaf --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-progress.ts @@ -0,0 +1,130 @@ +import { DEADLINE_GRACE_MS, takeoverAt } from "../timing"; +import { belongsTo } from "./board"; + +// How far the judge assigned to a presentation has got, for the organiser's +// live board. Pure: the router loads the rows, this decides. Organisers see +// judge names on the board already, but this says only whether the assigned +// judge or someone else is on the presentation, not who. +// +// From the presentation's start its assigned judge has until its takeover time +// (timing.ts takeoverAt) to begin. After that any judge who may grade it can +// take it over, which the judge desk allows; the assignment itself is never +// rewritten. An evaluation or a draft is the sign a judge has begun. +// +// Once the deadline and the grace a last write gets have passed, a draft still +// carrying its deadline is about to be submitted for its judge +// (evaluation-store reconcileDueDrafts). One that lost it never will be: it +// was empty, or could not be submitted, so it no longer counts as a start. + +/** + * Where a started presentation's grading stands against its assigned judge: + * "grading" (their draft is open), "graded" (they submitted), "taken_over" + * (another judge has a draft or an evaluation), "not_started" (nobody has + * begun and the takeover time has passed). + */ +export type JudgeProgress = "grading" | "graded" | "taken_over" | "not_started"; + +type Started = { judgeId: string }; +type Draft = Started & { updatedAt: Date; deadlineAt: Date | null }; + +/** + * Null when there is nothing to say yet: no assigned judge, a presentation + * that has not started, or one nobody has begun within the takeover window. + */ +export function judgeProgress( + input: { + startsAt: Date; + /** The presentation's deadline: judging for it ends here. */ + deadlineAt: Date; + assignedJudgeId: string | null; + /** Judges who submitted an evaluation of the presentation. */ + evaluations: readonly Started[]; + /** Judges with a draft of it open. */ + drafts: readonly Draft[]; + }, + now: Date, +): JudgeProgress | null { + const { assignedJudgeId } = input; + if (!assignedJudgeId || now < input.startsAt) return null; + + if (input.evaluations.length > 0) { + return input.evaluations.some((e) => e.judgeId === assignedJudgeId) + ? "graded" + : "taken_over"; + } + const ended = now.getTime() >= input.deadlineAt.getTime() + DEADLINE_GRACE_MS; + const drafts = ended + ? input.drafts.filter((draft) => draft.deadlineAt !== null) + : input.drafts; + // Drafts a judge left behind when another took over stay in the table, so + // the one saved last is the judge who has it. + const holder = drafts.reduce( + (latest, draft) => + !latest || draft.updatedAt > latest.updatedAt ? draft : latest, + null, + ); + if (holder) + return holder.judgeId === assignedJudgeId ? "grading" : "taken_over"; + + return now >= takeoverAt(input.startsAt) ? "not_started" : null; +} + +type Booked = { + id: string; + entryId: string; + challengeId: string; + startsAt: Date; + deadlineAt: Date; + assignedJudgeId: string | null; +}; + +type Row = { + entryId: string; + challengeId: string; + appointmentId: string | null; +}; + +/** + * The progress of every appointment that has something to say, by id. One pass + * over the edition's evaluations and drafts, matched to appointments the way + * the board matches them (board.ts belongsTo). + */ +export function judgeProgressById( + appointments: readonly Booked[], + evaluations: readonly (Row & Started)[], + drafts: readonly (Row & Draft)[], + rootIdOf: (challengeId: string) => string | null, + now: Date, +): Map { + const byEntry = (rows: readonly T[]) => { + const map = new Map(); + for (const row of rows) { + const list = map.get(row.entryId); + if (list) list.push(row); + else map.set(row.entryId, [row]); + } + return map; + }; + const evaluationsOf = byEntry(evaluations); + const draftsOf = byEntry(drafts); + + const result = new Map(); + for (const appointment of appointments) { + const mine = (rows: Map) => + (rows.get(appointment.entryId) ?? []).filter((row) => + belongsTo(appointment, row, rootIdOf), + ); + const progress = judgeProgress( + { + startsAt: appointment.startsAt, + deadlineAt: appointment.deadlineAt, + assignedJudgeId: appointment.assignedJudgeId, + evaluations: mine(evaluationsOf), + drafts: mine(draftsOf), + }, + now, + ); + if (progress) result.set(appointment.id, progress); + } + return result; +} diff --git a/packages/api/src/services/judging/schedule/judges.test.ts b/packages/api/src/services/judging/schedule/judges.test.ts new file mode 100644 index 00000000..dc66a53c --- /dev/null +++ b/packages/api/src/services/judging/schedule/judges.test.ts @@ -0,0 +1,532 @@ +import { describe, expect, it } from "vitest"; +import { + MAX_CONSECUTIVE, + assignJudges, + compareJudgeMetrics, + describeShortfall, + judgeShortfalls, + mayGrade, + scoreJudgeAssignment, + validateJudgeAssignment, +} from "./judges"; +import type { JudgeAssignment, JudgeInput } from "./judges"; +import type { Placement, PlanInputs } from "./inputs"; + +const inputs: PlanInputs = { + rooms: [ + { id: "g1", challengeId: "general", buildingId: "ENG" }, + { id: "g2", challengeId: "general", buildingId: "ENG" }, + { id: "g3", challengeId: "general", buildingId: "HEC" }, + { id: "s1", challengeId: "sponsor", buildingId: "HEC" }, + ], + tasks: [ + { entryId: "a", challengeId: "general", sponsor: false }, + { entryId: "b", challengeId: "general", sponsor: false }, + { entryId: "c", challengeId: "general", sponsor: false }, + { entryId: "d", challengeId: "general", sponsor: false }, + { entryId: "a", challengeId: "sponsor", sponsor: true }, + { entryId: "b", challengeId: "sponsor", sponsor: true }, + ], + slots: 12, + sameGap: 1, + crossGap: 2, +}; + +const placed = ( + entryId: string, + challengeId: string, + roomId: string, + slot: number, +): Placement => ({ entryId, challengeId, roomId, slot }); + +const member = (id: string, extra: Partial = {}): JudgeInput => ({ + id, + kind: "member", + challengeIds: ["general", "sponsor"], + roomId: null, + ...extra, +}); + +const guest = ( + id: string, + roomId: string | null, + challengeIds = ["sponsor"], +): JudgeInput => ({ id, kind: "guest", challengeIds, roomId }); + +const row = ( + entryId: string, + challengeId: string, + judgeId: string, +): JudgeAssignment => ({ entryId, challengeId, judgeId }); + +const placements: Placement[] = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g2", 0), + placed("c", "general", "g1", 3), + placed("d", "general", "g3", 3), + placed("a", "sponsor", "s1", 6), + placed("b", "sponsor", "s1", 7), +]; + +describe("mayGrade", () => { + it("lets a member grade the challenges they are given, in any room", () => { + const judge = member("m", { challengeIds: ["general"] }); + expect(mayGrade(judge, { challengeId: "general", roomId: "g2" })).toBe( + true, + ); + expect(mayGrade(judge, { challengeId: "sponsor", roomId: "s1" })).toBe( + false, + ); + }); + + it("keeps a guest to their own room, and a guest with no room out", () => { + expect( + mayGrade(guest("x", "s1"), { challengeId: "sponsor", roomId: "s1" }), + ).toBe(true); + expect( + mayGrade(guest("x", "s1"), { challengeId: "sponsor", roomId: "s2" }), + ).toBe(false); + expect( + mayGrade(guest("x", null), { challengeId: "sponsor", roomId: "s1" }), + ).toBe(false); + }); +}); + +describe("validateJudgeAssignment", () => { + const check = ( + judges: JudgeInput[], + assignment: JudgeAssignment[], + spots = placements, + ) => validateJudgeAssignment(inputs, spots, judges, assignment); + + it("accepts a judge who keeps every rule, and does not mind presentations left out", () => { + const judges = [member("m1"), guest("g", "s1")]; + const result = check(judges, [ + row("a", "general", "m1"), + row("c", "general", "m1"), + row("a", "sponsor", "g"), + ]); + expect(result).toEqual({ valid: true, problems: [] }); + }); + + it("refuses a challenge or a room the judge may not grade", () => { + const judges = [ + member("m1", { challengeIds: ["general"] }), + guest("g", "s1"), + ]; + const problems = check(judges, [ + row("a", "sponsor", "m1"), + row("a", "general", "g"), + ]).problems.join(" "); + expect(problems).toContain( + "Judge m1 may not grade a:sponsor: wrong challenge.", + ); + expect(problems).toContain( + "Judge g may not grade a:general: wrong challenge.", + ); + const wrongRoom = check( + [guest("g", "s1", ["general", "sponsor"])], + [row("a", "general", "g")], + ); + expect(wrongRoom.problems).toEqual([ + "Judge g may not grade a:general: wrong room.", + ]); + }); + + it("refuses unknown presentations and judges, and two judges for one presentation", () => { + const { problems } = check( + [member("m1"), member("m2")], + [ + row("zz", "general", "m1"), + row("a", "general", "nobody"), + row("c", "general", "m1"), + row("c", "general", "m2"), + ], + ); + expect(problems).toContain("No such presentation: zz:general."); + expect(problems).toContain("No such judge: nobody."); + expect(problems).toContain("c:general has more than one judge."); + }); + + it("refuses a judge in two rooms at once", () => { + const { problems } = check( + [member("m1")], + [row("a", "general", "m1"), row("b", "general", "m1")], + ); + expect(problems).toEqual(["Judge m1 is booked twice in slot 1."]); + }); + + it("needs the same-building break to change rooms and the longer one to change buildings", () => { + const both = (second: Placement) => + validateJudgeAssignment( + inputs, + [placed("a", "general", "g1", 0), second], + [member("m1")], + [ + row("a", "general", "m1"), + row(second.entryId, second.challengeId, "m1"), + ], + ); + + // g1 to g2 is one building: one free slot between (slots 0 and 2). + expect(both(placed("b", "general", "g2", 1)).problems).toHaveLength(1); + expect(both(placed("b", "general", "g2", 2)).valid).toBe(true); + // g1 to g3 is another building: two free slots between. + expect(both(placed("b", "general", "g3", 2)).problems).toHaveLength(1); + expect(both(placed("b", "general", "g3", 3)).valid).toBe(true); + }); + + it("lets a judge stay in one room without a break, up to the run limit", () => { + const wide: PlanInputs = { + ...inputs, + tasks: ["a", "b", "c", "d", "e", "f"].map((entryId) => ({ + entryId, + challengeId: "general", + sponsor: false, + })), + }; + const run = (count: number, judge: JudgeInput) => { + const spots = ["a", "b", "c", "d", "e", "f"] + .slice(0, count) + .map((entryId, slot) => placed(entryId, "general", "g1", slot)); + return validateJudgeAssignment( + wide, + spots, + [judge], + spots.map((p) => row(p.entryId, "general", judge.id)), + ); + }; + expect(MAX_CONSECUTIVE).toBe(4); + expect(run(4, member("m1")).valid).toBe(true); + expect(run(5, member("m1")).problems).toEqual([ + "Judge m1 grades more than 4 slots in a row.", + ]); + expect(run(5, member("m1", { maxConsecutive: 5 })).valid).toBe(true); + expect(run(6, member("m1", { maxConsecutive: 2 })).valid).toBe(false); + }); + + it("allows a free slot to end a run", () => { + const spots = [0, 1, 2, 3, 5].map((slot, i) => + placed(["a", "b", "c", "d", "e"][i]!, "general", "g1", slot), + ); + const wide: PlanInputs = { + ...inputs, + tasks: spots.map((p) => ({ + entryId: p.entryId, + challengeId: "general", + sponsor: false, + })), + }; + const result = validateJudgeAssignment( + wide, + spots, + [member("m1")], + spots.map((p) => row(p.entryId, "general", "m1")), + ); + expect(result.valid).toBe(true); + }); +}); + +describe("scoreJudgeAssignment", () => { + it("counts what is left out, the loads, the rooms and the parts of the day", () => { + const judges = [member("m1"), member("m2"), guest("g", "s1")]; + const metrics = scoreJudgeAssignment(inputs, placements, judges, [ + row("a", "general", "m1"), + row("c", "general", "m1"), + row("b", "general", "m2"), + row("a", "sponsor", "g"), + row("b", "sponsor", "m2"), + ]); + expect(metrics).toMatchObject({ + unassigned: 1, + memberOnSponsor: 1, + maxLoad: 2, + // Members: 2 against 2; the guest is a pool of one. + loadSpread: 0, + // m1 is in g1 twice. + repeats: 1, + loads: { m1: 2, m2: 2, g: 1 }, + }); + // m1: g1; m2: g2, s1; g: s1. + expect(metrics.avgDistinctRooms).toBeCloseTo(4 / 3); + }); + + it("ranks fewer left out ahead of everything else", () => { + const judges = [member("m1"), member("m2")]; + const some = scoreJudgeAssignment(inputs, placements, judges, [ + row("a", "general", "m1"), + ]); + const more = scoreJudgeAssignment(inputs, placements, judges, [ + row("a", "general", "m1"), + row("c", "general", "m1"), + ]); + expect(compareJudgeMetrics(more, some)).toBeLessThan(0); + expect(compareJudgeMetrics(some, some)).toBe(0); + }); +}); + +describe("assignJudges", () => { + it("gives every presentation a judge who may grade it, keeping the rules", () => { + const judges = [member("m1"), member("m2"), guest("g", "s1")]; + const { assignment, metrics } = assignJudges(inputs, placements, judges); + expect(assignment).toHaveLength(placements.length); + expect(metrics.unassigned).toBe(0); + expect( + validateJudgeAssignment(inputs, placements, judges, assignment), + ).toEqual({ + valid: true, + problems: [], + }); + }); + + it("leaves sponsor presentations to the sponsor's judge while that judge can take them", () => { + const judges = [member("m1"), member("m2"), guest("g", "s1")]; + const { assignment, metrics } = assignJudges(inputs, placements, judges); + const sponsor = assignment.filter((r) => r.challengeId === "sponsor"); + expect(sponsor.map((r) => r.judgeId)).toEqual(["g", "g"]); + expect(metrics.memberOnSponsor).toBe(0); + }); + + it("asks a table judge to cover the sponsor when no sponsor judge can", () => { + // Both sponsor presentations at once: the one guest can take only one. + const clash = [ + placed("a", "sponsor", "s1", 6), + placed("b", "sponsor", "s1", 7), + placed("a", "general", "g1", 0), + ]; + const tight = { ...inputs, sameGap: 1 }; + const judges = [member("m1"), guest("g", "s1")]; + const { assignment, metrics } = assignJudges(tight, clash, judges); + // Back to back in one room needs no break: the guest takes both. + expect(assignment.filter((r) => r.judgeId === "g")).toHaveLength(2); + expect(metrics.memberOnSponsor).toBe(0); + + const together = [ + placed("a", "sponsor", "s1", 6), + placed("b", "sponsor", "s2", 6), + ]; + const withTwoRooms: PlanInputs = { + ...tight, + rooms: [ + ...tight.rooms, + { id: "s2", challengeId: "sponsor", buildingId: "HEC" }, + ], + }; + const result = assignJudges(withTwoRooms, together, [ + member("m1"), + guest("g", "s1"), + ]); + expect(result.metrics.unassigned).toBe(0); + expect(result.metrics.memberOnSponsor).toBe(1); + expect(result.assignment.find((r) => r.entryId === "b")!.judgeId).toBe( + "m1", + ); + }); + + it("evens the load between judges of one pool", () => { + const only = placements.filter((p) => p.challengeId === "general"); + const judges = [member("m1"), member("m2")]; + const { metrics } = assignJudges(inputs, only, judges); + expect(metrics.loads).toEqual({ m1: 2, m2: 2 }); + expect(metrics.maxLoad).toBe(2); + expect(metrics.loadSpread).toBe(0); + }); + + it("spreads one judge over rooms and over the day", () => { + const wide: PlanInputs = { + ...inputs, + sameGap: 0, + crossGap: 0, + slots: 8, + tasks: ["a", "b", "c", "d"].map((entryId) => ({ + entryId, + challengeId: "general", + sponsor: false, + })), + }; + // Two presentations in each of two rooms, early then late. + const spots = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g2", 1), + placed("c", "general", "g1", 6), + placed("d", "general", "g2", 7), + ]; + const { metrics, assignment } = assignJudges(wide, spots, [ + member("m1"), + member("m2"), + ]); + expect(metrics.loads).toEqual({ m1: 2, m2: 2 }); + // Each judge sees both rooms and both ends of the day. + expect(metrics.repeats).toBe(0); + expect(metrics.avgDistinctRooms).toBe(2); + expect(metrics.dayPartsCovered).toBe(4); + expect(new Set(assignment.map((r) => r.judgeId)).size).toBe(2); + }); + + it("reports the slot where judges run out", () => { + const rush = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g2", 0), + placed("c", "general", "g3", 0), + ]; + const judges = [member("m1"), member("m2")]; + const { assignment, metrics } = assignJudges(inputs, rush, judges); + expect(metrics.unassigned).toBe(1); + const shortfalls = judgeShortfalls(inputs, rush, judges, assignment); + expect(shortfalls).toEqual([ + { slot: 0, challengeId: "general", presentations: 3, judgesFree: 2 }, + ]); + expect(describeShortfall(shortfalls[0]!, "General")).toBe( + "Slot 1: 3 presentations in General, only 2 judges free", + ); + expect( + describeShortfall( + { slot: 4, challengeId: "x", presentations: 1, judgesFree: 1 }, + "Acme", + ), + ).toBe("Slot 5: 1 presentation in Acme, only 1 judge free"); + }); + + it("does not count a judge who cannot walk there as free", () => { + // m1 grades in ENG at slot 0; slot 1 is HEC and needs two free slots. + const spots = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g3", 1), + ]; + const judges = [member("m1")]; + const { assignment } = assignJudges(inputs, spots, judges); + expect(assignment).toHaveLength(1); + const shortfalls = judgeShortfalls(inputs, spots, judges, assignment); + expect(shortfalls).toEqual([ + { + slot: assignment[0]!.entryId === "a" ? 1 : 0, + challengeId: "general", + presentations: 1, + judgesFree: 0, + }, + ]); + }); + + it("keeps assignments it is told to keep, and plans around them", () => { + const only = placements.filter((p) => p.challengeId === "general"); + const judges = [member("m1"), member("m2")]; + const keep = [row("a", "general", "m1"), row("c", "general", "m1")]; + const { assignment } = assignJudges(inputs, only, judges, { keep }); + expect(assignment).toHaveLength(4); + expect(assignment).toEqual(expect.arrayContaining(keep)); + // b and d share a slot with the kept ones, so only m2 is free for them. + const rest = assignment.filter( + (r) => r.entryId === "b" || r.entryId === "d", + ); + expect(rest.map((r) => r.judgeId)).toEqual(["m2", "m2"]); + }); + + it("gives the same answer however its inputs are ordered", () => { + const judges = [member("m1"), member("m2"), member("m3"), guest("g", "s1")]; + const first = assignJudges(inputs, placements, judges); + const second = assignJudges( + inputs, + [...placements].reverse(), + [...judges].reverse(), + ); + expect(second.assignment).toEqual(first.assignment); + expect(second.metrics).toEqual(first.metrics); + }); + + it("assigns nothing without judges", () => { + const result = assignJudges(inputs, placements, []); + expect(result.assignment).toEqual([]); + expect(result.metrics.unassigned).toBe(placements.length); + expect(result.metrics.avgDistinctRooms).toBe(0); + }); +}); + +// A small deterministic generator, so these cases never change between runs. +function lcg(seed: number) { + let state = seed >>> 0; + return (n: number) => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state % n; + }; +} + +function randomCase(seed: number) { + const next = lcg(seed); + const rooms = [ + { id: "r1", challengeId: "c1", buildingId: "A" }, + { id: "r2", challengeId: "c1", buildingId: "A" }, + { id: "r3", challengeId: "c1", buildingId: "B" }, + { id: "r4", challengeId: "c2", buildingId: "B" }, + ]; + const slots = 8; + const taken = new Set(); + const spots: Placement[] = []; + for (let n = 0; n < 7; n += 1) { + const room = rooms[next(rooms.length)]!; + const slot = next(slots); + if (taken.has(`${room.id}@${slot}`)) continue; + taken.add(`${room.id}@${slot}`); + spots.push(placed(`e${n}`, room.challengeId, room.id, slot)); + } + const judges: JudgeInput[] = [ + { id: "m1", kind: "member", challengeIds: ["c1", "c2"], roomId: null }, + { id: "m2", kind: "member", challengeIds: ["c1", "c2"], roomId: null }, + { id: "g4", kind: "guest", challengeIds: ["c2"], roomId: "r4" }, + ]; + const plan: PlanInputs = { + rooms, + tasks: spots.map((p) => ({ + entryId: p.entryId, + challengeId: p.challengeId, + sponsor: p.challengeId === "c2", + })), + slots, + sameGap: next(2), + crossGap: 1 + next(2), + }; + plan.crossGap = Math.max(plan.crossGap, plan.sameGap); + return { plan, spots, judges }; +} + +/** The fewest presentations any valid assignment leaves without a judge. */ +function fewestLeftOut( + plan: PlanInputs, + spots: Placement[], + judges: JudgeInput[], +): number { + let best = spots.length; + const rows: JudgeAssignment[] = []; + const go = (i: number) => { + if (i === spots.length) { + best = Math.min(best, spots.length - rows.length); + return; + } + go(i + 1); + const p = spots[i]!; + for (const judge of judges) { + if (!mayGrade(judge, p)) continue; + rows.push(row(p.entryId, p.challengeId, judge.id)); + if (validateJudgeAssignment(plan, spots, judges, rows).valid) go(i + 1); + rows.pop(); + } + }; + go(0); + return best; +} + +describe("assignJudges against every possibility", () => { + it("keeps the rules and leaves out no more than it must", () => { + for (let seed = 1; seed <= 40; seed += 1) { + const { plan, spots, judges } = randomCase(seed); + const result = assignJudges(plan, spots, judges); + expect( + validateJudgeAssignment(plan, spots, judges, result.assignment) + .problems, + `seed ${seed}`, + ).toEqual([]); + expect(result.metrics.unassigned, `seed ${seed}`).toBe( + fewestLeftOut(plan, spots, judges), + ); + } + }); +}); diff --git a/packages/api/src/services/judging/schedule/judges.ts b/packages/api/src/services/judging/schedule/judges.ts new file mode 100644 index 00000000..6a5f0310 --- /dev/null +++ b/packages/api/src/services/judging/schedule/judges.ts @@ -0,0 +1,784 @@ +import { taskKey } from "./inputs"; +import type { Placement, PlanInputs } from "./inputs"; + +// The second scheduling stage. Once every presentation has a time and a room, +// each one gets exactly one judge, so every project has a time, a place and a +// grader. A presentation is a project in one root challenge, and that +// challenge is graded once, by one judge. +// +// Table judges (members) handle general judging; a sponsor's judges are other +// people, tied to the sponsor's room, who grade only that sponsor's challenge. +// A judge is in one place per slot, needs the schedule's breaks to walk to +// another room, and takes a rest after a long run of presentations. Judge bias +// is calibrated on the assumption that every judge sees a random sample of +// projects, so after everything is graded the aim is an even load, then each +// judge spread over as many rooms and as much of the day as possible. +// +// Pure and deterministic, written apart from the CP-SAT script on purpose: the +// script's answer is re-checked here (validateJudgeAssignment), and a server +// without OR-Tools builds its assignment here (assignJudges). + +/** Most slots in a row a judge grades before at least one free slot. */ +export const MAX_CONSECUTIVE = 4; + +/** + * The day is cut into this many equal parts when asking whether a judge's + * presentations are spread across it. cpsat.py cuts it the same way. + */ +export const DAY_PARTS = 4; + +export type JudgeInput = { + id: string; + kind: "member" | "guest"; + /** The root challenges this judge may grade. */ + challengeIds: readonly string[]; + /** The one room a guest grades in; null for a member, who moves about. */ + roomId: string | null; + /** Defaults to MAX_CONSECUTIVE. */ + maxConsecutive?: number; +}; + +/** The judge of one presentation. */ +export type JudgeAssignment = { + entryId: string; + challengeId: string; + judgeId: string; +}; + +/** Lower is better, except the last, which counts what the judges cover. */ +export type JudgeMetrics = { + /** Presentations with no judge. */ + unassigned: number; + /** Sponsor presentations graded by a table judge rather than a sponsor's. */ + memberOnSponsor: number; + /** Presentations of the busiest judge. */ + maxLoad: number; + /** How far apart the judges of one pool are, summed over the pools. */ + loadSpread: number; + /** Times a judge grades in a room they have already graded in. */ + repeats: number; + /** Rooms a judge grades in, averaged over the judges who grade at all. */ + avgDistinctRooms: number; + /** Parts of the day a judge grades in, summed over the judges. Higher is better. */ + dayPartsCovered: number; + /** Presentations per judge, for every judge given. */ + loads: Record; +}; + +/** + * Judges with the same challenges and room share a pool: they are + * interchangeable, so only within a pool can the load be evened out. + */ +const poolKey = (judge: JudgeInput) => + `${[...judge.challengeIds].sort().join(",")}|${judge.roomId ?? ""}`; + +/** Whether `judge` may grade a presentation of `challengeId` held in `roomId`. */ +export function mayGrade( + judge: JudgeInput, + presentation: { challengeId: string; roomId: string }, +): boolean { + if (!judge.challengeIds.includes(presentation.challengeId)) return false; + // A guest grades only in their own room, and with no room grades nowhere. + if (judge.roomId !== null) return judge.roomId === presentation.roomId; + return judge.kind !== "guest"; +} + +// Same hash for every run, so ties break the same way each time. +function stableHash(text: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < text.length; i += 1) { + hash ^= text.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash; +} + +const byHash = (a: string, b: string) => + stableHash(a) - stableHash(b) || (a < b ? -1 : a > b ? 1 : 0); + +type Item = { + index: number; + key: string; + entryId: string; + challengeId: string; + roomId: string; + buildingId: string; + slot: number; + part: number; + sponsor: boolean; +}; + +type Prepared = { + input: JudgeInput; + max: number; + /** Judges are numbered in id order, the same for any input order. */ + pool: number; +}; + +type Context = { + items: Item[]; + judges: Prepared[]; + sameGap: number; + crossGap: number; + maxGap: number; + itemIndex: Map; + judgeIndex: Map; + /** Judges who may grade each item, in judge order. */ + eligible: number[][]; + pools: number[][]; +}; + +function contextOf( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], +): Context { + const buildings = new Map(inputs.rooms.map((r) => [r.id, r.buildingId])); + const sponsor = new Map( + inputs.tasks.map((t) => [taskKey(t.entryId, t.challengeId), t.sponsor]), + ); + + const items: Item[] = []; + const seen = new Set(); + for (const p of placements) { + const key = taskKey(p.entryId, p.challengeId); + const buildingId = buildings.get(p.roomId); + if (seen.has(key) || buildingId === undefined) continue; + seen.add(key); + items.push({ + index: 0, + key, + entryId: p.entryId, + challengeId: p.challengeId, + roomId: p.roomId, + buildingId, + slot: p.slot, + part: Math.min( + DAY_PARTS - 1, + Math.max( + 0, + Math.floor((p.slot * DAY_PARTS) / Math.max(1, inputs.slots)), + ), + ), + sponsor: sponsor.get(key) ?? false, + }); + } + items.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + items.forEach((item, index) => { + item.index = index; + }); + + const unique = [...new Map(judges.map((j) => [j.id, j])).values()].sort( + (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0), + ); + const poolIds = new Map(); + const prepared = unique.map((input) => { + const key = poolKey(input); + if (!poolIds.has(key)) poolIds.set(key, poolIds.size); + return { + input, + max: Math.max(1, input.maxConsecutive ?? MAX_CONSECUTIVE), + pool: poolIds.get(key)!, + }; + }); + const pools: number[][] = [...poolIds.keys()].map(() => []); + prepared.forEach((judge, j) => pools[judge.pool]!.push(j)); + + return { + items, + judges: prepared, + sameGap: inputs.sameGap, + crossGap: inputs.crossGap, + maxGap: Math.max(inputs.sameGap, inputs.crossGap), + itemIndex: new Map(items.map((item) => [item.key, item.index])), + judgeIndex: new Map(prepared.map((judge, j) => [judge.input.id, j])), + eligible: items.map((item) => + prepared.flatMap((judge, j) => (mayGrade(judge.input, item) ? [j] : [])), + ), + pools, + }; +} + +/** + * Whether one judge cannot do both presentations: they are at the same time, + * or the walk between two rooms needs more free slots than lie between them. + * Staying in one room needs no walk. + */ +function clash(context: Context, a: Item, b: Item): boolean { + const apart = Math.abs(a.slot - b.slot); + if (apart === 0) return true; + if (a.roomId === b.roomId) return false; + const gap = + a.buildingId === b.buildingId ? context.sameGap : context.crossGap; + return apart <= gap; +} + +/** Where the search stands: who grades what, and the counts the score needs. */ +class State { + readonly owner: number[]; + /** Per judge, the item they grade in each slot. */ + readonly slots: Map[]; + readonly load: number[]; + readonly rooms: Map[]; + readonly parts: number[][]; + unassigned: number; + memberOnSponsor = 0; + repeats = 0; + covered = 0; + + constructor(private readonly context: Context) { + this.owner = context.items.map(() => -1); + this.slots = context.judges.map(() => new Map()); + this.load = context.judges.map(() => 0); + this.rooms = context.judges.map(() => new Map()); + this.parts = context.judges.map(() => Array(DAY_PARTS).fill(0)); + this.unassigned = context.items.length; + } + + add(i: number, j: number) { + const item = this.context.items[i]!; + this.owner[i] = j; + this.slots[j]!.set(item.slot, i); + this.load[j]! += 1; + this.unassigned -= 1; + if (item.sponsor && this.context.judges[j]!.input.kind === "member") { + this.memberOnSponsor += 1; + } + const rooms = this.rooms[j]!; + const seen = rooms.get(item.roomId) ?? 0; + if (seen > 0) this.repeats += 1; + rooms.set(item.roomId, seen + 1); + if (this.parts[j]![item.part]! === 0) this.covered += 1; + this.parts[j]![item.part]! += 1; + } + + remove(i: number) { + const j = this.owner[i]!; + const item = this.context.items[i]!; + this.owner[i] = -1; + this.slots[j]!.delete(item.slot); + this.load[j]! -= 1; + this.unassigned += 1; + if (item.sponsor && this.context.judges[j]!.input.kind === "member") { + this.memberOnSponsor -= 1; + } + const rooms = this.rooms[j]!; + const left = rooms.get(item.roomId)! - 1; + if (left > 0) { + this.repeats -= 1; + rooms.set(item.roomId, left); + } else { + rooms.delete(item.roomId); + } + this.parts[j]![item.part]! -= 1; + if (this.parts[j]![item.part]! === 0) this.covered -= 1; + } + + /** + * Whether judge `j` could also take `item`: free in its slot, far enough + * from their other rooms, and not past their run limit. `ignore` is an + * item of theirs to leave out, as when it is about to be swapped away. + */ + fits(j: number, item: Item, ignore = -1): boolean { + const mine = this.slots[j]!; + const taken = (slot: number) => { + const other = mine.get(slot); + return other === undefined || other === ignore ? undefined : other; + }; + if (taken(item.slot) !== undefined) return false; + for (let d = 1; d <= this.context.maxGap; d += 1) { + for (const slot of [item.slot - d, item.slot + d]) { + const other = taken(slot); + if ( + other !== undefined && + clash(this.context, this.context.items[other]!, item) + ) { + return false; + } + } + } + let run = 1; + for (let s = item.slot - 1; taken(s) !== undefined; s -= 1) run += 1; + for (let s = item.slot + 1; taken(s) !== undefined; s += 1) run += 1; + return run <= this.context.judges[j]!.max; + } + + /** The score as a vector, compared left to right; lower is better. */ + vector(): number[] { + let maxLoad = 0; + for (const load of this.load) maxLoad = Math.max(maxLoad, load); + return [ + this.unassigned, + this.memberOnSponsor, + maxLoad, + this.spread(), + this.repeats, + -this.covered, + ]; + } + + spread(): number { + let total = 0; + for (const pool of this.context.pools) { + if (pool.length < 2) continue; + let low = Infinity; + let high = 0; + for (const j of pool) { + low = Math.min(low, this.load[j]!); + high = Math.max(high, this.load[j]!); + } + total += high - low; + } + return total; + } + + metrics(): JudgeMetrics { + const [unassigned, memberOnSponsor, maxLoad, loadSpread, repeats] = + this.vector() as [number, number, number, number, number]; + let graders = 0; + let distinct = 0; + const loads: Record = {}; + this.context.judges.forEach((judge, j) => { + loads[judge.input.id] = this.load[j]!; + if (this.load[j]! === 0) return; + graders += 1; + distinct += this.rooms[j]!.size; + }); + return { + unassigned, + memberOnSponsor, + maxLoad, + loadSpread, + repeats, + avgDistinctRooms: graders === 0 ? 0 : distinct / graders, + dayPartsCovered: this.covered, + loads, + }; + } +} + +function compareVectors(a: readonly number[], b: readonly number[]): number { + for (let i = 0; i < a.length; i += 1) { + if (a[i] !== b[i]) return a[i]! - b[i]!; + } + return 0; +} + +/** Lower is better, compared as the fields are listed in JudgeMetrics. */ +export function compareJudgeMetrics(a: JudgeMetrics, b: JudgeMetrics): number { + return compareVectors( + [ + a.unassigned, + a.memberOnSponsor, + a.maxLoad, + a.loadSpread, + a.repeats, + -a.dayPartsCovered, + ], + [ + b.unassigned, + b.memberOnSponsor, + b.maxLoad, + b.loadSpread, + b.repeats, + -b.dayPartsCovered, + ], + ); +} + +/** A state holding `assignment`, skipping rows that name nothing known. */ +function stateOf(context: Context, assignment: readonly JudgeAssignment[]) { + const state = new State(context); + for (const row of assignment) { + const i = context.itemIndex.get(taskKey(row.entryId, row.challengeId)); + const j = context.judgeIndex.get(row.judgeId); + if (i === undefined || j === undefined || state.owner[i]! !== -1) continue; + state.add(i, j); + } + return state; +} + +/** + * Checks an assignment against the hard rules. A presentation with no judge + * is not an error here (see judgeShortfalls); one with the wrong judge, or + * two, is. + */ +export function validateJudgeAssignment( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + assignment: readonly JudgeAssignment[], +) { + const problems = new Set(); + const context = contextOf(inputs, placements, judges); + const given = new Set(); + const mine = context.judges.map((): number[] => []); + + for (const row of assignment) { + const key = taskKey(row.entryId, row.challengeId); + const i = context.itemIndex.get(key); + const j = context.judgeIndex.get(row.judgeId); + if (i === undefined) { + problems.add(`No such presentation: ${key}.`); + continue; + } + if (j === undefined) { + problems.add(`No such judge: ${row.judgeId}.`); + continue; + } + if (given.has(key)) problems.add(`${key} has more than one judge.`); + given.add(key); + const judge = context.judges[j]!.input; + const item = context.items[i]!; + if (!judge.challengeIds.includes(item.challengeId)) { + problems.add(`Judge ${judge.id} may not grade ${key}: wrong challenge.`); + } else if (!mayGrade(judge, item)) { + problems.add(`Judge ${judge.id} may not grade ${key}: wrong room.`); + } + mine[j]!.push(i); + } + + context.judges.forEach((judge, j) => { + const items = mine[j]!.map((i) => context.items[i]!).sort( + (a, b) => a.slot - b.slot, + ); + for (let a = 0; a < items.length; a += 1) { + for (let b = a + 1; b < items.length; b += 1) { + if (items[b]!.slot - items[a]!.slot > context.maxGap) break; + if (clash(context, items[a]!, items[b]!)) { + problems.add( + items[a]!.slot === items[b]!.slot + ? `Judge ${judge.input.id} is booked twice in slot ${items[a]!.slot + 1}.` + : `Judge ${judge.input.id} has too little time between ${items[a]!.key} and ${items[b]!.key}.`, + ); + } + } + } + let run = 0; + let previous = -2; + for (const item of items) { + if (item.slot === previous) continue; + run = item.slot === previous + 1 ? run + 1 : 1; + previous = item.slot; + if (run === judge.max + 1) { + problems.add( + `Judge ${judge.input.id} grades more than ${judge.max} slots in a row.`, + ); + } + } + }); + + return { valid: problems.size === 0, problems: [...problems] }; +} + +/** How even and how mixed an assignment is, and how many presentations it leaves out. */ +export function scoreJudgeAssignment( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + assignment: readonly JudgeAssignment[], +): JudgeMetrics { + const context = contextOf(inputs, placements, judges); + return stateOf(context, assignment).metrics(); +} + +/** A slot where some presentations of one challenge have no judge. */ +export type JudgeShortfall = { + slot: number; + challengeId: string; + presentations: number; + /** How many of them judges could cover at that time. */ + judgesFree: number; +}; + +const plural = (n: number, one: string) => `${n} ${n === 1 ? one : `${one}s`}`; + +/** The shortfall as a sentence; `challenge` is the challenge's label. */ +export function describeShortfall( + shortfall: JudgeShortfall, + challenge: string, +): string { + return `Slot ${shortfall.slot + 1}: ${plural(shortfall.presentations, "presentation")} in ${challenge}, only ${plural(shortfall.judgesFree, "judge")} free`; +} + +/** + * Each slot and challenge with a presentation that has no judge, earliest + * first. A judge counts as free for a challenge when they may grade it there, + * are not grading another challenge in that slot, and could walk there. + */ +export function judgeShortfalls( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + assignment: readonly JudgeAssignment[], +): JudgeShortfall[] { + const context = contextOf(inputs, placements, judges); + const state = stateOf(context, assignment); + + const groups = new Map(); + for (const item of context.items) { + const key = `${item.slot}|${item.challengeId}`; + const list = groups.get(key); + if (list) list.push(item.index); + else groups.set(key, [item.index]); + } + + const found: JudgeShortfall[] = []; + for (const members of groups.values()) { + if (!members.some((i) => state.owner[i]! === -1)) continue; + const first = context.items[members[0]!]!; + // Presentations of this group each need a different free judge: match them. + const options = members.map((i) => + context.eligible[i]!.filter((j) => { + const held = state.slots[j]!.get(first.slot); + const own = held !== undefined && members.includes(held) ? held : -1; + return ( + (held === undefined || own !== -1) && + state.fits(j, context.items[i]!, own) + ); + }), + ); + const taker = new Map(); + const place = (m: number, tried: Set): boolean => { + for (const j of options[m]!) { + if (tried.has(j)) continue; + tried.add(j); + const holder = taker.get(j); + if (holder === undefined || place(holder, tried)) { + taker.set(j, m); + return true; + } + } + return false; + }; + let covered = 0; + for (let m = 0; m < members.length; m += 1) { + if (place(m, new Set())) covered += 1; + } + found.push({ + slot: first.slot, + challengeId: first.challengeId, + presentations: members.length, + judgesFree: covered, + }); + } + return found.sort( + (a, b) => a.slot - b.slot || (a.challengeId < b.challengeId ? -1 : 1), + ); +} + +export type JudgeAssignmentResult = { + /** Every presentation that has a judge, the kept ones included. */ + assignment: JudgeAssignment[]; + metrics: JudgeMetrics; +}; + +/** + * Moves and swaps tried, and blocking items moved out of the way, before the + * search stops, however large the event. + */ +const MAX_EVALUATIONS = 50_000; +const MAX_DISPLACEMENTS = 100_000; +const MAX_PASSES = 12; + +/** + * Assigns a judge to every presentation it can: a greedy pass, each + * presentation to the judge that leaves the score best, from three orders of + * going through them, then a bounded local search on the best of the three + * that moves, swaps and displaces. The same inputs give the same answer, + * whatever order they come in: ties break on a hash of the ids. `keep` is + * assignments that stay as they are, and count towards loads and breaks. + */ +export function assignJudges( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + options: { keep?: readonly JudgeAssignment[] } = {}, +): JudgeAssignmentResult { + const context = contextOf(inputs, placements, judges); + const start = stateOf(context, options.keep ?? []); + const locked = new Set( + context.items + .filter((item) => start.owner[item.index]! !== -1) + .map((i) => i.index), + ); + const rank = (j: number) => stableHash(context.judges[j]!.input.id); + const free = context.items.filter((item) => !locked.has(item.index)); + + // Judges to try for an item, in a fixed order that does not favour the + // first by id. + const order = (i: number) => + [...context.eligible[i]!].sort((a, b) => rank(a) - rank(b) || a - b); + + /** + * The judge for `item` that leaves the score best, or -1. With `stay`, one + * who was in this room the slot before comes first: staying costs a judge + * no break, where moving on spends one. + */ + const bestJudge = (state: State, item: Item, skip = -1, stay = false) => { + let best = -1; + let bestVector: number[] = []; + for (const j of order(item.index)) { + if (j === skip || !state.fits(j, item)) continue; + const before = state.slots[j]!.get(item.slot - 1); + const staying = + before !== undefined && context.items[before]!.roomId === item.roomId; + state.add(item.index, j); + const vector = [stay && !staying ? 1 : 0, ...state.vector()]; + state.remove(item.index); + if (best === -1 || compareVectors(vector, bestVector) < 0) { + best = j; + bestVector = vector; + } + } + return best; + }; + + // Three starting points, the best of them is improved below: most + // constrained first, which is what a short supply of judges needs; and + // through the day in order, with or without judges staying where they are, + // which is what a long run of crowded slots needs. + const construct = (sort: (a: Item, b: Item) => number, stay: boolean) => { + const state = stateOf(context, options.keep ?? []); + for (const item of [...free].sort(sort)) { + const j = bestJudge(state, item, -1, stay); + if (j !== -1) state.add(item.index, j); + } + return state; + }; + const tightFirst = (a: Item, b: Item) => + context.eligible[a.index]!.length - context.eligible[b.index]!.length || + a.slot - b.slot || + byHash(a.key, b.key); + const inOrder = (a: Item, b: Item) => a.slot - b.slot || byHash(a.key, b.key); + const state = [ + construct(tightFirst, false), + construct(inOrder, true), + construct(inOrder, false), + ].reduce((best, next) => + compareVectors(next.vector(), best.vector()) < 0 ? next : best, + ); + + let evaluations = 0; + let displacements = 0; + + /** Moves `i` to judge `to` if that scores better than `current`, and says so. */ + const tryMove = (i: number, to: number, current: number[]) => { + const from = state.owner[i]!; + evaluations += 1; + state.remove(i); + state.add(i, to); + if (compareVectors(state.vector(), current) < 0) return true; + state.remove(i); + state.add(i, from); + return false; + }; + + /** Swaps the judges of `i` and `q` if that scores better than `current`, and says so. */ + const trySwap = (i: number, q: number, current: number[]) => { + const a = state.owner[i]!; + const b = state.owner[q]!; + evaluations += 1; + state.remove(i); + state.remove(q); + if (state.fits(b, context.items[i]!) && state.fits(a, context.items[q]!)) { + state.add(i, b); + state.add(q, a); + if (compareVectors(state.vector(), current) < 0) return true; + state.remove(i); + state.remove(q); + } + state.add(i, a); + state.add(q, b); + return false; + }; + + /** + * Gives `item` a judge, moving blocking items of that judge to other + * judges, `depth` moves deep. Changes nothing if it cannot. + */ + const displace = (item: Item, depth: number): boolean => { + const direct = bestJudge(state, item); + if (direct !== -1) { + state.add(item.index, direct); + return true; + } + if (depth === 0) return false; + for (const j of order(item.index)) { + const reach = Math.max(context.maxGap, context.judges[j]!.max); + for (const q of [...state.slots[j]!.values()]) { + if ( + locked.has(q) || + Math.abs(context.items[q]!.slot - item.slot) > reach + ) { + continue; + } + if (displacements >= MAX_DISPLACEMENTS) return false; + displacements += 1; + state.remove(q); + if (state.fits(j, item)) { + state.add(item.index, j); + if (displace(context.items[q]!, depth - 1)) return true; + state.remove(item.index); + } + state.add(q, j); + } + } + return false; + }; + + for ( + let pass = 0; + pass < MAX_PASSES && evaluations < MAX_EVALUATIONS; + pass += 1 + ) { + let improved = false; + + // A presentation with no judge: clear its way by sending an item that + // blocks a judge to somebody else, who may in turn send one on. + for (const item of free) { + if (state.owner[item.index]! !== -1) continue; + if (displacements >= MAX_DISPLACEMENTS) break; + if (displace(item, 4)) improved = true; + } + + // A presentation that scores better with another judge, alone or swapped + // with one of theirs. + for (const item of free) { + const from = state.owner[item.index]!; + if (from === -1) continue; + const current = state.vector(); + for (const to of order(item.index)) { + if (evaluations >= MAX_EVALUATIONS) break; + if (to === from) continue; + if (!state.slots[to]!.has(item.slot) && state.fits(to, item)) { + if (tryMove(item.index, to, current)) { + improved = true; + break; + } + } + let swapped = false; + for (const q of [...state.slots[to]!.values()]) { + if (evaluations >= MAX_EVALUATIONS) break; + if (locked.has(q) || !context.eligible[q]!.includes(from)) continue; + if (trySwap(item.index, q, current)) { + improved = true; + swapped = true; + break; + } + } + if (swapped) break; + } + } + if (!improved) break; + } + + const assignment = context.items + .filter((item) => state.owner[item.index]! !== -1) + .map((item) => ({ + entryId: item.entryId, + challengeId: item.challengeId, + judgeId: context.judges[state.owner[item.index]!]!.input.id, + })); + return { assignment, metrics: state.metrics() }; +} diff --git a/packages/api/src/services/judging/scope.test.ts b/packages/api/src/services/judging/scope.test.ts index dc6737f7..4bad80cd 100644 --- a/packages/api/src/services/judging/scope.test.ts +++ b/packages/api/src/services/judging/scope.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { descendantsOf, effectiveMode, + isSponsorChallenge, normalizeLabel, rootOf, sameLabel, @@ -108,3 +109,10 @@ describe("labels", () => { expect(sameLabel("AI", "A I")).toBe(false); }); }); + +describe("isSponsorChallenge", () => { + it("is every challenge but the general ones", () => { + expect(isSponsorChallenge({ isGeneral: false })).toBe(true); + expect(isSponsorChallenge({ isGeneral: true })).toBe(false); + }); +}); diff --git a/packages/api/src/services/judging/scope.ts b/packages/api/src/services/judging/scope.ts index 7fd40cb4..d8691ed5 100644 --- a/packages/api/src/services/judging/scope.ts +++ b/packages/api/src/services/judging/scope.ts @@ -35,6 +35,14 @@ export function rootOf( return null; } +/** + * Whether a challenge is a sponsor's: every challenge but the general ones. + * The schedule and the judge desk both read the flag this way. + */ +export function isSponsorChallenge(challenge: { isGeneral: boolean }): boolean { + return !challenge.isGeneral; +} + /** How a challenge is judged: always its root's mode. Null when unknown. */ export function effectiveMode( challenges: readonly ChallengeNode[], diff --git a/packages/api/src/services/judging/scoring-sql.test.ts b/packages/api/src/services/judging/scoring-sql.test.ts new file mode 100644 index 00000000..94c440b4 --- /dev/null +++ b/packages/api/src/services/judging/scoring-sql.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { judgingEvaluations } from "@query/db"; +import { holdsEveryRating } from "./scoring-sql"; + +const HACK = "11111111-1111-4111-8111-111111111111"; + +// Built with a handle that has no connection: only the SQL text is looked at. +const db = drizzle.mock(); + +/** The SQL with its spacing and parameter numbers evened out. */ +const flat = (text: string) => + text + .replace(/\s+/g, " ") + .replace(/\( /g, "(") + .replace(/ \)/g, ")") + .replace(/\$\d+/g, "$?"); + +// An evaluation holds every rating when its ratings are at least as many as +// the rubric's rating items: scoring.hasEveryRating, in SQL. +const RULE = + 'select count(*) from "judging_evaluation_rating" as r where r."evaluation_id" = "judging_evaluation"."id"'; +const ITEMS = + 'select count(*) from "judging_rubric_item" as i where i."hackathon_id" = $? and i."kind" = \'rating\''; +const HOLDS = `(${RULE}) >= (${ITEMS})`; + +describe("holdsEveryRating", () => { + it("counts the evaluation's ratings against the rubric's rating items", () => { + const query = db + .select({ id: judgingEvaluations.id }) + .from(judgingEvaluations) + .where( + and(eq(judgingEvaluations.hackathonId, HACK), holdsEveryRating(HACK)), + ) + .toSQL(); + + const text = flat(query.sql); + expect(text).toContain(HOLDS); + expect(query.params).toEqual([HACK, HACK]); + // Whether a written response was left empty is not looked at. + expect(text).not.toContain("judging_evaluation_response"); + expect(text).not.toContain("is_complete"); + }); + + it("keeps its table names in a select from the one table, which strips them from columns", () => { + // A sql field over a single table loses the table name from any column in + // it. The names here are not columns, so the subquery still correlates on + // the evaluation's id and not on its own. + const counted = db + .select({ + total: sql`count(*)::int`, + complete: sql`count(*) filter (where ${holdsEveryRating(HACK)})::int`, + holds: holdsEveryRating(HACK), + }) + .from(judgingEvaluations) + .where(eq(judgingEvaluations.hackathonId, HACK)) + .toSQL(); + + const text = flat(counted.sql); + expect(text.split(HOLDS)).toHaveLength(3); + expect(counted.params).toEqual([HACK, HACK, HACK]); + }); + + it("stands beside other conditions on the same evaluations", () => { + const query = db + .select({ id: judgingEvaluations.id }) + .from(judgingEvaluations) + .where( + and( + inArray(judgingEvaluations.entryId, ["a", "b"]), + holdsEveryRating(HACK), + ), + ) + .toSQL(); + expect(query.sql).toContain('"judging_evaluation"."entry_id" in ($1, $2)'); + expect(query.params).toEqual(["a", "b", HACK]); + }); +}); diff --git a/packages/api/src/services/judging/scoring-sql.ts b/packages/api/src/services/judging/scoring-sql.ts new file mode 100644 index 00000000..5b108af4 --- /dev/null +++ b/packages/api/src/services/judging/scoring-sql.ts @@ -0,0 +1,33 @@ +import { getTableName, sql } from "drizzle-orm"; +import { + judgingEvaluationRatings, + judgingEvaluations, + judgingRubricItems, +} from "@query/db"; + +// scoring.hasEveryRating for the queries that count evaluations in the +// database instead of loading them. The same rule, so a count an organiser or +// a hacker reads is the number placings rank on: an evaluation counts once it +// holds a rating for every rating item of the rubric, whether or not a written +// response was left empty. +// +// Names are spelled out rather than passed as columns. A select from one table +// with no joins drops the table name from any column in a sql expression, and +// the subquery would then compare its own columns with each other. + +/** + * True for a judging_evaluation row of the edition that holds every rating. + * The query must read judging_evaluation under its own name, as every query + * that uses this does. + */ +export const holdsEveryRating = (hackathonId: string) => { + const evaluation = sql.identifier(getTableName(judgingEvaluations)); + return sql`( + select count(*) from ${judgingEvaluationRatings} as r + where r.${sql.identifier(judgingEvaluationRatings.evaluationId.name)} = ${evaluation}.${sql.identifier(judgingEvaluations.id.name)} +) >= ( + select count(*) from ${judgingRubricItems} as i + where i.${sql.identifier(judgingRubricItems.hackathonId.name)} = ${hackathonId} + and i.${sql.identifier(judgingRubricItems.kind.name)} = 'rating' +)`; +}; diff --git a/packages/api/src/services/judging/scoring.test.ts b/packages/api/src/services/judging/scoring.test.ts index 6b88aab2..0e0162e6 100644 --- a/packages/api/src/services/judging/scoring.test.ts +++ b/packages/api/src/services/judging/scoring.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { evaluationScore, + hasEveryRating, meanOrNull, overallScore, rankByScore, @@ -19,6 +20,23 @@ describe("meanOrNull", () => { }); }); +describe("hasEveryRating", () => { + it("is whether the evaluation rated every rating item", () => { + expect(hasEveryRating([5, 4, 3], 3)).toBe(true); + expect(hasEveryRating([5, 4], 3)).toBe(false); + expect(hasEveryRating([], 3)).toBe(false); + }); + + it("does not need to be told about written responses", () => { + // Four rating items, however many responses the rubric also asks for. + expect(hasEveryRating([1, 2, 3, 4], 4)).toBe(true); + }); + + it("has nothing to miss on a rubric with no rating items", () => { + expect(hasEveryRating([], 0)).toBe(true); + }); +}); + describe("evaluationScore", () => { it("is the mean of the ratings", () => { expect(evaluationScore([5, 4, 3])).toBe(4); @@ -31,15 +49,15 @@ describe("evaluationScore", () => { }); const evals: ScoredEvaluation[] = [ - { challengeId: "general", isComplete: true, ratings: [5, 5, 5] }, // 5 - { challengeId: "general", isComplete: true, ratings: [3, 3, 3] }, // 3 - { challengeId: "general", isComplete: false, ratings: [1, 1] }, // ignored - { challengeId: "mongo", isComplete: true, ratings: [4, 2] }, // 3 - { challengeId: "mongo", isComplete: true, ratings: [] }, // no score + { challengeId: "general", ratingsComplete: true, ratings: [5, 5, 5] }, // 5 + { challengeId: "general", ratingsComplete: true, ratings: [3, 3, 3] }, // 3 + { challengeId: "general", ratingsComplete: false, ratings: [1, 1] }, // short of a rating: ignored + { challengeId: "mongo", ratingsComplete: true, ratings: [4, 2] }, // 3 + { challengeId: "mongo", ratingsComplete: true, ratings: [] }, // no score ]; describe("scopedScore", () => { - it("averages complete evaluations in that challenge only", () => { + it("averages evaluations with every rating in that challenge only", () => { expect(scopedScore(evals, "general")).toEqual({ score: 4, count: 2 }); expect(scopedScore(evals, "mongo")).toEqual({ score: 3, count: 1 }); }); @@ -48,7 +66,7 @@ describe("scopedScore", () => { expect(scopedScore(evals, "other")).toEqual({ score: null, count: 0 }); expect( scopedScore( - [{ challengeId: "general", isComplete: false, ratings: [5] }], + [{ challengeId: "general", ratingsComplete: false, ratings: [5] }], "general", ), ).toEqual({ score: null, count: 0 }); @@ -56,7 +74,7 @@ describe("scopedScore", () => { }); describe("overallScore", () => { - it("weighs every complete evaluation the same, across challenges", () => { + it("weighs every counted evaluation the same, across challenges", () => { // (5 + 3 + 3) / 3 expect(overallScore(evals)).toEqual({ score: 11 / 3, count: 3 }); }); @@ -66,6 +84,78 @@ describe("overallScore", () => { }); }); +describe("which evaluations count", () => { + const score = (over: Partial) => + overallScore([{ challengeId: "general", ratings: [4, 2], ...over }]); + + it("counts one with every rating, though a written response is missing", () => { + // An auto-submit at the hard end: not complete in the database, but all + // its ratings are there. + expect(score({ ratingsComplete: true, isComplete: false })).toEqual({ + score: 3, + count: 1, + }); + }); + + it("leaves out one short of a rating, though it was submitted whole", () => { + expect(score({ ratingsComplete: false, isComplete: true })).toEqual({ + score: null, + count: 0, + }); + }); + + it("reads isComplete from a caller that gives no ratingsComplete", () => { + expect(score({ isComplete: true })).toEqual({ score: 3, count: 1 }); + expect(score({ isComplete: false })).toEqual({ score: null, count: 0 }); + }); + + it("counts nothing when told nothing", () => { + expect(score({})).toEqual({ score: null, count: 0 }); + }); +}); + +describe("an adjusted score", () => { + const adjusted: ScoredEvaluation[] = [ + { + challengeId: "general", + ratingsComplete: true, + ratings: [5, 5], + adjustedScore: 4, + }, + { + challengeId: "general", + ratingsComplete: true, + ratings: [3, 3], + adjustedScore: 3.5, + }, + // No adjusted score: the mean of its ratings counts. + { challengeId: "general", ratingsComplete: true, ratings: [2, 4] }, + // Short of a rating is still left out, adjusted or not. + { + challengeId: "general", + ratingsComplete: false, + ratings: [1], + adjustedScore: 1, + }, + ]; + + it("counts in place of the mean of the ratings", () => { + // (4 + 3.5 + 3) / 3 + expect(scopedScore(adjusted, "general")).toEqual({ score: 3.5, count: 3 }); + expect(overallScore(adjusted)).toEqual({ score: 3.5, count: 3 }); + }); + + it("leaves evaluations without one on the raw mean", () => { + const raw = adjusted.map(({ challengeId, ratingsComplete, ratings }) => ({ + challengeId, + ratingsComplete, + ratings, + })); + // (5 + 3 + 3) / 3 + expect(overallScore(raw)).toEqual({ score: 11 / 3, count: 3 }); + }); +}); + describe("rankByScore", () => { it("ranks highest first", () => { const ranked = rankByScore([ diff --git a/packages/api/src/services/judging/scoring.ts b/packages/api/src/services/judging/scoring.ts index ec77cc1b..0cd35bdc 100644 --- a/packages/api/src/services/judging/scoring.ts +++ b/packages/api/src/services/judging/scoring.ts @@ -1,18 +1,33 @@ // Pure scoring for room judging. A judge's evaluation scores the mean of its // 1–5 ratings; a project's score in a scope is the mean of those evaluation -// scores, counting complete evaluations only. Every evaluation weighs the -// same, whoever wrote it. +// scores, counting evaluations that hold every rating. A written response left +// empty does not stop one counting: a draft submitted at the hard end is often +// short of a response and is the project's only grade. Every evaluation weighs +// the same, whoever wrote it. Placements feed in the calibrated score of each +// evaluation (calibration.ts) in place of the raw mean; the judge desk does not. /** An evaluation as scoring sees it. */ export type ScoredEvaluation = { challengeId: string; - isComplete: boolean; + /** + * Holds a rating for every rating item of the rubric (hasEveryRating). It + * counts when this is true, whatever was left unwritten. + */ + ratingsComplete?: boolean; + /** + * For a caller that only knows the evaluation was submitted whole, which + * implies every rating is there. Read only when ratingsComplete is not + * given. + */ + isComplete?: boolean; /** The 1–5 rating values it holds. */ ratings: readonly number[]; + /** Counted in place of the mean of the ratings when set: the calibrated score. */ + adjustedScore?: number; }; export type ScoreSummary = { - /** Null when no complete evaluation has a rating. */ + /** Null when no counted evaluation has a rating. */ score: number | null; /** How many evaluations the score is the mean of. */ count: number; @@ -30,10 +45,28 @@ export function evaluationScore(values: readonly number[]): number | null { return meanOrNull(values); } +/** + * Whether an evaluation rated every rating item. The rubric is fixed once an + * evaluation exists, and a rating is stored once per item, so counting them + * against the rubric's rating items is enough. Responses are not looked at. + */ +export function hasEveryRating( + ratings: readonly number[], + ratingItems: number, +): boolean { + return ratings.length >= ratingItems; +} + +const counts = (evaluation: ScoredEvaluation) => + evaluation.ratingsComplete ?? evaluation.isComplete ?? false; + function summarize(evals: readonly ScoredEvaluation[]): ScoreSummary { const scores = evals - .filter((evaluation) => evaluation.isComplete) - .map((evaluation) => evaluationScore(evaluation.ratings)) + .filter(counts) + .map( + (evaluation) => + evaluation.adjustedScore ?? evaluationScore(evaluation.ratings), + ) .filter((score): score is number => score !== null); return { score: meanOrNull(scores), count: scores.length }; } @@ -46,7 +79,7 @@ export function scopedScore( return summarize(evals.filter((e) => e.challengeId === challengeId)); } -/** A project's overall score: every complete evaluation, in any challenge. */ +/** A project's overall score: every counted evaluation, in any challenge. */ export function overallScore(evals: readonly ScoredEvaluation[]): ScoreSummary { return summarize(evals); } diff --git a/packages/api/src/services/judging/timing.test.ts b/packages/api/src/services/judging/timing.test.ts index 69ae34f3..f3b4276a 100644 --- a/packages/api/src/services/judging/timing.test.ts +++ b/packages/api/src/services/judging/timing.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from "vitest"; -import { evaluationTiming } from "./timing"; +import { + DEADLINE_GRACE_MS, + TAKEOVER_AFTER_MS, + evaluationTiming, + reservedUntil, + takeoverAt, +} from "./timing"; import type { TimingInput } from "./timing"; const startsAt = new Date("2027-02-08T15:00:00Z"); @@ -9,6 +15,7 @@ const during = new Date("2027-02-08T15:03:00Z"); const afterDeadline = new Date("2027-02-08T15:09:00Z"); const scheduled = (over: Partial = {}): TimingInput => ({ + judgeId: "judge-a", hasSchedule: true, mode: "scheduled", challengeId: "general", @@ -43,16 +50,76 @@ describe("evaluationTiming", () => { } }); - it("keeps a submitted evaluation editable, whatever the clock says", () => { + it("keeps a submitted evaluation editable until the deadline", () => { + for (const now of [before, during]) { + expect( + evaluationTiming( + scheduled({ + evaluation: { appointmentId: "appt-1" }, + presenceChallengeId: null, + }), + now, + ), + ).toEqual({ canEdit: true, deadlineAt, appointmentId: "appt-1" }); + } + }); + + it("gives a write DEADLINE_GRACE_MS past the deadline, and a read none", () => { + const inGrace = new Date(deadlineAt.getTime() + DEADLINE_GRACE_MS - 1); + const pastGrace = new Date(deadlineAt.getTime() + DEADLINE_GRACE_MS); + for (const evaluation of [null, { appointmentId: "appt-1" }]) { + const write = scheduled({ evaluation, graceMs: DEADLINE_GRACE_MS }); + expect(evaluationTiming(write, inGrace)).toMatchObject({ + canEdit: true, + }); + expect(evaluationTiming(write, pastGrace)).toMatchObject({ + canEdit: false, + ended: true, + }); + expect( + evaluationTiming(scheduled({ evaluation }), inGrace), + ).toMatchObject({ canEdit: false, ended: true }); + } + }); + + it("still reports the deadline to a write in the grace window", () => { + // The deadline is already past, but a draft first saved now still has to + // carry it, or nothing submits the draft for the judge. + const inGrace = new Date(deadlineAt.getTime() + DEADLINE_GRACE_MS - 1); + for (const evaluation of [null, { appointmentId: "appt-1" }]) { + const timing = evaluationTiming( + scheduled({ evaluation, graceMs: DEADLINE_GRACE_MS }), + inGrace, + ); + expect(timing).toEqual({ + canEdit: true, + deadlineAt, + appointmentId: "appt-1", + }); + } + }); + + it("locks a submitted evaluation once the deadline passes", () => { + for (const now of [deadlineAt, afterDeadline]) { + const timing = evaluationTiming( + scheduled({ evaluation: { appointmentId: "appt-1" } }), + now, + ); + expect(timing).toMatchObject({ canEdit: false, ended: true }); + if (!timing.canEdit) expect(timing.reason).toMatch(/final/); + } + }); + + it("keeps an untimed evaluation editable after any clock", () => { expect( evaluationTiming( scheduled({ - evaluation: { appointmentId: "appt-1" }, - presenceChallengeId: null, + mode: "remote", + evaluation: { appointmentId: null }, }), - before, + afterDeadline, ), - ).toEqual({ canEdit: true, deadlineAt: null, appointmentId: "appt-1" }); + ).toEqual({ canEdit: true, deadlineAt: null, appointmentId: null }); }); it("runs against the deadline during the presentation", () => { @@ -67,15 +134,18 @@ describe("evaluationTiming", () => { }); }); - it("drops the deadline once it has passed, still tied to the slot", () => { - expect(evaluationTiming(scheduled(), afterDeadline)).toEqual({ - canEdit: true, - deadlineAt: null, - appointmentId: "appt-1", - }); - expect(evaluationTiming(scheduled(), deadlineAt)).toMatchObject({ - deadlineAt: null, - }); + it("refuses a first evaluation once the deadline passes", () => { + for (const now of [deadlineAt, afterDeadline]) { + expect(evaluationTiming(scheduled(), now)).toMatchObject({ + canEdit: false, + opensAt: null, + ended: true, + }); + } + // The deadline wins over the room: there is nothing left to join. + expect( + evaluationTiming(scheduled({ presenceChallengeId: null }), afterDeadline), + ).toMatchObject({ canEdit: false, ended: true }); }); it("waits for the presentation to start", () => { @@ -89,7 +159,7 @@ describe("evaluationTiming", () => { for (const presenceChallengeId of [null, "mlh"]) { expect( evaluationTiming(scheduled({ presenceChallengeId }), during), - ).toMatchObject({ canEdit: false, opensAt: null }); + ).toMatchObject({ canEdit: false, opensAt: null, ended: false }); } }); @@ -99,3 +169,149 @@ describe("evaluationTiming", () => { if (!timing.canEdit) expect(timing.reason).toMatch(/no presentation slot/); }); }); + +describe("TAKEOVER_AFTER_MS", () => { + it("is five minutes", () => { + expect(TAKEOVER_AFTER_MS).toBe(5 * 60_000); + expect(takeoverAt(startsAt)).toEqual(new Date("2027-02-08T15:05:00Z")); + }); +}); + +describe("reservedUntil", () => { + const assigned = { startsAt, assignedJudgeId: "judge-b" }; + const takeover = takeoverAt(startsAt); + + it("is the takeover time while a presentation is kept for another judge", () => { + for (const now of [before, startsAt, during]) { + expect(reservedUntil(assigned, "judge-a", now)).toEqual(takeover); + } + }); + + it("is over from the takeover time on", () => { + expect( + reservedUntil(assigned, "judge-a", new Date(takeover.getTime() - 1)), + ).toEqual(takeover); + expect(reservedUntil(assigned, "judge-a", takeover)).toBeNull(); + expect(reservedUntil(assigned, "judge-a", afterDeadline)).toBeNull(); + }); + + it("keeps nothing from the judge it is assigned to", () => { + expect(reservedUntil(assigned, "judge-b", during)).toBeNull(); + }); + + it("keeps nothing when nobody is assigned", () => { + expect(reservedUntil({ startsAt }, "judge-a", during)).toBeNull(); + expect( + reservedUntil({ startsAt, assignedJudgeId: null }, "judge-a", during), + ).toBeNull(); + expect(reservedUntil(null, "judge-a", during)).toBeNull(); + }); +}); + +describe("evaluationTiming for an assigned presentation", () => { + const appointment = (assignedJudgeId: string | null) => ({ + id: "appt-1", + startsAt, + deadlineAt, + assignedJudgeId, + }); + const mine = scheduled({ appointment: appointment("judge-a") }); + const theirs = scheduled({ appointment: appointment("judge-b") }); + const takeover = takeoverAt(startsAt); + + it("is the assigned judge's from the start", () => { + expect(evaluationTiming(mine, startsAt)).toEqual({ + canEdit: true, + deadlineAt, + appointmentId: "appt-1", + }); + expect(evaluationTiming(mine, before)).toMatchObject({ + canEdit: false, + opensAt: startsAt, + }); + }); + + it("keeps another judge out until the takeover time, and says when it opens", () => { + for (const now of [before, startsAt, during]) { + const timing = evaluationTiming(theirs, now); + expect(timing).toMatchObject({ + canEdit: false, + opensAt: takeover, + ended: false, + }); + } + }); + + it("tells another judge why without saying who", () => { + const timing = evaluationTiming(theirs, during); + if (timing.canEdit) throw new Error("expected a refusal"); + expect(timing.reason).toBe( + "Another judge is assigned to this presentation. If they have not started 5 minutes in, you can take it.", + ); + expect(timing.reason).not.toContain("judge-b"); + }); + + it("lets another judge take it from the takeover time", () => { + expect(evaluationTiming(theirs, takeover)).toEqual({ + canEdit: true, + deadlineAt, + appointmentId: "appt-1", + }); + expect( + evaluationTiming(theirs, new Date(takeover.getTime() - 1)), + ).toMatchObject({ canEdit: false, opensAt: takeover }); + }); + + it("still needs the judge who takes over in a room serving the challenge", () => { + expect( + evaluationTiming({ ...theirs, presenceChallengeId: "mlh" }, takeover), + ).toMatchObject({ canEdit: false, opensAt: null, ended: false }); + }); + + it("names the reservation before the room: joining one does not help yet", () => { + expect( + evaluationTiming({ ...theirs, presenceChallengeId: null }, during), + ).toMatchObject({ canEdit: false, opensAt: takeover }); + }); + + it("is a first-come presentation when nobody is assigned", () => { + const open = scheduled({ appointment: appointment(null) }); + expect(evaluationTiming(open, startsAt)).toMatchObject({ canEdit: true }); + }); + + it("lets the end win over the reservation", () => { + const short = scheduled({ + appointment: { + id: "appt-1", + startsAt, + deadlineAt: new Date("2027-02-08T15:03:00Z"), + assignedJudgeId: "judge-b", + }, + }); + expect( + evaluationTiming(short, new Date("2027-02-08T15:04:00Z")), + ).toMatchObject({ canEdit: false, ended: true }); + }); + + it("leaves a judge's own evaluation theirs, whoever it was assigned to", () => { + expect( + evaluationTiming( + { ...theirs, evaluation: { appointmentId: "appt-1" } }, + during, + ), + ).toEqual({ canEdit: true, deadlineAt, appointmentId: "appt-1" }); + }); + + it("ignores an assignment where nothing is timed", () => { + for (const over of [ + { mode: "remote" as const }, + { mode: "unscheduled" as const }, + { hasSchedule: false }, + ]) { + expect(evaluationTiming({ ...theirs, ...over }, during)).toMatchObject({ + canEdit: true, + deadlineAt: null, + }); + } + }); +}); diff --git a/packages/api/src/services/judging/timing.ts b/packages/api/src/services/judging/timing.ts index ab529460..b36a1c75 100644 --- a/packages/api/src/services/judging/timing.ts +++ b/packages/api/src/services/judging/timing.ts @@ -9,13 +9,32 @@ import type { ChallengeMode } from "./scope"; // sitting in a room that serves the challenge, and the presentation has to // have started. Until the slot's deadline (setup plus judging minutes in) the // answers run against that deadline, and a draft still open when it passes is -// submitted for the judge. After the deadline the judge may still submit, -// untimed: a late evaluation is better than a missed one. +// submitted for the judge. The deadline is a hard end: after it the judge can +// neither submit a first evaluation nor edit one they already submitted. +// Writes get DEADLINE_GRACE_MS past it, so an answer sent in the last seconds +// is not refused for the time it spent on the network. // -// An evaluation the judge already submitted is always editable while judging -// is open; the clock only governs the first submission. +// Where nothing is timed, a submitted evaluation stays editable while judging +// is open. +// +// A presentation can have one judge assigned to it. From its start until +// TAKEOVER_AFTER_MS later only that judge may start grading it. After that any +// judge who may grade it there can, as long as the assigned judge has not +// started (their draft holds the project, see grading-hold.ts). The assignment +// is never rewritten to hand the project over. + +/** How long after a deadline a write sent before it still lands. */ +export const DEADLINE_GRACE_MS = 15_000; + +// The assigned judge may be walking over from another room, so the first +// minutes are theirs. After that a project is better graded by someone in the +// room than not at all. +/** How long after a presentation starts only its assigned judge may grade it. */ +export const TAKEOVER_AFTER_MS = 5 * 60_000; export type TimingInput = { + /** The judge asking. */ + judgeId: string; /** Whether the edition has a saved schedule. */ hasSchedule: boolean; /** The challenge's effective mode (its root's). */ @@ -25,15 +44,26 @@ export type TimingInput = { /** This judge's submitted evaluation of the project for the challenge. */ evaluation: { appointmentId: string | null } | null; /** The project's presentation for the challenge in the saved schedule. */ - appointment: { id: string; startsAt: Date; deadlineAt: Date } | null; + appointment: { + id: string; + startsAt: Date; + deadlineAt: Date; + /** The judge the presentation is assigned to; null or absent for none. */ + assignedJudgeId?: string | null; + } | null; /** The challenge served by the room the judge is sitting in, if any. */ presenceChallengeId: string | null; + /** DEADLINE_GRACE_MS for a write; reads leave it at 0. */ + graceMs?: number; }; export type EvaluationTiming = | { canEdit: true; - /** Set while a presentation is running: the draft is submitted then. */ + /** + * Set while a presentation has not ended: the draft is submitted then. + * A write in the grace window gets a deadline that has just passed. + */ deadlineAt: Date | null; appointmentId: string | null; } @@ -42,21 +72,61 @@ export type EvaluationTiming = reason: string; /** When the judge can start, if waiting is all it takes. */ opensAt: Date | null; + /** Shut because the presentation's deadline has passed. */ + ended: boolean; }; +const ENDED = + "Judging for this presentation ended at its deadline, so scores for it are final."; + +// Never names the judge. +const RESERVED = `Another judge is assigned to this presentation. If they have not started ${TAKEOVER_AFTER_MS / 60_000} minutes in, you can take it.`; + +/** When a presentation assigned to someone else opens to everyone else. */ +export function takeoverAt(startsAt: Date): Date { + return new Date(startsAt.getTime() + TAKEOVER_AFTER_MS); +} + +/** + * When a presentation opens to `judgeId`, if it is held back for another judge + * until then. Null when nobody is assigned, the judge is the one assigned, or + * the takeover time has passed. Whether the assigned judge has started is not + * decided here: a draft of theirs holds the project (grading-hold.ts). + */ +export function reservedUntil( + appointment: { startsAt: Date; assignedJudgeId?: string | null } | null, + judgeId: string, + now: Date, +): Date | null { + const assigned = appointment?.assignedJudgeId ?? null; + if (!appointment || !assigned || assigned === judgeId) return null; + const opens = takeoverAt(appointment.startsAt); + return now < opens ? opens : null; +} + export function evaluationTiming( input: TimingInput, now: Date, ): EvaluationTiming { + const timed = input.hasSchedule && input.mode === "scheduled"; + const deadline = timed ? (input.appointment?.deadlineAt ?? null) : null; + const ended = + deadline !== null && + now.getTime() >= deadline.getTime() + (input.graceMs ?? 0); + if (input.evaluation) { + if (ended) { + return { canEdit: false, reason: ENDED, opensAt: null, ended: true }; + } return { canEdit: true, - deadlineAt: null, + // Counts down the time left to edit; nothing is submitted for the judge. + deadlineAt: deadline, appointmentId: input.evaluation.appointmentId, }; } - if (!input.hasSchedule || input.mode !== "scheduled") { + if (!timed) { return { canEdit: true, deadlineAt: null, appointmentId: null }; } @@ -66,6 +136,22 @@ export function evaluationTiming( reason: "This project has no presentation slot for this challenge yet. Ask an organiser to place it on the schedule.", opensAt: null, + ended: false, + }; + } + + if (ended) { + return { canEdit: false, reason: ENDED, opensAt: null, ended: true }; + } + + // Before the room: nothing the judge does now helps until the takeover time. + const reserved = reservedUntil(input.appointment, input.judgeId, now); + if (reserved) { + return { + canEdit: false, + reason: RESERVED, + opensAt: reserved, + ended: false, }; } @@ -75,6 +161,7 @@ export function evaluationTiming( reason: "Join the room or table that judges this challenge to evaluate this project.", opensAt: null, + ended: false, }; } @@ -84,13 +171,16 @@ export function evaluationTiming( reason: "This project has not presented yet. You can judge it once its presentation starts.", opensAt: input.appointment.startsAt, + ended: false, }; } + // Not ended, so the deadline is still ahead or inside the write grace. A + // draft first saved in the grace window must still carry it, or nothing + // submits it for the judge. return { canEdit: true, - deadlineAt: - now < input.appointment.deadlineAt ? input.appointment.deadlineAt : null, + deadlineAt: input.appointment.deadlineAt, appointmentId: input.appointment.id, }; } diff --git a/packages/api/src/services/points.test.ts b/packages/api/src/services/points.test.ts index 8b175e73..3b7341fa 100644 --- a/packages/api/src/services/points.test.ts +++ b/packages/api/src/services/points.test.ts @@ -3,8 +3,11 @@ import { TRPCError } from "@trpc/server"; import { CHECK_IN_CODE_ALPHABET, CHECK_IN_CODE_LENGTH, + CHECK_IN_GAP_MS, alreadyCheckedInMessage, assignCheckInCode, + attendanceConflict, + attendanceConflictMessage, canSupply, fullName, generateCheckInCode, @@ -18,8 +21,9 @@ import { rankLeaderboard, recordEventAttendance, selfCheckInWindow, + CheckInRuleBreak, } from "./points"; -import type { PointsDb } from "./points"; +import type { EarlierAttendance, PointsDb } from "./points"; import type { DrizzleDB } from "@query/db"; // Drizzle wraps the pg error, so the SQLSTATE sits on `.cause`. @@ -341,27 +345,296 @@ describe("selfCheckInWindow", () => { }); }); +const MIN = 60 * 1000; + +describe("attendanceConflict", () => { + const T0 = new Date("2026-10-10T18:00:00Z").getTime(); + const at = (minutes: number) => new Date(T0 + minutes * MIN); + + const ev = ( + id: string, + startMin: number, + endMin: number, + type = "workshop", + ) => ({ + id, + type, + startTime: at(startMin), + endTime: at(endMin), + }); + + // An earlier check-in. Defaults to long ago, so only the overlap rule can + // fire unless a test says otherwise. + const attended = ( + id: string, + startMin: number, + endMin: number, + opts: { type?: string; checkedInMin?: number; name?: string } = {}, + ): EarlierAttendance => ({ + event: { + ...ev(id, startMin, endMin, opts.type), + name: opts.name ?? `Event ${id}`, + }, + checkedInAt: at(opts.checkedInMin ?? -600), + }); + + it("allows a first check-in", () => { + expect(attendanceConflict(ev("a", 0, 60), [], at(0))).toBeNull(); + }); + + it("refuses an event that runs at the same time as one already attended", () => { + const bny = attended("bny", 0, 60, { name: "BNY Workshop" }); + expect(attendanceConflict(ev("fs", 30, 90), [bny], at(30))).toEqual({ + kind: "overlap", + other: { id: "bny", name: "BNY Workshop" }, + }); + // Contained and containing count too. + expect(attendanceConflict(ev("fs", 10, 20), [bny], at(30))?.kind).toBe( + "overlap", + ); + expect(attendanceConflict(ev("fs", -30, 120), [bny], at(30))?.kind).toBe( + "overlap", + ); + }); + + it("lets back-to-back events through", () => { + const first = attended("a", 0, 60); + expect(attendanceConflict(ev("b", 60, 120), [first], at(60))).toBeNull(); + const second = attended("c", 60, 120); + expect(attendanceConflict(ev("d", 0, 60), [second], at(30))).toBeNull(); + }); + + it("refuses a second check-in 29 minutes after the first and allows it at 30", () => { + const first = attended("a", 0, 60, { checkedInMin: 0, name: "Workshop" }); + // Back-to-back events, so only the gap is in play. + const next = ev("b", 60, 120); + expect(attendanceConflict(next, [first], at(29))).toEqual({ + kind: "too_soon", + other: { id: "a", name: "Workshop" }, + minutesAgo: 29, + }); + expect(attendanceConflict(next, [first], at(30))).toBeNull(); + expect(CHECK_IN_GAP_MS).toBe(30 * MIN); + }); + + it("reports whole minutes, zero for a check-in just now", () => { + const first = attended("a", 0, 60, { checkedInMin: 0 }); + const next = ev("b", 60, 120); + expect( + attendanceConflict(next, [first], new Date(T0 + 59 * 1000))?.minutesAgo, + ).toBe(0); + expect( + attendanceConflict(next, [first], new Date(T0 + 61 * 1000))?.minutesAgo, + ).toBe(1); + }); + + it("does not report a negative wait when the clock is behind the database", () => { + const first = attended("a", 0, 60, { checkedInMin: 1 }); + expect( + attendanceConflict(ev("b", 60, 120), [first], at(0))?.minutesAgo, + ).toBe(0); + }); + + it("names the most recent check-in when several are too recent", () => { + const older = attended("a", 0, 10, { checkedInMin: 0, name: "Older" }); + const newer = attended("b", 20, 30, { checkedInMin: 10, name: "Newer" }); + const expected = { + kind: "too_soon", + other: { id: "b", name: "Newer" }, + minutesAgo: 10, + }; + expect(attendanceConflict(ev("c", 60, 90), [newer, older], at(20))).toEqual( + expected, + ); + expect(attendanceConflict(ev("c", 60, 90), [older, newer], at(20))).toEqual( + expected, + ); + }); + + it("does not block a meal, and a meal does not block anything", () => { + const workshop = attended("w", 0, 60, { checkedInMin: 0 }); + expect( + attendanceConflict(ev("lunch", 10, 50, "meal"), [workshop], at(10)), + ).toBeNull(); + + const lunch = attended("lunch", 0, 60, { type: "meal", checkedInMin: 0 }); + expect(attendanceConflict(ev("w", 10, 50), [lunch], at(10))).toBeNull(); + }); + + it("ignores the same event, which the unique index handles", () => { + const same = attended("a", 0, 60, { checkedInMin: 0 }); + expect(attendanceConflict(ev("a", 0, 60), [same], at(5))).toBeNull(); + // A check-in elsewhere since then does not turn a repeat into "too soon": + // the true answer is still "already checked in". + const since = attended("b", 60, 120, { checkedInMin: 50 }); + expect( + attendanceConflict(ev("a", 0, 60), [same, since], at(55)), + ).toBeNull(); + }); + + it("reports an overlap before a recent check-in", () => { + const overlapping = attended("a", 0, 60, { + checkedInMin: 10, + name: "Overlaps", + }); + const recent = attended("b", -120, -90, { + checkedInMin: 14, + name: "Recent", + }); + expect( + attendanceConflict(ev("c", 30, 90), [recent, overlapping], at(15)), + ).toEqual({ kind: "overlap", other: { id: "a", name: "Overlaps" } }); + }); +}); + +describe("attendanceConflictMessage", () => { + const input = { + event: { name: "Full Sail Fireside" }, + participant: { name: "Ada Lovelace" }, + }; + const overlap = { + kind: "overlap", + other: { id: "bny", name: "BNY Workshop" }, + } as const; + const soon = (minutesAgo: number) => + ({ + kind: "too_soon", + other: { id: "bny", name: "BNY Workshop" }, + minutesAgo, + }) as const; + + it("speaks to the hacker who entered the code about an overlap", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, overlap), + ).toBe( + "You're already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. You can check into one event at a time.", + ); + }); + + it("tells staff to remove the participant from the other event", () => { + for (const method of ["scan", "manual"] as const) { + expect(attendanceConflictMessage({ ...input, method }, overlap)).toBe( + "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", + ); + } + }); + + it("tells the hacker how long to wait", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(12)), + ).toBe( + "You checked into BNY Workshop 12 minutes ago. Event check-ins have to be at least 30 minutes apart, so try again in 18 minutes.", + ); + }); + + it("tells staff the same without a countdown", () => { + for (const method of ["scan", "manual"] as const) { + expect(attendanceConflictMessage({ ...input, method }, soon(12))).toBe( + "Ada Lovelace checked into BNY Workshop 12 minutes ago. Event check-ins have to be at least 30 minutes apart. If that was a mistake, remove them from BNY Workshop first.", + ); + } + }); + + it("uses singular wording at one minute", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(1)), + ).toBe( + "You checked into BNY Workshop 1 minute ago. Event check-ins have to be at least 30 minutes apart, so try again in 29 minutes.", + ); + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(29)), + ).toContain("so try again in 1 minute."); + }); + + it("says less than a minute for zero", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(0)), + ).toBe( + "You checked into BNY Workshop less than a minute ago. Event check-ins have to be at least 30 minutes apart, so try again in 30 minutes.", + ); + expect( + attendanceConflictMessage({ ...input, method: "manual" }, soon(0)), + ).toContain("checked into BNY Workshop less than a minute ago."); + }); +}); + describe("recordEventAttendance", () => { - const fakeDb = (fail?: () => Error) => { + // Stands in for the pool: a transaction hands back a builder that answers the + // participant lock, the attendance read and the insert, noting the order. + const fakeDb = ( + opts: { fail?: () => Error; earlier?: EarlierAttendance[] } = {}, + ) => { const inserted: unknown[] = []; - const db = { + const calls: string[] = []; + const tx = { + select: () => ({ + from: () => ({ + where: () => ({ + for: async (mode: string) => { + calls.push(`lock:${mode}`); + return [{ id: "part-1" }]; + }, + }), + innerJoin: () => ({ + where: async () => { + calls.push("read"); + return opts.earlier ?? []; + }, + }), + }), + }), insert: () => ({ values: async (values: unknown) => { - if (fail) throw fail(); + calls.push("insert"); + if (opts.fail) throw opts.fail(); inserted.push(values); }, }), }; - return { db: db as unknown as PointsDb, inserted }; + const db = { + transaction: async (run: (tx: unknown) => unknown) => run(tx), + }; + return { db: db as unknown as PointsDb, inserted, calls }; }; + // A meal, so the rules stay out of the way unless a test switches to the + // workshop below. const input = { - event: { id: "event-1", name: "Lunch", points: 15 }, + event: { + id: "event-1", + name: "Lunch", + points: 15, + type: "meal", + startTime: new Date(Date.now() - 30 * MIN), + endTime: new Date(Date.now() + 30 * MIN), + }, participant: { id: "part-1", name: "Ada Lovelace" }, method: "manual" as const, actorId: "staff-1", }; + const workshop = { + ...input.event, + id: "event-2", + name: "Workshop", + type: "workshop", + }; + + const earlierRow = ( + event: Partial = {}, + checkedInAt = new Date(Date.now() - 300 * MIN), + ): EarlierAttendance => ({ + event: { + id: "event-9", + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 30 * MIN), + endTime: new Date(Date.now() + 30 * MIN), + ...event, + }, + checkedInAt, + }); + it("copies the event's points and records how and by whom", async () => { const { db, inserted } = fakeDb(); await expect(recordEventAttendance(db, input)).resolves.toEqual({ @@ -374,12 +647,81 @@ describe("recordEventAttendance", () => { pointsAwarded: 15, method: "manual", checkedInById: "staff-1", + checkedInAt: expect.any(Date), }, ]); }); + it("locks the participant before it reads their attendance", async () => { + const { db, calls } = fakeDb(); + await recordEventAttendance(db, input); + expect(calls).toEqual(["lock:no key update", "read", "insert"]); + }); + + it("refuses a check-in into an event that overlaps one already attended", async () => { + const { db, inserted } = fakeDb({ earlier: [earlierRow()] }); + const error = await recordEventAttendance(db, { + ...input, + event: workshop, + method: "code", + }).catch((e: unknown) => e); + expect(error).toBeInstanceOf(TRPCError); + expect((error as TRPCError).code).toBe("FORBIDDEN"); + expect((error as TRPCError).message).toBe( + "You're already checked into BNY Workshop, which runs at the same time as Workshop. You can check into one event at a time.", + ); + // What a caller needs to record the attempt once this has rolled back. + const cause = (error as TRPCError).cause; + expect(cause).toBeInstanceOf(CheckInRuleBreak); + expect(cause).toMatchObject({ + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-9", + rule: "overlap", + method: "code", + actorId: "staff-1", + at: expect.any(Date), + }); + expect(inserted).toEqual([]); + }); + + it("refuses a check-in under 30 minutes after the last one", async () => { + const { db, inserted } = fakeDb({ + earlier: [ + earlierRow( + { + startTime: new Date(Date.now() - 300 * MIN), + endTime: new Date(Date.now() - 240 * MIN), + }, + new Date(Date.now() - 10 * MIN), + ), + ], + }); + const error = await recordEventAttendance(db, { + ...input, + event: workshop, + }).catch((e: unknown) => e); + expect((error as TRPCError).code).toBe("FORBIDDEN"); + expect((error as TRPCError).message).toMatch( + /^Ada Lovelace checked into BNY Workshop 10 minutes ago\./, + ); + expect((error as TRPCError).cause).toMatchObject({ + eventId: "event-2", + conflictEventId: "event-9", + rule: "too_soon", + method: "manual", + }); + expect(inserted).toEqual([]); + }); + + it("lets a meal through alongside an overlapping event", async () => { + const { db, inserted } = fakeDb({ earlier: [earlierRow()] }); + await recordEventAttendance(db, input); + expect(inserted).toHaveLength(1); + }); + it("reads a duplicate as already checked in", async () => { - const { db } = fakeDb(uniqueViolation); + const { db } = fakeDb({ fail: uniqueViolation }); const error = await recordEventAttendance(db, input).catch( (e: unknown) => e, ); @@ -390,15 +732,52 @@ describe("recordEventAttendance", () => { ); }); + it("reads a repeat as already checked in after a later check-in elsewhere", async () => { + const { db } = fakeDb({ + fail: uniqueViolation, + earlier: [ + earlierRow( + { id: workshop.id, name: workshop.name }, + new Date(Date.now() - 40 * MIN), + ), + // Starts as the workshop ends, checked into five minutes ago. + earlierRow( + { + startTime: workshop.endTime, + endTime: new Date(Date.now() + 90 * MIN), + }, + new Date(Date.now() - 5 * MIN), + ), + ], + }); + const error = await recordEventAttendance(db, { + ...input, + event: workshop, + method: "code", + }).catch((e: unknown) => e); + expect((error as TRPCError).code).toBe("CONFLICT"); + expect((error as TRPCError).message).toBe( + "You are already checked into Workshop.", + ); + }); + it("speaks to the hacker when they entered the code", async () => { - const { db } = fakeDb(uniqueViolation); + const { db } = fakeDb({ fail: uniqueViolation }); await expect( recordEventAttendance(db, { ...input, method: "code" }), ).rejects.toThrow("You are already checked into Lunch."); }); + it("leaves a duplicate without a rule-break cause", async () => { + const { db } = fakeDb({ fail: uniqueViolation }); + const error = await recordEventAttendance(db, input).catch( + (e: unknown) => e, + ); + expect((error as TRPCError).cause).not.toBeInstanceOf(CheckInRuleBreak); + }); + it("passes other failures through", async () => { - const { db } = fakeDb(() => new Error("connection reset")); + const { db } = fakeDb({ fail: () => new Error("connection reset") }); await expect(recordEventAttendance(db, input)).rejects.toThrow( "connection reset", ); diff --git a/packages/api/src/services/points.ts b/packages/api/src/services/points.ts index a2ad9dd4..2004613a 100644 --- a/packages/api/src/services/points.ts +++ b/packages/api/src/services/points.ts @@ -240,6 +240,126 @@ export function selfCheckInWindow( // --------------------------------------------------------------------------- // Attendance +// Two events at once, or two check-ins in quick succession, mean somebody is +// collecting points for events they are not at. Meals are the exception: people +// eat alongside whatever else is on, and a meal scan only counts food. +export const CHECK_IN_GAP_MS = 30 * 60 * 1000; + +type AttendedEvent = { + id: string; + type: string; + startTime: Date; + endTime: Date; +}; + +/** One of the participant's existing check-ins, with the event it was for. */ +export type EarlierAttendance = { + event: AttendedEvent & { name: string }; + checkedInAt: Date; +}; + +export type AttendanceConflict = { + /** overlap: the events run at the same time. too_soon: under the gap. */ + kind: "overlap" | "too_soon"; + /** The earlier check-in that stands in the way. */ + other: { id: string; name: string }; + /** Whole minutes since the earlier check-in. Only on too_soon. */ + minutesAgo?: number; +}; + +/** + * Why `event` cannot be checked into now, given the participant's earlier + * check-ins, or null when it can. Overlap is reported before too_soon because + * it is the firmer answer: waiting does not fix it. Pure, so the rule is + * testable without a database. + */ +export function attendanceConflict( + event: AttendedEvent, + earlier: readonly EarlierAttendance[], + now: Date, +): AttendanceConflict | null { + if (event.type === "meal") return null; + + // Already in this event: the duplicate case, which the unique index answers + // with "already checked in". Checking the others too would answer a repeat + // code with "try again in 20 minutes" after a later check-in elsewhere. + if (earlier.some((row) => row.event.id === event.id)) return null; + + const others = earlier.filter((row) => row.event.type !== "meal"); + + // Strict, so an event that ends at 2:00 does not clash with one that starts + // at 2:00. + const overlapping = others.find( + (row) => + row.event.startTime < event.endTime && + event.startTime < row.event.endTime, + ); + if (overlapping) { + return { + kind: "overlap", + other: { id: overlapping.event.id, name: overlapping.event.name }, + }; + } + + let latest: EarlierAttendance | null = null; + for (const row of others) { + if (now.getTime() - row.checkedInAt.getTime() >= CHECK_IN_GAP_MS) continue; + if (!latest || row.checkedInAt > latest.checkedInAt) latest = row; + } + if (!latest) return null; + + return { + kind: "too_soon", + other: { id: latest.event.id, name: latest.event.name }, + // A clock a little behind the database can put the check-in in the future. + minutesAgo: Math.max( + 0, + Math.floor((now.getTime() - latest.checkedInAt.getTime()) / 60_000), + ), + }; +} + +export type CheckInRuleBreakDetails = { + participantId: string; + /** The event they tried to enter. */ + eventId: string; + /** The event they were already in. */ + conflictEventId: string; + rule: AttendanceConflict["kind"]; + method: AttendanceMethod; + /** The hacker on a code, staff otherwise. */ + actorId: string | null; + /** The clock the rule was judged on. */ + at: Date; +}; + +/** + * Rides on the `cause` of the refusal recordEventAttendance throws, so a caller + * outside the transaction can record the attempt after the refusal has rolled + * back. It never reaches a client: tRPC does not serialise `cause`. + */ +export class CheckInRuleBreak extends Error implements CheckInRuleBreakDetails { + readonly participantId: string; + readonly eventId: string; + readonly conflictEventId: string; + readonly rule: AttendanceConflict["kind"]; + readonly method: AttendanceMethod; + readonly actorId: string | null; + readonly at: Date; + + constructor(details: CheckInRuleBreakDetails) { + super(`Check-in rule broken: ${details.rule}`); + this.name = "CheckInRuleBreak"; + this.participantId = details.participantId; + this.eventId = details.eventId; + this.conflictEventId = details.conflictEventId; + this.rule = details.rule; + this.method = details.method; + this.actorId = details.actorId; + this.at = details.at; + } +} + /** * The one writer of attendance rows: pass scans, event codes and manual * check-in all come through here, so every row carries the event's points at @@ -248,25 +368,91 @@ export function selfCheckInWindow( * attend. The unique index on (event, participant) is the duplicate guard; * two stations racing both pass any earlier read, and the loser gets the same * sentence a single duplicate would. + * + * It also holds the bad-actor rules (see attendanceConflict): nobody is in two + * events that run at the same time, or in two within CHECK_IN_GAP_MS of each + * other. Those are read-then-write, so the participant's row is locked first: + * two check-ins for one person at the same moment (two tabs, two stations) + * queue up, and the second one sees the first. A refusal is FORBIDDEN, not + * CONFLICT: the self check-in page reads CONFLICT as "already checked in". + * The refusal rolls this transaction back, so it carries a CheckInRuleBreak as + * its cause for the caller to record (see check-in-flags.ts). */ export async function recordEventAttendance( db: PointsDb, input: { - event: { id: string; name: string; points: number }; + event: { + id: string; + name: string; + points: number; + type: string; + startTime: Date; + endTime: Date; + }; participant: { id: string; name: string }; method: AttendanceMethod; actorId: string | null; }, ): Promise<{ pointsAwarded: number }> { const { event, participant, method, actorId } = input; + // One clock for the rule and the row it writes. Left to the database's + // now(), checked_in_at (no time zone) follows the session's TimeZone and + // would not compare with this one. + const now = new Date(); try { - await db.insert(hackathonEventAttendees).values({ - eventId: event.id, - participantId: participant.id, - pointsAwarded: event.points, - method, - checkedInById: actorId, + await db.transaction(async (tx) => { + // No key update: enough to queue this person's check-ins behind each + // other without holding up other writes that reference the row. + await tx + .select({ id: hackathonParticipants.id }) + .from(hackathonParticipants) + .where(eq(hackathonParticipants.id, participant.id)) + .for("no key update"); + + const earlier = await tx + .select({ + event: { + id: hackathonEvents.id, + name: hackathonEvents.name, + type: hackathonEvents.type, + startTime: hackathonEvents.startTime, + endTime: hackathonEvents.endTime, + }, + checkedInAt: hackathonEventAttendees.checkedInAt, + }) + .from(hackathonEventAttendees) + .innerJoin( + hackathonEvents, + eq(hackathonEvents.id, hackathonEventAttendees.eventId), + ) + .where(eq(hackathonEventAttendees.participantId, participant.id)); + + const conflict = attendanceConflict(event, earlier, now); + if (conflict) { + throw new TRPCError({ + code: "FORBIDDEN", + message: attendanceConflictMessage(input, conflict), + cause: new CheckInRuleBreak({ + participantId: participant.id, + eventId: event.id, + conflictEventId: conflict.other.id, + rule: conflict.kind, + method, + actorId, + at: now, + }), + }); + } + + await tx.insert(hackathonEventAttendees).values({ + eventId: event.id, + participantId: participant.id, + pointsAwarded: event.points, + method, + checkedInById: actorId, + checkedInAt: now, + }); }); } catch (error) { if (isUniqueViolation(error)) { @@ -293,6 +479,39 @@ export function alreadyCheckedInMessage(input: { : `${input.participant.name} is already checked into ${input.event.name}.`; } +const minutesLabel = (n: number) => `${n} minute${n === 1 ? "" : "s"}`; + +/** The refusal sentence for an attendanceConflict, worded like the one above. */ +export function attendanceConflictMessage( + input: { + event: { name: string }; + participant: { name: string }; + method: AttendanceMethod; + }, + conflict: AttendanceConflict, +): string { + const { event, participant, method } = input; + const other = conflict.other.name; + const self = method === "code"; + + if (conflict.kind === "overlap") { + return self + ? `You're already checked into ${other}, which runs at the same time as ${event.name}. You can check into one event at a time.` + : `${participant.name} is already checked into ${other}, which runs at the same time as ${event.name}. If that was a mistake, remove them from ${other} first.`; + } + + const ago = conflict.minutesAgo ?? 0; + const when = + ago === 0 ? "less than a minute ago" : `${minutesLabel(ago)} ago`; + const rule = `Event check-ins have to be at least ${CHECK_IN_GAP_MS / 60_000} minutes apart`; + if (!self) { + return `${participant.name} checked into ${other} ${when}. ${rule}. If that was a mistake, remove them from ${other} first.`; + } + // minutesAgo is floored, so what is left of the gap rounds up on its own. + const left = Math.max(1, CHECK_IN_GAP_MS / 60_000 - ago); + return `You checked into ${other} ${when}. ${rule}, so try again in ${minutesLabel(left)}.`; +} + // --------------------------------------------------------------------------- // Balances diff --git a/packages/db/src/schemas/hackathons.ts b/packages/db/src/schemas/hackathons.ts index 56d0fc95..c5fc4603 100644 --- a/packages/db/src/schemas/hackathons.ts +++ b/packages/db/src/schemas/hackathons.ts @@ -504,6 +504,77 @@ export const hackathonEventAttendeesRelations = relations( }), ); +// A check-in the rules refused (two events at the same time, or two under 30 +// minutes apart). The refusal itself rolls back, so this is the only trace: +// organisers read it as a flag on the hacker's team in judging and decide +// whether the team stays in the running for prizes. Never shown to hackers. +// One row per (person, event tried, event already in) with a count, so a hacker +// retrying a rejected code adds an attempt rather than a screen of duplicates. +// It points at the participant, not the project, because a person can change +// teams or have their project promoted later; judging resolves it to an entry +// when it is read. +export const hackathonCheckInFlags = pgTable( + "hackathon_check_in_flag", + { + id: uuid("id").defaultRandom().primaryKey(), + hackathonId: uuid("hackathon_id") + .notNull() + .references(() => hackathons.id, { onDelete: "cascade" }), + // The participant and event FKs are declared in the table extras with + // explicit names. The generated participant one is 66 characters, past + // Postgres's 63-char limit, and the conflict event one lands on exactly 63; + // the event one is named to match. + participantId: uuid("participant_id").notNull(), + // The event they tried to enter. Cleared if the event is deleted: the flag + // is the only record of the rule break, and prize review still needs it + // after a session is cancelled. Always set when the flag is written. + eventId: uuid("event_id"), + // The event they were already in. Cleared the same way. + conflictEventId: uuid("conflict_event_id"), + rule: text("rule", { enum: ["overlap", "too_soon"] }).notNull(), + method: text("method", { enum: ["scan", "code", "manual"] }).notNull(), + // The hacker on a code, staff otherwise. + attemptedById: text("attempted_by_id").references(() => users.id, { + onDelete: "set null", + }), + attempts: integer("attempts").notNull().default(1), + // Written from the app clock, like checked_in_at: the rule compares against + // it, and the database's now() follows the session's TimeZone. + createdAt: timestamp("created_at").defaultNow().notNull(), + lastAttemptAt: timestamp("last_attempt_at").defaultNow().notNull(), + // An organiser reviewed the flag and cleared it. It stays on record. + dismissedAt: timestamp("dismissed_at"), + dismissedById: text("dismissed_by_id").references(() => users.id, { + onDelete: "set null", + }), + }, + (table) => [ + // Nulls are distinct in a unique index, so clearing a deleted event's id + // never makes two flags collide here. + uniqueIndex("check_in_flag_attempt_idx").on( + table.participantId, + table.eventId, + table.conflictEventId, + ), + index("check_in_flag_hackathon_idx").on(table.hackathonId), + foreignKey({ + name: "check_in_flag_participant_fk", + columns: [table.participantId], + foreignColumns: [hackathonParticipants.id], + }).onDelete("cascade"), + foreignKey({ + name: "check_in_flag_event_fk", + columns: [table.eventId], + foreignColumns: [hackathonEvents.id], + }).onDelete("set null"), + foreignKey({ + name: "check_in_flag_conflict_event_fk", + columns: [table.conflictEventId], + foreignColumns: [hackathonEvents.id], + }).onDelete("set null"), + ], +); + export const hackathonProjectsRelations = relations( hackathonProjects, ({ one }) => ({ diff --git a/packages/db/src/schemas/judging.ts b/packages/db/src/schemas/judging.ts index e35141ac..0ab2b1a6 100644 --- a/packages/db/src/schemas/judging.ts +++ b/packages/db/src/schemas/judging.ts @@ -90,6 +90,14 @@ export const judgingEntries = pgTable( deletedById: text("deleted_by_id").references(() => users.id, { onDelete: "set null", }), + // An organiser pulled this entry from prizes, usually after a check-in rule + // break. Judges still score it, so nothing about the room schedule changes; + // only placements and results leave it out. + prizeIneligibleAt: timestamp("prize_ineligible_at"), + prizeIneligibleById: text("prize_ineligible_by_id").references( + () => users.id, + { onDelete: "set null" }, + ), createdAt: timestamp("created_at").defaultNow().notNull(), updatedAt: timestamp("updated_at").defaultNow().notNull(), }, @@ -455,6 +463,12 @@ export const judgingAppointments = pgTable( endsAt: timestamp("ends_at").notNull(), // Compare-and-swap for moves made from two organiser screens at once. revision: integer("revision").notNull().default(0), + // The one judge who grades this presentation. Null until judges are + // assigned, and again if that judge is removed. + assignedJudgeId: uuid("assigned_judge_id").references( + () => judgingJudges.id, + { onDelete: "set null" }, + ), }, (table) => [ uniqueIndex("judging_appointment_task_idx").on( @@ -468,6 +482,7 @@ export const judgingAppointments = pgTable( table.startsAt, ), index("judging_appointment_entry_idx").on(table.entryId), + index("judging_appointment_judge_idx").on(table.assignedJudgeId), foreignKey({ name: "judging_appointment_challenge_fk", columns: [table.challengeId], @@ -758,7 +773,11 @@ export const judgingPlacements = pgTable( scope: text("scope").notNull().default("overall"), scopeLabel: text("scope_label").notNull().default("Overall"), placement: integer("placement").notNull(), + // Calibrated for judge bias: what the placing is ranked on. score: numeric("score", { precision: 4, scale: 2 }), + // The plain mean of the same evaluations. Null on rows computed before + // calibration existed. + rawScore: numeric("raw_score", { precision: 4, scale: 2 }), evaluationCount: integer("evaluation_count").notNull().default(0), publishedAt: timestamp("published_at"), computedAt: timestamp("computed_at").defaultNow().notNull(), diff --git a/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx b/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx index 1c699f39..588bc39f 100644 --- a/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx +++ b/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx @@ -112,6 +112,11 @@ export default function AdminEditionJudgingPage() { {stationCount(data.counts.rooms.active, data.counts.rooms.tables)} {plural(data.counts.judgesSeated, "judge")} seated + {data.counts.checkInFlags.open > 0 && ( + + {plural(data.counts.checkInFlags.open, "open check-in flag")} + + )} {data.config.emergencyMode && ( Emergency mode on )} diff --git a/sites/mainweb/app/(portal)/submit/page.tsx b/sites/mainweb/app/(portal)/submit/page.tsx index cbc0130b..8b6a7353 100644 --- a/sites/mainweb/app/(portal)/submit/page.tsx +++ b/sites/mainweb/app/(portal)/submit/page.tsx @@ -59,6 +59,9 @@ function SubmitPortalContent() { const [selectedHackathonId, setSelectedHackathonId] = useState( urlHackathonId || "", ); + // The id can come straight from the URL or the picker, so it is encoded + // wherever it becomes part of a link path. + const hackathonPath = `/hackathons/${encodeURIComponent(selectedHackathonId)}`; const [confirmWithdraw, setConfirmWithdraw] = useState(false); const [projectName, setProjectName] = useState(""); @@ -353,7 +356,7 @@ function SubmitPortalContent() { )} {currentReg && (currentReg.team || admitted) && ( {currentReg.team ? "Manage team" : "Create or join a team"} @@ -372,7 +375,7 @@ function SubmitPortalContent() { Your project is in judging. Your team presents to judges for each challenge it entered, and the{" "} Judging tab @@ -659,7 +662,7 @@ function SubmitPortalContent() { submit points there. */} {projectSubmitted ? ( Go to Judging diff --git a/sites/mainweb/components/admin/judging/ChecklistSection.tsx b/sites/mainweb/components/admin/judging/ChecklistSection.tsx index 47cda90f..727942d0 100644 --- a/sites/mainweb/components/admin/judging/ChecklistSection.tsx +++ b/sites/mainweb/components/admin/judging/ChecklistSection.tsx @@ -91,7 +91,7 @@ export function ChecklistSection({ { label: "Evaluations", value: counts.evaluations.complete, - detail: `complete of ${counts.evaluations.total} · ${plural(counts.evaluations.drafts, "draft")}`, + detail: `counted of ${counts.evaluations.total} · ${plural(counts.evaluations.drafts, "draft")}`, }, ]; diff --git a/sites/mainweb/components/admin/judging/DangerZone.test.tsx b/sites/mainweb/components/admin/judging/DangerZone.test.tsx new file mode 100644 index 00000000..f0214659 --- /dev/null +++ b/sites/mainweb/components/admin/judging/DangerZone.test.tsx @@ -0,0 +1,120 @@ +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DangerZone } from "./DangerZone"; +import type { Overview } from "./shared"; + +// Rendered to markup with the tRPC hooks stubbed, as in EntriesList.test.tsx: +// a mutation's onSuccess is read off the options its hook was given and called +// directly. + +Object.assign(globalThis, { React }); + +type MutationOptions = { onSuccess?: (result: unknown) => void }; + +const mocks = vi.hoisted(() => ({ + invalidateJudging: vi.fn(), + invalidateResults: vi.fn(), + options: {} as Record, +})); + +vi.mock("@/lib/use-portal-context", () => ({ + READ_ONLY_TITLE: "Read-only", + useReadOnly: () => false, +})); + +vi.mock("@/lib/trpc", () => { + const mutation = (name: string) => (options: MutationOptions) => { + mocks.options[name] = options; + return { + isPending: false, + error: null, + mutate: () => undefined, + reset: () => undefined, + }; + }; + return { + trpc: { + useUtils: () => ({ + judging: { invalidate: mocks.invalidateJudging }, + hackathon: { getResults: { invalidate: mocks.invalidateResults } }, + }), + judging: { + admin: Object.fromEntries( + [ + "dropEvaluations", + "resetLaunch", + "dropRooms", + "resetProjects", + "resetSetup", + "resetAll", + ].map((name) => [name, { useMutation: mutation(name) }]), + ), + }, + }, + }; +}); + +const overview = { + hackathon: { name: "Spring Build" }, + schedule: null, + counts: { + evaluations: { total: 0, drafts: 0 }, + rooms: { active: 0, archived: 0 }, + }, +} as unknown as Overview; + +// Every count a summary line reads, so each onSuccess can build its text. +const result = { + evaluations: 0, + drafts: 0, + placements: 0, + claimLinks: 0, + unclaimedMembers: 0, + rooms: 0, + announcements: 0, + entries: 0, + challenges: 0, + groups: 0, + rubricItems: 0, + judges: 0, + scheduleDropped: false, +}; + +const succeed = (name: string) => { + renderToStaticMarkup( + , + ); + mocks.options[name]?.onSuccess?.(result); +}; + +beforeEach(() => { + mocks.invalidateJudging.mockClear(); + mocks.invalidateResults.mockClear(); + mocks.options = {}; +}); + +describe("DangerZone public results", () => { + it.each(["dropEvaluations", "resetProjects", "resetAll"])( + "refetches the public results after %s", + (name) => { + succeed(name); + + expect(mocks.invalidateJudging).toHaveBeenCalledTimes(1); + expect(mocks.invalidateResults).toHaveBeenCalledTimes(1); + }, + ); + + it.each(["resetLaunch", "dropRooms", "resetSetup"])( + "leaves the public results alone after %s", + (name) => { + succeed(name); + + expect(mocks.invalidateJudging).toHaveBeenCalledTimes(1); + expect(mocks.invalidateResults).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/sites/mainweb/components/admin/judging/DangerZone.tsx b/sites/mainweb/components/admin/judging/DangerZone.tsx index dee2f7bc..85d1fd1e 100644 --- a/sites/mainweb/components/admin/judging/DangerZone.tsx +++ b/sites/mainweb/components/admin/judging/DangerZone.tsx @@ -28,6 +28,12 @@ type ResetId = | "resetSetup" | "resetAll"; +const CLEARS_RESULTS = new Set([ + "dropEvaluations", + "resetProjects", + "resetAll", +]); + /** The way back, each reset behind the edition's name typed out. */ export function DangerZone({ hackathonId, @@ -38,6 +44,7 @@ export function DangerZone({ }) { const readOnly = useReadOnly(); const refresh = useRefreshJudging(); + const utils = trpc.useUtils(); const [open, setOpen] = useState(null); const [typed, setTyped] = useState(""); const [summary, setSummary] = useState<{ id: ResetId; text: string } | null>( @@ -49,6 +56,9 @@ export function DangerZone({ setTyped(""); setSummary({ id, text }); void refresh(); + // These clear the public results too, and the portal keeps that query for + // minutes. + if (CLEARS_RESULTS.has(id)) void utils.hackathon.getResults.invalidate(); }; const mutations = { diff --git a/sites/mainweb/components/admin/judging/EntriesList.test.tsx b/sites/mainweb/components/admin/judging/EntriesList.test.tsx new file mode 100644 index 00000000..7cdd189d --- /dev/null +++ b/sites/mainweb/components/admin/judging/EntriesList.test.tsx @@ -0,0 +1,121 @@ +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { EntriesList } from "./EntriesList"; + +// Rendered to markup with the tRPC hooks stubbed: the web app has no DOM test +// environment, so a mutation's onSuccess is read off the options each hook was +// given and called directly, rather than through a click. + +// The root vitest config compiles JSX to React.createElement, and components +// written for Next's automatic runtime do not import React. +Object.assign(globalThis, { React }); + +type MutationOptions = { onSuccess?: () => void }; + +const mocks = vi.hoisted(() => ({ + invalidateJudging: vi.fn(), + invalidateResults: vi.fn(), + options: {} as Record, + entry: { + id: "entry-1", + source: "portal", + title: "Tidewater", + submissionUrl: null, + deletedAt: null, + prizeIneligible: false, + appointments: 0, + completeEvaluations: 0, + members: [], + challenges: [], + checkInFlags: [], + }, +})); + +vi.mock("@/lib/use-portal-context", () => ({ + READ_ONLY_TITLE: "Read-only", + useReadOnly: () => false, +})); + +vi.mock("@/lib/trpc", () => { + const mutation = (name: string) => (options: MutationOptions) => { + mocks.options[name] = options; + return { + isPending: false, + variables: undefined, + error: null, + mutate: () => undefined, + }; + }; + return { + trpc: { + useUtils: () => ({ + judging: { invalidate: mocks.invalidateJudging }, + hackathon: { getResults: { invalidate: mocks.invalidateResults } }, + }), + judging: { + admin: { + listEntries: { + useQuery: () => ({ + data: { + entries: [mocks.entry], + total: 1, + page: 1, + pageSize: 50, + }, + isLoading: false, + isError: false, + isPlaceholderData: false, + error: null, + refetch: () => undefined, + }), + }, + listChallenges: { useQuery: () => ({ data: { challenges: [] } }) }, + setEntryChallenges: { useMutation: mutation("setEntryChallenges") }, + removeEntry: { useMutation: mutation("removeEntry") }, + restoreEntry: { useMutation: mutation("restoreEntry") }, + dismissCheckInFlag: { useMutation: mutation("dismissCheckInFlag") }, + setPrizeEligibility: { useMutation: mutation("setPrizeEligibility") }, + }, + }, + }, + }; +}); + +const succeed = (name: string) => { + renderToStaticMarkup( + , + ); + mocks.options[name]?.onSuccess?.(); +}; + +beforeEach(() => { + mocks.invalidateJudging.mockClear(); + mocks.invalidateResults.mockClear(); + mocks.options = {}; +}); + +describe("EntriesList public results", () => { + // Each of these changes what hackathon.getResults returns, and the portal + // keeps that query fresh for minutes, so the public copy has to be dropped + // along with the judging screens (as the Results tab does for eligibility). + it.each(["setPrizeEligibility", "removeEntry", "restoreEntry"])( + "refetches the public results after %s", + (name) => { + succeed(name); + + expect(mocks.invalidateJudging).toHaveBeenCalledTimes(1); + expect(mocks.invalidateResults).toHaveBeenCalledTimes(1); + }, + ); + + it.each(["setEntryChallenges", "dismissCheckInFlag"])( + "leaves the public results alone after %s", + (name) => { + succeed(name); + + expect(mocks.invalidateJudging).toHaveBeenCalledTimes(1); + expect(mocks.invalidateResults).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/sites/mainweb/components/admin/judging/EntriesList.tsx b/sites/mainweb/components/admin/judging/EntriesList.tsx index b37c2da0..6d4630ef 100644 --- a/sites/mainweb/components/admin/judging/EntriesList.tsx +++ b/sites/mainweb/components/admin/judging/EntriesList.tsx @@ -23,12 +23,14 @@ import { import { ErrorLine, plural, + shortTime, useRefreshJudging, } from "@/components/admin/judging/shared"; import type { Challenge } from "@/components/admin/judging/shared"; type Entry = RouterOutputs["judging"]["admin"]["listEntries"]["entries"][number]; +type CheckInFlag = Entry["checkInFlags"][number]; const PAGE_SIZE = 50; @@ -44,6 +46,30 @@ const STATUSES = [ { id: "all", label: "Both" }, ] as const; +const FLAG_METHOD = { + code: "event code", + scan: "badge scan", + manual: "manual check-in", +} as const; + +/** + * One refused check-in as an organiser reads it. The data does not keep how + * many minutes apart the two check-ins were, so the too-soon rule says what + * the limit is. + */ +function flagLine(flag: CheckInFlag) { + const gap = flag.rule === "overlap" ? "same time" : "under 30 minutes apart"; + return [ + `${flag.participantName} tried ${flag.eventName} while checked into ${flag.conflictEventName} (${gap})`, + FLAG_METHOD[flag.method], + flag.attempts > 1 ? plural(flag.attempts, "try", "tries") : null, + shortTime(flag.lastAttemptAt), + flag.dismissed ? "dismissed" : null, + ] + .filter(Boolean) + .join(" · "); +} + /** The projects in judging, with their challenges and remove/restore. */ export function EntriesList({ hackathonId }: { hackathonId: string }) { const [search, setSearch] = useState(""); @@ -51,6 +77,7 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { const [source, setSource] = useState<(typeof SOURCES)[number]["id"]>("all"); const [state, setState] = useState<(typeof STATUSES)[number]["id"]>("live"); const [challengeId, setChallengeId] = useState(""); + const [flagged, setFlagged] = useState(false); const [page, setPage] = useState(1); useEffect(() => { @@ -65,6 +92,7 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { source, status: state, challengeId: challengeId || undefined, + flagged: flagged || undefined, page, pageSize: PAGE_SIZE, }, @@ -77,7 +105,8 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { const tree = (challenges.data?.challenges ?? []).filter((c) => !c.parentId); const data = entries.data; const pages = data ? Math.max(1, Math.ceil(data.total / data.pageSize)) : 1; - const filtered = !!query || source !== "all" || state !== "live" || !!challengeId; + const filtered = + !!query || source !== "all" || state !== "live" || !!challengeId || flagged; return (
@@ -172,6 +201,21 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { {option.label} ))} +
@@ -203,6 +247,7 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { {data.entries.map((entry) => ( @@ -242,14 +287,17 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { } function EntryRow({ + hackathonId, entry, challenges, }: { + hackathonId: string; entry: Entry; challenges: Challenge[]; }) { const readOnly = useReadOnly(); const refresh = useRefreshJudging(); + const utils = trpc.useUtils(); const [editing, setEditing] = useState(false); const [chosen, setChosen] = useState>(new Set()); const [error, setError] = useState(null); @@ -258,6 +306,13 @@ function EntryRow({ setError(null); void refresh(); }; + // Removing or restoring a project and changing its prize eligibility all + // change the public results. The portal keeps that query for minutes, so + // drop its copy too, as the Results tab does for eligibility. + const doneResults = () => { + done(); + void utils.hackathon.getResults.invalidate(); + }; const setChallenges = trpc.judging.admin.setEntryChallenges.useMutation({ onSuccess: () => { setEditing(false); @@ -266,19 +321,34 @@ function EntryRow({ onError: (e) => setError(trpcErrorMessage(e, "Could not save the challenges.")), }); const remove = trpc.judging.admin.removeEntry.useMutation({ - onSuccess: done, + onSuccess: doneResults, onError: (e) => setError(trpcErrorMessage(e, "Could not remove the project.")), }); const restore = trpc.judging.admin.restoreEntry.useMutation({ - onSuccess: done, + onSuccess: doneResults, onError: (e) => setError(trpcErrorMessage(e, "Could not restore the project.")), }); + const dismiss = trpc.judging.admin.dismissCheckInFlag.useMutation({ + onSuccess: done, + onError: (e) => setError(trpcErrorMessage(e, "Could not update the flag.")), + }); + const setEligible = trpc.judging.admin.setPrizeEligibility.useMutation({ + onSuccess: doneResults, + onError: (e) => + setError(trpcErrorMessage(e, "Could not change prize eligibility.")), + }); const optIns = entry.challenges.filter((c) => c.isOptIn); const derived = entry.challenges.filter((c) => !c.isOptIn); const linked = entry.members.filter((m) => m.linked).length; const removed = !!entry.deletedAt; - const busy = setChallenges.isPending || remove.isPending || restore.isPending; + const openFlags = entry.checkInFlags.filter((flag) => !flag.dismissed).length; + const busy = + setChallenges.isPending || + remove.isPending || + restore.isPending || + dismiss.isPending || + setEligible.isPending; const openEditor = () => { setChosen(new Set(optIns.map((c) => c.challengeId))); @@ -293,6 +363,14 @@ function EntryRow({

{entry.source === "devpost" ? "Devpost" : "Portal"} {removed && Removed} + {openFlags > 0 && ( + + {plural(openFlags, "check-in flag")} + + )} + {entry.prizeIneligible && ( + Not eligible for prizes + )}

{entry.title} @@ -327,9 +405,41 @@ function EntryRow({ {(entry.appointments > 0 || entry.completeEvaluations > 0) && (

{plural(entry.appointments, "presentation")} scheduled ·{" "} - {plural(entry.completeEvaluations, "evaluation")} complete + {plural(entry.completeEvaluations, "evaluation")} counted

)} + {entry.checkInFlags.length > 0 && ( +
    + {entry.checkInFlags.map((flag) => ( +
  • + {flagLine(flag)} + +
    • + ))} +
+ )}
{!removed && ( @@ -344,6 +454,47 @@ function EntryRow({ Challenges )} + {entry.prizeIneligible ? ( + + ) : ( + !removed && ( + + ) + )} {removed ? ( +
+ )} {score(row.score)} {" "} + {row.rawScore !== null && ( + + raw {row.rawScore.toFixed(2)}{" "} + + )} · {row.evaluationCount}{" "} {row.evaluationCount === 1 ? "eval" : "evals"} @@ -336,9 +473,42 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { )} + {calibration.length > 0 && ( +
+

Judge calibration

+

+ Each project is graded once, so a strict or generous judge moves + every project they graded. Scores above are adjusted for that, + comparing judges only within their own challenge. Judges and hackers + see none of this. +

+
+ {calibration.map((challenge) => ( + + setOpenCalibration((previous) => { + const next = new Set(previous); + if (next.has(challenge.challengeId)) { + next.delete(challenge.challengeId); + } else { + next.add(challenge.challengeId); + } + return next; + }) + } + /> + ))} +
+
+ )} +
setEntryFilter(null)} /> @@ -347,15 +517,65 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { ); } +function CalibrationChallengeRow({ + challenge, + open, + onToggle, +}: { + challenge: CalibrationChallenge; + open: boolean; + onToggle: () => void; +}) { + return ( +
+ + {open && ( +
    + {challenge.judges.map((judge) => ( +
  • +

    + {judge.judgeName} + {judge.kind === "guest" && ( + (guest) + )} +

    +

    {describeJudge(judge).join(" · ")}

    +

    {judgeNumbers(judge)}

    +
    • + ))} +
+ )} +
+ ); +} + function EvaluationAudit({ hackathonId, + published, entryFilter, onClearFilter, }: { hackathonId: string; + published: boolean; entryFilter: { id: string; title: string } | null; onClearFilter: () => void; }) { + const readOnly = useReadOnly(); + const refreshJudging = useRefreshJudging(); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); const [shown, setShown] = useState(AUDIT_PAGE); const [expanded, setExpanded] = useState<{ id: string; @@ -367,6 +587,24 @@ function EvaluationAudit({ { refetchInterval: POLL_MS, placeholderData: (previous) => previous }, ); + // Deleting the grade changes the project list and the judges' queues as + // well as this one, so refresh all of judging. + const reopen = trpc.judging.admin.reopenGrade.useMutation({ + onSuccess: (result) => { + setError(null); + setNotice( + result.placings > 0 + ? "The grade is deleted and another judge can grade the project. The draft placings were ranked with it, so they are cleared. Compute results again once the project is graded." + : "The grade is deleted and another judge can grade the project. Compute results again once it is graded.", + ); + void refreshJudging(); + }, + onError: (e) => { + setNotice(null); + setError(trpcErrorMessage(e, "That did not go through. Try again.")); + }, + }); + const toggle = (id: string, view: "appointment" | "history") => setExpanded((current) => current?.id === id && current.view === view ? null : { id, view }, @@ -394,6 +632,17 @@ function EvaluationAudit({

)} + {error && ( +
+ {error} +
+ )} + {notice && ( +
+ {notice} +
+ )} + {audit.isLoading ? (

Loading evaluations…

) : audit.isError ? ( @@ -424,6 +673,28 @@ function EvaluationAudit({ row={row} expanded={expanded?.id === row.id ? expanded.view : null} onToggle={(view) => toggle(row.id, view)} + reopenDisabled={readOnly || published || reopen.isPending} + reopenTitle={ + readOnly + ? READ_ONLY_TITLE + : published + ? "Unpublish first to reopen a grade." + : undefined + } + reopening={ + reopen.isPending && reopen.variables?.evaluationId === row.id + } + onReopen={() => { + if ( + !window.confirm( + `Reopen ${row.title} for grading? ${row.judgeName}'s grade is deleted, with its ratings, responses and history, and another judge can grade the project. The draft placings are cleared until you compute results again. This cannot be undone.`, + ) + ) + return; + setError(null); + setNotice(null); + reopen.mutate({ hackathonId, evaluationId: row.id }); + }} /> ))} @@ -447,12 +718,24 @@ function AuditItem({ row, expanded, onToggle, + reopenDisabled, + reopenTitle, + reopening, + onReopen, }: { hackathonId: string; row: AuditRow; expanded: "appointment" | "history" | null; onToggle: (view: "appointment" | "history") => void; + reopenDisabled: boolean; + reopenTitle: string | undefined; + reopening: boolean; + onReopen: () => void; }) { + const badge = gradeBadge( + row.isComplete ? "complete" : "incomplete", + row.counted, + ); return (
  • @@ -472,9 +755,7 @@ function AuditItem({ {score(row.score)} - - {row.isComplete ? "Complete" : "Incomplete"} - + {badge.text} {row.appointmentId && ( +
    {expanded === "appointment" && row.appointmentId && ( @@ -531,7 +821,7 @@ function AppointmentDetail({ ); } - const { appointment, average, completeCount, hasResultsElsewhere, judges } = + const { appointment, average, countedCount, hasResultsElsewhere, judges } = detail.data; const where = [appointment.building, appointment.room] .filter(Boolean) @@ -548,13 +838,13 @@ function AppointmentDetail({ {score(average)} {" "} - from {completeCount} complete{" "} - {completeCount === 1 ? "evaluation" : "evaluations"}. + from {countedCount} counted{" "} + {countedCount === 1 ? "evaluation" : "evaluations"}.

    {hasResultsElsewhere && (

    - This project also has complete evaluations for this challenge outside - this slot. + This project also has evaluations for this challenge outside this + slot.

    )} {judges.length === 0 ? ( @@ -572,18 +862,10 @@ function AppointmentDetail({

    - {judge.status === "complete" - ? "Complete" - : judge.status === "incomplete" - ? "Incomplete" - : "Still writing"} + {gradeBadge(judge.status, judge.counted).text} {judge.score !== null && ( diff --git a/sites/mainweb/components/admin/judging/results/calibration.test.ts b/sites/mainweb/components/admin/judging/results/calibration.test.ts new file mode 100644 index 00000000..0e921cc0 --- /dev/null +++ b/sites/mainweb/components/admin/judging/results/calibration.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { describeJudge, judgeNumbers, signed } from "./calibration"; + +const judge = (over: Partial[0]> = {}) => ({ + grades: 12, + offset: 0, + spread: 1, + drift: 0, + ...over, +}); + +describe("signed", () => { + it("shows the direction of a number", () => { + expect(signed(0.384)).toBe("+0.38"); + expect(signed(-0.451)).toBe("-0.45"); + expect(signed(0)).toBe("0.00"); + }); +}); + +describe("describeJudge", () => { + it("calls a judge in line when nothing stands out", () => { + expect(describeJudge(judge())).toEqual(["In line with the other judges"]); + expect( + describeJudge(judge({ offset: 0.1, spread: 0.9, drift: -0.15 })), + ).toEqual(["In line with the other judges"]); + }); + + it("says which way a judge's marks sit and by how much", () => { + expect(describeJudge(judge({ offset: -0.4 }))).toEqual([ + "Stricter than average by 0.40", + ]); + expect(describeJudge(judge({ offset: 0.55 }))).toEqual([ + "More lenient than average by 0.55", + ]); + }); + + it("says when scores are bunched or spread wide", () => { + expect(describeJudge(judge({ spread: 0.6 }))).toEqual([ + "Scores bunched together", + ]); + expect(describeJudge(judge({ spread: 1.4 }))).toEqual([ + "Scores spread wider than average", + ]); + }); + + it("says when a judge changed over the day", () => { + expect(describeJudge(judge({ drift: -0.4 }))).toEqual([ + "Got harsher through the day", + ]); + expect(describeJudge(judge({ drift: 0.3 }))).toEqual([ + "Got more lenient through the day", + ]); + }); + + it("does not read a habit into what chance alone gives a judge", () => { + expect( + describeJudge(judge({ grades: 30, offset: -0.15, drift: 0.15 })), + ).toEqual(["In line with the other judges"]); + }); + + it("lists everything that applies", () => { + expect( + describeJudge(judge({ offset: -0.3, spread: 0.7, drift: -0.25 })), + ).toEqual([ + "Stricter than average by 0.30", + "Scores bunched together", + "Got harsher through the day", + ]); + }); + + it("does not read much into a judge with few grades", () => { + expect(describeJudge(judge({ grades: 2, offset: -0.4 }))).toEqual([ + "Too few grades to adjust much", + ]); + expect(describeJudge(judge({ grades: 1 }))).toEqual([ + "Too few grades to adjust much", + ]); + }); +}); + +describe("judgeNumbers", () => { + it("prints the grades and the three numbers", () => { + expect( + judgeNumbers(judge({ offset: -0.38, spread: 0.72, drift: -0.45 })), + ).toBe("12 grades · offset -0.38 · spread 0.72 · drift -0.45"); + }); + + it("is singular for one grade", () => { + expect(judgeNumbers(judge({ grades: 1 }))).toBe( + "1 grade · offset 0.00 · spread 1.00 · drift 0.00", + ); + }); +}); diff --git a/sites/mainweb/components/admin/judging/results/calibration.ts b/sites/mainweb/components/admin/judging/results/calibration.ts new file mode 100644 index 00000000..97c00739 --- /dev/null +++ b/sites/mainweb/components/admin/judging/results/calibration.ts @@ -0,0 +1,60 @@ +import type { RouterOutputs } from "@query/api"; + +export type CalibrationChallenge = + RouterOutputs["judging"]["results"]["list"]["calibration"][number]; +export type CalibratedJudge = CalibrationChallenge["judges"][number]; + +type Numbers = Pick; + +// How far a number has to sit from "no correction" before it is worth saying +// out loud. Offset and drift are in rating points; spread is a multiple of the +// pool's, so 1 is the same. Chance alone moves them for a judge with no habit +// at all (which projects they drew, and in what order), by about 0.1 for the +// offset over thirty grades and under 0.1 for the drift (the API shrinks it to +// that, and fits none below twenty grades), so these sit about twice that +// above zero. +const OFFSET_NOTICED = 0.2; +const SPREAD_NOTICED = 0.15; +const DRIFT_NOTICED = 0.2; +// With fewer grades than this the correction is mostly shrunk away. +const FEW_GRADES = 3; + +/** +0.38, -0.45, 0.00: the sign shown so a direction reads at a glance. */ +export function signed(value: number) { + const text = Math.abs(value).toFixed(2); + return value > 0 ? `+${text}` : value < 0 ? `-${text}` : text; +} + +/** What a judge's numbers say about how they mark, in plain words. */ +export function describeJudge(judge: Numbers): string[] { + if (judge.grades < FEW_GRADES) return ["Too few grades to adjust much"]; + + const phrases: string[] = []; + const size = Math.abs(judge.offset).toFixed(2); + if (judge.offset <= -OFFSET_NOTICED) { + phrases.push(`Stricter than average by ${size}`); + } else if (judge.offset >= OFFSET_NOTICED) { + phrases.push(`More lenient than average by ${size}`); + } + if (judge.spread <= 1 - SPREAD_NOTICED) { + phrases.push("Scores bunched together"); + } else if (judge.spread >= 1 + SPREAD_NOTICED) { + phrases.push("Scores spread wider than average"); + } + if (judge.drift <= -DRIFT_NOTICED) { + phrases.push("Got harsher through the day"); + } else if (judge.drift >= DRIFT_NOTICED) { + phrases.push("Got more lenient through the day"); + } + return phrases.length > 0 ? phrases : ["In line with the other judges"]; +} + +/** The numbers behind the words: 12 grades · offset -0.38 · spread 0.72 · drift -0.45 */ +export function judgeNumbers(judge: Numbers) { + return [ + `${judge.grades} ${judge.grades === 1 ? "grade" : "grades"}`, + `offset ${signed(judge.offset)}`, + `spread ${judge.spread.toFixed(2)}`, + `drift ${signed(judge.drift)}`, + ].join(" · "); +} diff --git a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx index b5016dda..92009679 100644 --- a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx +++ b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx @@ -120,6 +120,7 @@ function solution(overrides: Record = {}) { reducedBreaks: [], candidateKey: "a".repeat(64), solverStatus: "optimal", + judging: { judges: 3, presentations: 2, unassigned: 0, source: "cp-sat" }, solution: "{}", ...overrides, }; @@ -226,6 +227,25 @@ describe("CpsatPanel", () => { expect(mixed).toContain("Rooms or tables"); }); + it("says who assigned the judges, or that there are none yet", () => { + mocks.solved = solution(); + expect(render()).toContain("CP-SAT gave 2 of 2 presentations a judge, from 3 judges."); + + mocks.reported = false; + mocks.solved = solution({ + judging: { judges: 1, presentations: 2, unassigned: 1, source: "in-app" }, + }); + expect(render()).toContain( + "The in-app assigner gave 1 of 2 presentations a judge, from 1 judge.", + ); + + mocks.reported = false; + mocks.solved = solution({ + judging: { judges: 0, presentations: 2, unassigned: 2, source: "in-app" }, + }); + expect(render()).toContain("No judge has joined yet."); + }); + it("says when the time limit ended the search", () => { mocks.solved = solution({ solverStatus: "feasible" }); const html = render(); diff --git a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx index bc80df91..c69fb376 100644 --- a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx +++ b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx @@ -16,7 +16,7 @@ import { sectionRule, status, } from "@/components/portal/ui"; -import { ErrorLine } from "@/components/admin/judging/shared"; +import { ErrorLine, plural } from "@/components/admin/judging/shared"; import type { CpsatPreview, ScheduleBoard, Timing } from "./format"; import { PreviewFigures, @@ -264,7 +264,8 @@ export function CpsatPanel({

    CP-SAT is searching. It can take up to {seconds} seconds of solving, or twice that if no schedule keeps the full break between buildings, - plus up to a minute of setup on large events. + and as long again to pick judges once some have joined, plus up to a + minute of setup on large events.

    ) : check.isPending ? (

    @@ -294,6 +295,19 @@ export function CpsatPanel({ ); } +/** What a CP-SAT schedule says about its judges. */ +function judgesLine({ + judges, + presentations, + unassigned, + source, +}: CpsatPreview["judging"]) { + if (judges === 0) { + return "No judge has joined yet. Assign judges from the schedule once they have."; + } + return `${source === "cp-sat" ? "CP-SAT" : "The in-app assigner"} gave ${presentations - unassigned} of ${plural(presentations, "presentation")} a judge, from ${plural(judges, "judge")}.`; +} + /** One CP-SAT schedule, shown like the preview, and saving it. */ function CpsatResult({ hackathonId, @@ -342,6 +356,8 @@ function CpsatResult({ +

    {judgesLine(result.judging)}

    + {result.relaxed && ( void; +}) { + const utils = trpc.useUtils(); + const readOnly = useReadOnly(); + const [picked, setPicked] = useState(null); + + const query = trpc.judging.schedule.judgeChoices.useQuery( + { hackathonId, appointmentId }, + { retry: false }, + ); + const refresh = () => utils.judging.schedule.get.invalidate({ hackathonId }); + const save = trpc.judging.schedule.setAssignedJudge.useMutation({ + onSuccess: async () => { + await refresh(); + onClose(); + }, + // Someone else took the judge, or the presentation started: both the + // board and the choices are out of date. + onError: () => { + void refresh(); + void query.refetch(); + }, + }); + + const data = query.data; + const current = data?.appointment.assignedJudgeId ?? ""; + const selected = picked ?? current; + const choices = data?.choices ?? []; + + return ( + { + if (!save.isPending) onClose(); + }} + maxWidth="lg" + label="Choose a judge" + > +
    +
    +

    Choose a judge

    +

    + {data?.appointment.entryTitle ?? "Appointment"} +

    + {data && ( +

    + {[ + data.appointment.challengeLabel, + data.appointment.roomName, + slotTime(data.appointment.startsAt, windowStart), + ].join(" · ")} +

    + )} +
    + + {query.isLoading ? ( +

    + Finding judges… +

    + ) : query.error ? ( + + {trpcErrorMessage(query.error, "Could not load the judges.")} + + ) : ( +
    + + + {data && !data.appointment.canAssignJudge ? ( +

    + This presentation has started, or a judge has begun on it, so + its judge stays. +

    + ) : choices.length === 0 ? ( +

    + No judge can grade this yet. A guest grades only at their own + room or table, and a table judge appears once they have joined. +

    + ) : null} +
    + )} + + {save.error && ( + + {trpcErrorMessage(save.error, "Could not set the judge.")} + + )} + +
    + + +
    +
    +     + ); +} diff --git a/sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx b/sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx new file mode 100644 index 00000000..c4b2bc30 --- /dev/null +++ b/sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx @@ -0,0 +1,280 @@ +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { JudgeDialog } from "./JudgeDialog"; +import { JudgesPanel } from "./JudgesPanel"; + +// Rendered to markup with the tRPC hooks stubbed, as CpsatPanel.test.tsx does: +// these check what each state shows and disables, not clicks. +Object.assign(globalThis, { React }); + +const mocks = vi.hoisted(() => ({ + readOnly: false, + pending: false, + assigned: null as Record | null, + choices: null as Record | null, +})); + +vi.mock("@/lib/use-portal-context", () => ({ + READ_ONLY_TITLE: "Read-only", + useReadOnly: () => mocks.readOnly, +})); + +vi.mock("@/lib/trpc", () => { + const idle = () => ({ + isPending: false, + error: null, + mutate: () => undefined, + reset: () => undefined, + }); + return { + trpc: { + useUtils: () => ({}), + judging: { + schedule: { + assignJudges: { + // Static rendering cannot click, so a finished run is simply + // there as the mutation's data. + useMutation: () => ({ + ...idle(), + data: mocks.assigned ?? undefined, + isPending: mocks.pending, + }), + }, + setAssignedJudge: { useMutation: idle }, + judgeChoices: { + useQuery: () => ({ + isLoading: mocks.choices === null, + error: null, + data: mocks.choices ?? undefined, + refetch: () => undefined, + }), + }, + }, + }, + }, + }; +}); + +const windowStart = new Date(2026, 10, 1, 10, 0); + +const judging = (overrides: Record = {}) => ({ + judges: 3, + assigned: 5, + unassigned: 0, + loads: [ + { + judgeId: "m1", + name: "Ada", + kind: "member", + roomName: null, + available: true, + load: 3, + }, + { + judgeId: "gs", + name: "Grace", + kind: "guest", + roomName: "Acme 1", + available: true, + load: 2, + }, + { + judgeId: "old", + name: "Linus", + kind: "guest", + roomName: null, + available: false, + load: 0, + }, + ], + warnings: [], + ...overrides, +}); + +function panel(over: Record = {}) { + return renderToStaticMarkup( + , + ); +} + +const button = (html: string, text: string) => + html.match(new RegExp(`]*>${text}`))?.[0] ?? ""; +const DISABLED = ' disabled=""'; + +beforeEach(() => { + mocks.readOnly = false; + mocks.pending = false; + mocks.assigned = null; + mocks.choices = null; +}); + +describe("JudgesPanel", () => { + it("states the rule and what is assigned, with a button to assign again", () => { + const html = panel(); + expect(html).toContain("graded once, by one judge"); + expect(html).toContain( + "5 presentations with a judge. Every upcoming one has its judge.", + ); + expect(button(html, "Assign judges")).not.toContain(DISABLED); + }); + + it("counts the presentations still waiting for a judge", () => { + const html = panel({ unassigned: 2 }); + expect(html).toContain( + "5 presentations with a judge, 2 still waiting for one.", + ); + }); + + it("lists each judge with their load, and says who has no room or table", () => { + const html = panel(); + expect(html).toContain("Presentations per judge"); + expect(html).toContain("Ada"); + expect(html).toContain("Table judge"); + expect(html).toContain("Guest · Acme 1"); + expect(html).toContain("Linus"); + expect(html).toContain("has no room or table now"); + }); + + it("warns of the slots where judges run short, with their times", () => { + const html = panel({ + warnings: [ + { + slot: 9, + startsAt: new Date(2026, 10, 1, 11, 30), + challengeId: "general", + challengeLabel: "General", + presentations: 2, + judgesFree: 1, + message: "Slot 10: 2 presentations in General, only 1 judge free", + }, + ], + }); + expect(html).toContain("1 slot has too few judges"); + expect(html).toContain( + "Slot 10: 2 presentations in General, only 1 judge free, at 11:30", + ); + }); + + it("cannot assign before any judge has joined", () => { + const html = panel({ judges: 0, assigned: 0, unassigned: 4, loads: [] }); + expect(html).toContain("No judge has joined a room or table yet"); + expect(button(html, "Assign judges")).toContain(DISABLED); + }); + + it("is read-only for a read-only organiser, and busy while it works", () => { + mocks.readOnly = true; + expect(button(panel(), "Assign judges")).toContain(DISABLED); + + mocks.readOnly = false; + mocks.pending = true; + expect(button(panel(), "Assigning…")).toContain(DISABLED); + }); + + it("reports what an assignment changed and who worked it out", () => { + mocks.assigned = { changed: 4, solver: "cp-sat" }; + expect(panel()).toContain( + "4 presentations given a judge, worked out by CP-SAT.", + ); + + mocks.assigned = { changed: 0, solver: "in-app" }; + expect(panel()).toContain("Nothing changed"); + }); +}); + +const dialog = () => + renderToStaticMarkup( + undefined} + />, + ); + +const choices = (over: Record = {}) => ({ + appointment: { + id: "appointment-1", + entryTitle: "Pathfinder", + challengeLabel: "General", + roomName: "Table G1", + startsAt: new Date(2026, 10, 1, 11, 30), + assignedJudgeId: "m1", + canAssignJudge: true, + }, + choices: [ + { + judgeId: "m2", + name: "Bea", + kind: "member", + load: 2, + bookedWith: null, + booked: false, + }, + { + judgeId: "m1", + name: "Ada", + kind: "member", + load: 3, + bookedWith: null, + booked: false, + }, + { + judgeId: "gs", + name: "Grace", + kind: "guest", + load: 1, + bookedWith: "Pathfinder", + booked: true, + }, + ], + ...over, +}); + +describe("JudgeDialog", () => { + it("waits for the judges", () => { + expect(dialog()).toContain("Finding judges…"); + }); + + it("offers only the judges the server says may grade it, and no judge", () => { + mocks.choices = choices(); + const html = dialog(); + expect(html).toContain("Pathfinder"); + expect(html).toContain("General · Table G1 · 11:30"); + expect(html).toContain(''); + expect(html).toContain("Bea · 2 presentations"); + expect(html).toContain( + "Grace (guest) · 1 presentation · grading Pathfinder then", + ); + }); + + it("starts on the current judge, and cannot save without a change", () => { + mocks.choices = choices(); + const html = dialog(); + expect(html).toMatch(/