From fd6537947771a475e1b3bfe1479eccdaa7d88f01 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 11:11:31 -0400 Subject: [PATCH 01/27] fix(points): refuse event-code check-in for banned hackers points.me.checkIn writes attendance and awards points, so it now sits behind notHackathonBanned like the other participation writes. The summary, leaderboard and store reads stay open. Adds a test that a banned hacker's check-in is refused before any participant lookup or insert. --- .../hackathon-admin-edge.test.ts | 19 +++++++++++++++++++ packages/api/src/middleware/procedures.ts | 2 +- packages/api/src/routers/points/me.ts | 8 ++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts index 3ab02749..905e20ef 100644 --- a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts +++ b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts @@ -546,6 +546,25 @@ describe("Hackathon admin management edge cases", () => { ).rejects.toThrow(refused); }); + // A banned hacker can keep a registration, so the code check-in is a way + // of taking part that has to stop at the gate, before any lookup. + it("refuses a banned hacker's event code check-in", async () => { + const caller = participant(true); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: expect.stringMatching(/can't take part in Hacklytics/), + }); + + expect(mockFindFirst).not.toHaveBeenCalledWith( + "hackathonParticipants", + expect.anything(), + ); + expect(mockInsert).not.toHaveBeenCalled(); + }); + it("lets everyone else through the ban gate", async () => { const caller = participant(false); const result = await caller.hackathon diff --git a/packages/api/src/middleware/procedures.ts b/packages/api/src/middleware/procedures.ts index 67392057..db6ce2f7 100644 --- a/packages/api/src/middleware/procedures.ts +++ b/packages/api/src/middleware/procedures.ts @@ -195,7 +195,7 @@ export const hackathonBanCacheKey = (email: string) => // Refuses people banned from hackathons. Goes on the procedures where someone // takes part — registering, the interest list, teams, submitting, applying to -// judge — and not on reads or on the ways out (withdraw, leave), so a banned +// judge, event-code check-in — and not on reads or on the ways out (withdraw, leave), so a banned // person can still see and undo what they already did. The ban is by email, // so it holds across providers and new accounts on the same address. Cached // 60s, the negative answer too; ban and unban clear the key. diff --git a/packages/api/src/routers/points/me.ts b/packages/api/src/routers/points/me.ts index 1a688067..1fb62242 100644 --- a/packages/api/src/routers/points/me.ts +++ b/packages/api/src/routers/points/me.ts @@ -9,6 +9,7 @@ import { } from "@query/db"; import type { DrizzleDB } from "@query/db"; import { createTRPCRouter, protectedProcedure } from "../../trpc"; +import { notHackathonBanned } from "../../middleware/procedures"; import { rateLimit } from "../../middleware/security"; import { isSoldOut, @@ -213,8 +214,11 @@ export const pointsMeRouter = createTRPCRouter({ }), // Self check-in with the code shown at the event. Every refusal says what - // to do instead, since the hacker is standing in a doorway reading it. - checkIn: protectedProcedure + // to do instead, since the hacker is standing in a doorway reading it. Behind + // the ban gate because it writes: a banned hacker who kept their + // registration could otherwise keep collecting points. The reads above stay + // open, as with the rest of the API. + checkIn: notHackathonBanned .input(scope.extend({ code: z.string().max(32) })) .mutation(async ({ ctx, input }) => { // There are 31^6 codes; this keeps anyone from walking the space. Per From a746a6877969318aa0729778985466100a863431 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 11:11:33 -0400 Subject: [PATCH 02/27] fix(announcements): stop counting in-app posts as emailed listAnnouncements counted every stamped sentAt as sent, but an in-app post is written with sentAt already set. Only sendBatch claims rows, so sent now counts rows that were claimed and sent, pending is counted directly in SQL, and a new inAppOnly flag marks posts that went to event pages without email. --- .../src/.internal-tests/announcements.test.ts | 38 +++++++++++++++++++ .../api/src/routers/hackathon/announce.ts | 17 +++++---- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/packages/api/src/.internal-tests/announcements.test.ts b/packages/api/src/.internal-tests/announcements.test.ts index 0f2b2d0f..39df9907 100644 --- a/packages/api/src/.internal-tests/announcements.test.ts +++ b/packages/api/src/.internal-tests/announcements.test.ts @@ -148,6 +148,7 @@ vi.mock("@query/db", () => { email: "email", sentAt: "sent_at", failedAt: "failed_at", + claimedAt: "claimed_at", }, }; }); @@ -513,6 +514,43 @@ describe("Announcements", () => { expect(res.inAppOnly).toBe(false); expect(recipientRowsOf()[0]).not.toHaveProperty("sentAt"); }); + + /** + * The counts are SQL aggregates, so the rows here are canned as the query + * would return them. What this pins is the router's side: an all-stamped, + * never-claimed announcement is reported as in-app with nothing pending, + * and `pending` is taken from the query rather than derived from + * total - sent - failed, which would read 2 here and put it under + * "Unfinished sends". + */ + it("lists an in-app announcement as published, not as an unfinished send", async () => { + asAdmin(); + mockSelectRows.mockReturnValue([ + { + id: ANNOUNCEMENT, + subject: "Registration is open", + audience: "interested", + createdAt: new Date("2026-02-01T12:00:00Z"), + total: 2, + sent: 0, + failed: 0, + pending: 0, + inAppOnly: true, + }, + ]); + + const [first] = await callerFor(ADMIN).hackathon.listAnnouncements({ + hackathonId: HACK, + }); + + expect(first).toMatchObject({ + total: 2, + sent: 0, + failed: 0, + pending: 0, + inAppOnly: true, + }); + }); }); /** diff --git a/packages/api/src/routers/hackathon/announce.ts b/packages/api/src/routers/hackathon/announce.ts index a837e3bf..a37feafa 100644 --- a/packages/api/src/routers/hackathon/announce.ts +++ b/packages/api/src/routers/hackathon/announce.ts @@ -171,20 +171,28 @@ export const hackathonAnnounceRouter = createTRPCRouter({ // Announcements for this edition and how far each got, so a reopened tab can // pick an unfinished send back up instead of starting a duplicate. + // + // `sentAt` alone does not mean "emailed": an in-app post is born with it + // stamped. Only sendBatch sets `claimedAt`, so `sent` counts rows with both, + // and `inAppOnly` is an announcement whose every row is stamped but never + // claimed. `pending` is counted directly, not total - sent - failed, which + // would leave in-app rows looking like an unfinished send. listAnnouncements: isAdmin .input(z.object({ hackathonId: z.string().uuid() })) .query(async ({ ctx, input }) => { const db = ctx.db as DrizzleDB; - const rows = await db + return await db .select({ id: hackathonAnnouncements.id, subject: hackathonAnnouncements.subject, audience: hackathonAnnouncements.audience, createdAt: hackathonAnnouncements.createdAt, total: sql`count(${hackathonAnnouncementRecipients.id})::int`, - sent: sql`count(${hackathonAnnouncementRecipients.sentAt})::int`, + sent: sql`count(${hackathonAnnouncementRecipients.id}) filter (where ${hackathonAnnouncementRecipients.sentAt} is not null and ${hackathonAnnouncementRecipients.claimedAt} is not null)::int`, failed: sql`count(${hackathonAnnouncementRecipients.failedAt})::int`, + pending: sql`count(${hackathonAnnouncementRecipients.id}) filter (where ${hackathonAnnouncementRecipients.sentAt} is null and ${hackathonAnnouncementRecipients.failedAt} is null)::int`, + inAppOnly: sql`count(${hackathonAnnouncementRecipients.id}) > 0 and count(${hackathonAnnouncementRecipients.id}) filter (where ${hackathonAnnouncementRecipients.sentAt} is not null and ${hackathonAnnouncementRecipients.claimedAt} is null) = count(${hackathonAnnouncementRecipients.id})`, }) .from(hackathonAnnouncements) .leftJoin( @@ -198,11 +206,6 @@ export const hackathonAnnounceRouter = createTRPCRouter({ .groupBy(hackathonAnnouncements.id) .orderBy(desc(hackathonAnnouncements.createdAt)) .limit(50); - - return rows.map((row) => ({ - ...row, - pending: row.total - row.sent - row.failed, - })); }), // Freezes a message and its audience. Sends nothing. The recipient rows are From 1c42309ccb7b3d704e001d43a01c4e76e7d1dead Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 11:11:35 -0400 Subject: [PATCH 03/27] fix(submit): encode the hackathon id in submit page links The id can come from the URL or the picker, so it is passed through encodeURIComponent once and every link on the page uses that path. --- sites/mainweb/app/(portal)/submit/page.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sites/mainweb/app/(portal)/submit/page.tsx b/sites/mainweb/app/(portal)/submit/page.tsx index cbc0130b..8b6a7353 100644 --- a/sites/mainweb/app/(portal)/submit/page.tsx +++ b/sites/mainweb/app/(portal)/submit/page.tsx @@ -59,6 +59,9 @@ function SubmitPortalContent() { const [selectedHackathonId, setSelectedHackathonId] = useState( urlHackathonId || "", ); + // The id can come straight from the URL or the picker, so it is encoded + // wherever it becomes part of a link path. + const hackathonPath = `/hackathons/${encodeURIComponent(selectedHackathonId)}`; const [confirmWithdraw, setConfirmWithdraw] = useState(false); const [projectName, setProjectName] = useState(""); @@ -353,7 +356,7 @@ function SubmitPortalContent() { )} {currentReg && (currentReg.team || admitted) && ( {currentReg.team ? "Manage team" : "Create or join a team"} @@ -372,7 +375,7 @@ function SubmitPortalContent() { Your project is in judging. Your team presents to judges for each challenge it entered, and the{" "} Judging tab @@ -659,7 +662,7 @@ function SubmitPortalContent() { submit points there. */} {projectSubmitted ? ( Go to Judging From 4bec2297c6ea2c007fdb9549e5fdd8682853e68e Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 11:11:37 -0400 Subject: [PATCH 04/27] fix(registration): keep sensitive answers out of the saved draft The application draft in localStorage no longer holds gender, race or the emergency contact. The portal is used on shared and borrowed laptops; the fields still work in the form and are still submitted. --- sites/mainweb/components/hackathon/InfoTab.tsx | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sites/mainweb/components/hackathon/InfoTab.tsx b/sites/mainweb/components/hackathon/InfoTab.tsx index 8731b18b..a494ae03 100644 --- a/sites/mainweb/components/hackathon/InfoTab.tsx +++ b/sites/mainweb/components/hackathon/InfoTab.tsx @@ -247,14 +247,15 @@ export function InfoTab({ // Switching tabs unmounts this one, so the answers are kept as they change. // The consents are left out on purpose: they are ticked fresh at submit. + // So are gender, race and the emergency contact: this portal is used on + // shared and borrowed laptops, and sensitive personal details should not + // sit in localStorage in clear text. const draft = JSON.stringify({ firstName, lastName, phone, age, - gender, pronouns, - race, underrepresented, school, major, @@ -269,8 +270,6 @@ export function InfoTab({ whyAttend, shirtSize, dietary, - emergencyContact, - emergencyPhone, needsHardware, }); useEffect(() => { @@ -293,9 +292,7 @@ export function InfoTab({ lastName: setLastName, phone: setPhone, age: setAge, - gender: setGender, pronouns: setPronouns, - race: setRace, school: setSchool, major: setMajor, graduationYear: setGraduationYear, @@ -305,8 +302,6 @@ export function InfoTab({ linkedinUrl: setLinkedinUrl, githubUrl: setGithubUrl, whyAttend: setWhyAttend, - emergencyContact: setEmergencyContact, - emergencyPhone: setEmergencyPhone, }; for (const [key, set] of Object.entries(text)) { const value = saved[key]; From 1c867da592ef4129383c2bcaa2869b6171b9976e Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 12:21:58 -0400 Subject: [PATCH 05/27] feat(points): refuse overlapping and back-to-back event check-ins Organisers' rules for bad actors: nobody is checked into two events that run at the same time, or into two less than 30 minutes apart. Both live in recordEventAttendance, the one writer of attendance rows, so pass scans, event codes and manual check-in all follow them. - Meals are exempt on both sides; back-to-back events do not overlap. - A repeat of an event already attended still reads "already checked in". - The participant row is locked (FOR NO KEY UPDATE) before the read, so two check-ins for one person at the same moment cannot both pass. - The rule and the row use one app clock rather than the database's now(), which follows the session time zone. - Refusals are FORBIDDEN with a sentence for the hacker or for staff; staff are told to remove the earlier check-in if it was a mistake. --- docs/architecture.md | 1 + .../api/src/.internal-tests/_db-tx-mock.ts | 8 + .../hackathon-admin-edge.test.ts | 176 ++++++++- .../src/.internal-tests/qr-checkin.test.ts | 47 ++- packages/api/src/routers/hackathon/admin.ts | 13 +- packages/api/src/routers/points/me.ts | 1 + packages/api/src/services/points.test.ts | 367 +++++++++++++++++- packages/api/src/services/points.ts | 177 ++++++++- 8 files changed, 770 insertions(+), 20 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 5f6fa53b..17426854 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,6 +57,7 @@ Edition-scoped event operations. Everything hangs off `hackathon`. - Judge roster (`judge`, `judge_assignment`): applications and approval - Judging: room-based judging with Devpost import — `judging_*` tables (`judging.ts`). The older pool-dispatch tables (`judging_project`, `judge_vote`, `judge_queue`, `hackathon_result`) are retired and kept for past editions' results - Points: `hackathon_event` points, staff adjustments and the point store (`points.ts`) +- Event check-in rules: nobody is checked into two events that run at the same time, or into two less than 30 minutes apart (meals are exempt). `recordEventAttendance` enforces this for pass scans, event codes and manual check-in - Announcements and acceptance waves Hackathon participation is **open to non-members**. Membership is not a registration requirement. diff --git a/packages/api/src/.internal-tests/_db-tx-mock.ts b/packages/api/src/.internal-tests/_db-tx-mock.ts index 9af27af6..804f78dc 100644 --- a/packages/api/src/.internal-tests/_db-tx-mock.ts +++ b/packages/api/src/.internal-tests/_db-tx-mock.ts @@ -147,6 +147,14 @@ export const createTransactionMock = ({ if (select) { tx.select = (...selectArgs: any[]) => ({ from: (...fromArgs: any[]) => ({ + // A join is a plain read, never a locking one; the hook gets the + // joined table as a fifth argument. + innerJoin: (...joinArgs: any[]) => ({ + where: (...wArgs: any[]) => + Promise.resolve( + select("select", selectArgs, fromArgs, wArgs, joinArgs), + ), + }), // Lazy: `.for("update")` must be the thing that takes the lock, and // eagerly building the unlocked promise would run the query twice. where: (...wArgs: any[]) => { diff --git a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts index 905e20ef..1777c6b6 100644 --- a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts +++ b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts @@ -19,6 +19,11 @@ const mockDelete = vi.fn(); const mockCount = vi.fn(); /** maxParticipants as the capacity check reads it; undefined = uncapped. */ const mockMaxParticipants = vi.fn<() => number | undefined>(() => undefined); +/** + * A participant's earlier check-ins as recordEventAttendance reads them: + * attendance joined to its event. Empty unless a test says otherwise. + */ +const mockEarlierAttendance = vi.fn<() => unknown[]>(() => []); // Acceptance emails are sent through a dynamic import; intercept it so we can // see exactly who a mass-approval actually mailed. const mockSendAcceptanceEmail = vi.fn(); @@ -111,7 +116,7 @@ vi.mock("@query/db", () => { select: vi.fn().mockImplementation(() => ({ from: (tbl: any) => { const count = mockCount(tbl?._t) ?? 0; - return thenable([ + const node = thenable([ { count, n: count, @@ -119,6 +124,10 @@ vi.mock("@query/db", () => { tbl?._t === "hackathons" ? mockMaxParticipants() : undefined, }, ]); + if (tbl?._t === "hackathonEventAttendees") { + node.innerJoin = () => thenable(mockEarlierAttendance()); + } + return node; }, })), }, @@ -292,6 +301,7 @@ describe("Hackathon admin management edge cases", () => { cache.clear(); mockCount.mockReturnValue(0); mockMaxParticipants.mockImplementation(() => undefined); + mockEarlierAttendance.mockReturnValue([]); mockFindMany.mockReturnValue([]); mockInsert.mockReturnValue([{}]); mockUpdate.mockReturnValue([{}]); @@ -1293,6 +1303,170 @@ describe("Hackathon admin management edge cases", () => { }); }); + // ===================================================================== + // Organisers' rules for bad actors: nobody is in two events that run at the + // same time, or in two within 30 minutes. They live in recordEventAttendance, + // so every way of writing attendance is held to them. + describe("Event check-in rules", () => { + const MINUTE = 60 * 1000; + + const person = { + id: PART_A1, + hackathonId: HACK_A, + registrationStatus: "checked_in", + firstName: "Ada", + lastName: "Lovelace", + user: { name: "Ada Lovelace", email: "ada@example.com" }, + }; + const fireside = { + id: EVENT_A, + hackathonId: HACK_A, + name: "Full Sail Fireside", + type: "sponsor_session", + points: 10, + selfCheckIn: true, + startTime: new Date(Date.now() - HOUR), + endTime: new Date(Date.now() + HOUR), + }; + + // A check-in for another event, as the attendance read returns it. + const earlier = (event: Record, minutesAgo: number) => ({ + event: { + id: "99999999-9999-4999-8999-999999999999", + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 5 * HOUR), + endTime: new Date(Date.now() - 4 * HOUR), + ...event, + }, + checkedInAt: new Date(Date.now() - minutesAgo * MINUTE), + }); + + // The hacker's own session, past the ban gate, with the event live. + const hackerCaller = () => { + mockFindFirst.mockImplementation((table: string) => { + if (table === "users") return { email: "ada@example.com" }; + if (table === "hackathonParticipants") return person; + if (table === "hackathonEvents") return fireside; + return undefined; + }); + return appRouter.createCaller(createMockCtx("rules-hacker")); + }; + + const attendanceWrites = () => + mockInsert.mock.calls.filter((c) => c[1][0] === hackathonEventAttendees); + + it("refuses a code check-in into an event that overlaps one already attended", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([ + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 20, + ), + ]); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "You're already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. You can check into one event at a time.", + }); + expect(attendanceWrites()).toHaveLength(0); + }); + + it("lets a code check-in through when nothing overlaps and the last was long ago", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([earlier({}, 120)]); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).resolves.toMatchObject({ eventName: "Full Sail Fireside" }); + expect(attendanceWrites()).toHaveLength(1); + }); + + it("refuses a pass scan within 30 minutes of the last check-in", async () => { + const caller = adminCaller({ + hackathonParticipants: { ...person, registrationStatus: "approved" }, + hackathonEvents: fireside, + hackathonEventAttendees: undefined, + }); + mockEarlierAttendance.mockReturnValue([earlier({}, 10)]); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "Ada Lovelace checked into BNY Workshop 10 minutes ago. Event check-ins have to be at least 30 minutes apart. If that was a mistake, remove them from BNY Workshop first.", + }); + // The whole scan rolls back: no attendance, and no arrival either. + expect(attendanceWrites()).toHaveLength(0); + expect(mockUpdate).not.toHaveBeenCalled(); + }); + + it("refuses a manual check-in into an event that overlaps one already attended", async () => { + const caller = adminCaller({ + hackathonParticipants: person, + hackathonEvents: fireside, + }); + mockEarlierAttendance.mockReturnValue([ + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 45, + ), + ]); + + await expect( + caller.hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", + }); + expect(attendanceWrites()).toHaveLength(0); + }); + + it("does not hold a meal to the rules", async () => { + const caller = adminCaller({ + hackathonParticipants: person, + hackathonEvents: { ...fireside, name: "Dinner", type: "meal" }, + }); + mockEarlierAttendance.mockReturnValue([ + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 5, + ), + ]); + + await expect( + caller.hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).resolves.toMatchObject({ eventName: "Dinner" }); + expect(attendanceWrites()).toHaveLength(1); + }); + }); + // ===================================================================== describe("Chasing unconfirmed seats", () => { const whereOf = () => diff --git a/packages/api/src/.internal-tests/qr-checkin.test.ts b/packages/api/src/.internal-tests/qr-checkin.test.ts index d3e9af21..db3024b6 100644 --- a/packages/api/src/.internal-tests/qr-checkin.test.ts +++ b/packages/api/src/.internal-tests/qr-checkin.test.ts @@ -21,6 +21,9 @@ const mockInsert = vi.fn(); const mockUpdate = vi.fn(); const mockDelete = vi.fn(); const mockSelect = vi.fn(); +// A participant's earlier check-ins, which recordEventAttendance reads as +// attendance joined to its event. Empty unless a test says otherwise. +const mockEarlierAttendance = vi.fn<() => unknown[]>(() => []); vi.mock("@query/db", async () => { const { createTransactionMock } = await import("./_db-tx-mock"); @@ -261,7 +264,10 @@ describe("QR check-in", () => { cache.clear(); // Aggregate reads default to "nothing there"; the concurrency tests below // override this to hand back the live row a FOR UPDATE would have locked. - mockSelect.mockReturnValue([{ count: 0 }]); + mockSelect.mockImplementation((_op, _sel, _from, _where, joined) => + joined ? mockEarlierAttendance() : [{ count: 0 }], + ); + mockEarlierAttendance.mockReturnValue([]); }); const createMockCtx = (userId?: string) => @@ -898,6 +904,45 @@ describe("QR check-in", () => { expect(err.message).toMatch(/is already checked into/); }); + it("refuses a scan into an event that runs at the same time as one already attended", async () => { + // Named and typed outright: the default fixture reads as a meal, and + // meals are exempt. + const caller = scannerCtx({ + event: hackEvent({ + name: "Full Sail Fireside", + type: "sponsor_session", + endTime: new Date(Date.now() + HOUR), + }), + }); + mockEarlierAttendance.mockReturnValue([ + { + event: { + id: CLUB_EVENT, + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 2 * HOUR), + endTime: new Date(Date.now() + 2 * HOUR), + }, + checkedInAt: new Date(Date.now() - 90 * 60 * 1000), + }, + ]); + + const err: any = await caller.hackathon + .scanParticipantPass({ + hackathonId: HACK_A, + eventId: HACK_EVENT, + participantId: PARTICIPANT, + }) + .catch((e: unknown) => e); + + expect(err.code).toBe("FORBIDDEN"); + expect(err.message).toBe( + "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", + ); + expect(mockInsert).not.toHaveBeenCalled(); + expect(mockUpdate).not.toHaveBeenCalled(); + }); + // BUG: scanParticipantPass matches on (id, hackathonId) only // (admin.ts:287-298) and never compares startTime/endTime to now, so one // pass of the scanner can check a participant into the whole weekend. diff --git a/packages/api/src/routers/hackathon/admin.ts b/packages/api/src/routers/hackathon/admin.ts index a8b951d7..ac10fe55 100644 --- a/packages/api/src/routers/hackathon/admin.ts +++ b/packages/api/src/routers/hackathon/admin.ts @@ -272,7 +272,9 @@ async function checkInParticipant( // 4. Record attendance. The unique index is the real guard, and // recordEventAttendance turns a racing scanner's 23505 into the same - // conflict as the read above. + // conflict as the read above. It also refuses an event that overlaps, or + // comes too soon after, one this person is already in; that refusal ends the + // scan before the promotion below. // Attendance and the roster promotion are one transaction. Split, a promotion // failing after the attendance row committed left the scan half-done — and // the retry hits the duplicate guard, so that attendee could never submit. @@ -1313,7 +1315,14 @@ export const hackathonAdminRouter = createTRPCRouter({ eq(hackathonEvents.id, input.eventId), eq(hackathonEvents.hackathonId, input.hackathonId), ), - columns: { id: true, name: true, points: true, endTime: true }, + columns: { + id: true, + name: true, + type: true, + points: true, + startTime: true, + endTime: true, + }, }), ]); diff --git a/packages/api/src/routers/points/me.ts b/packages/api/src/routers/points/me.ts index 1fb62242..2aa5f314 100644 --- a/packages/api/src/routers/points/me.ts +++ b/packages/api/src/routers/points/me.ts @@ -267,6 +267,7 @@ export const pointsMeRouter = createTRPCRouter({ columns: { id: true, name: true, + type: true, points: true, startTime: true, endTime: true, diff --git a/packages/api/src/services/points.test.ts b/packages/api/src/services/points.test.ts index 8b175e73..e6f67dbc 100644 --- a/packages/api/src/services/points.test.ts +++ b/packages/api/src/services/points.test.ts @@ -3,8 +3,11 @@ import { TRPCError } from "@trpc/server"; import { CHECK_IN_CODE_ALPHABET, CHECK_IN_CODE_LENGTH, + CHECK_IN_GAP_MS, alreadyCheckedInMessage, assignCheckInCode, + attendanceConflict, + attendanceConflictMessage, canSupply, fullName, generateCheckInCode, @@ -19,7 +22,7 @@ import { recordEventAttendance, selfCheckInWindow, } from "./points"; -import type { PointsDb } from "./points"; +import type { EarlierAttendance, PointsDb } from "./points"; import type { DrizzleDB } from "@query/db"; // Drizzle wraps the pg error, so the SQLSTATE sits on `.cause`. @@ -341,27 +344,293 @@ describe("selfCheckInWindow", () => { }); }); +const MIN = 60 * 1000; + +describe("attendanceConflict", () => { + const T0 = new Date("2026-10-10T18:00:00Z").getTime(); + const at = (minutes: number) => new Date(T0 + minutes * MIN); + + const ev = ( + id: string, + startMin: number, + endMin: number, + type = "workshop", + ) => ({ + id, + type, + startTime: at(startMin), + endTime: at(endMin), + }); + + // An earlier check-in. Defaults to long ago, so only the overlap rule can + // fire unless a test says otherwise. + const attended = ( + id: string, + startMin: number, + endMin: number, + opts: { type?: string; checkedInMin?: number; name?: string } = {}, + ): EarlierAttendance => ({ + event: { + ...ev(id, startMin, endMin, opts.type), + name: opts.name ?? `Event ${id}`, + }, + checkedInAt: at(opts.checkedInMin ?? -600), + }); + + it("allows a first check-in", () => { + expect(attendanceConflict(ev("a", 0, 60), [], at(0))).toBeNull(); + }); + + it("refuses an event that runs at the same time as one already attended", () => { + const bny = attended("bny", 0, 60, { name: "BNY Workshop" }); + expect(attendanceConflict(ev("fs", 30, 90), [bny], at(30))).toEqual({ + kind: "overlap", + other: { name: "BNY Workshop" }, + }); + // Contained and containing count too. + expect(attendanceConflict(ev("fs", 10, 20), [bny], at(30))?.kind).toBe( + "overlap", + ); + expect(attendanceConflict(ev("fs", -30, 120), [bny], at(30))?.kind).toBe( + "overlap", + ); + }); + + it("lets back-to-back events through", () => { + const first = attended("a", 0, 60); + expect(attendanceConflict(ev("b", 60, 120), [first], at(60))).toBeNull(); + const second = attended("c", 60, 120); + expect(attendanceConflict(ev("d", 0, 60), [second], at(30))).toBeNull(); + }); + + it("refuses a second check-in 29 minutes after the first and allows it at 30", () => { + const first = attended("a", 0, 60, { checkedInMin: 0, name: "Workshop" }); + // Back-to-back events, so only the gap is in play. + const next = ev("b", 60, 120); + expect(attendanceConflict(next, [first], at(29))).toEqual({ + kind: "too_soon", + other: { name: "Workshop" }, + minutesAgo: 29, + }); + expect(attendanceConflict(next, [first], at(30))).toBeNull(); + expect(CHECK_IN_GAP_MS).toBe(30 * MIN); + }); + + it("reports whole minutes, zero for a check-in just now", () => { + const first = attended("a", 0, 60, { checkedInMin: 0 }); + const next = ev("b", 60, 120); + expect( + attendanceConflict(next, [first], new Date(T0 + 59 * 1000))?.minutesAgo, + ).toBe(0); + expect( + attendanceConflict(next, [first], new Date(T0 + 61 * 1000))?.minutesAgo, + ).toBe(1); + }); + + it("does not report a negative wait when the clock is behind the database", () => { + const first = attended("a", 0, 60, { checkedInMin: 1 }); + expect( + attendanceConflict(ev("b", 60, 120), [first], at(0))?.minutesAgo, + ).toBe(0); + }); + + it("names the most recent check-in when several are too recent", () => { + const older = attended("a", 0, 10, { checkedInMin: 0, name: "Older" }); + const newer = attended("b", 20, 30, { checkedInMin: 10, name: "Newer" }); + const expected = { + kind: "too_soon", + other: { name: "Newer" }, + minutesAgo: 10, + }; + expect(attendanceConflict(ev("c", 60, 90), [newer, older], at(20))).toEqual( + expected, + ); + expect(attendanceConflict(ev("c", 60, 90), [older, newer], at(20))).toEqual( + expected, + ); + }); + + it("does not block a meal, and a meal does not block anything", () => { + const workshop = attended("w", 0, 60, { checkedInMin: 0 }); + expect( + attendanceConflict(ev("lunch", 10, 50, "meal"), [workshop], at(10)), + ).toBeNull(); + + const lunch = attended("lunch", 0, 60, { type: "meal", checkedInMin: 0 }); + expect(attendanceConflict(ev("w", 10, 50), [lunch], at(10))).toBeNull(); + }); + + it("ignores the same event, which the unique index handles", () => { + const same = attended("a", 0, 60, { checkedInMin: 0 }); + expect(attendanceConflict(ev("a", 0, 60), [same], at(5))).toBeNull(); + // A check-in elsewhere since then does not turn a repeat into "too soon": + // the true answer is still "already checked in". + const since = attended("b", 60, 120, { checkedInMin: 50 }); + expect( + attendanceConflict(ev("a", 0, 60), [same, since], at(55)), + ).toBeNull(); + }); + + it("reports an overlap before a recent check-in", () => { + const overlapping = attended("a", 0, 60, { + checkedInMin: 10, + name: "Overlaps", + }); + const recent = attended("b", -120, -90, { + checkedInMin: 14, + name: "Recent", + }); + expect( + attendanceConflict(ev("c", 30, 90), [recent, overlapping], at(15)), + ).toEqual({ kind: "overlap", other: { name: "Overlaps" } }); + }); +}); + +describe("attendanceConflictMessage", () => { + const input = { + event: { name: "Full Sail Fireside" }, + participant: { name: "Ada Lovelace" }, + }; + const overlap = { kind: "overlap", other: { name: "BNY Workshop" } } as const; + const soon = (minutesAgo: number) => + ({ + kind: "too_soon", + other: { name: "BNY Workshop" }, + minutesAgo, + }) as const; + + it("speaks to the hacker who entered the code about an overlap", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, overlap), + ).toBe( + "You're already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. You can check into one event at a time.", + ); + }); + + it("tells staff to remove the participant from the other event", () => { + for (const method of ["scan", "manual"] as const) { + expect(attendanceConflictMessage({ ...input, method }, overlap)).toBe( + "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", + ); + } + }); + + it("tells the hacker how long to wait", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(12)), + ).toBe( + "You checked into BNY Workshop 12 minutes ago. Event check-ins have to be at least 30 minutes apart, so try again in 18 minutes.", + ); + }); + + it("tells staff the same without a countdown", () => { + for (const method of ["scan", "manual"] as const) { + expect(attendanceConflictMessage({ ...input, method }, soon(12))).toBe( + "Ada Lovelace checked into BNY Workshop 12 minutes ago. Event check-ins have to be at least 30 minutes apart. If that was a mistake, remove them from BNY Workshop first.", + ); + } + }); + + it("uses singular wording at one minute", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(1)), + ).toBe( + "You checked into BNY Workshop 1 minute ago. Event check-ins have to be at least 30 minutes apart, so try again in 29 minutes.", + ); + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(29)), + ).toContain("so try again in 1 minute."); + }); + + it("says less than a minute for zero", () => { + expect( + attendanceConflictMessage({ ...input, method: "code" }, soon(0)), + ).toBe( + "You checked into BNY Workshop less than a minute ago. Event check-ins have to be at least 30 minutes apart, so try again in 30 minutes.", + ); + expect( + attendanceConflictMessage({ ...input, method: "manual" }, soon(0)), + ).toContain("checked into BNY Workshop less than a minute ago."); + }); +}); + describe("recordEventAttendance", () => { - const fakeDb = (fail?: () => Error) => { + // Stands in for the pool: a transaction hands back a builder that answers the + // participant lock, the attendance read and the insert, noting the order. + const fakeDb = ( + opts: { fail?: () => Error; earlier?: EarlierAttendance[] } = {}, + ) => { const inserted: unknown[] = []; - const db = { + const calls: string[] = []; + const tx = { + select: () => ({ + from: () => ({ + where: () => ({ + for: async (mode: string) => { + calls.push(`lock:${mode}`); + return [{ id: "part-1" }]; + }, + }), + innerJoin: () => ({ + where: async () => { + calls.push("read"); + return opts.earlier ?? []; + }, + }), + }), + }), insert: () => ({ values: async (values: unknown) => { - if (fail) throw fail(); + calls.push("insert"); + if (opts.fail) throw opts.fail(); inserted.push(values); }, }), }; - return { db: db as unknown as PointsDb, inserted }; + const db = { + transaction: async (run: (tx: unknown) => unknown) => run(tx), + }; + return { db: db as unknown as PointsDb, inserted, calls }; }; + // A meal, so the rules stay out of the way unless a test switches to the + // workshop below. const input = { - event: { id: "event-1", name: "Lunch", points: 15 }, + event: { + id: "event-1", + name: "Lunch", + points: 15, + type: "meal", + startTime: new Date(Date.now() - 30 * MIN), + endTime: new Date(Date.now() + 30 * MIN), + }, participant: { id: "part-1", name: "Ada Lovelace" }, method: "manual" as const, actorId: "staff-1", }; + const workshop = { + ...input.event, + id: "event-2", + name: "Workshop", + type: "workshop", + }; + + const earlierRow = ( + event: Partial = {}, + checkedInAt = new Date(Date.now() - 300 * MIN), + ): EarlierAttendance => ({ + event: { + id: "event-9", + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 30 * MIN), + endTime: new Date(Date.now() + 30 * MIN), + ...event, + }, + checkedInAt, + }); + it("copies the event's points and records how and by whom", async () => { const { db, inserted } = fakeDb(); await expect(recordEventAttendance(db, input)).resolves.toEqual({ @@ -374,12 +643,63 @@ describe("recordEventAttendance", () => { pointsAwarded: 15, method: "manual", checkedInById: "staff-1", + checkedInAt: expect.any(Date), }, ]); }); + it("locks the participant before it reads their attendance", async () => { + const { db, calls } = fakeDb(); + await recordEventAttendance(db, input); + expect(calls).toEqual(["lock:no key update", "read", "insert"]); + }); + + it("refuses a check-in into an event that overlaps one already attended", async () => { + const { db, inserted } = fakeDb({ earlier: [earlierRow()] }); + const error = await recordEventAttendance(db, { + ...input, + event: workshop, + method: "code", + }).catch((e: unknown) => e); + expect(error).toBeInstanceOf(TRPCError); + expect((error as TRPCError).code).toBe("FORBIDDEN"); + expect((error as TRPCError).message).toBe( + "You're already checked into BNY Workshop, which runs at the same time as Workshop. You can check into one event at a time.", + ); + expect(inserted).toEqual([]); + }); + + it("refuses a check-in under 30 minutes after the last one", async () => { + const { db, inserted } = fakeDb({ + earlier: [ + earlierRow( + { + startTime: new Date(Date.now() - 300 * MIN), + endTime: new Date(Date.now() - 240 * MIN), + }, + new Date(Date.now() - 10 * MIN), + ), + ], + }); + const error = await recordEventAttendance(db, { + ...input, + event: workshop, + }).catch((e: unknown) => e); + expect((error as TRPCError).code).toBe("FORBIDDEN"); + expect((error as TRPCError).message).toMatch( + /^Ada Lovelace checked into BNY Workshop 10 minutes ago\./, + ); + expect(inserted).toEqual([]); + }); + + it("lets a meal through alongside an overlapping event", async () => { + const { db, inserted } = fakeDb({ earlier: [earlierRow()] }); + await recordEventAttendance(db, input); + expect(inserted).toHaveLength(1); + }); + it("reads a duplicate as already checked in", async () => { - const { db } = fakeDb(uniqueViolation); + const { db } = fakeDb({ fail: uniqueViolation }); const error = await recordEventAttendance(db, input).catch( (e: unknown) => e, ); @@ -390,15 +710,44 @@ describe("recordEventAttendance", () => { ); }); + it("reads a repeat as already checked in after a later check-in elsewhere", async () => { + const { db } = fakeDb({ + fail: uniqueViolation, + earlier: [ + earlierRow( + { id: workshop.id, name: workshop.name }, + new Date(Date.now() - 40 * MIN), + ), + // Starts as the workshop ends, checked into five minutes ago. + earlierRow( + { + startTime: workshop.endTime, + endTime: new Date(Date.now() + 90 * MIN), + }, + new Date(Date.now() - 5 * MIN), + ), + ], + }); + const error = await recordEventAttendance(db, { + ...input, + event: workshop, + method: "code", + }).catch((e: unknown) => e); + expect((error as TRPCError).code).toBe("CONFLICT"); + expect((error as TRPCError).message).toBe( + "You are already checked into Workshop.", + ); + }); + it("speaks to the hacker when they entered the code", async () => { - const { db } = fakeDb(uniqueViolation); + const { db } = fakeDb({ fail: uniqueViolation }); await expect( recordEventAttendance(db, { ...input, method: "code" }), ).rejects.toThrow("You are already checked into Lunch."); }); it("passes other failures through", async () => { - const { db } = fakeDb(() => new Error("connection reset")); + const { db } = fakeDb({ fail: () => new Error("connection reset") }); await expect(recordEventAttendance(db, input)).rejects.toThrow( "connection reset", ); diff --git a/packages/api/src/services/points.ts b/packages/api/src/services/points.ts index a2ad9dd4..9e1e0303 100644 --- a/packages/api/src/services/points.ts +++ b/packages/api/src/services/points.ts @@ -240,6 +240,81 @@ export function selfCheckInWindow( // --------------------------------------------------------------------------- // Attendance +// Two events at once, or two check-ins in quick succession, mean somebody is +// collecting points for events they are not at. Meals are the exception: people +// eat alongside whatever else is on, and a meal scan only counts food. +export const CHECK_IN_GAP_MS = 30 * 60 * 1000; + +type AttendedEvent = { + id: string; + type: string; + startTime: Date; + endTime: Date; +}; + +/** One of the participant's existing check-ins, with the event it was for. */ +export type EarlierAttendance = { + event: AttendedEvent & { name: string }; + checkedInAt: Date; +}; + +export type AttendanceConflict = { + /** overlap: the events run at the same time. too_soon: under the gap. */ + kind: "overlap" | "too_soon"; + other: { name: string }; + /** Whole minutes since the earlier check-in. Only on too_soon. */ + minutesAgo?: number; +}; + +/** + * Why `event` cannot be checked into now, given the participant's earlier + * check-ins, or null when it can. Overlap is reported before too_soon because + * it is the firmer answer: waiting does not fix it. Pure, so the rule is + * testable without a database. + */ +export function attendanceConflict( + event: AttendedEvent, + earlier: readonly EarlierAttendance[], + now: Date, +): AttendanceConflict | null { + if (event.type === "meal") return null; + + // Already in this event: the duplicate case, which the unique index answers + // with "already checked in". Checking the others too would answer a repeat + // code with "try again in 20 minutes" after a later check-in elsewhere. + if (earlier.some((row) => row.event.id === event.id)) return null; + + const others = earlier.filter((row) => row.event.type !== "meal"); + + // Strict, so an event that ends at 2:00 does not clash with one that starts + // at 2:00. + const overlapping = others.find( + (row) => + row.event.startTime < event.endTime && + event.startTime < row.event.endTime, + ); + if (overlapping) { + return { kind: "overlap", other: { name: overlapping.event.name } }; + } + + let latest: EarlierAttendance | null = null; + for (const row of others) { + if (now.getTime() - row.checkedInAt.getTime() >= CHECK_IN_GAP_MS) continue; + if (!latest || row.checkedInAt > latest.checkedInAt) latest = row; + } + if (!latest) return null; + + return { + kind: "too_soon", + other: { name: latest.event.name }, + // A clock a little behind the database can put the check-in in the future. + minutesAgo: Math.max( + 0, + Math.floor((now.getTime() - latest.checkedInAt.getTime()) / 60_000), + ), + }; +} + /** * The one writer of attendance rows: pass scans, event codes and manual * check-in all come through here, so every row carries the event's points at @@ -248,25 +323,80 @@ export function selfCheckInWindow( * attend. The unique index on (event, participant) is the duplicate guard; * two stations racing both pass any earlier read, and the loser gets the same * sentence a single duplicate would. + * + * It also holds the bad-actor rules (see attendanceConflict): nobody is in two + * events that run at the same time, or in two within CHECK_IN_GAP_MS of each + * other. Those are read-then-write, so the participant's row is locked first: + * two check-ins for one person at the same moment (two tabs, two stations) + * queue up, and the second one sees the first. A refusal is FORBIDDEN, not + * CONFLICT: the self check-in page reads CONFLICT as "already checked in". */ export async function recordEventAttendance( db: PointsDb, input: { - event: { id: string; name: string; points: number }; + event: { + id: string; + name: string; + points: number; + type: string; + startTime: Date; + endTime: Date; + }; participant: { id: string; name: string }; method: AttendanceMethod; actorId: string | null; }, ): Promise<{ pointsAwarded: number }> { const { event, participant, method, actorId } = input; + // One clock for the rule and the row it writes. Left to the database's + // now(), checked_in_at (no time zone) follows the session's TimeZone and + // would not compare with this one. + const now = new Date(); try { - await db.insert(hackathonEventAttendees).values({ - eventId: event.id, - participantId: participant.id, - pointsAwarded: event.points, - method, - checkedInById: actorId, + await db.transaction(async (tx) => { + // No key update: enough to queue this person's check-ins behind each + // other without holding up other writes that reference the row. + await tx + .select({ id: hackathonParticipants.id }) + .from(hackathonParticipants) + .where(eq(hackathonParticipants.id, participant.id)) + .for("no key update"); + + const earlier = await tx + .select({ + event: { + id: hackathonEvents.id, + name: hackathonEvents.name, + type: hackathonEvents.type, + startTime: hackathonEvents.startTime, + endTime: hackathonEvents.endTime, + }, + checkedInAt: hackathonEventAttendees.checkedInAt, + }) + .from(hackathonEventAttendees) + .innerJoin( + hackathonEvents, + eq(hackathonEvents.id, hackathonEventAttendees.eventId), + ) + .where(eq(hackathonEventAttendees.participantId, participant.id)); + + const conflict = attendanceConflict(event, earlier, now); + if (conflict) { + throw new TRPCError({ + code: "FORBIDDEN", + message: attendanceConflictMessage(input, conflict), + }); + } + + await tx.insert(hackathonEventAttendees).values({ + eventId: event.id, + participantId: participant.id, + pointsAwarded: event.points, + method, + checkedInById: actorId, + checkedInAt: now, + }); }); } catch (error) { if (isUniqueViolation(error)) { @@ -293,6 +423,39 @@ export function alreadyCheckedInMessage(input: { : `${input.participant.name} is already checked into ${input.event.name}.`; } +const minutesLabel = (n: number) => `${n} minute${n === 1 ? "" : "s"}`; + +/** The refusal sentence for an attendanceConflict, worded like the one above. */ +export function attendanceConflictMessage( + input: { + event: { name: string }; + participant: { name: string }; + method: AttendanceMethod; + }, + conflict: AttendanceConflict, +): string { + const { event, participant, method } = input; + const other = conflict.other.name; + const self = method === "code"; + + if (conflict.kind === "overlap") { + return self + ? `You're already checked into ${other}, which runs at the same time as ${event.name}. You can check into one event at a time.` + : `${participant.name} is already checked into ${other}, which runs at the same time as ${event.name}. If that was a mistake, remove them from ${other} first.`; + } + + const ago = conflict.minutesAgo ?? 0; + const when = + ago === 0 ? "less than a minute ago" : `${minutesLabel(ago)} ago`; + const rule = `Event check-ins have to be at least ${CHECK_IN_GAP_MS / 60_000} minutes apart`; + if (!self) { + return `${participant.name} checked into ${other} ${when}. ${rule}. If that was a mistake, remove them from ${other} first.`; + } + // minutesAgo is floored, so what is left of the gap rounds up on its own. + const left = Math.max(1, CHECK_IN_GAP_MS / 60_000 - ago); + return `You checked into ${other} ${when}. ${rule}, so try again in ${minutesLabel(left)}.`; +} + // --------------------------------------------------------------------------- // Balances From b1957293300c8f37c297f56e9e5e641d2be23670 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 15:12:51 -0400 Subject: [PATCH 06/27] feat(points): record refused check-ins as flags for organisers A check-in the rules refuse (two events at once, or two under 30 minutes apart) rolls back, so the attempt is now written afterwards on the pool to a new hackathon_check_in_flag table. Repeats of the same pair add to an attempt count. Writing the flag is best effort and never changes the refusal the person sees. All three check-in paths (event codes, pass scans, manual check-in) are wrapped. Judging entries gain nullable prize_ineligible_at and prize_ineligible_by_id columns. Schema changes are additive only. --- docs/architecture.md | 1 + docs/operations/testing.md | 1 + docs/packages/db.md | 2 +- .../hackathon-admin-edge.test.ts | 196 +++++ .../src/.internal-tests/qr-checkin.test.ts | 81 +- packages/api/src/routers/hackathon/admin.ts | 24 +- packages/api/src/routers/points/me.ts | 21 +- .../api/src/services/check-in-flags.test.ts | 693 ++++++++++++++++++ packages/api/src/services/check-in-flags.ts | 405 ++++++++++ packages/api/src/services/points.test.ts | 42 +- packages/api/src/services/points.ts | 62 +- packages/db/src/schemas/hackathons.ts | 67 ++ packages/db/src/schemas/judging.ts | 8 + 13 files changed, 1578 insertions(+), 25 deletions(-) create mode 100644 packages/api/src/services/check-in-flags.test.ts create mode 100644 packages/api/src/services/check-in-flags.ts diff --git a/docs/architecture.md b/docs/architecture.md index 17426854..a87d2b6f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,6 +58,7 @@ Edition-scoped event operations. Everything hangs off `hackathon`. - Judging: room-based judging with Devpost import — `judging_*` tables (`judging.ts`). The older pool-dispatch tables (`judging_project`, `judge_vote`, `judge_queue`, `hackathon_result`) are retired and kept for past editions' results - Points: `hackathon_event` points, staff adjustments and the point store (`points.ts`) - Event check-in rules: nobody is checked into two events that run at the same time, or into two less than 30 minutes apart (meals are exempt). `recordEventAttendance` enforces this for pass scans, event codes and manual check-in +- Check-in rule-break flags: each refused attempt is recorded (`hackathon_check_in_flag`, `services/check-in-flags.ts`) and shown in judging on the hacker's team, in full to organisers and as a bare marker to judges. Organisers decide whether to pull the team from prizes (`judging_entry.prize_ineligible_at`). Hackers never see flags, and they stay out of public and cached payloads - Announcements and acceptance waves Hackathon participation is **open to non-members**. Membership is not a registration requirement. diff --git a/docs/operations/testing.md b/docs/operations/testing.md index 63c81425..d6f0f425 100644 --- a/docs/operations/testing.md +++ b/docs/operations/testing.md @@ -28,6 +28,7 @@ Config: `packages/api/vitest.config.ts`. Includes `src/**/*.test.ts` and `src/.i | Hackathon flow | `hackathon-flow.test.ts`, `hackathon-admin-edge.test.ts`, `hackathon-interest.test.ts` | | Payments | `stripe-payments.test.ts` | | QR / check-in | `qr-checkin.test.ts` | +| Check-in rule-break flags | `services/check-in-flags.test.ts`, `.internal-tests/check-in-flags-*.test.ts` | | Security / sanitizer / rate limit | `security.test.ts`, `resilience.test.ts` | | Judges / participants / initiatives / bootcamp / announcements | matching `*-edge.test.ts` / `bootcamp.test.ts` / `announcements.test.ts` | | Portal context | `services/portal-context.test.ts`, `routers/user/portal-context.test.ts` | diff --git a/docs/packages/db.md b/docs/packages/db.md index d928531e..31b19450 100644 --- a/docs/packages/db.md +++ b/docs/packages/db.md @@ -27,7 +27,7 @@ Files in `src/schemas/`, re-exported from `schemas/index.ts`. `drizzle.config.ts | `auth.ts` | `user`, `account`, `session`, `verificationToken` | | `members.ts` | `user_profile`, `member`, `membership_history` | | `admins.ts` | `admin` | -| `hackathons.ts` | `hackathon`, `hackathon_team`, `hackathon_participant`, `hackathon_project`, `hackathon_interest`, `hackathon_event`, `hackathon_event_attendee`, `hackathon_announcement`, `hackathon_announcement_recipient` | +| `hackathons.ts` | `hackathon`, `hackathon_team`, `hackathon_participant`, `hackathon_project`, `hackathon_interest`, `hackathon_event`, `hackathon_event_attendee`, `hackathon_check_in_flag`, `hackathon_announcement`, `hackathon_announcement_recipient` | | `judge.ts` | `judge`, `judge_assignment` (live roster); `judging_project`, `judge_vote`, `judge_queue`, `hackathon_result` (retired, kept for past editions) | | `judging.ts` | `judging_*`: config, entries and members, challenges, buildings, rooms and room links, judges, guest sessions, presence, schedule and appointments, rubric, evaluations, deliberation, placements | | `points.ts` | `hackathon_point_adjustment`, `hackathon_store_item`, `hackathon_store_purchase` | diff --git a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts index 1777c6b6..5612dc42 100644 --- a/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts +++ b/packages/api/src/.internal-tests/hackathon-admin-edge.test.ts @@ -1,11 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { appRouter } from "../root"; +import { errorFormatter } from "../trpc"; import { cache } from "../middleware/cache"; import { db, hackathons, hackathonParticipants, hackathonEventAttendees, + hackathonCheckInFlags, } from "@query/db"; // Fully mock the DB at the file level. vi.mock factories are hoisted and @@ -195,6 +197,16 @@ vi.mock("@query/db", () => { participantId: "participant_id", checkedInAt: "checked_in_at", }, + hackathonCheckInFlags: { + _t: "hackathonCheckInFlags", + id: "id", + hackathonId: "hackathon_id", + participantId: "participant_id", + eventId: "event_id", + conflictEventId: "conflict_event_id", + attempts: "attempts", + lastAttemptAt: "last_attempt_at", + }, members: { _t: "members", id: "id", @@ -1356,6 +1368,14 @@ describe("Hackathon admin management edge cases", () => { const attendanceWrites = () => mockInsert.mock.calls.filter((c) => c[1][0] === hackathonEventAttendees); + // Each refused attempt leaves a flag for organisers; the values are the + // third element of the mock's recorded call. + const flagWrites = () => + mockInsert.mock.calls.filter((c) => c[1][0] === hackathonCheckInFlags); + + // The event the fixtures' earlier check-ins are for. + const BNY = "99999999-9999-4999-8999-999999999999"; + it("refuses a code check-in into an event that overlaps one already attended", async () => { const caller = hackerCaller(); mockEarlierAttendance.mockReturnValue([ @@ -1441,6 +1461,182 @@ describe("Hackathon admin management edge cases", () => { expect(attendanceWrites()).toHaveLength(0); }); + describe("flags for organisers", () => { + const overlapping = () => + earlier( + { + startTime: new Date(Date.now() - 30 * MINUTE), + endTime: new Date(Date.now() + 30 * MINUTE), + }, + 20, + ); + + it("records a refused code check-in against the hacker", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([overlapping()]); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(flagWrites()).toHaveLength(1); + expect(flagWrites()[0]![2][0]).toEqual({ + hackathonId: HACK_A, + participantId: PART_A1, + eventId: EVENT_A, + conflictEventId: BNY, + rule: "overlap", + method: "code", + attemptedById: "rules-hacker", + attempts: 1, + createdAt: expect.any(Date), + lastAttemptAt: expect.any(Date), + }); + }); + + it("records a refused pass scan, with the staff member who scanned", async () => { + const caller = adminCaller({ + hackathonParticipants: { ...person, registrationStatus: "approved" }, + hackathonEvents: fireside, + hackathonEventAttendees: undefined, + }); + mockEarlierAttendance.mockReturnValue([earlier({}, 10)]); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(flagWrites()).toHaveLength(1); + expect(flagWrites()[0]![2][0]).toMatchObject({ + hackathonId: HACK_A, + participantId: PART_A1, + eventId: EVENT_A, + conflictEventId: BNY, + rule: "too_soon", + method: "scan", + attemptedById: ADMIN_USER, + }); + }); + + it("records a refused manual check-in", async () => { + const caller = adminCaller({ + hackathonParticipants: person, + hackathonEvents: fireside, + }); + mockEarlierAttendance.mockReturnValue([overlapping()]); + + await expect( + caller.hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(flagWrites()).toHaveLength(1); + expect(flagWrites()[0]![2][0]).toMatchObject({ + participantId: PART_A1, + rule: "overlap", + method: "manual", + attemptedById: ADMIN_USER, + }); + }); + + it("records nothing for a check-in that goes through", async () => { + mockEarlierAttendance.mockReturnValue([earlier({}, 120)]); + + await hackerCaller().points.me.checkIn({ + hackathonId: HACK_A, + code: "ABC234", + }); + await adminCaller({ + hackathonParticipants: person, + hackathonEvents: fireside, + }).hackathon.manualEventCheckIn({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }); + + expect(attendanceWrites()).toHaveLength(2); + expect(flagWrites()).toHaveLength(0); + }); + + it("records nothing for a duplicate scan", async () => { + const caller = adminCaller({ + hackathonParticipants: { ...person, registrationStatus: "approved" }, + hackathonEvents: fireside, + hackathonEventAttendees: { id: "already" }, + }); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: EVENT_A, + participantId: PART_A1, + }), + ).rejects.toMatchObject({ code: "CONFLICT" }); + + expect(flagWrites()).toHaveLength(0); + }); + + it("gives the hacker the same refusal when the flag cannot be written", async () => { + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([overlapping()]); + mockInsert.mockImplementation((_op, [table]) => { + if (table === hackathonCheckInFlags) throw new Error("db is down"); + return [{}]; + }); + + await expect( + caller.points.me.checkIn({ hackathonId: HACK_A, code: "ABC234" }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "You're already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. You can check into one event at a time.", + }); + expect(consoleError).toHaveBeenCalled(); + consoleError.mockRestore(); + }); + + it("keeps the flag out of what the hacker is sent", async () => { + const caller = hackerCaller(); + mockEarlierAttendance.mockReturnValue([overlapping()]); + + const error: any = await caller.points.me + .checkIn({ hackathonId: HACK_A, code: "ABC234" }) + .catch((e: unknown) => e); + + // The record rides on the error server side only; the formatter is what + // builds the response. + const response = JSON.stringify( + errorFormatter({ + shape: { + message: error.message, + code: -32003, + data: { + code: "FORBIDDEN", + httpStatus: 403, + path: "points.me.checkIn", + }, + }, + error, + }), + ); + expect(response).not.toContain(PART_A1); + expect(response).not.toContain(BNY); + expect(response).not.toContain("CheckInRuleBreak"); + expect(response).not.toContain("conflictEventId"); + }); + }); + it("does not hold a meal to the rules", async () => { const caller = adminCaller({ hackathonParticipants: person, diff --git a/packages/api/src/.internal-tests/qr-checkin.test.ts b/packages/api/src/.internal-tests/qr-checkin.test.ts index db3024b6..c402667a 100644 --- a/packages/api/src/.internal-tests/qr-checkin.test.ts +++ b/packages/api/src/.internal-tests/qr-checkin.test.ts @@ -1,7 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { appRouter } from "../root"; import { cache } from "../middleware/cache"; -import { db, events, eventCheckIns, hackathonEventAttendees } from "@query/db"; +import { + db, + events, + eventCheckIns, + hackathonCheckInFlags, + hackathonEventAttendees, +} from "@query/db"; // The real rule: a hardcoded "2026-fall" would fail every January. import { currentTerm } from "@query/db/services/membership"; import { __onRollback } from "./_db-tx-mock"; @@ -191,6 +197,12 @@ vi.mock("@query/db", async () => { eventId: "event_id", participantId: "participant_id", }, + hackathonCheckInFlags: { + participantId: "participant_id", + eventId: "event_id", + conflictEventId: "conflict_event_id", + attempts: "attempts", + }, members: { id: "id", userId: "user_id", hackathonId: "hackathon_id" }, events: { id: "id", @@ -939,10 +951,75 @@ describe("QR check-in", () => { expect(err.message).toBe( "Ada Lovelace is already checked into BNY Workshop, which runs at the same time as Full Sail Fireside. If that was a mistake, remove them from BNY Workshop first.", ); - expect(mockInsert).not.toHaveBeenCalled(); + // No attendance and no arrival, but the attempt is on record for + // organisers: it is the one write that survives the refusal. + const inserts = mockInsert.mock.calls as any[][]; + expect( + inserts.filter((c) => c[1][0] === hackathonEventAttendees), + ).toHaveLength(0); + const flags = inserts.filter((c) => c[1][0] === hackathonCheckInFlags); + expect(flags).toHaveLength(1); + expect(flags[0]![2][0]).toMatchObject({ + hackathonId: HACK_A, + participantId: PARTICIPANT, + eventId: HACK_EVENT, + conflictEventId: CLUB_EVENT, + rule: "overlap", + method: "scan", + attemptedById: "admin_user_id", + attempts: 1, + }); expect(mockUpdate).not.toHaveBeenCalled(); }); + it("writes the flag after the refused scan has rolled back, not inside it", async () => { + const caller = scannerCtx({ + event: hackEvent({ + name: "Full Sail Fireside", + type: "sponsor_session", + endTime: new Date(Date.now() + HOUR), + }), + }); + mockEarlierAttendance.mockReturnValue([ + { + event: { + id: CLUB_EVENT, + name: "BNY Workshop", + type: "workshop", + startTime: new Date(Date.now() - 2 * HOUR), + endTime: new Date(Date.now() + 2 * HOUR), + }, + checkedInAt: new Date(Date.now() - 90 * 60 * 1000), + }, + ]); + // A write made through a transaction registers an undo that runs when the + // refusal throws; one made on the pool registers nothing. + let flagRolledBack = false; + mockInsert.mockImplementation((_op, [table]) => { + if (table === hackathonCheckInFlags) { + __onRollback(() => { + flagRolledBack = true; + }); + } + return [{}]; + }); + + await expect( + caller.hackathon.scanParticipantPass({ + hackathonId: HACK_A, + eventId: HACK_EVENT, + participantId: PARTICIPANT, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect( + (mockInsert.mock.calls as any[][]).filter( + (c) => c[1][0] === hackathonCheckInFlags, + ), + ).toHaveLength(1); + expect(flagRolledBack).toBe(false); + }); + // BUG: scanParticipantPass matches on (id, hackathonId) only // (admin.ts:287-298) and never compares startTime/endTime to now, so one // pass of the scanner can check a participant into the whole weekend. diff --git a/packages/api/src/routers/hackathon/admin.ts b/packages/api/src/routers/hackathon/admin.ts index ac10fe55..4737c9f8 100644 --- a/packages/api/src/routers/hackathon/admin.ts +++ b/packages/api/src/routers/hackathon/admin.ts @@ -14,6 +14,7 @@ import { recordEventAttendance, searchParticipants, } from "../../services/points"; +import { flaggingRuleBreaks } from "../../services/check-in-flags"; import { CacheKeys } from "../../middleware/cache"; import { MASS_EMAIL_BATCH } from "../../services/email-limits"; import { @@ -281,7 +282,7 @@ async function checkInParticipant( // Rolling the attendance back makes a rescan the fix. let promotedToCheckedIn = false; - await ctx.db.transaction(async (tx) => { + const attendAndPromote = () => ctx.db.transaction(async (tx) => { await recordEventAttendance(tx, attendance); // The first scan is the check-in. `checked_in` previously had no writer @@ -308,6 +309,10 @@ async function checkInParticipant( promotedToCheckedIn = !!promoted; }); + // A refusal for the check-in rules rolls that transaction back, so the flag + // for organisers is written here, on the pool, once it has. + await flaggingRuleBreaks(ctx.db, input.hackathonId, attendAndPromote); + // A scan changes one event's attendee count and, on the first one, this // attendee's roster row. This runs at every door all weekend, so eviction // stays narrow — the old blanket pattern wiped every cached registration. @@ -1358,12 +1363,17 @@ export const hackathonAdminRouter = createTRPCRouter({ }); } - const { pointsAwarded } = await recordEventAttendance(db, { - event, - participant: { id: participant.id, name }, - method: "manual", - actorId: ctx.userId, - }); + const { pointsAwarded } = await flaggingRuleBreaks( + db, + input.hackathonId, + () => + recordEventAttendance(db, { + event, + participant: { id: participant.id, name }, + method: "manual", + actorId: ctx.userId, + }), + ); ctx.cache.delete(`hackathon:${input.hackathonId}:events`); diff --git a/packages/api/src/routers/points/me.ts b/packages/api/src/routers/points/me.ts index 2aa5f314..2e665a7a 100644 --- a/packages/api/src/routers/points/me.ts +++ b/packages/api/src/routers/points/me.ts @@ -11,6 +11,7 @@ import type { DrizzleDB } from "@query/db"; import { createTRPCRouter, protectedProcedure } from "../../trpc"; import { notHackathonBanned } from "../../middleware/procedures"; import { rateLimit } from "../../middleware/security"; +import { flaggingRuleBreaks } from "../../services/check-in-flags"; import { isSoldOut, leaderboardName, @@ -304,12 +305,20 @@ export const pointsMeRouter = createTRPCRouter({ }); } - const { pointsAwarded } = await recordEventAttendance(db, { - event, - participant: { id: participant.id, name: "You" }, - method: "code", - actorId: ctx.userId, - }); + // A refusal for the check-in rules is recorded for organisers, on the + // pool and outside the transaction it rolled back. The hacker sees only + // the refusal. + const { pointsAwarded } = await flaggingRuleBreaks( + db, + input.hackathonId, + () => + recordEventAttendance(db, { + event, + participant: { id: participant.id, name: "You" }, + method: "code", + actorId: ctx.userId, + }), + ); // The public schedule shows how many people came. ctx.cache.delete(`hackathon:${input.hackathonId}:events`); diff --git a/packages/api/src/services/check-in-flags.test.ts b/packages/api/src/services/check-in-flags.test.ts new file mode 100644 index 00000000..e3b3e03c --- /dev/null +++ b/packages/api/src/services/check-in-flags.test.ts @@ -0,0 +1,693 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TRPCError } from "@trpc/server"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + hackathonCheckInFlags, + hackathonEvents, + judgingEntries, + judgingEntryMembers, +} from "@query/db"; +import type { DrizzleDB } from "@query/db"; +import { + checkInFlagsForEntries, + entriesForFlags, + flaggedEntryIds, + flaggingRuleBreaks, + openCheckInFlagCount, + recordCheckInFlag, +} from "./check-in-flags"; +import { CheckInRuleBreak } from "./points"; +import type { CheckInRuleBreakDetails } from "./points"; + +const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; +const AT = new Date("2027-02-06T18:00:00Z"); + +const breach: CheckInRuleBreakDetails = { + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + actorId: "user-1", + at: AT, +}; + +const render = (condition: unknown) => + new PgDialect().sqlToQuery(condition as never); + +// --------------------------------------------------------------------------- +// Writing + +describe("recordCheckInFlag", () => { + const fakeInsertDb = () => { + const writes: { + table: unknown; + values: Record; + conflict: { target: unknown[]; set: Record }; + }[] = []; + const db = { + insert: (table: unknown) => ({ + values: (values: Record) => ({ + onConflictDoUpdate: async (conflict: { + target: unknown[]; + set: Record; + }) => { + writes.push({ table, values, conflict }); + }, + }), + }), + }; + return { db: db as unknown as DrizzleDB, writes }; + }; + + it("inserts one row for the person, the event tried and the event they were in", async () => { + const { db, writes } = fakeInsertDb(); + await recordCheckInFlag(db, HACK, breach); + + expect(writes).toHaveLength(1); + expect(writes[0]!.table).toBe(hackathonCheckInFlags); + // Both timestamps come from the clock the rule was judged on, not from the + // database's now(). + expect(writes[0]!.values).toEqual({ + hackathonId: HACK, + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attemptedById: "user-1", + attempts: 1, + createdAt: AT, + lastAttemptAt: AT, + }); + }); + + it("counts a repeat instead of adding a row, and leaves the dismissal alone", async () => { + const { db, writes } = fakeInsertDb(); + await recordCheckInFlag(db, HACK, breach); + + const { target, set } = writes[0]!.conflict; + expect(target).toEqual([ + hackathonCheckInFlags.participantId, + hackathonCheckInFlags.eventId, + hackathonCheckInFlags.conflictEventId, + ]); + expect(Object.keys(set).sort()).toEqual(["attempts", "lastAttemptAt"]); + expect(set.lastAttemptAt).toBe(AT); + expect(render(set.attempts).sql).toBe( + '"hackathon_check_in_flag"."attempts" + 1', + ); + }); + + it("keeps the staff member as the one who tried when there is no hacker", async () => { + const { db, writes } = fakeInsertDb(); + await recordCheckInFlag(db, HACK, { + ...breach, + method: "scan", + rule: "too_soon", + actorId: null, + }); + expect(writes[0]!.values).toMatchObject({ + method: "scan", + rule: "too_soon", + attemptedById: null, + }); + }); +}); + +describe("flaggingRuleBreaks", () => { + afterEach(() => vi.restoreAllMocks()); + + const refusal = () => + new TRPCError({ + code: "FORBIDDEN", + message: "You're already checked into BNY Workshop.", + cause: new CheckInRuleBreak(breach), + }); + + // A pool whose insert chain ends in `onConflictDoUpdate`, noting each flag. + const pool = (opts: { failWith?: Error } = {}) => { + const flags: unknown[] = []; + const db = { + insert: () => ({ + values: (values: unknown) => ({ + onConflictDoUpdate: async () => { + if (opts.failWith) throw opts.failWith; + flags.push(values); + }, + }), + }), + }; + return { db: db as unknown as DrizzleDB, flags }; + }; + + it("returns what the write returned and records nothing", async () => { + const { db, flags } = pool(); + await expect( + flaggingRuleBreaks(db, HACK, async () => ({ pointsAwarded: 5 })), + ).resolves.toEqual({ pointsAwarded: 5 }); + expect(flags).toEqual([]); + }); + + it("records a refusal for the check-in rules and rethrows the same error", async () => { + const { db, flags } = pool(); + const error = refusal(); + + const caught = await flaggingRuleBreaks(db, HACK, async () => { + throw error; + }).catch((e: unknown) => e); + + expect(caught).toBe(error); + expect(flags).toEqual([ + expect.objectContaining({ + hackathonId: HACK, + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attemptedById: "user-1", + }), + ]); + }); + + it("ignores every other failure", async () => { + for (const error of [ + new Error("connection reset"), + new TRPCError({ code: "CONFLICT", message: "Already checked in." }), + new TRPCError({ + code: "FORBIDDEN", + message: "Not accepted.", + cause: new Error("something else"), + }), + // A cause shaped like a rule break is not one. + new TRPCError({ + code: "FORBIDDEN", + message: "Lookalike.", + cause: Object.assign(new Error("lookalike"), { ...breach }), + }), + ]) { + const { db, flags } = pool(); + const caught = await flaggingRuleBreaks(db, HACK, async () => { + throw error; + }).catch((e: unknown) => e); + expect(caught).toBe(error); + expect(flags).toEqual([]); + } + }); + + it("still throws the original refusal when the flag cannot be written", async () => { + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const { db } = pool({ failWith: new Error("database is down") }); + const error = refusal(); + + const caught = await flaggingRuleBreaks(db, HACK, async () => { + throw error; + }).catch((e: unknown) => e); + + // The person at the door reads the same sentence either way. + expect(caught).toBe(error); + expect((caught as TRPCError).code).toBe("FORBIDDEN"); + expect((caught as TRPCError).message).toBe( + "You're already checked into BNY Workshop.", + ); + expect(consoleError).toHaveBeenCalledTimes(1); + }); + + it("is not thrown off when the pool itself is unusable", async () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const db = { + insert: () => { + throw new Error("pool is closed"); + }, + } as unknown as DrizzleDB; + const error = refusal(); + + await expect( + flaggingRuleBreaks(db, HACK, async () => { + throw error; + }), + ).rejects.toBe(error); + }); +}); + +// --------------------------------------------------------------------------- +// Reading + +describe("entriesForFlags", () => { + const flag = ( + id: string, + over: Partial<{ + participantId: string; + teamId: string | null; + userId: string; + }> = {}, + ) => ({ + id, + participantId: `part-${id}`, + teamId: null as string | null, + userId: `user-${id}`, + ...over, + }); + + it("puts a hacker's flag on their team's portal entry", () => { + const result = entriesForFlags( + [flag("f1", { teamId: "team-1" })], + [ + { entryId: "e-team", teamId: "team-1", submittedById: "part-cap" }, + { entryId: "e-other", teamId: "team-2", submittedById: "part-x" }, + ], + [], + ); + expect(result.get("f1")).toEqual(new Set(["e-team"])); + }); + + it("matches a solo project by who filed it", () => { + const result = entriesForFlags( + [flag("f1")], + [ + { entryId: "e-solo", teamId: null, submittedById: "part-f1" }, + { entryId: "e-not-me", teamId: null, submittedById: "part-other" }, + ], + [], + ); + expect(result.get("f1")).toEqual(new Set(["e-solo"])); + }); + + it("does not treat a former captain as a member of the team's project", () => { + // They filed it, then moved to team-2: only the team link counts for a + // project that has a team. + const result = entriesForFlags( + [flag("f1", { teamId: "team-2" })], + [{ entryId: "e-team-1", teamId: "team-1", submittedById: "part-f1" }], + [], + ); + expect(result.get("f1")).toEqual(new Set()); + }); + + it("does not match teamless hackers to teamless projects by their missing team", () => { + const result = entriesForFlags( + [flag("f1", { teamId: null })], + [{ entryId: "e-solo", teamId: null, submittedById: "part-other" }], + [], + ); + expect(result.get("f1")).toEqual(new Set()); + }); + + it("matches a member by account, which covers Devpost entries and claimed teammates", () => { + const result = entriesForFlags( + [flag("f1")], + [], + [ + { entryId: "e-devpost", userId: "user-f1" }, + { entryId: "e-unclaimed", userId: null }, + { entryId: "e-else", userId: "user-else" }, + ], + ); + expect(result.get("f1")).toEqual(new Set(["e-devpost"])); + }); + + it("counts an entry once however many ways the hacker reaches it", () => { + const result = entriesForFlags( + [flag("f1", { teamId: "team-1" })], + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + [ + { entryId: "e1", userId: "user-f1" }, + { entryId: "e2", userId: "user-f1" }, + ], + ); + expect(result.get("f1")).toEqual(new Set(["e1", "e2"])); + }); + + it("gives every flag an answer, empty when it is on nobody's entry", () => { + const result = entriesForFlags( + [flag("f1"), flag("f2", { teamId: "team-1" })], + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + [], + ); + expect(result.get("f1")).toEqual(new Set()); + expect(result.get("f2")).toEqual(new Set(["e1"])); + }); +}); + +describe("reading flags", () => { + // A db that answers by the table a query reads from, and notes each query so + // a test can say how many there were and what they asked for. It does not + // evaluate any SQL, so the conditions are checked by rendering them. + const fakeReadDb = (rows: Map) => { + const calls: { + table: unknown; + fields: Record; + where?: unknown; + }[] = []; + const db = { + select: (fields: Record) => ({ + from: (table: unknown) => { + const call: (typeof calls)[number] = { table, fields }; + calls.push(call); + // Chainable and awaitable, like a drizzle query. + type Query = { + innerJoin: () => Query; + where: (condition: unknown) => Query; + then: ( + resolve: (value: unknown) => unknown, + reject: (reason: unknown) => unknown, + ) => Promise; + }; + const node: Query = { + innerJoin: () => node, + where: (condition) => { + call.where = condition; + return node; + }, + then: (resolve, reject) => + Promise.resolve(rows.get(table) ?? []).then(resolve, reject), + }; + return node; + }, + }), + }; + return { db: db as unknown as DrizzleDB, calls }; + }; + + const flagRow = (id: string, over: Record = {}) => ({ + id, + participantId: `part-${id}`, + eventId: "event-2", + conflictEventId: "event-1", + rule: "overlap", + method: "code", + attempts: 1, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T18:00:00Z"), + dismissedAt: null, + teamId: null, + userId: `user-${id}`, + firstName: "Ada", + lastName: "Lovelace", + accountName: "ada l", + email: "ada@example.com", + ...over, + }); + + const events = [ + { id: "event-1", name: "BNY Workshop" }, + { id: "event-2", name: "Full Sail Fireside" }, + ]; + + const table = (...entries: [unknown, unknown[]][]) => new Map(entries); + + describe("checkInFlagsForEntries", () => { + it("asks nothing for an empty list of entries", async () => { + const { db, calls } = fakeReadDb(table()); + const result = await checkInFlagsForEntries(db, HACK, []); + expect(result.size).toBe(0); + expect(calls).toEqual([]); + }); + + it("stops after one read when the edition has no flags", async () => { + const { db, calls } = fakeReadDb(table()); + const result = await checkInFlagsForEntries(db, HACK, null); + expect(result.size).toBe(0); + expect(calls).toHaveLength(1); + }); + + it("describes each flag with names for organisers", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("f1", { + teamId: "team-1", + rule: "too_soon", + method: "scan", + attempts: 3, + }), + ], + ], + [hackathonEvents, events], + [ + judgingEntries, + [{ entryId: "e1", teamId: "team-1", submittedById: null }], + ], + ), + ); + + const result = await checkInFlagsForEntries(db, HACK, ["e1"]); + + expect(result.get("e1")).toEqual([ + { + id: "f1", + participantName: "Ada Lovelace", + rule: "too_soon", + eventName: "Full Sail Fireside", + conflictEventName: "BNY Workshop", + method: "scan", + attempts: 3, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T18:00:00Z"), + dismissed: false, + }, + ]); + }); + + it("falls back to the account name, then the email, when the registration has none", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("f1", { firstName: null, lastName: null }), + flagRow("f2", { + firstName: null, + lastName: null, + accountName: null, + }), + ], + ], + [hackathonEvents, events], + [ + judgingEntryMembers, + [ + { entryId: "e1", userId: "user-f1" }, + { entryId: "e1", userId: "user-f2" }, + ], + ], + ), + ); + + const flags = (await checkInFlagsForEntries(db, HACK, null)).get("e1")!; + expect(flags.map((f) => f.participantName).sort()).toEqual([ + "ada l", + "ada@example.com", + ]); + }); + + it("lists open flags before cleared ones, newest attempt first", async () => { + const t = (minute: number) => new Date(Date.UTC(2027, 1, 6, 18, minute)); + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + flagRow("old-open", { lastAttemptAt: t(1) }), + flagRow("cleared", { + lastAttemptAt: t(30), + dismissedAt: t(40), + }), + flagRow("new-open", { lastAttemptAt: t(20) }), + ], + ], + [hackathonEvents, events], + [ + judgingEntryMembers, + ["old-open", "cleared", "new-open"].map((id) => ({ + entryId: "e1", + userId: `user-${id}`, + })), + ], + ), + ); + + const flags = (await checkInFlagsForEntries(db, HACK, ["e1"])).get("e1")!; + expect(flags.map((f) => [f.id, f.dismissed])).toEqual([ + ["new-open", false], + ["old-open", false], + ["cleared", true], + ]); + }); + + it("keeps to the entries asked for, and to every entry when asked for null", async () => { + const rows = table( + [hackathonCheckInFlags, [flagRow("f1", { teamId: "team-1" })]], + [hackathonEvents, events], + [ + judgingEntries, + [ + { entryId: "e1", teamId: "team-1", submittedById: null }, + { entryId: "e2", teamId: "team-1", submittedById: null }, + ], + ], + ); + + const some = await checkInFlagsForEntries(fakeReadDb(rows).db, HACK, [ + "e2", + "e-unflagged", + ]); + expect([...some.keys()]).toEqual(["e2"]); + + const all = await checkInFlagsForEntries(fakeReadDb(rows).db, HACK, null); + expect([...all.keys()].sort()).toEqual(["e1", "e2"]); + // One flag, one description, wherever it shows. + expect(all.get("e1")![0]).toBe(all.get("e2")![0]); + }); + + it("leaves out a flag whose hacker is on no entry", async () => { + const { db } = fakeReadDb( + table( + [hackathonCheckInFlags, [flagRow("f1")]], + [hackathonEvents, events], + ), + ); + const result = await checkInFlagsForEntries(db, HACK, null); + expect(result.size).toBe(0); + }); + + it("reads in four batched queries however many flags there are", async () => { + const many = Array.from({ length: 40 }, (_, i) => + flagRow(`f${i}`, { teamId: `team-${i}` }), + ); + const { db, calls } = fakeReadDb( + table([hackathonCheckInFlags, many], [hackathonEvents, events]), + ); + + await checkInFlagsForEntries(db, HACK, null); + + expect(calls.map((c) => c.table)).toEqual([ + hackathonCheckInFlags, + hackathonEvents, + judgingEntries, + judgingEntryMembers, + ]); + }); + + it("looks for live entries of this edition only", async () => { + const { db, calls } = fakeReadDb( + table( + [hackathonCheckInFlags, [flagRow("f1", { teamId: "team-1" })]], + [hackathonEvents, events], + ), + ); + await checkInFlagsForEntries(db, HACK, null); + + const portal = render( + calls.find((c) => c.table === judgingEntries)!.where, + ); + const members = render( + calls.find((c) => c.table === judgingEntryMembers)!.where, + ); + for (const query of [portal, members]) { + expect(query.sql).toContain('"deleted_at" is null'); + expect(query.params).toContain(HACK); + } + expect(portal.params).toContain("portal"); + expect(portal.params).toContain("team-1"); + expect(members.params).toContain("user-f1"); + }); + }); + + describe("flaggedEntryIds", () => { + const target = (id: string, over: Record = {}) => ({ + id, + participantId: `part-${id}`, + teamId: null, + userId: `user-${id}`, + ...over, + }); + + it("asks nothing for an empty list of entries", async () => { + const { db, calls } = fakeReadDb(table()); + expect(await flaggedEntryIds(db, HACK, [])).toEqual(new Set()); + expect(calls).toEqual([]); + }); + + it("returns only the asked-for entries that carry a flag", async () => { + const { db } = fakeReadDb( + table( + [hackathonCheckInFlags, [target("f1", { teamId: "team-1" })]], + [ + judgingEntries, + [ + { entryId: "e1", teamId: "team-1", submittedById: null }, + { entryId: "e-not-asked", teamId: "team-1", submittedById: null }, + ], + ], + ), + ); + const flagged = await flaggedEntryIds(db, HACK, ["e1", "e2"]); + expect(flagged).toEqual(new Set(["e1"])); + }); + + it("reads open flags only, and never who or why", async () => { + const { db, calls } = fakeReadDb(table()); + await flaggedEntryIds(db, HACK, ["e1"]); + + const flags = calls.find((c) => c.table === hackathonCheckInFlags)!; + const where = render(flags.where); + expect(where.sql).toContain('"dismissed_at" is null'); + expect(where.params).toContain(HACK); + // Ids to find the team by, and nothing a judge could read. + expect(Object.keys(flags.fields).sort()).toEqual([ + "id", + "participantId", + "teamId", + "userId", + ]); + }); + }); + + describe("openCheckInFlagCount", () => { + it("is zero without reading entries when nothing is open", async () => { + const { db, calls } = fakeReadDb(table()); + expect(await openCheckInFlagCount(db, HACK)).toBe(0); + expect(calls).toHaveLength(1); + }); + + it("counts each open flag that sits on an entry once, and skips the rest", async () => { + const { db } = fakeReadDb( + table( + [ + hackathonCheckInFlags, + [ + { id: "f1", participantId: "p1", teamId: "team-1", userId: "u1" }, + { id: "f2", participantId: "p2", teamId: null, userId: "u2" }, + { id: "f3", participantId: "p3", teamId: null, userId: "u3" }, + ], + ], + [ + judgingEntries, + [ + { entryId: "e1", teamId: "team-1", submittedById: null }, + { entryId: "e2", teamId: "team-1", submittedById: null }, + ], + ], + [judgingEntryMembers, [{ entryId: "e3", userId: "u2" }]], + ), + ); + // f1 is on two entries and counts once, f2 reaches one through its + // account, f3 is on nobody's. + expect(await openCheckInFlagCount(db, HACK)).toBe(2); + }); + + it("reads open flags of this edition only", async () => { + const { db, calls } = fakeReadDb(table()); + await openCheckInFlagCount(db, HACK); + const where = render(calls[0]!.where); + expect(where.sql).toContain('"dismissed_at" is null'); + expect(where.params).toContain(HACK); + }); + }); +}); diff --git a/packages/api/src/services/check-in-flags.ts b/packages/api/src/services/check-in-flags.ts new file mode 100644 index 00000000..15135ca7 --- /dev/null +++ b/packages/api/src/services/check-in-flags.ts @@ -0,0 +1,405 @@ +import { TRPCError } from "@trpc/server"; +import { and, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { + hackathonCheckInFlags, + hackathonEvents, + hackathonParticipants, + hackathonProjects, + judgingEntries, + judgingEntryMembers, + users, +} from "@query/db"; +import type { DrizzleDB } from "@query/db"; +import type { JudgingDb } from "./judging/config"; +import { CheckInRuleBreak, fullName, personName } from "./points"; +import type { CheckInRuleBreakDetails } from "./points"; + +// Check-in rule breaks, as organisers and judges meet them. recordEventAttendance +// refuses an overlapping or too-soon check-in and the refusal rolls its +// transaction back, so the attempt is written here, on the pool, afterwards. +// A flag belongs to a person; judging shows it on the team they are on, found +// when it is read (see entriesForFlags) because people change teams and +// projects are promoted late. Hackers never read any of this, and nothing here +// may reach a cached or public payload. + +/** One refused attempt, as organisers read it. Judges get only an entry id. */ +export type CheckInFlagView = { + id: string; + participantName: string; + rule: "overlap" | "too_soon"; + /** The event they tried to enter. */ + eventName: string; + /** The event they were already in. */ + conflictEventName: string; + method: "scan" | "code" | "manual"; + attempts: number; + createdAt: Date; + lastAttemptAt: Date; + /** An organiser reviewed it and cleared it. */ + dismissed: boolean; +}; + +/** + * Stores a refused attempt. The same person trying the same pair of events + * again adds to `attempts` instead of a second row; a dismissal stays as it + * is, so a cleared flag does not reopen itself but still shows the retry count. + */ +export async function recordCheckInFlag( + db: DrizzleDB, + hackathonId: string, + breach: CheckInRuleBreakDetails, +): Promise { + await db + .insert(hackathonCheckInFlags) + .values({ + hackathonId, + participantId: breach.participantId, + eventId: breach.eventId, + conflictEventId: breach.conflictEventId, + rule: breach.rule, + method: breach.method, + attemptedById: breach.actorId, + attempts: 1, + createdAt: breach.at, + lastAttemptAt: breach.at, + }) + .onConflictDoUpdate({ + target: [ + hackathonCheckInFlags.participantId, + hackathonCheckInFlags.eventId, + hackathonCheckInFlags.conflictEventId, + ], + set: { + attempts: sql`${hackathonCheckInFlags.attempts} + 1`, + lastAttemptAt: breach.at, + }, + }); +} + +/** + * Runs an attendance write and, if the check-in rules refuse it, records the + * attempt on the pool before rethrowing. `db` has to be the pool and not the + * transaction the refusal rolled back: a row written inside it would go with + * it. Writing the flag is best effort. The person at the door must see the same + * refusal whether or not the flag landed, so a failure here is logged and the + * original error still goes out. + */ +export async function flaggingRuleBreaks( + db: DrizzleDB, + hackathonId: string, + run: () => Promise, +): Promise { + try { + return await run(); + } catch (error) { + if (error instanceof TRPCError && error.cause instanceof CheckInRuleBreak) { + try { + await recordCheckInFlag(db, hackathonId, error.cause); + } catch (flagError) { + // Deliberate server-side logging: the refusal is the answer the person + // gets, so the console is the only place a lost flag can show up. + // eslint-disable-next-line no-console + console.error( + "[CheckInFlags] Failed to record a rule break:", + flagError, + ); + } + } + throw error; + } +} + +// --------------------------------------------------------------------------- +// Reading flags against judging entries + +/** What finding a flag's entries needs from the flag and its participant. */ +export type FlagTarget = { + id: string; + participantId: string; + teamId: string | null; + userId: string; +}; + +/** A portal entry and the project it was promoted from. */ +export type PortalEntryLink = { + entryId: string; + teamId: string | null; + submittedById: string | null; +}; + +/** A person on an entry, claimed or not. */ +export type MemberEntryLink = { entryId: string; userId: string | null }; + +const pushTo = (map: Map, key: string, value: string) => { + const list = map.get(key); + if (list) list.push(value); + else map.set(key, [value]); +}; + +/** + * The entries each flag belongs to, by flag id. A participant is on an entry + * when it is a portal project filed by their team, or by them alone when it + * has no team, or when they are one of its members (which covers Devpost + * entries and claimed teammates). Reaching the same entry by more than one + * route counts once. A flag on nobody's entry maps to an empty set. Pure, so + * the matching is testable without a database. + */ +export function entriesForFlags( + flags: readonly FlagTarget[], + portal: readonly PortalEntryLink[], + members: readonly MemberEntryLink[], +): Map> { + const byTeam = new Map(); + const bySoloSubmitter = new Map(); + for (const link of portal) { + if (link.teamId) pushTo(byTeam, link.teamId, link.entryId); + else if (link.submittedById) { + pushTo(bySoloSubmitter, link.submittedById, link.entryId); + } + } + const byUser = new Map(); + for (const link of members) { + if (link.userId) pushTo(byUser, link.userId, link.entryId); + } + + const result = new Map>(); + for (const flag of flags) { + const entries = new Set(); + const sources = [ + flag.teamId ? byTeam.get(flag.teamId) : undefined, + bySoloSubmitter.get(flag.participantId), + byUser.get(flag.userId), + ]; + for (const ids of sources) for (const id of ids ?? []) entries.add(id); + result.set(flag.id, entries); + } + return result; +} + +/** Resolves flags to live entries of the edition in two batched reads. */ +async function entriesFor( + db: JudgingDb, + hackathonId: string, + flags: readonly FlagTarget[], +): Promise>> { + if (flags.length === 0) return new Map(); + + const teamIds = [ + ...new Set(flags.flatMap((flag) => (flag.teamId ? [flag.teamId] : []))), + ]; + const participantIds = [...new Set(flags.map((flag) => flag.participantId))]; + const userIds = [...new Set(flags.map((flag) => flag.userId))]; + + const [portal, members] = await Promise.all([ + db + .select({ + entryId: judgingEntries.id, + teamId: hackathonProjects.teamId, + submittedById: hackathonProjects.submittedById, + }) + .from(judgingEntries) + .innerJoin( + hackathonProjects, + eq(hackathonProjects.id, judgingEntries.sourceProjectId), + ) + .where( + and( + eq(judgingEntries.hackathonId, hackathonId), + eq(judgingEntries.source, "portal"), + isNull(judgingEntries.deletedAt), + or( + teamIds.length > 0 + ? inArray(hackathonProjects.teamId, teamIds) + : undefined, + and( + isNull(hackathonProjects.teamId), + inArray(hackathonProjects.submittedById, participantIds), + ), + ), + ), + ), + db + .select({ + entryId: judgingEntryMembers.entryId, + userId: judgingEntryMembers.userId, + }) + .from(judgingEntryMembers) + .innerJoin( + judgingEntries, + eq(judgingEntries.id, judgingEntryMembers.entryId), + ) + .where( + and( + eq(judgingEntryMembers.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + inArray(judgingEntryMembers.userId, userIds), + ), + ), + ]); + + return entriesForFlags(flags, portal, members); +} + +/** The open flags of an edition with just enough to find their entries. */ +async function openFlagTargets( + db: JudgingDb, + hackathonId: string, +): Promise { + return db + .select({ + id: hackathonCheckInFlags.id, + participantId: hackathonCheckInFlags.participantId, + teamId: hackathonParticipants.teamId, + userId: hackathonParticipants.userId, + }) + .from(hackathonCheckInFlags) + .innerJoin( + hackathonParticipants, + eq(hackathonParticipants.id, hackathonCheckInFlags.participantId), + ) + .where( + and( + eq(hackathonCheckInFlags.hackathonId, hackathonId), + isNull(hackathonCheckInFlags.dismissedAt), + ), + ); +} + +/** + * Every flag on the given entries, grouped by entry, for organisers: names and + * events included. Open flags come first, then the most recent attempt. Null + * entry ids means every live entry of the edition (the "Flagged" filter). An + * entry with no flags is not in the map. Never pass the result to a judge or + * put it in a cached payload. + */ +export async function checkInFlagsForEntries( + db: JudgingDb, + hackathonId: string, + entryIds: string[] | null, +): Promise> { + const result = new Map(); + if (entryIds && entryIds.length === 0) return result; + + const rows = await db + .select({ + id: hackathonCheckInFlags.id, + participantId: hackathonCheckInFlags.participantId, + eventId: hackathonCheckInFlags.eventId, + conflictEventId: hackathonCheckInFlags.conflictEventId, + rule: hackathonCheckInFlags.rule, + method: hackathonCheckInFlags.method, + attempts: hackathonCheckInFlags.attempts, + createdAt: hackathonCheckInFlags.createdAt, + lastAttemptAt: hackathonCheckInFlags.lastAttemptAt, + dismissedAt: hackathonCheckInFlags.dismissedAt, + teamId: hackathonParticipants.teamId, + userId: hackathonParticipants.userId, + firstName: hackathonParticipants.firstName, + lastName: hackathonParticipants.lastName, + accountName: users.name, + email: users.email, + }) + .from(hackathonCheckInFlags) + .innerJoin( + hackathonParticipants, + eq(hackathonParticipants.id, hackathonCheckInFlags.participantId), + ) + .innerJoin(users, eq(users.id, hackathonParticipants.userId)) + .where(eq(hackathonCheckInFlags.hackathonId, hackathonId)); + if (rows.length === 0) return result; + + const eventIds = [ + ...new Set(rows.flatMap((row) => [row.eventId, row.conflictEventId])), + ]; + const [events, entriesByFlag] = await Promise.all([ + db + .select({ id: hackathonEvents.id, name: hackathonEvents.name }) + .from(hackathonEvents) + .where(inArray(hackathonEvents.id, eventIds)), + entriesFor(db, hackathonId, rows), + ]); + const eventName = new Map(events.map((event) => [event.id, event.name])); + + const wanted = entryIds ? new Set(entryIds) : null; + const views: [CheckInFlagView, string[]][] = []; + for (const row of rows) { + const entries = [...(entriesByFlag.get(row.id) ?? [])].filter( + (id) => !wanted || wanted.has(id), + ); + if (entries.length === 0) continue; + views.push([ + { + id: row.id, + participantName: fullName(personName(row), row.email), + rule: row.rule, + eventName: eventName.get(row.eventId) ?? "A deleted event", + conflictEventName: + eventName.get(row.conflictEventId) ?? "A deleted event", + method: row.method, + attempts: row.attempts, + createdAt: row.createdAt, + lastAttemptAt: row.lastAttemptAt, + dismissed: row.dismissedAt !== null, + }, + entries, + ]); + } + + // One order for every entry: open before cleared, newest attempt first, and + // the id last so equal timestamps never shuffle between polls. + views.sort( + ([a], [b]) => + Number(a.dismissed) - Number(b.dismissed) || + b.lastAttemptAt.getTime() - a.lastAttemptAt.getTime() || + (a.id < b.id ? -1 : a.id > b.id ? 1 : 0), + ); + for (const [view, entries] of views) { + for (const entryId of entries) { + const list = result.get(entryId); + if (list) list.push(view); + else result.set(entryId, [view]); + } + } + return result; +} + +/** + * Which of the given entries carry at least one flag nobody has dismissed. + * Ids only, which is all a judge is told: that the team was flagged, not who + * or why. + */ +export async function flaggedEntryIds( + db: JudgingDb, + hackathonId: string, + entryIds: string[], +): Promise> { + if (entryIds.length === 0) return new Set(); + + const targets = await openFlagTargets(db, hackathonId); + const entriesByFlag = await entriesFor(db, hackathonId, targets); + + const wanted = new Set(entryIds); + const flagged = new Set(); + for (const entries of entriesByFlag.values()) { + for (const id of entries) if (wanted.has(id)) flagged.add(id); + } + return flagged; +} + +/** + * Open flags in the edition that sit on a live entry, for the organiser + * header. A flag whose hacker is on no entry is left out: there is nowhere in + * judging to review it, and a count of things nobody can open is noise. + */ +export async function openCheckInFlagCount( + db: JudgingDb, + hackathonId: string, +): Promise { + const targets = await openFlagTargets(db, hackathonId); + const entriesByFlag = await entriesFor(db, hackathonId, targets); + + let count = 0; + for (const entries of entriesByFlag.values()) + if (entries.size > 0) count += 1; + return count; +} diff --git a/packages/api/src/services/points.test.ts b/packages/api/src/services/points.test.ts index e6f67dbc..3b7341fa 100644 --- a/packages/api/src/services/points.test.ts +++ b/packages/api/src/services/points.test.ts @@ -21,6 +21,7 @@ import { rankLeaderboard, recordEventAttendance, selfCheckInWindow, + CheckInRuleBreak, } from "./points"; import type { EarlierAttendance, PointsDb } from "./points"; import type { DrizzleDB } from "@query/db"; @@ -385,7 +386,7 @@ describe("attendanceConflict", () => { const bny = attended("bny", 0, 60, { name: "BNY Workshop" }); expect(attendanceConflict(ev("fs", 30, 90), [bny], at(30))).toEqual({ kind: "overlap", - other: { name: "BNY Workshop" }, + other: { id: "bny", name: "BNY Workshop" }, }); // Contained and containing count too. expect(attendanceConflict(ev("fs", 10, 20), [bny], at(30))?.kind).toBe( @@ -409,7 +410,7 @@ describe("attendanceConflict", () => { const next = ev("b", 60, 120); expect(attendanceConflict(next, [first], at(29))).toEqual({ kind: "too_soon", - other: { name: "Workshop" }, + other: { id: "a", name: "Workshop" }, minutesAgo: 29, }); expect(attendanceConflict(next, [first], at(30))).toBeNull(); @@ -439,7 +440,7 @@ describe("attendanceConflict", () => { const newer = attended("b", 20, 30, { checkedInMin: 10, name: "Newer" }); const expected = { kind: "too_soon", - other: { name: "Newer" }, + other: { id: "b", name: "Newer" }, minutesAgo: 10, }; expect(attendanceConflict(ev("c", 60, 90), [newer, older], at(20))).toEqual( @@ -482,7 +483,7 @@ describe("attendanceConflict", () => { }); expect( attendanceConflict(ev("c", 30, 90), [recent, overlapping], at(15)), - ).toEqual({ kind: "overlap", other: { name: "Overlaps" } }); + ).toEqual({ kind: "overlap", other: { id: "a", name: "Overlaps" } }); }); }); @@ -491,11 +492,14 @@ describe("attendanceConflictMessage", () => { event: { name: "Full Sail Fireside" }, participant: { name: "Ada Lovelace" }, }; - const overlap = { kind: "overlap", other: { name: "BNY Workshop" } } as const; + const overlap = { + kind: "overlap", + other: { id: "bny", name: "BNY Workshop" }, + } as const; const soon = (minutesAgo: number) => ({ kind: "too_soon", - other: { name: "BNY Workshop" }, + other: { id: "bny", name: "BNY Workshop" }, minutesAgo, }) as const; @@ -666,6 +670,18 @@ describe("recordEventAttendance", () => { expect((error as TRPCError).message).toBe( "You're already checked into BNY Workshop, which runs at the same time as Workshop. You can check into one event at a time.", ); + // What a caller needs to record the attempt once this has rolled back. + const cause = (error as TRPCError).cause; + expect(cause).toBeInstanceOf(CheckInRuleBreak); + expect(cause).toMatchObject({ + participantId: "part-1", + eventId: "event-2", + conflictEventId: "event-9", + rule: "overlap", + method: "code", + actorId: "staff-1", + at: expect.any(Date), + }); expect(inserted).toEqual([]); }); @@ -689,6 +705,12 @@ describe("recordEventAttendance", () => { expect((error as TRPCError).message).toMatch( /^Ada Lovelace checked into BNY Workshop 10 minutes ago\./, ); + expect((error as TRPCError).cause).toMatchObject({ + eventId: "event-2", + conflictEventId: "event-9", + rule: "too_soon", + method: "manual", + }); expect(inserted).toEqual([]); }); @@ -746,6 +768,14 @@ describe("recordEventAttendance", () => { ).rejects.toThrow("You are already checked into Lunch."); }); + it("leaves a duplicate without a rule-break cause", async () => { + const { db } = fakeDb({ fail: uniqueViolation }); + const error = await recordEventAttendance(db, input).catch( + (e: unknown) => e, + ); + expect((error as TRPCError).cause).not.toBeInstanceOf(CheckInRuleBreak); + }); + it("passes other failures through", async () => { const { db } = fakeDb({ fail: () => new Error("connection reset") }); await expect(recordEventAttendance(db, input)).rejects.toThrow( diff --git a/packages/api/src/services/points.ts b/packages/api/src/services/points.ts index 9e1e0303..2004613a 100644 --- a/packages/api/src/services/points.ts +++ b/packages/api/src/services/points.ts @@ -261,7 +261,8 @@ export type EarlierAttendance = { export type AttendanceConflict = { /** overlap: the events run at the same time. too_soon: under the gap. */ kind: "overlap" | "too_soon"; - other: { name: string }; + /** The earlier check-in that stands in the way. */ + other: { id: string; name: string }; /** Whole minutes since the earlier check-in. Only on too_soon. */ minutesAgo?: number; }; @@ -294,7 +295,10 @@ export function attendanceConflict( event.startTime < row.event.endTime, ); if (overlapping) { - return { kind: "overlap", other: { name: overlapping.event.name } }; + return { + kind: "overlap", + other: { id: overlapping.event.id, name: overlapping.event.name }, + }; } let latest: EarlierAttendance | null = null; @@ -306,7 +310,7 @@ export function attendanceConflict( return { kind: "too_soon", - other: { name: latest.event.name }, + other: { id: latest.event.id, name: latest.event.name }, // A clock a little behind the database can put the check-in in the future. minutesAgo: Math.max( 0, @@ -315,6 +319,47 @@ export function attendanceConflict( }; } +export type CheckInRuleBreakDetails = { + participantId: string; + /** The event they tried to enter. */ + eventId: string; + /** The event they were already in. */ + conflictEventId: string; + rule: AttendanceConflict["kind"]; + method: AttendanceMethod; + /** The hacker on a code, staff otherwise. */ + actorId: string | null; + /** The clock the rule was judged on. */ + at: Date; +}; + +/** + * Rides on the `cause` of the refusal recordEventAttendance throws, so a caller + * outside the transaction can record the attempt after the refusal has rolled + * back. It never reaches a client: tRPC does not serialise `cause`. + */ +export class CheckInRuleBreak extends Error implements CheckInRuleBreakDetails { + readonly participantId: string; + readonly eventId: string; + readonly conflictEventId: string; + readonly rule: AttendanceConflict["kind"]; + readonly method: AttendanceMethod; + readonly actorId: string | null; + readonly at: Date; + + constructor(details: CheckInRuleBreakDetails) { + super(`Check-in rule broken: ${details.rule}`); + this.name = "CheckInRuleBreak"; + this.participantId = details.participantId; + this.eventId = details.eventId; + this.conflictEventId = details.conflictEventId; + this.rule = details.rule; + this.method = details.method; + this.actorId = details.actorId; + this.at = details.at; + } +} + /** * The one writer of attendance rows: pass scans, event codes and manual * check-in all come through here, so every row carries the event's points at @@ -330,6 +375,8 @@ export function attendanceConflict( * two check-ins for one person at the same moment (two tabs, two stations) * queue up, and the second one sees the first. A refusal is FORBIDDEN, not * CONFLICT: the self check-in page reads CONFLICT as "already checked in". + * The refusal rolls this transaction back, so it carries a CheckInRuleBreak as + * its cause for the caller to record (see check-in-flags.ts). */ export async function recordEventAttendance( db: PointsDb, @@ -386,6 +433,15 @@ export async function recordEventAttendance( throw new TRPCError({ code: "FORBIDDEN", message: attendanceConflictMessage(input, conflict), + cause: new CheckInRuleBreak({ + participantId: participant.id, + eventId: event.id, + conflictEventId: conflict.other.id, + rule: conflict.kind, + method, + actorId, + at: now, + }), }); } diff --git a/packages/db/src/schemas/hackathons.ts b/packages/db/src/schemas/hackathons.ts index 56d0fc95..5e70ebe2 100644 --- a/packages/db/src/schemas/hackathons.ts +++ b/packages/db/src/schemas/hackathons.ts @@ -504,6 +504,73 @@ export const hackathonEventAttendeesRelations = relations( }), ); +// A check-in the rules refused (two events at the same time, or two under 30 +// minutes apart). The refusal itself rolls back, so this is the only trace: +// organisers read it as a flag on the hacker's team in judging and decide +// whether the team stays in the running for prizes. Never shown to hackers. +// One row per (person, event tried, event already in) with a count, so a hacker +// retrying a rejected code adds an attempt rather than a screen of duplicates. +// It points at the participant, not the project, because a person can change +// teams or have their project promoted later; judging resolves it to an entry +// when it is read. +export const hackathonCheckInFlags = pgTable( + "hackathon_check_in_flag", + { + id: uuid("id").defaultRandom().primaryKey(), + hackathonId: uuid("hackathon_id") + .notNull() + .references(() => hackathons.id, { onDelete: "cascade" }), + // The participant and event FKs are declared in the table extras with + // explicit names. The generated participant one is 66 characters, past + // Postgres's 63-char limit, and the conflict event one lands on exactly 63; + // the event one is named to match. + participantId: uuid("participant_id").notNull(), + // The event they tried to enter. + eventId: uuid("event_id").notNull(), + // The event they were already in. + conflictEventId: uuid("conflict_event_id").notNull(), + rule: text("rule", { enum: ["overlap", "too_soon"] }).notNull(), + method: text("method", { enum: ["scan", "code", "manual"] }).notNull(), + // The hacker on a code, staff otherwise. + attemptedById: text("attempted_by_id").references(() => users.id, { + onDelete: "set null", + }), + attempts: integer("attempts").notNull().default(1), + // Written from the app clock, like checked_in_at: the rule compares against + // it, and the database's now() follows the session's TimeZone. + createdAt: timestamp("created_at").defaultNow().notNull(), + lastAttemptAt: timestamp("last_attempt_at").defaultNow().notNull(), + // An organiser reviewed the flag and cleared it. It stays on record. + dismissedAt: timestamp("dismissed_at"), + dismissedById: text("dismissed_by_id").references(() => users.id, { + onDelete: "set null", + }), + }, + (table) => [ + uniqueIndex("check_in_flag_attempt_idx").on( + table.participantId, + table.eventId, + table.conflictEventId, + ), + index("check_in_flag_hackathon_idx").on(table.hackathonId), + foreignKey({ + name: "check_in_flag_participant_fk", + columns: [table.participantId], + foreignColumns: [hackathonParticipants.id], + }).onDelete("cascade"), + foreignKey({ + name: "check_in_flag_event_fk", + columns: [table.eventId], + foreignColumns: [hackathonEvents.id], + }).onDelete("cascade"), + foreignKey({ + name: "check_in_flag_conflict_event_fk", + columns: [table.conflictEventId], + foreignColumns: [hackathonEvents.id], + }).onDelete("cascade"), + ], +); + export const hackathonProjectsRelations = relations( hackathonProjects, ({ one }) => ({ diff --git a/packages/db/src/schemas/judging.ts b/packages/db/src/schemas/judging.ts index e35141ac..35291e40 100644 --- a/packages/db/src/schemas/judging.ts +++ b/packages/db/src/schemas/judging.ts @@ -90,6 +90,14 @@ export const judgingEntries = pgTable( deletedById: text("deleted_by_id").references(() => users.id, { onDelete: "set null", }), + // An organiser pulled this entry from prizes, usually after a check-in rule + // break. Judges still score it, so nothing about the room schedule changes; + // only placements and results leave it out. + prizeIneligibleAt: timestamp("prize_ineligible_at"), + prizeIneligibleById: text("prize_ineligible_by_id").references( + () => users.id, + { onDelete: "set null" }, + ), createdAt: timestamp("created_at").defaultNow().notNull(), updatedAt: timestamp("updated_at").defaultNow().notNull(), }, From a79cd7f7480320e4f341c353249ca2be1b094f0d Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 15:12:52 -0400 Subject: [PATCH 07/27] feat(judging): show check-in flags to organisers in the projects list listEntries returns each entry's flags (who, which events, how, how many attempts) and a Flagged filter; overview counts open flags for the page header. New dismissCheckInFlag and setPrizeEligibility mutations let organisers clear a flag or pull a team from prizes, both audited. --- .../check-in-flags-organizer.test.ts | 783 ++++++++++++++++++ packages/api/src/routers/judging/admin.ts | 146 +++- .../admin/judging/[hackathonId]/page.tsx | 5 + .../components/admin/judging/EntriesList.tsx | 147 +++- 4 files changed, 1077 insertions(+), 4 deletions(-) create mode 100644 packages/api/src/.internal-tests/check-in-flags-organizer.test.ts diff --git a/packages/api/src/.internal-tests/check-in-flags-organizer.test.ts b/packages/api/src/.internal-tests/check-in-flags-organizer.test.ts new file mode 100644 index 00000000..eb1b60a3 --- /dev/null +++ b/packages/api/src/.internal-tests/check-in-flags-organizer.test.ts @@ -0,0 +1,783 @@ +/** + * Check-in flags in the organiser's judging console: the project list carries + * each project's flags and whether it was pulled from prizes, can be narrowed + * to flagged projects, and the header counts open flags. Two writes close the + * loop: dismissing a flag and pulling (or returning) a project from prizes. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way judge-edge.test.ts does. It keeps + * the rows of the tables a write touches and applies the conditions the router + * asks for, so a missing edition condition lets another edition's row through + * and fails the test, rather than passing on a canned answer. The flag + * service has its own tests; here it is a stub, which also lets these tests + * see exactly what the router asks it for. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + auditLogs, + hackathonCheckInFlags, + hackathons, + judgingConfigs, + judgingEntries, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; +import type * as CheckInFlags from "../services/check-in-flags"; + +const mockFlagsForEntries = vi.fn(); +const mockFlaggedIds = vi.fn(); +const mockOpenCount = vi.fn(); + +vi.mock("../services/check-in-flags", async (importActual) => ({ + ...(await importActual()), + checkInFlagsForEntries: (...args: unknown[]) => mockFlagsForEntries(...args), + flaggedEntryIds: (...args: unknown[]) => mockFlaggedIds(...args), + openCheckInFlagCount: (...args: unknown[]) => mockOpenCount(...args), +})); + +const HACK_A = "11111111-1111-4111-8111-111111111111"; +const HACK_B = "22222222-2222-4222-8222-222222222222"; +const ENTRY_1 = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1"; +const ENTRY_2 = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa2"; +const ENTRY_3 = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa3"; +const ENTRY_OTHER = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbb1"; +const FLAG_1 = "cccccccc-cccc-4ccc-8ccc-ccccccccccc1"; +const FLAG_OTHER = "dddddddd-dddd-4ddd-8ddd-ddddddddddd1"; +const NO_SUCH = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); + +type Row = Record; +type SelectCall = { + fields: Record | undefined; + table: unknown; + where: unknown; +}; + +const camel = (name: string) => + name.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase()); + +const TERM = /"[a-z_]+"\."([a-z_]+)" = \$(\d+)/g; + +/** + * A row filter from a where made only of `column = value` terms joined by + * `and`. Anything else throws, so the fake never quietly ignores a condition + * it does not understand. + */ +function matcher(where: unknown) { + const { sql: text, params } = render(where); + const terms = [...text.matchAll(TERM)].map( + (match) => [camel(match[1]!), params[Number(match[2]) - 1]] as const, + ); + if (text.replace(TERM, "").replace(/[()\s]|and/g, "")) { + throw new Error(`The fake database cannot evaluate: ${text}`); + } + return (row: Row) => terms.every(([key, value]) => row[key] === value); +} + +// --------------------------------------------------------------------------- +// The stand-in database + +let adminRow: Row | undefined; +const store = new Map(); +const selects: SelectCall[] = []; +const updates: { table: unknown; values: Row }[] = []; +const inserts: { table: unknown; values: Row }[] = []; +// Answers a select outright; undefined hands it to the stored rows. +let route: ((call: SelectCall) => unknown[] | undefined) | null = null; + +const matching = (table: unknown, where: unknown) => { + const rows = store.get(table) ?? []; + return where ? rows.filter(matcher(where)) : [...rows]; +}; + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: (fields?: Record) => { + const call: SelectCall = { fields, table: null, where: null }; + selects.push(call); + const chain: any = { + then: (onOk: any, onErr: any) => + Promise.resolve(route?.(call) ?? matching(call.table, call.where)).then( + onOk, + onErr, + ), + }; + chain.from = (table: unknown) => { + call.table = table; + return chain; + }; + chain.where = (where: unknown) => { + call.where = where; + return chain; + }; + for (const method of [ + "innerJoin", + "leftJoin", + "groupBy", + "orderBy", + "limit", + "offset", + "for", + ]) { + chain[method] = () => chain; + } + return chain; + }, + insert: (table: unknown) => ({ + values: (values: Row) => { + inserts.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + // The config row already exists, so a create-if-missing finds nothing + // to return and reads the stored one. + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [values], + }); + }, + }), + update: (table: unknown) => ({ + set: (values: Row) => ({ + where: (where: unknown) => { + const hit = matching(table, where); + for (const row of hit) Object.assign(row, values); + updates.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => hit.map((row) => ({ ...row })), + }); + }, + }), + }), + delete: () => ({ where: async () => [] }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string | undefined) => + appRouter.createCaller({ + db: fakeDb, + session: userId ? { user: { id: userId } } : null, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); + +const admin = () => callerAs("admin_user"); + +const audits = () => + inserts.filter((write) => write.table === auditLogs).map((w) => w.values); + +const flagView = ( + over: Partial = {}, +): CheckInFlags.CheckInFlagView => ({ + id: FLAG_1, + participantName: "Ada Lovelace", + rule: "overlap", + eventName: "BNY Workshop", + conflictEventName: "Full Sail Fireside", + method: "code", + attempts: 3, + createdAt: new Date("2027-02-06T18:00:00Z"), + lastAttemptAt: new Date("2027-02-06T19:41:00Z"), + dismissed: false, + ...over, +}); + +const entryRow = (id: string, over: Row = {}) => ({ + id, + source: "portal", + sourceProjectId: null, + title: `Project ${id.slice(-1)}`, + submissionUrl: null, + description: "", + demoLinks: [], + videoUrl: null, + technologies: [], + universities: [], + prizeCategories: [], + participantCount: 2, + submittedAt: null, + deletedAt: null, + prizeIneligible: false, + createdAt: new Date("2027-02-06T12:00:00Z"), + ...over, +}); + +beforeEach(() => { + vi.clearAllMocks(); + cache.clear(); + selects.length = 0; + updates.length = 0; + inserts.length = 0; + route = null; + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + mockFlagsForEntries.mockReset().mockResolvedValue(new Map()); + mockFlaggedIds.mockReset().mockResolvedValue(new Set()); + mockOpenCount.mockReset().mockResolvedValue(0); + + store.clear(); + store.set(hackathons, [ + { id: HACK_A, name: "Hacklytics 2027", startDate: null, endDate: null }, + { id: HACK_B, name: "Hacklytics 2026", startDate: null, endDate: null }, + ]); + store.set(judgingConfigs, [ + { + hackathonId: HACK_A, + state: "draft", + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: false, + emergencyMode: false, + claimsStartedAt: null, + }, + { + hackathonId: HACK_B, + state: "draft", + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: false, + emergencyMode: false, + claimsStartedAt: null, + }, + ]); + store.set(hackathonCheckInFlags, [ + { + id: FLAG_1, + hackathonId: HACK_A, + dismissedAt: null, + dismissedById: null, + }, + { + id: FLAG_OTHER, + hackathonId: HACK_B, + dismissedAt: null, + dismissedById: null, + }, + ]); + store.set(judgingEntries, [ + { + id: ENTRY_1, + hackathonId: HACK_A, + prizeIneligibleAt: null, + prizeIneligibleById: null, + }, + { + id: ENTRY_OTHER, + hackathonId: HACK_B, + prizeIneligibleAt: null, + prizeIneligibleById: null, + }, + ]); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.listEntries", () => { + /** The page of projects the database returns, whatever filter it was given. */ + const wirePage = (rows: Row[], liveIds: string[] = rows.map((r) => r.id)) => { + route = (call) => { + if (call.table !== judgingEntries) return []; + if (call.fields && "value" in call.fields) + return [{ value: rows.length }]; + if (call.fields && "title" in call.fields) return rows; + return liveIds.map((id) => ({ id })); + }; + }; + + const pageQuery = () => + selects.find((call) => call.fields && "title" in call.fields)!; + const totalQuery = () => + selects.find((call) => call.fields && "value" in call.fields)!; + + it("puts each project's flags and its prize status on the project", async () => { + wirePage([entryRow(ENTRY_1, { prizeIneligible: true }), entryRow(ENTRY_2)]); + const flag = flagView(); + mockFlagsForEntries.mockResolvedValue(new Map([[ENTRY_1, [flag]]])); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + }); + + expect(result.entries.map((e) => e.id)).toEqual([ENTRY_1, ENTRY_2]); + expect(result.entries[0]).toMatchObject({ + checkInFlags: [flag], + prizeIneligible: true, + }); + expect(result.entries[1]).toMatchObject({ + checkInFlags: [], + prizeIneligible: false, + }); + // Asked for this page's projects only, in this edition. + expect(mockFlagsForEntries).toHaveBeenCalledTimes(1); + expect(mockFlagsForEntries).toHaveBeenCalledWith(fakeDb, HACK_A, [ + ENTRY_1, + ENTRY_2, + ]); + }); + + it("reads prize status from the entry's prize_ineligible_at column", async () => { + wirePage([entryRow(ENTRY_1)]); + + await admin().judging.admin.listEntries({ hackathonId: HACK_A }); + + expect(render(pageQuery().fields!.prizeIneligible).sql).toBe( + '"judging_entry"."prize_ineligible_at" is not null', + ); + }); + + it("does not look up flags for an empty page", async () => { + wirePage([]); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + }); + + expect(result.entries).toEqual([]); + expect(mockFlagsForEntries).not.toHaveBeenCalled(); + }); + + it("applies no flag condition unless asked", async () => { + wirePage([entryRow(ENTRY_1)]); + + await admin().judging.admin.listEntries({ hackathonId: HACK_A }); + await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: false, + }); + + expect(mockFlaggedIds).not.toHaveBeenCalled(); + for (const call of selects.filter((c) => c.fields && "title" in c.fields)) { + expect(render(call.where).sql).not.toMatch(/ in \(/); + } + }); + + it("narrows the page and the total to projects with an open flag", async () => { + wirePage([entryRow(ENTRY_2)], [ENTRY_1, ENTRY_2, ENTRY_3]); + mockFlaggedIds.mockResolvedValue(new Set([ENTRY_2])); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: true, + }); + + // Open flags are looked for among this edition's live projects. + expect(mockFlaggedIds).toHaveBeenCalledWith(fakeDb, HACK_A, [ + ENTRY_1, + ENTRY_2, + ENTRY_3, + ]); + const liveRead = selects.find( + (call) => + call.table === judgingEntries && + call.fields && + !("title" in call.fields) && + !("value" in call.fields), + )!; + const live = render(liveRead.where); + expect(live.sql).toContain('"judging_entry"."deleted_at" is null'); + expect(live.params).toContain(HACK_A); + + // The same condition sits in the query that pages and the one that counts, + // so the total is the flagged total and not the edition's. + for (const call of [pageQuery(), totalQuery()]) { + const { sql, params } = render(call.where); + expect(sql).toMatch(/"judging_entry"\."id" in \(\$\d+\)/); + expect(params).toContain(ENTRY_2); + expect(params).not.toContain(ENTRY_1); + expect(params).not.toContain(ENTRY_3); + } + expect(result.total).toBe(1); + }); + + it("matches nothing when no project has an open flag", async () => { + wirePage([], [ENTRY_1, ENTRY_2]); + + const result = await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: true, + }); + + for (const call of [pageQuery(), totalQuery()]) { + expect(render(call.where).sql).toMatch(/ and false\)$/); + } + expect(result).toMatchObject({ total: 0, entries: [] }); + }); + + it("combines the flag filter with the others", async () => { + wirePage([], [ENTRY_1]); + mockFlaggedIds.mockResolvedValue(new Set([ENTRY_1])); + + await admin().judging.admin.listEntries({ + hackathonId: HACK_A, + flagged: true, + source: "portal", + status: "live", + }); + + const { sql, params } = render(pageQuery().where); + expect(sql).toContain('"judging_entry"."source" = $'); + expect(sql).toMatch(/"judging_entry"\."id" in \(/); + expect(params).toEqual(expect.arrayContaining([HACK_A, "portal", ENTRY_1])); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.overview", () => { + const overview = () => + admin().judging.admin.overview({ hackathonId: HACK_A }); + + // Every other count reads from tables this test leaves empty. + beforeEach(() => { + route = (call) => + call.table === hackathons || call.table === judgingConfigs + ? undefined + : []; + }); + + it("counts the open check-in flags and nothing about them", async () => { + mockOpenCount.mockResolvedValue(4); + + const result = await overview(); + + expect(result.counts.checkInFlags).toEqual({ open: 4 }); + expect(mockOpenCount).toHaveBeenCalledWith(fakeDb, HACK_A); + }); + + it("reports zero when there are none", async () => { + const result = await overview(); + + expect(result.counts.checkInFlags).toEqual({ open: 0 }); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.dismissCheckInFlag", () => { + const flagRow = (id: string) => + store.get(hackathonCheckInFlags)!.find((row) => row.id === id)!; + + it("dismisses a flag and records who", async () => { + const result = await admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: true, + }); + + expect(result).toEqual({ dismissed: true }); + expect(flagRow(FLAG_1)).toMatchObject({ + dismissedAt: expect.any(Date), + dismissedById: "admin_user", + }); + expect(updates).toHaveLength(1); + expect(updates[0]!.table).toBe(hackathonCheckInFlags); + expect(Object.keys(updates[0]!.values).sort()).toEqual([ + "dismissedAt", + "dismissedById", + ]); + expect(audits()).toEqual([ + expect.objectContaining({ + userId: "admin_user", + action: "judging.dismissCheckInFlag", + resourceId: FLAG_1, + severity: "info", + metadata: { hackathonId: HACK_A }, + }), + ]); + }); + + it("opens a dismissed flag again", async () => { + Object.assign(flagRow(FLAG_1), { + dismissedAt: new Date(), + dismissedById: "someone", + }); + + const result = await admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: false, + }); + + expect(result).toEqual({ dismissed: false }); + expect(flagRow(FLAG_1)).toMatchObject({ + dismissedAt: null, + dismissedById: null, + }); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.reopenCheckInFlag", + resourceId: FLAG_1, + }), + ]); + }); + + it("refuses a flag that belongs to another hackathon", async () => { + await expect( + admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_OTHER, + dismissed: true, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(flagRow(FLAG_OTHER).dismissedAt).toBeNull(); + expect(audits()).toEqual([]); + }); + + it("refuses a flag that does not exist", async () => { + await expect( + admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: NO_SUCH, + dismissed: true, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(audits()).toEqual([]); + }); + + it("leaves the cached public results alone", async () => { + cache.set(`hackathon:${HACK_A}:results`, [{ id: "placing" }], 60); + + await admin().judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: true, + }); + + expect(cache.get(`hackathon:${HACK_A}:results`)).toEqual([ + { id: "placing" }, + ]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.setPrizeEligibility", () => { + const entry = (id: string) => + store.get(judgingEntries)!.find((row) => row.id === id)!; + const resultsKey = (hackathonId: string) => + `hackathon:${hackathonId}:results`; + + it("pulls a project from prizes and clears the cached results at once", async () => { + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + cache.set(resultsKey(HACK_B), [{ id: "last year" }], 60); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: false, + }); + + expect(result).toEqual({ changed: true }); + expect(entry(ENTRY_1)).toMatchObject({ + prizeIneligibleAt: expect.any(Date), + prizeIneligibleById: "admin_user", + }); + expect(updates).toHaveLength(1); + expect(updates[0]!.table).toBe(judgingEntries); + expect(cache.get(resultsKey(HACK_A))).toBeNull(); + // Only this edition's results. + expect(cache.get(resultsKey(HACK_B))).toEqual([{ id: "last year" }]); + expect(audits()).toEqual([ + expect.objectContaining({ + userId: "admin_user", + action: "judging.setPrizeEligibility", + resourceId: ENTRY_1, + severity: "warn", + metadata: { hackathonId: HACK_A, eligible: false }, + }), + ]); + }); + + it("puts a project back in the running and clears the cached results", async () => { + Object.assign(entry(ENTRY_1), { + prizeIneligibleAt: new Date(), + prizeIneligibleById: "someone", + }); + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: true, + }); + + expect(result).toEqual({ changed: true }); + expect(entry(ENTRY_1)).toMatchObject({ + prizeIneligibleAt: null, + prizeIneligibleById: null, + }); + expect(cache.get(resultsKey(HACK_A))).toBeNull(); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.setPrizeEligibility", + severity: "warn", + metadata: { hackathonId: HACK_A, eligible: true }, + }), + ]); + }); + + it("does nothing when the project is already in that state", async () => { + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: true, + }); + + expect(result).toEqual({ changed: false }); + expect(updates).toEqual([]); + expect(audits()).toEqual([]); + expect(cache.get(resultsKey(HACK_A))).toEqual([{ id: "placing" }]); + }); + + it("does not move the pull date when asked to pull again", async () => { + const pulledAt = new Date("2027-02-06T20:00:00Z"); + Object.assign(entry(ENTRY_1), { + prizeIneligibleAt: pulledAt, + prizeIneligibleById: "first_admin", + }); + + const result = await admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: false, + }); + + expect(result).toEqual({ changed: false }); + expect(entry(ENTRY_1)).toMatchObject({ + prizeIneligibleAt: pulledAt, + prizeIneligibleById: "first_admin", + }); + }); + + it("refuses a project that belongs to another hackathon", async () => { + cache.set(resultsKey(HACK_A), [{ id: "placing" }], 60); + cache.set(resultsKey(HACK_B), [{ id: "last year" }], 60); + + await expect( + admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_OTHER, + eligible: false, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(entry(ENTRY_OTHER).prizeIneligibleAt).toBeNull(); + expect(updates).toEqual([]); + expect(audits()).toEqual([]); + expect(cache.get(resultsKey(HACK_A))).toEqual([{ id: "placing" }]); + expect(cache.get(resultsKey(HACK_B))).toEqual([{ id: "last year" }]); + }); + + it("refuses a project that does not exist", async () => { + await expect( + admin().judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: NO_SUCH, + eligible: false, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(updates).toEqual([]); + }); + + it("refuses an edition that does not exist", async () => { + await expect( + admin().judging.admin.setPrizeEligibility({ + hackathonId: NO_SUCH, + entryId: ENTRY_1, + eligible: false, + }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(entry(ENTRY_1).prizeIneligibleAt).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +describe("who may use them", () => { + const writes = (caller: ReturnType) => [ + () => + caller.judging.admin.dismissCheckInFlag({ + hackathonId: HACK_A, + flagId: FLAG_1, + dismissed: true, + }), + () => + caller.judging.admin.setPrizeEligibility({ + hackathonId: HACK_A, + entryId: ENTRY_1, + eligible: false, + }), + ]; + const reads = (caller: ReturnType) => [ + () => caller.judging.admin.listEntries({ hackathonId: HACK_A }), + () => caller.judging.admin.overview({ hackathonId: HACK_A }), + ]; + + const untouched = () => { + expect(updates).toEqual([]); + expect( + store + .get(hackathonCheckInFlags)! + .every((row) => row.dismissedAt === null), + ).toBe(true); + expect( + store.get(judgingEntries)!.every((row) => row.prizeIneligibleAt === null), + ).toBe(true); + expect(mockFlagsForEntries).not.toHaveBeenCalled(); + expect(mockOpenCount).not.toHaveBeenCalled(); + }; + + it("refuses someone who is not an admin, reads and writes alike", async () => { + adminRow = undefined; + const caller = callerAs("hacker_user"); + + for (const run of [...reads(caller), ...writes(caller)]) { + await expect(run()).rejects.toMatchObject({ code: "FORBIDDEN" }); + } + untouched(); + }); + + it("refuses a volunteer, who only staffs the scanner", async () => { + adminRow = { userId: "vol_user", isActive: true, role: "volunteer" }; + const caller = callerAs("vol_user"); + + for (const run of [...reads(caller), ...writes(caller)]) { + await expect(run()).rejects.toMatchObject({ code: "FORBIDDEN" }); + } + untouched(); + }); + + it("refuses someone who is not signed in", async () => { + const caller = callerAs(undefined); + + for (const run of [...reads(caller), ...writes(caller)]) { + await expect(run()).rejects.toMatchObject({ code: "UNAUTHORIZED" }); + } + untouched(); + }); + + it("lets a bug tester read but not write", async () => { + adminRow = { userId: "qa_user", isActive: true, role: "bug_tester" }; + const caller = callerAs("qa_user"); + route = (call) => + call.table === hackathons || call.table === judgingConfigs + ? undefined + : []; + + for (const run of reads(caller)) await run(); + for (const run of writes(caller)) { + await expect(run()).rejects.toMatchObject({ code: "FORBIDDEN" }); + } + + expect(updates).toEqual([]); + expect(audits()).toEqual([]); + }); +}); diff --git a/packages/api/src/routers/judging/admin.ts b/packages/api/src/routers/judging/admin.ts index 241d4637..d596e6f0 100644 --- a/packages/api/src/routers/judging/admin.ts +++ b/packages/api/src/routers/judging/admin.ts @@ -17,6 +17,7 @@ import { sql, } from "drizzle-orm"; import { + hackathonCheckInFlags, hackathonProjects, judgingAnnouncements, judgingAppointments, @@ -90,6 +91,12 @@ import { stateChangeProblem, } from "../../services/judging/checklist"; import { notifyItineraryPosted } from "../../services/judging/notify"; +import { + checkInFlagsForEntries, + flaggedEntryIds, + openCheckInFlagCount, +} from "../../services/check-in-flags"; +import type { CheckInFlagView } from "../../services/check-in-flags"; // Organiser setup and live control of room judging: getting projects in, // shaping challenges, rooms and the rubric, the switches flipped on judging @@ -151,6 +158,25 @@ const evictResults = ( hackathonId: string, ) => cache.delete(`hackathon:${hackathonId}:results`); +/** Live projects of the edition that carry a check-in flag nobody dismissed. */ +async function entriesWithOpenFlags(db: JudgingDb, hackathonId: string) { + const live = await db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .where( + and( + eq(judgingEntries.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + ), + ); + const flagged = await flaggedEntryIds( + db, + hackathonId, + live.map((row) => row.id), + ); + return [...flagged]; +} + // Each of these finds the row's edition, takes the edition lock, then reads // the row again so the checks see it as it stands under the lock. @@ -285,6 +311,7 @@ export const judgingAdminRouter = createTRPCRouter({ [drafts], [announcements], [seated], + openCheckInFlags, ] = await Promise.all([ db .select({ @@ -375,6 +402,7 @@ export const judgingAdminRouter = createTRPCRouter({ gte(judgingPresence.lastSeenAt, seatedSince), ), ), + openCheckInFlagCount(db, id), ]); const hasEvaluations = (evaluations?.total ?? 0) + (drafts?.total ?? 0) > 0; @@ -422,6 +450,9 @@ export const judgingAdminRouter = createTRPCRouter({ }, announcements: announcements?.total ?? 0, judgesSeated: seated?.total ?? 0, + // Only how many wait for a decision. Who and why stay in the project + // list, which is organiser-only. + checkInFlags: { open: openCheckInFlags }, }, schedule: schedule ?? null, locks: { @@ -541,6 +572,8 @@ export const judgingAdminRouter = createTRPCRouter({ source: z.enum(["all", "devpost", "portal"]).default("all"), status: z.enum(["live", "removed", "all"]).default("live"), challengeId: uuid.optional(), + // Only projects with a check-in flag nobody has dismissed. + flagged: z.boolean().optional(), page: z.number().int().min(1).default(1), pageSize: z.number().int().min(1).max(200).default(50), }), @@ -549,6 +582,12 @@ export const judgingAdminRouter = createTRPCRouter({ const db = ctx.db as DrizzleDB; const pattern = input.search ? `%${escapeLike(input.search)}%` : null; + // Settled before the query so the filter narrows the page and the total + // together instead of thinning out a page afterwards. + const flaggedIds = input.flagged + ? await entriesWithOpenFlags(db, input.hackathonId) + : null; + const where = and( eq(judgingEntries.hackathonId, input.hackathonId), input.source === "all" @@ -559,6 +598,11 @@ export const judgingAdminRouter = createTRPCRouter({ : input.status === "removed" ? isNotNull(judgingEntries.deletedAt) : undefined, + flaggedIds + ? flaggedIds.length > 0 + ? inArray(judgingEntries.id, flaggedIds) + : sql`false` + : undefined, input.challengeId ? exists( db @@ -611,6 +655,7 @@ export const judgingAdminRouter = createTRPCRouter({ participantCount: judgingEntries.participantCount, submittedAt: judgingEntries.submittedAt, deletedAt: judgingEntries.deletedAt, + prizeIneligible: sql`${judgingEntries.prizeIneligibleAt} is not null`, createdAt: judgingEntries.createdAt, }) .from(judgingEntries) @@ -621,9 +666,9 @@ export const judgingAdminRouter = createTRPCRouter({ ]); const ids = rows.map((row) => row.id); - const [members, memberships, appointments, evaluations] = + const [members, memberships, appointments, evaluations, flags] = ids.length === 0 - ? [[], [], [], []] + ? [[], [], [], [], new Map()] : await Promise.all([ db .select({ @@ -675,6 +720,9 @@ export const judgingAdminRouter = createTRPCRouter({ ), ) .groupBy(judgingEvaluations.entryId), + // Names and events of other people: organiser-only, which this + // procedure is, and never part of a cached payload. + checkInFlagsForEntries(db, input.hackathonId, ids), ]); const countFor = (list: { entryId: string; value: number }[], id: string) => @@ -694,6 +742,7 @@ export const judgingAdminRouter = createTRPCRouter({ .map(({ entryId: _entryId, ...membership }) => membership), appointments: countFor(appointments, row.id), completeEvaluations: countFor(evaluations, row.id), + checkInFlags: flags.get(row.id) ?? [], })), }; }), @@ -935,6 +984,99 @@ export const judgingAdminRouter = createTRPCRouter({ return { restored: true }; }), + /** + * Clears a check-in flag an organiser has looked into, or opens it again. + * The flag stays on record either way; judges stop seeing the marker while it + * is dismissed. The edition is part of the match so a flag id from another + * edition cannot be touched from this one. + */ + dismissCheckInFlag: isAdmin + .input(hackathonInput.extend({ flagId: uuid, dismissed: z.boolean() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const [flag] = await db + .update(hackathonCheckInFlags) + .set( + input.dismissed + ? { dismissedAt: new Date(), dismissedById: ctx.userId } + : { dismissedAt: null, dismissedById: null }, + ) + .where( + and( + eq(hackathonCheckInFlags.id, input.flagId), + eq(hackathonCheckInFlags.hackathonId, input.hackathonId), + ), + ) + .returning({ id: hackathonCheckInFlags.id }); + if (!flag) throw notFound("That flag is gone. Refresh the list."); + + await recordAdminAction(db, { + userId: ctx.userId, + action: input.dismissed + ? "judging.dismissCheckInFlag" + : "judging.reopenCheckInFlag", + resourceId: flag.id, + metadata: { hackathonId: input.hackathonId }, + }); + return { dismissed: input.dismissed }; + }), + + /** + * Pulls a project from prizes, or puts it back. Judges still score it and + * the room schedule is untouched; only placements and public results leave + * it out, so the cached results go at once. + */ + setPrizeEligibility: isAdmin + .input(hackathonInput.extend({ entryId: uuid, eligible: z.boolean() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const changed = await db.transaction(async (tx) => { + await lockEdition(tx, input.hackathonId); + const [entry] = await tx + .select({ prizeIneligibleAt: judgingEntries.prizeIneligibleAt }) + .from(judgingEntries) + .where( + and( + eq(judgingEntries.id, input.entryId), + eq(judgingEntries.hackathonId, input.hackathonId), + ), + ) + .limit(1); + if (!entry) throw notFound("That project is gone. Refresh the list."); + if ((entry.prizeIneligibleAt === null) === input.eligible) return false; + + const now = new Date(); + await tx + .update(judgingEntries) + .set( + input.eligible + ? { + prizeIneligibleAt: null, + prizeIneligibleById: null, + updatedAt: now, + } + : { + prizeIneligibleAt: now, + prizeIneligibleById: ctx.userId, + updatedAt: now, + }, + ) + .where(eq(judgingEntries.id, input.entryId)); + return true; + }); + if (!changed) return { changed: false }; + + evictResults(ctx.cache, input.hackathonId); + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.setPrizeEligibility", + resourceId: input.entryId, + severity: "warn", + metadata: { hackathonId: input.hackathonId, eligible: input.eligible }, + }); + return { changed: true }; + }), + // --------------------------------------------------------------------- // Challenges diff --git a/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx b/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx index 1c699f39..588bc39f 100644 --- a/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx +++ b/sites/mainweb/app/(portal)/admin/judging/[hackathonId]/page.tsx @@ -112,6 +112,11 @@ export default function AdminEditionJudgingPage() { {stationCount(data.counts.rooms.active, data.counts.rooms.tables)} {plural(data.counts.judgesSeated, "judge")} seated + {data.counts.checkInFlags.open > 0 && ( + + {plural(data.counts.checkInFlags.open, "open check-in flag")} + + )} {data.config.emergencyMode && ( Emergency mode on )} diff --git a/sites/mainweb/components/admin/judging/EntriesList.tsx b/sites/mainweb/components/admin/judging/EntriesList.tsx index b37c2da0..d40387d9 100644 --- a/sites/mainweb/components/admin/judging/EntriesList.tsx +++ b/sites/mainweb/components/admin/judging/EntriesList.tsx @@ -23,12 +23,14 @@ import { import { ErrorLine, plural, + shortTime, useRefreshJudging, } from "@/components/admin/judging/shared"; import type { Challenge } from "@/components/admin/judging/shared"; type Entry = RouterOutputs["judging"]["admin"]["listEntries"]["entries"][number]; +type CheckInFlag = Entry["checkInFlags"][number]; const PAGE_SIZE = 50; @@ -44,6 +46,30 @@ const STATUSES = [ { id: "all", label: "Both" }, ] as const; +const FLAG_METHOD = { + code: "event code", + scan: "badge scan", + manual: "manual check-in", +} as const; + +/** + * One refused check-in as an organiser reads it. The data does not keep how + * many minutes apart the two check-ins were, so the too-soon rule says what + * the limit is. + */ +function flagLine(flag: CheckInFlag) { + const gap = flag.rule === "overlap" ? "same time" : "under 30 minutes apart"; + return [ + `${flag.participantName} tried ${flag.eventName} while checked into ${flag.conflictEventName} (${gap})`, + FLAG_METHOD[flag.method], + flag.attempts > 1 ? plural(flag.attempts, "try", "tries") : null, + shortTime(flag.lastAttemptAt), + flag.dismissed ? "dismissed" : null, + ] + .filter(Boolean) + .join(" · "); +} + /** The projects in judging, with their challenges and remove/restore. */ export function EntriesList({ hackathonId }: { hackathonId: string }) { const [search, setSearch] = useState(""); @@ -51,6 +77,7 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { const [source, setSource] = useState<(typeof SOURCES)[number]["id"]>("all"); const [state, setState] = useState<(typeof STATUSES)[number]["id"]>("live"); const [challengeId, setChallengeId] = useState(""); + const [flagged, setFlagged] = useState(false); const [page, setPage] = useState(1); useEffect(() => { @@ -65,6 +92,7 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { source, status: state, challengeId: challengeId || undefined, + flagged: flagged || undefined, page, pageSize: PAGE_SIZE, }, @@ -77,7 +105,8 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { const tree = (challenges.data?.challenges ?? []).filter((c) => !c.parentId); const data = entries.data; const pages = data ? Math.max(1, Math.ceil(data.total / data.pageSize)) : 1; - const filtered = !!query || source !== "all" || state !== "live" || !!challengeId; + const filtered = + !!query || source !== "all" || state !== "live" || !!challengeId || flagged; return (
@@ -172,6 +201,21 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { {option.label} ))} +
@@ -203,6 +247,7 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { {data.entries.map((entry) => ( @@ -242,9 +287,11 @@ export function EntriesList({ hackathonId }: { hackathonId: string }) { } function EntryRow({ + hackathonId, entry, challenges, }: { + hackathonId: string; entry: Entry; challenges: Challenge[]; }) { @@ -273,12 +320,27 @@ function EntryRow({ onSuccess: done, onError: (e) => setError(trpcErrorMessage(e, "Could not restore the project.")), }); + const dismiss = trpc.judging.admin.dismissCheckInFlag.useMutation({ + onSuccess: done, + onError: (e) => setError(trpcErrorMessage(e, "Could not update the flag.")), + }); + const setEligible = trpc.judging.admin.setPrizeEligibility.useMutation({ + onSuccess: done, + onError: (e) => + setError(trpcErrorMessage(e, "Could not change prize eligibility.")), + }); const optIns = entry.challenges.filter((c) => c.isOptIn); const derived = entry.challenges.filter((c) => !c.isOptIn); const linked = entry.members.filter((m) => m.linked).length; const removed = !!entry.deletedAt; - const busy = setChallenges.isPending || remove.isPending || restore.isPending; + const openFlags = entry.checkInFlags.filter((flag) => !flag.dismissed).length; + const busy = + setChallenges.isPending || + remove.isPending || + restore.isPending || + dismiss.isPending || + setEligible.isPending; const openEditor = () => { setChosen(new Set(optIns.map((c) => c.challengeId))); @@ -293,6 +355,14 @@ function EntryRow({

{entry.source === "devpost" ? "Devpost" : "Portal"} {removed && Removed} + {openFlags > 0 && ( + + {plural(openFlags, "check-in flag")} + + )} + {entry.prizeIneligible && ( + Not eligible for prizes + )}

{entry.title} @@ -330,6 +400,38 @@ function EntryRow({ {plural(entry.completeEvaluations, "evaluation")} complete

)} + {entry.checkInFlags.length > 0 && ( +
    + {entry.checkInFlags.map((flag) => ( +
  • + {flagLine(flag)} + +
    • + ))} +
+ )}
{!removed && ( @@ -344,6 +446,47 @@ function EntryRow({ Challenges )} + {entry.prizeIneligible ? ( + + ) : ( + !removed && ( + + ) + )} {removed ? ( +
+ )} From d0fb5f287fa1641a1605c5410fb68161c42234a1 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 15:12:55 -0400 Subject: [PATCH 09/27] feat(judging): mark flagged teams on the judge desk Judges see a bare Flagged marker and a note that organisers decide prize eligibility. Who was flagged and why stays with organisers. --- .../check-in-flags-desk.test.ts | 434 ++++++++++++++++++ packages/api/src/routers/judging/desk.ts | 8 + .../components/judge/EvaluationSheet.tsx | 11 + .../mainweb/components/judge/ProjectQueue.tsx | 20 +- 4 files changed, 472 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/.internal-tests/check-in-flags-desk.test.ts diff --git a/packages/api/src/.internal-tests/check-in-flags-desk.test.ts b/packages/api/src/.internal-tests/check-in-flags-desk.test.ts new file mode 100644 index 00000000..8727deac --- /dev/null +++ b/packages/api/src/.internal-tests/check-in-flags-desk.test.ts @@ -0,0 +1,434 @@ +/** + * Check-in rule-break flags at the judge desk. A judge, member or guest, is + * told only that a team is flagged: one boolean on each project in the list and + * on the project being judged. The hacker, the rule and the events stay with + * organisers, so these tests read the whole serialized response for them. + * + * The router runs for real against a stand-in database that answers each + * select by the tables it reads, which is also how the tests see what the desk + * did and did not read. + */ +import { describe, it, expect } from "vitest"; +import { getTableName } from "drizzle-orm"; +import { + hackathonCheckInFlags, + hackathonEvents, + hackathonParticipants, + hackathonProjects, + judgingChallenges, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingGuestSessions, + judgingJudges, + judges, + judgingRoomLinks, + judgingRooms, + users, +} from "@query/db"; +import { judgingDeskRouter } from "../routers/judging/desk"; + +const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; +const CHALLENGE = "0c4a8e10-2b3d-4c5e-9f60-7a8b9c0d1e2f"; +const JUDGE_ID = "1d5b9f21-3c4e-4d6f-8a70-8b9c0d1e2f3a"; +const ROOM = "2e6ca032-4d5f-4e70-9b81-9c0d1e2f3a4b"; + +// The two projects on the desk. One belongs to the flagged hacker's team. +const FLAGGED_ENTRY = "3f7db143-5e60-4f81-8c92-0d1e2f3a4b5c"; +const CLEAN_ENTRY = "40e8c254-6f71-4092-9da3-1e2f3a4b5c6d"; + +// Everything an organiser would see about the flag. None of it may reach a +// judge. +const FLAG_ID = "51f9d365-7082-41a3-8eb4-2f3a4b5c6d7e"; +const PARTICIPANT_ID = "62a0e476-8193-42b4-9fc5-3a4b5c6d7e8f"; +const HACKER_USER_ID = "73b1f587-92a4-43c5-80d6-4b5c6d7e8f90"; +const FLAGGED_TEAM = "84c2a698-a3b5-44d6-91e7-5c6d7e8f90a1"; +const OTHER_TEAM = "95d3b7a9-b4c6-45e7-a2f8-6d7e8f90a1b2"; +const HACKER_EMAIL = "mia.okafor@example.com"; +const HACKER_NAME = "Mia Okafor"; +const TRIED_EVENT = "Opening Ceremony"; +const CONFLICT_EVENT = "BNY Mellon Workshop"; + +const challenges = [ + { + id: CHALLENGE, + parentId: null, + mode: "unscheduled", + label: "General", + isGeneral: true, + isGroup: false, + }, +]; + +const entry = (id: string, title: string, participantCount: number) => ({ + id, + title, + description: "", + submissionUrl: null, + demoLinks: [], + videoUrl: null, + technologies: [], + participantCount, +}); + +const flaggedEntry = entry(FLAGGED_ENTRY, "Tidewater", 3); +const cleanEntry = entry(CLEAN_ENTRY, "Quiet Garden", 2); + +/** The key a stand-in answer is filed under: the table read, then its joins. */ +const via = (from: unknown, ...joins: unknown[]) => + [from, ...joins].map((table) => getTableName(table as never)).join(">"); + +type Answers = Record; + +/** + * A database whose selects resolve from `answers`, looked up by the tables the + * query reads and defaulting to no rows. Every select is kept, so a test can + * see which tables were read and how often. + */ +function fakeDb(answers: Answers) { + const reads: { key: string; from: string; fields: string[] }[] = []; + const db = { + select: (fields: Record) => { + const joined: unknown[] = []; + let from: unknown; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (error: unknown) => unknown, + ) => { + const key = via(from, ...joined); + reads.push({ + key, + from: getTableName(from as never), + fields: Object.keys(fields ?? {}), + }); + return Promise.resolve(answers[key] ?? []).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown) => { + joined.push(table); + return chain; + }; + } + for (const method of ["where", "orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }, + }; + return { db, reads }; +} + +/** What the desk reads before the project list or the project itself. */ +const deskBasics = (): Answers => ({ + [via(judges)]: [{ name: "Grace Hopper" }], + [via(judgingJudges)]: [{ id: JUDGE_ID, displayName: "Grace Hopper" }], + [via(judgingChallenges)]: challenges, +}); + +/** + * One open flag from a hacker on FLAGGED_TEAM. The rows for the tables only an + * organiser's view reads are filled in too, so a desk that went and fetched + * them would have something to leak. + */ +const flagRows = ( + opts: { portal?: unknown[]; members?: unknown[] } = {}, +): Answers => ({ + [via(hackathonCheckInFlags, hackathonParticipants)]: [ + { + id: FLAG_ID, + participantId: PARTICIPANT_ID, + teamId: FLAGGED_TEAM, + userId: HACKER_USER_ID, + }, + ], + [via(judgingEntries, hackathonProjects)]: opts.portal ?? [ + { entryId: FLAGGED_ENTRY, teamId: FLAGGED_TEAM, submittedById: null }, + { entryId: CLEAN_ENTRY, teamId: OTHER_TEAM, submittedById: null }, + ], + [via(judgingEntryMembers, judgingEntries)]: opts.members ?? [], + [via(users)]: [{ id: HACKER_USER_ID, email: HACKER_EMAIL }], + [via(hackathonEvents)]: [ + { id: "event-a", name: TRIED_EVENT }, + { id: "event-b", name: CONFLICT_EVENT }, + ], +}); + +const memberCtx = (db: unknown) => + ({ + db, + session: { user: { id: "judge-user", name: "Grace Hopper" } }, + userId: "judge-user", + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + }) as never; + +const guestAnswers = (): Answers => ({ + [via(judgingGuestSessions, judgingRoomLinks, judgingRooms, judgingJudges)]: [ + { + sessionId: "session-1", + hackathonId: HACK, + linkId: "link-1", + roomId: ROOM, + roomChallengeId: CHALLENGE, + judgeId: JUDGE_ID, + displayName: "Guest Judge", + }, + ], + [via(judgingChallenges)]: challenges, +}); + +const guestCtx = (db: unknown) => + ({ + db, + session: null, + userId: undefined, + clientIp: "127.0.0.2", + req: { + headers: { + get: (name: string) => + name.toLowerCase() === "cookie" + ? "query_judging_guest=guest-token" + : null, + }, + }, + }) as never; + +/** Words and ids that only organisers may see. */ +const organiserOnly = [ + "email", + "userId", + FLAG_ID, + PARTICIPANT_ID, + HACKER_USER_ID, + FLAGGED_TEAM, + HACKER_EMAIL, + HACKER_NAME, + TRIED_EVENT, + CONFLICT_EVENT, + "overlap", + "too_soon", + "attempts", +]; + +const expectNothingOrganiserOnly = (response: unknown) => { + const serialized = JSON.stringify(response); + for (const secret of organiserOnly) expect(serialized).not.toContain(secret); +}; + +/** The tables a desk read took data from, for the privacy checks. */ +const tablesRead = (reads: { from: string }[]) => + new Set(reads.map((read) => read.from)); + +const flagReads = (reads: T[]) => + reads.filter((read) => read.from === getTableName(hackathonCheckInFlags)); + +const judgesUnderTest = [ + { + kind: "member judge", + answers: deskBasics, + ctx: memberCtx, + }, + { + kind: "guest judge", + answers: guestAnswers, + ctx: guestCtx, + }, +] as const; + +describe.each(judgesUnderTest)( + "check-in flags at the desk for a $kind", + ({ answers, ctx }) => { + describe("desk.projects", () => { + const listAnswers = (flags: Answers): Answers => ({ + ...answers(), + [via(judgingEntries, judgingEntryChallenges)]: [ + { id: FLAGGED_ENTRY, title: "Tidewater", participantCount: 3 }, + { id: CLEAN_ENTRY, title: "Quiet Garden", participantCount: 2 }, + ], + ...flags, + }); + + it("marks the flagged team's project true and the other false", async () => { + const { db } = fakeDb(listAnswers(flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + const flagged = Object.fromEntries( + result.projects.map((project) => [project.id, project.flagged]), + ); + expect(flagged).toEqual({ + [FLAGGED_ENTRY]: true, + [CLEAN_ENTRY]: false, + }); + }); + + it("marks a project flagged through a member row, as a Devpost entry is", async () => { + const { db } = fakeDb( + listAnswers( + flagRows({ + portal: [], + members: [{ entryId: CLEAN_ENTRY, userId: HACKER_USER_ID }], + }), + ), + ); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect( + result.projects.find((project) => project.id === CLEAN_ENTRY) + ?.flagged, + ).toBe(true); + expect( + result.projects.find((project) => project.id === FLAGGED_ENTRY) + ?.flagged, + ).toBe(false); + }); + + it("is false for every project when nobody is flagged", async () => { + const { db } = fakeDb(listAnswers({})); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect(result.projects).toHaveLength(2); + expect(result.projects.map((project) => project.flagged)).toEqual([ + false, + false, + ]); + }); + + it("adds a boolean to each project and nothing else about the flag", async () => { + const { db } = fakeDb(listAnswers(flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + for (const project of result.projects) { + expect(typeof project.flagged).toBe("boolean"); + expect(Object.keys(project).sort()).toEqual([ + "challenges", + "flagged", + "hasDraft", + "id", + "myEvaluation", + "participantCount", + "presentation", + "title", + ]); + } + expectNothingOrganiserOnly(result); + }); + + it("asks for flags once for the whole list, not once per project", async () => { + const { db, reads } = fakeDb(listAnswers(flagRows())); + + await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect(flagReads(reads)).toHaveLength(1); + }); + + it("never reads the people or events behind a flag", async () => { + const { db, reads } = fakeDb(listAnswers(flagRows())); + + await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + const read = tablesRead(reads); + expect(read.has(getTableName(users))).toBe(false); + expect(read.has(getTableName(hackathonEvents))).toBe(false); + // Only the flag's own ids are selected, never a name or an email. + for (const select of flagReads(reads)) { + expect(select.fields).toEqual([ + "id", + "participantId", + "teamId", + "userId", + ]); + } + }); + + it("does not look for flags when there are no projects", async () => { + const { db, reads } = fakeDb({ ...answers(), ...flagRows() }); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .projects({ hackathonId: HACK, inMyRoomOnly: false }); + + expect(result.projects).toEqual([]); + expect(flagReads(reads)).toHaveLength(0); + }); + }); + + describe("desk.project", () => { + const sheetAnswers = ( + projectEntry: ReturnType, + flags: Answers, + ): Answers => ({ + ...answers(), + [via(judgingEntries, judgingEntryChallenges)]: [projectEntry], + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + ...flags, + }); + + it("marks the flagged team's project true", async () => { + const { db } = fakeDb(sheetAnswers(flaggedEntry, flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: FLAGGED_ENTRY }); + + expect(result.entry.flagged).toBe(true); + }); + + it("marks another team's project false", async () => { + const { db } = fakeDb(sheetAnswers(cleanEntry, flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: CLEAN_ENTRY }); + + expect(result.entry.flagged).toBe(false); + }); + + it("is false when nobody is flagged", async () => { + const { db } = fakeDb(sheetAnswers(flaggedEntry, {})); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: FLAGGED_ENTRY }); + + expect(result.entry.flagged).toBe(false); + }); + + it("tells the judge nothing more than the boolean", async () => { + const { db, reads } = fakeDb(sheetAnswers(flaggedEntry, flagRows())); + + const result = await judgingDeskRouter + .createCaller(ctx(db)) + .project({ hackathonId: HACK, entryId: FLAGGED_ENTRY }); + + expect(result.entry.flagged).toBe(true); + expectNothingOrganiserOnly(result); + const read = tablesRead(reads); + expect(read.has(getTableName(users))).toBe(false); + expect(read.has(getTableName(hackathonEvents))).toBe(false); + expect(flagReads(reads)).toHaveLength(1); + }); + }); + }, +); diff --git a/packages/api/src/routers/judging/desk.ts b/packages/api/src/routers/judging/desk.ts index 849147db..8bbe14db 100644 --- a/packages/api/src/routers/judging/desk.ts +++ b/packages/api/src/routers/judging/desk.ts @@ -24,6 +24,7 @@ import { } from "@query/db"; import type { DrizzleDB } from "@query/db"; import type { JudgingTx as Tx } from "../../services/judging/config"; +import { flaggedEntryIds } from "../../services/check-in-flags"; import { createTRPCRouter, publicProcedure } from "../../trpc"; import { isUniqueViolation } from "../../middleware/db-errors"; import { @@ -1052,6 +1053,9 @@ export const judgingDeskRouter = createTRPCRouter({ ), ); const tags = await tagsOf(db, judge, challenges, entryIds); + // One yes or no per project and nothing else. Who was flagged and why + // is for organisers; this list is polled by every judge, guests included. + const flagged = await flaggedEntryIds(db, input.hackathonId, entryIds); const presentationOf = new Map(presentations.map((p) => [p.entryId, p])); const evaluationOf = new Map(mine.map((e) => [e.entryId, e])); @@ -1081,6 +1085,7 @@ export const judgingDeskRouter = createTRPCRouter({ return { ...entry, challenges: tags.get(entry.id) ?? [], + flagged: flagged.has(entry.id), presentation: presentation ? { id: presentation.id, @@ -1150,6 +1155,8 @@ export const judgingDeskRouter = createTRPCRouter({ asc(judgingEntryMembers.createdAt), ); const tags = await tagsOf(db, judge, challenges, [entry.id]); + // A bare marker, as in the project list: never who or why. + const flagged = await flaggedEntryIds(db, input.hackathonId, [entry.id]); const rubric = await loadRubric(db, input.hackathonId); const switches = await judgingSwitches(db, input.hackathonId); @@ -1246,6 +1253,7 @@ export const judgingDeskRouter = createTRPCRouter({ ...entry, members: members.map((member) => member.name), challenges: tags.get(entry.id) ?? [], + flagged: flagged.has(entry.id), }, challenge: { id: challenge.id, diff --git a/sites/mainweb/components/judge/EvaluationSheet.tsx b/sites/mainweb/components/judge/EvaluationSheet.tsx index 6b36008e..7605d6b9 100644 --- a/sites/mainweb/components/judge/EvaluationSheet.tsx +++ b/sites/mainweb/components/judge/EvaluationSheet.tsx @@ -430,6 +430,17 @@ function EvaluationForm({ + {entry.flagged && ( +
+

Flagged by organisers

+

+ This team was flagged for a check-in rule. Judge the project as you + normally would. Organisers decide whether the team stays in the + prize running. +

+
+ )} +
{ diff --git a/sites/mainweb/components/judge/ProjectQueue.tsx b/sites/mainweb/components/judge/ProjectQueue.tsx index 5d4d0f7e..16537b12 100644 --- a/sites/mainweb/components/judge/ProjectQueue.tsx +++ b/sites/mainweb/components/judge/ProjectQueue.tsx @@ -188,7 +188,17 @@ export function ProjectQueue({

)}

{next.title}

-

{where(next, false)}

+
+

{where(next, false)}

+ {next.flagged && ( + + Flagged + + )} +
+ )} + {none?.challengeId === challenge.id && ( +

+ {none.needsRoom ? ( + <> + You are not seated where {none.label} is judged.{" "} + {onPickRoom && ( + + )} + + ) : ( + `Nothing left to grade in ${none.label}.` + )} +

+ )} + {findError && ( +

+ {findError} +

+ )} +

+ Each project is graded once per challenge, by one judge. +

+ + )} +
{context.challenges.length > 1 && (
@@ -333,7 +430,7 @@ export function ProjectQueue({ >

{mixed ? ( - `${rest.length - judged} ${next ? "more " : ""}to judge · ${judged} judged` + `${rest.length - judged - taken} ${next ? "more " : ""}to judge${judged > 0 ? ` · ${judged} judged` : ""}${taken > 0 ? ` · ${taken} with other judges` : ""}` ) : ( <> {rest.length} {next ? "more " : ""} From 6e735a03b3ed91155126787f9648901082df4c38 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 18:44:18 -0400 Subject: [PATCH 14/27] feat(judging): rank placings on judge-calibrated scores With every project graded once, one strict, lenient, bunched or drifting judge decides each project's score. calibration.ts fits an offset, a spread and a drift for each judge within each root challenge, so sponsor judges are only compared with their own challenge's judges, and judges with few grades are shrunk toward no correction. computePlacements ranks on the adjusted scores of live, prize-eligible entries. The model assumes the desk queue hands each judge a random sample of projects. judging_placement gains a nullable raw_score column holding the plain mean (additive; drizzle-kit push applies it). Only the organisers' judging.results.list returns it, with a per-judge calibration summary; public results and the judge desk are unchanged. The results tab captions adjusted lists, shows the raw mean beside each score, and describes each judge's habits in plain words. --- docs/packages/api.md | 2 + .../check-in-flags-results.test.ts | 283 +++++++- packages/api/src/routers/judging/results.ts | 167 ++++- .../src/services/judging/calibration.test.ts | 647 ++++++++++++++++++ .../api/src/services/judging/calibration.ts | 234 +++++++ .../src/services/judging/placements.test.ts | 206 +++++- .../api/src/services/judging/placements.ts | 62 +- .../api/src/services/judging/scoring.test.ts | 42 ++ packages/api/src/services/judging/scoring.ts | 10 +- packages/db/src/schemas/judging.ts | 4 + .../admin/judging/results/ResultsTab.tsx | 110 ++- .../admin/judging/results/calibration.test.ts | 93 +++ .../admin/judging/results/calibration.ts | 59 ++ 13 files changed, 1851 insertions(+), 68 deletions(-) create mode 100644 packages/api/src/services/judging/calibration.test.ts create mode 100644 packages/api/src/services/judging/calibration.ts create mode 100644 sites/mainweb/components/admin/judging/results/calibration.test.ts create mode 100644 sites/mainweb/components/admin/judging/results/calibration.ts diff --git a/docs/packages/api.md b/docs/packages/api.md index cf0f16b2..6be97624 100644 --- a/docs/packages/api.md +++ b/docs/packages/api.md @@ -93,6 +93,8 @@ Batch status updates allow up to **2500** ids. The global array cap in `scrubMar Judging itself is the `judging` namespace; its shared helpers are in `services/judging/` (`room-link`, `scope`, `scoring`, `config`). `hackathon.getResults` returns published placings from both the retired `hackathon_result` table and `judging_placement`, in one shape. It leaves out entries pulled from judging (`deleted_at`) or from prizes (`prize_ineligible_at`, set by `judging.admin.setPrizeEligibility`). Check-in rule-break flags (`services/check-in-flags.ts`) reach organisers through `judging.admin.listEntries` and `judging.results.list`, and judges through `judging.desk` as a bare `flagged` boolean; no hacker-facing, public or cached procedure returns them. +Placings rank on scores adjusted for judge bias (`services/judging/calibration.ts`): each evaluation is calibrated within its root challenge for its judge's strictness, spread and drift, on the assumption that `judging.desk.nextProject` (`services/judging/queue.ts`) hands every judge a random sample of projects. `judging_placement.raw_score` keeps the plain mean; only `judging.results.list` (organisers) returns it and the per-judge calibration, and `judging.desk` stays on raw means. + ## Club procedures (selected) **Members:** `me`, `register`, `update`, `history`, `myPass`, `rotatePass`, `checkStatus`; admin `list`, `getById`, `adminSearch`, `adminHistory`, `adminGrant`, `adminRevoke`. diff --git a/packages/api/src/.internal-tests/check-in-flags-results.test.ts b/packages/api/src/.internal-tests/check-in-flags-results.test.ts index 99a66221..d7454f2b 100644 --- a/packages/api/src/.internal-tests/check-in-flags-results.test.ts +++ b/packages/api/src/.internal-tests/check-in-flags-results.test.ts @@ -4,6 +4,11 @@ * how many flags are open on each placed team, and the public hackathon.getResults * drops a pulled team at once without ever carrying flag data. * + * It also covers what calibration adds to those procedures: compute stores the + * judge-adjusted score with the raw mean beside it, the organiser's list returns + * both and a per-judge summary, and the public payload carries neither the raw + * score nor anything about a judge. + * * The schema is the real one; only the database handle is a stand-in, handed * to the router through the context the way judge-edge.test.ts does. Its * selects answer by the table they read from, and the entries query applies @@ -24,6 +29,7 @@ import { judgingEvaluationDrafts, judgingEvaluationRatings, judgingEvaluations, + judgingJudges, judgingPlacements, } from "@query/db"; import { appRouter } from "../root"; @@ -35,7 +41,12 @@ const ADMIN_ROW = { userId: "admin_user", isActive: true, role: "admin" }; const render = (condition: unknown) => new PgDialect().sqlToQuery(condition as never); -type Query = { table: unknown; wheres: unknown[] }; +type Query = { + table: unknown; + wheres: unknown[]; + /** The columns the select asked for, to tell two reads of one table apart. */ + fields?: Record; +}; const mockSelect = vi.fn<(query: Query) => unknown[]>(); const mockFindMany = vi.fn(); @@ -44,8 +55,8 @@ const deletes: unknown[] = []; const settled = (value: T) => Object.assign(Promise.resolve(value), {}); -const selectChain = () => { - const query: Query = { table: null, wheres: [] }; +const selectChain = (fields?: Record) => { + const query: Query = { table: null, wheres: [], fields }; const chain: any = { then: (onOk: any, onErr: any) => Promise.resolve(mockSelect(query)).then(onOk, onErr), @@ -83,7 +94,7 @@ const fakeDb: any = { findMany: (...args: unknown[]) => mockFindMany(...args), }, }, - select: () => selectChain(), + select: (fields?: Record) => selectChain(fields), transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), insert: (table: unknown) => ({ values: (values: unknown) => { @@ -144,10 +155,26 @@ describe("judging.results.compute", () => { { id: "entry-b", deletedAt: null, prizeIneligibleAt: new Date() }, { id: "entry-c", deletedAt: null, prizeIneligibleAt: null }, ]; + const evaluationRow = ( + id: string, + entryId: string, + judgeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + isComplete: true, + }); + // One judge marked all three. With the pulled team left out of the + // calibration she has two grades, a mean of 3.5 and the pool's own spread, so + // nothing about her scores moves; with it counted, they would. const scored = [ - { id: "eval-a", entryId: "entry-a", challengeId: "ch-1", isComplete: true }, - { id: "eval-b", entryId: "entry-b", challengeId: "ch-1", isComplete: true }, - { id: "eval-c", entryId: "entry-c", challengeId: "ch-1", isComplete: true }, + evaluationRow("eval-a", "entry-a", "judge-1", 0), + evaluationRow("eval-b", "entry-b", "judge-1", 1), + evaluationRow("eval-c", "entry-c", "judge-1", 2), ]; const ratings = [ { evaluationId: "eval-a", value: 4 }, @@ -206,6 +233,7 @@ describe("judging.results.compute", () => { ["entry-a", "overall", 1, "4.00"], ["entry-c", "overall", 2, "3.00"], ]); + expect(placed.map((row) => row.rawScore)).toEqual(["4.00", "3.00"]); expect(result.placements).toBe(2); }); @@ -244,6 +272,82 @@ describe("judging.results.compute", () => { }); }); +describe("judging.results.compute with judge calibration", () => { + const entries = ["e1", "e2", "e3", "e4"].map((id) => ({ + id, + deletedAt: null, + prizeIneligibleAt: null, + })); + const evaluation = ( + id: string, + entryId: string, + judgeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + isComplete: true, + }); + + // A strict judge marked e1 and e2, a lenient one e3 and e4. Raw, e2 and e3 + // tie on 4.00 and the lenient judge's e4 runs away with first. + const wire = () => + mockSelect.mockImplementation(({ table }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + if (table === judgingPlacements) return [{ n: 0 }]; + if (table === judgingEntries) return entries; + if (table === judgingEvaluations) { + return [ + evaluation("v1", "e1", "strict", 0), + evaluation("v2", "e2", "strict", 1), + evaluation("v3", "e3", "lenient", 2), + evaluation("v4", "e4", "lenient", 3), + ]; + } + if (table === judgingEvaluationRatings) { + return [ + { evaluationId: "v1", value: 3 }, + { evaluationId: "v2", value: 4 }, + { evaluationId: "v3", value: 4 }, + { evaluationId: "v4", value: 5 }, + ]; + } + return []; + }); + + it("stores the judge-adjusted score with the raw mean beside it", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + // Pool mean 4. Two grades each gives an offset of 2/5 of the 0.5 gap: the + // strict judge is lifted 0.2, the lenient one lowered 0.2. Both spread + // their marks by the same 0.5, so neither is stretched. + expect( + placementsInserted().map((row) => [ + row.entryId, + row.placement, + row.score, + row.rawScore, + ]), + ).toEqual([ + ["e4", 1, "4.80", "5.00"], + ["e2", 2, "4.20", "4.00"], + ["e3", 3, "3.80", "4.00"], + ["e1", 4, "3.20", "3.00"], + ]); + }); +}); + // --------------------------------------------------------------------------- describe("judging.results.list", () => { const COMPUTED_AT = new Date("2027-02-07T09:00:00Z"); @@ -291,10 +395,23 @@ describe("judging.results.list", () => { placements: unknown[]; flags?: unknown[]; portal?: unknown[]; + /** Live entries, as the calibration summary reads them. */ + live?: unknown[]; + /** Complete evaluations, as the calibration summary reads them. */ + evaluations?: unknown[]; + ratings?: unknown[]; + judges?: unknown[]; + challenges?: unknown[]; }) => { - mockSelect.mockImplementation(({ table }) => { + mockSelect.mockImplementation(({ table, fields }) => { if (table === judgingPlacements) return opts.placements; - if (table === judgingEvaluations) return [{ n: 0 }]; + // The stale count reads one column; the calibration summary reads many. + if (table === judgingEvaluations) { + return fields && "n" in fields ? [{ n: 0 }] : (opts.evaluations ?? []); + } + if (table === judgingEvaluationRatings) return opts.ratings ?? []; + if (table === judgingJudges) return opts.judges ?? []; + if (table === judgingChallenges) return opts.challenges ?? []; if (table === hackathonCheckInFlags) return opts.flags ?? []; if (table === hackathonEvents) { return [ @@ -302,7 +419,7 @@ describe("judging.results.list", () => { { id: "event-2", name: "Sponsor Talk" }, ]; } - if (table === judgingEntries) return opts.portal ?? []; + if (table === judgingEntries) return opts.portal ?? opts.live ?? []; if (table === judgingEntryMembers) return []; return []; }); @@ -383,12 +500,154 @@ describe("judging.results.list", () => { "openCheckInFlags", "placement", "prizeIneligible", + "rawScore", "score", "title", "withdrawn", ]); }); + it("returns the raw score beside the adjusted one, and null on older rows", async () => { + wire({ + placements: [ + placement("p1", "entry-a", { score: "4.20", rawScore: "3.90" }), + placement("p2", "entry-b", { placement: 2, rawScore: null }), + ], + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const byEntry = Object.fromEntries( + rowsOf(data).map((row) => [row.entryId, row]), + ) as Record; + expect(byEntry["entry-a"].score).toBe(4.2); + expect(byEntry["entry-a"].rawScore).toBe(3.9); + expect(byEntry["entry-b"].rawScore).toBeNull(); + }); + + describe("judge calibration summary", () => { + const graded = ( + id: string, + entryId: string, + judgeId: string, + challengeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId, + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + isComplete: true, + }); + const challenges = [ + { id: "ch-sponsor", label: "Sponsor Prize" }, + { id: "ch-general", label: "General" }, + ]; + const judges = [ + { id: "strict", displayName: "Sam Strict", kind: "member" }, + { id: "lenient", displayName: "Lee Lenient", kind: "guest" }, + { id: "sponsor-guest", displayName: "Gus Guest", kind: "guest" }, + ]; + // Six table grades by two judges, and two by the sponsor's own guest. + const table = [ + ["g1", "strict", 3], + ["g2", "strict", 3], + ["g3", "strict", 4], + ["g4", "lenient", 4], + ["g5", "lenient", 5], + ["g6", "lenient", 5], + ] as const; + const evaluations = [ + ...table.map(([id, judgeId], i) => + graded(id, `entry-${i}`, judgeId, "ch-general", i), + ), + graded("s1", "entry-0", "sponsor-guest", "ch-sponsor", 10), + graded("s2", "entry-1", "sponsor-guest", "ch-sponsor", 11), + ]; + const ratings = [ + ...table.map(([id, , value]) => ({ evaluationId: id, value })), + { evaluationId: "s1", value: 2 }, + { evaluationId: "s2", value: 2 }, + ]; + const live = Array.from({ length: 6 }, (_, i) => ({ id: `entry-${i}` })); + + it("is computed on read, per challenge, one row for each judge", async () => { + wire({ + placements: [placement("p1", "entry-0")], + evaluations, + ratings, + judges, + challenges, + live, + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + expect(data.calibration.map((c) => c.challenge)).toEqual([ + "Sponsor Prize", + "General", + ]); + const [sponsor, general] = data.calibration; + // The sponsor's guest is compared with no one at the tables. + expect(sponsor!.judges).toHaveLength(1); + expect(sponsor!.judges[0]).toMatchObject({ + judgeName: "Gus Guest", + kind: "guest", + grades: 2, + offset: 0, + spread: 1, + drift: 0, + }); + expect(general!.judges.map((j) => [j.judgeName, j.grades])).toEqual([ + ["Lee Lenient", 3], + ["Sam Strict", 3], + ]); + const [lee, sam] = general!.judges; + expect(lee!.offset).toBeGreaterThan(0); + expect(sam!.offset).toBeLessThan(0); + // Rounded to two places. + for (const judge of general!.judges) { + for (const value of [judge.offset, judge.spread, judge.drift]) { + expect(value).toBe(Math.round(value * 100) / 100); + } + } + }); + + it("leaves a team pulled from prizes out of the summary", async () => { + wire({ + placements: [placement("p1", "entry-0")], + evaluations, + ratings, + judges, + challenges, + // Only the first four teams are live and prize-eligible. + live: live.slice(0, 4), + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const general = data.calibration.find((c) => c.challenge === "General"); + expect(general!.judges.map((j) => j.grades)).toEqual([1, 3]); + }); + + it("is empty before anything is computed", async () => { + wire({ placements: [], evaluations, ratings, judges, challenges, live }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + expect(data.calibration).toEqual([]); + }); + }); + it("reads no flags when nothing is placed", async () => { wire({ placements: [] }); @@ -410,6 +669,8 @@ describe("hackathon.getResults", () => { scopeLabel: "Overall", placement: 1, score: "4.25", + // Stored beside the score; it must never leave in the public payload. + rawScore: "3.75", entry: { id: `entry-${id}`, title: `Project ${id}`, @@ -481,6 +742,8 @@ describe("hackathon.getResults", () => { const text = JSON.stringify(results).toLowerCase(); expect(text).not.toContain("flag"); expect(text).not.toContain("ineligible"); + expect(text).not.toContain("3.75"); + expect(text).not.toContain("calibrat"); // The cached payload is built from placements alone: no select, so no // flag table, ran for it. expect(mockSelect).not.toHaveBeenCalled(); diff --git a/packages/api/src/routers/judging/results.ts b/packages/api/src/routers/judging/results.ts index 575d50ed..87c925d4 100644 --- a/packages/api/src/routers/judging/results.ts +++ b/packages/api/src/routers/judging/results.ts @@ -37,6 +37,7 @@ import { assertHackathonExists } from "../../services/judging/config"; import { reconcileDueDrafts } from "../../services/judging/evaluation-store"; import type { JudgingDb } from "../../services/judging/config"; import { lockJudgingConfig } from "../../services/judging/claims"; +import { calibrate } from "../../services/judging/calibration"; import { checkInFlagsForEntries } from "../../services/check-in-flags"; import { notifyResultsPublished, @@ -83,6 +84,123 @@ async function ratingsOf(db: JudgingDb, evaluationIds: string[]) { ); } +/** + * Every complete evaluation in the edition with its ratings: what compute ranks + * and the judge calibration summary reads. The ratings are joined rather than + * listed by id, since an edition's evaluations run to thousands. + */ +async function completeEvaluations(db: JudgingDb, hackathonId: string) { + const evaluations = await db + .select({ + id: judgingEvaluations.id, + entryId: judgingEvaluations.entryId, + challengeId: judgingEvaluations.challengeId, + judgeId: judgingEvaluations.judgeId, + submittedAt: judgingEvaluations.submittedAt, + isComplete: judgingEvaluations.isComplete, + }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.hackathonId, hackathonId), + eq(judgingEvaluations.isComplete, true), + ), + ); + const ratingRows = await db + .select({ + evaluationId: judgingEvaluationRatings.evaluationId, + value: judgingEvaluationRatings.value, + }) + .from(judgingEvaluationRatings) + .innerJoin( + judgingEvaluations, + eq(judgingEvaluations.id, judgingEvaluationRatings.evaluationId), + ) + .where( + and( + eq(judgingEvaluations.hackathonId, hackathonId), + eq(judgingEvaluations.isComplete, true), + ), + ); + const ratings = groupRatings(ratingRows); + return evaluations.map((evaluation) => ({ + ...evaluation, + ratings: ratings.get(evaluation.id) ?? [], + })); +} + +// Two decimals, and never a negative zero. +const round2 = (value: number) => Math.round(value * 100) / 100 || 0; + +/** + * How each judge's scores are being adjusted, grouped by challenge: the + * organiser's window onto what compute does. Read from the current evaluations + * of live, prize-eligible entries, the same ones compute calibrates. Judges and + * hackers never get this; only the organiser's results list returns it. + */ +async function judgeCalibrations(db: JudgingDb, hackathonId: string) { + const evaluations = await completeEvaluations(db, hackathonId); + const entries = await db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .where( + and( + eq(judgingEntries.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + isNull(judgingEntries.prizeIneligibleAt), + ), + ); + const live = new Set(entries.map((entry) => entry.id)); + const { judges: calibrated } = calibrate( + evaluations.filter((evaluation) => live.has(evaluation.entryId)), + ); + if (calibrated.length === 0) return []; + + const [judges, challenges] = await Promise.all([ + db + .select({ + id: judgingJudges.id, + displayName: judgingJudges.displayName, + kind: judgingJudges.kind, + }) + .from(judgingJudges) + .where(eq(judgingJudges.hackathonId, hackathonId)), + db + .select({ id: judgingChallenges.id, label: judgingChallenges.label }) + .from(judgingChallenges) + .where(eq(judgingChallenges.hackathonId, hackathonId)) + .orderBy( + asc(judgingChallenges.displayOrder), + asc(judgingChallenges.label), + ), + ]); + const judgeOf = new Map(judges.map((judge) => [judge.id, judge])); + + return challenges + .map((challenge) => ({ + challengeId: challenge.id, + challenge: challenge.label, + judges: calibrated + .filter( + (row) => row.challengeId === challenge.id && judgeOf.has(row.judgeId), + ) + .map((row) => { + const judge = judgeOf.get(row.judgeId)!; + return { + judgeId: row.judgeId, + judgeName: judge.displayName, + kind: judge.kind, + grades: row.evaluations, + offset: round2(row.offset), + spread: round2(row.spread), + drift: round2(row.drift), + }; + }) + .sort((a, b) => a.judgeName.localeCompare(b.judgeName)), + })) + .filter((challenge) => challenge.judges.length > 0); +} + // Computing and publishing placements, and the organiser's view of what each // judge submitted. export const judgingResultsRouter = createTRPCRouter({ @@ -145,48 +263,14 @@ export const judgingResultsRouter = createTRPCRouter({ asc(judgingChallenges.displayOrder), asc(judgingChallenges.label), ); - const evaluations = await tx - .select({ - id: judgingEvaluations.id, - entryId: judgingEvaluations.entryId, - challengeId: judgingEvaluations.challengeId, - isComplete: judgingEvaluations.isComplete, - }) - .from(judgingEvaluations) - .where( - and( - eq(judgingEvaluations.hackathonId, input.hackathonId), - eq(judgingEvaluations.isComplete, true), - ), - ); - // Joined rather than listed by id: an edition's evaluations run to - // thousands, and the filter is the same one as above. - const ratingRows = await tx - .select({ - evaluationId: judgingEvaluationRatings.evaluationId, - value: judgingEvaluationRatings.value, - }) - .from(judgingEvaluationRatings) - .innerJoin( - judgingEvaluations, - eq(judgingEvaluations.id, judgingEvaluationRatings.evaluationId), - ) - .where( - and( - eq(judgingEvaluations.hackathonId, input.hackathonId), - eq(judgingEvaluations.isComplete, true), - ), - ); - const ratings = groupRatings(ratingRows); + const evaluations = await completeEvaluations(tx, input.hackathonId); + // Each evaluation is calibrated for its judge before it is scored. const placements = computePlacements({ entryIds: entries.map((entry) => entry.id), memberships, challenges, - evaluations: evaluations.map((evaluation) => ({ - ...evaluation, - ratings: ratings.get(evaluation.id) ?? [], - })), + evaluations, }); await tx @@ -201,6 +285,7 @@ export const judgingResultsRouter = createTRPCRouter({ scopeLabel: placement.scopeLabel, placement: placement.placement, score: formatScore(placement.score), + rawScore: formatScore(placement.rawScore), evaluationCount: placement.evaluationCount, computedAt: now, })), @@ -311,6 +396,7 @@ export const judgingResultsRouter = createTRPCRouter({ scopeLabel: judgingPlacements.scopeLabel, placement: judgingPlacements.placement, score: judgingPlacements.score, + rawScore: judgingPlacements.rawScore, evaluationCount: judgingPlacements.evaluationCount, publishedAt: judgingPlacements.publishedAt, computedAt: judgingPlacements.computedAt, @@ -376,7 +462,10 @@ export const judgingResultsRouter = createTRPCRouter({ entryId: string; title: string; placement: number; + /** Adjusted for judge bias; this is what the placing ranks on. */ score: number | null; + /** The plain mean of the same evaluations. Null on older rows. */ + rawScore: number | null; evaluationCount: number; /** Pulled from judging since; getResults already hides it. */ withdrawn: boolean; @@ -399,6 +488,7 @@ export const judgingResultsRouter = createTRPCRouter({ title: row.title, placement: row.placement, score: toScore(row.score), + rawScore: toScore(row.rawScore), evaluationCount: row.evaluationCount, withdrawn: !!row.deletedAt, openCheckInFlags: openFlags.get(row.entryId) ?? 0, @@ -411,6 +501,9 @@ export const judgingResultsRouter = createTRPCRouter({ computedAt, publishedAt, evaluationsSinceCompute, + calibration: computedAt + ? await judgeCalibrations(db, input.hackathonId) + : [], scopes: [...scopes.values()].sort( (a, b) => Number(b.scope === OVERALL_SCOPE) - diff --git a/packages/api/src/services/judging/calibration.test.ts b/packages/api/src/services/judging/calibration.test.ts new file mode 100644 index 00000000..2ec47272 --- /dev/null +++ b/packages/api/src/services/judging/calibration.test.ts @@ -0,0 +1,647 @@ +import { describe, expect, it } from "vitest"; +import { K_A, K_B, K_D, calibrate } from "./calibration"; +import type { CalibrationEvaluation } from "./calibration"; + +// A small seeded generator (mulberry32), so every dataset below is the same on +// every run. No Math.random. +function seeded(seed: number) { + let state = seed >>> 0; + const uniform = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + // Roughly normal: the sum of four uniforms, centred and scaled to sd 1. + const normal = () => + (uniform() + uniform() + uniform() + uniform() - 2) * Math.sqrt(3); + const shuffle = (items: readonly T[]) => { + const copy = [...items]; + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(uniform() * (i + 1)); + [copy[i], copy[j]] = [copy[j] as T, copy[i] as T]; + } + return copy; + }; + return { uniform, normal, shuffle }; +} + +const DAY = Date.UTC(2027, 1, 6, 10, 0, 0); + +const evaluation = ( + id: string, + judgeId: string, + score: number, + over: Partial = {}, +): CalibrationEvaluation => ({ + id, + judgeId, + challengeId: "general", + entryId: `entry-${id}`, + submittedAt: new Date(DAY), + isComplete: true, + ratings: [score], + ...over, +}); + +const clampRating = (value: number) => Math.min(5, Math.max(1, value)); + +type Habit = { + judgeId: string; + /** Added to every score. */ + offset?: number; + /** Multiplies the distance from 3. */ + spread?: number; + /** Points added per grade, in grading order. */ + drift?: number; + noise?: number; +}; + +/** + * A pool of projects with a known quality, dealt out at random to judges with + * known habits. Returns the evaluations and each project's true quality. + */ +function pool(options: { + projects: number; + judges: Habit[]; + seed: number; + challengeId?: string; +}) { + const random = seeded(options.seed); + const quality = Array.from( + { length: options.projects }, + () => 1.8 + 2.4 * random.uniform(), + ); + const order = random.shuffle(quality.map((_, index) => index)); + const evaluations: CalibrationEvaluation[] = []; + const truth = new Map(); + + order.forEach((project, position) => { + const habit = options.judges[position % options.judges.length]!; + const turn = Math.floor(position / options.judges.length); + const raw = + 3 + + (habit.spread ?? 1) * ((quality[project] as number) - 3) + + (habit.offset ?? 0) + + (habit.drift ?? 0) * turn + + (habit.noise ?? 0.2) * random.normal(); + const id = `${options.challengeId ?? "general"}-${habit.judgeId}-${turn}`; + truth.set(id, quality[project] as number); + evaluations.push( + evaluation(id, habit.judgeId, clampRating(raw), { + challengeId: options.challengeId ?? "general", + entryId: `project-${project}`, + submittedAt: new Date(DAY + turn * 60_000), + }), + ); + }); + return { evaluations, truth }; +} + +function ranks(values: readonly number[]) { + const sorted = values + .map((value, index) => ({ value, index })) + .sort((a, b) => a.value - b.value); + const result = new Array(values.length); + sorted.forEach(({ index }, position) => { + result[index] = position; + }); + return result; +} + +/** Spearman rank correlation. */ +function spearman(a: readonly number[], b: readonly number[]) { + const ra = ranks(a); + const rb = ranks(b); + const mean = (ra.length - 1) / 2; + let top = 0; + let sa = 0; + let sb = 0; + for (let i = 0; i < ra.length; i++) { + top += (ra[i]! - mean) * (rb[i]! - mean); + sa += (ra[i]! - mean) ** 2; + sb += (rb[i]! - mean) ** 2; + } + return top / Math.sqrt(sa * sb); +} + +const scoreOf = (e: CalibrationEvaluation) => + e.ratings.reduce((sum, rating) => sum + rating, 0) / e.ratings.length; + +function sdOf(values: readonly number[]) { + const mean = values.reduce((sum, value) => sum + value, 0) / values.length; + return Math.sqrt( + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / values.length, + ); +} + +/** Least-squares slope of the values on their position. */ +function slopeOf(values: readonly number[]) { + const centre = (values.length - 1) / 2; + const mean = values.reduce((sum, value) => sum + value, 0) / values.length; + let along = 0; + let across = 0; + values.forEach((value, index) => { + along += (index - centre) * (value - mean); + across += (index - centre) ** 2; + }); + return along / across; +} + +describe("calibrate", () => { + it("uses the documented pseudo-evaluation counts", () => { + expect([K_A, K_B, K_D]).toEqual([3, 6, 10]); + }); + + it("returns nothing for no evaluations", () => { + const result = calibrate([]); + expect(result.adjusted.size).toBe(0); + expect(result.judges).toEqual([]); + }); + + it("leaves one evaluation as it is", () => { + const result = calibrate([evaluation("only", "judge", 4.25)]); + expect(result.adjusted.get("only")).toBeCloseTo(4.25, 10); + expect(result.judges).toEqual([ + { + judgeId: "judge", + challengeId: "general", + evaluations: 1, + offset: 0, + spread: 1, + drift: 0, + }, + ]); + }); + + it("scores an evaluation as the mean of its ratings", () => { + const result = calibrate([ + evaluation("a", "judge", 0, { ratings: [4, 5, 3, 4] }), + ]); + expect(result.adjusted.get("a")).toBeCloseTo(4, 10); + }); + + it("ignores incomplete evaluations and ones with no ratings", () => { + const result = calibrate([ + evaluation("done", "judge", 4), + evaluation("draft", "judge", 1, { isComplete: false }), + evaluation("blank", "judge", 0, { ratings: [] }), + ]); + expect([...result.adjusted.keys()]).toEqual(["done"]); + expect(result.judges).toHaveLength(1); + expect(result.judges[0]?.evaluations).toBe(1); + }); + + it("does not correct the offset of the only judge in a pool", () => { + const { evaluations } = pool({ + projects: 30, + judges: [{ judgeId: "alone", offset: 0.9 }], + seed: 5, + }); + const result = calibrate(evaluations); + // Nobody to compare them with: no offset and no spread. + expect(result.judges[0]?.offset).toBeCloseTo(0, 10); + expect(result.judges[0]?.spread).toBeCloseTo(1, 10); + for (const e of evaluations) { + expect(result.adjusted.get(e.id)).toBeCloseTo(scoreOf(e), 0); + } + }); + + it("matches the model on a small pool worked by hand", () => { + // Pool scores 2, 4, 4, 5: mu = 3.75. Taking each judge's mean out leaves + // -1, 1 and -0.5, 0.5, so the sd a judge is measured against is + // sqrt(2.5 / 4), not the plain pool sd of sqrt(1.1875). + const mu = 3.75; + const sd = Math.sqrt(2.5 / 4); + // Judge a has 2 and 4: n = 2, so no drift. Mean 3, population sd 1. + const offset = (2 / (2 + K_A)) * (3 - mu); + const spread = 1 + (2 / (2 + K_B)) * (1 / sd - 1); + const result = calibrate([ + evaluation("a1", "a", 2, { submittedAt: new Date(DAY) }), + evaluation("a2", "a", 4, { submittedAt: new Date(DAY + 60_000) }), + evaluation("b1", "b", 4, { submittedAt: new Date(DAY) }), + evaluation("b2", "b", 5, { submittedAt: new Date(DAY + 60_000) }), + ]); + expect(result.adjusted.get("a1")).toBeCloseTo( + mu + (2 - mu - offset) / spread, + 10, + ); + expect(result.adjusted.get("a2")).toBeCloseTo( + mu + (4 - mu - offset) / spread, + 10, + ); + const a = result.judges.find((j) => j.judgeId === "a"); + expect(a?.offset).toBeCloseTo(offset, 10); + expect(a?.spread).toBeCloseTo(spread, 10); + expect(a?.drift).toBe(0); + }); + + describe("strict and lenient judges", () => { + const { evaluations, truth } = pool({ + projects: 80, + judges: [ + { judgeId: "strict", offset: -0.8 }, + { judgeId: "lenient", offset: 0.8 }, + ], + seed: 11, + }); + const result = calibrate(evaluations); + + it("rank projects far closer to their true quality than raw means do", () => { + const quality = evaluations.map((e) => truth.get(e.id) as number); + const raw = spearman(quality, evaluations.map(scoreOf)); + const adjusted = spearman( + quality, + evaluations.map((e) => result.adjusted.get(e.id) as number), + ); + expect(raw).toBeLessThan(0.65); + expect(adjusted).toBeGreaterThan(0.9); + expect(adjusted - raw).toBeGreaterThan(0.25); + }); + + it("moves their scores by roughly the size of their bias", () => { + const moves = evaluations.map((e) => + Math.abs((result.adjusted.get(e.id) as number) - scoreOf(e)), + ); + const average = moves.reduce((sum, move) => sum + move, 0) / moves.length; + expect(average).toBeGreaterThan(0.6); + expect(average).toBeLessThan(1); + }); + + it("reports which one is strict and which lenient", () => { + const strict = result.judges.find((j) => j.judgeId === "strict"); + const lenient = result.judges.find((j) => j.judgeId === "lenient"); + expect(strict?.offset).toBeLessThan(-0.5); + expect(lenient?.offset).toBeGreaterThan(0.5); + expect(strict?.evaluations).toBe(40); + }); + + it("does not mistake the gap between them for bunched scores", () => { + // Both mark with an ordinary spread. The pool's plain spread also holds + // the gap between their means; measured against that, both would look + // bunched and every score would be stretched. + for (const judge of result.judges) { + expect(judge.spread).toBeGreaterThan(0.9); + expect(judge.spread).toBeLessThan(1.1); + } + const quality = evaluations.map((e) => truth.get(e.id) as number); + const adjusted = evaluations.map( + (e) => result.adjusted.get(e.id) as number, + ); + expect(sdOf(adjusted)).toBeLessThan(sdOf(quality) * 1.15); + }); + + it("brings the two judges' average scores together", () => { + const meanOf = ( + judgeId: string, + score: (e: CalibrationEvaluation) => number, + ) => { + const mine = evaluations.filter((e) => e.judgeId === judgeId); + return mine.reduce((sum, e) => sum + score(e), 0) / mine.length; + }; + const adjusted = (e: CalibrationEvaluation) => + result.adjusted.get(e.id) as number; + expect( + meanOf("lenient", scoreOf) - meanOf("strict", scoreOf), + ).toBeGreaterThan(1.3); + expect( + Math.abs(meanOf("lenient", adjusted) - meanOf("strict", adjusted)), + ).toBeLessThan(0.2); + }); + }); + + describe("a bunched judge", () => { + const { evaluations } = pool({ + projects: 80, + judges: [{ judgeId: "bunched", spread: 0.3 }, { judgeId: "normal" }], + seed: 23, + }); + const result = calibrate(evaluations); + const of = (judgeId: string) => + evaluations.filter((e) => e.judgeId === judgeId); + + it("is stretched toward the pool's spread", () => { + const bunched = of("bunched"); + const before = sdOf(bunched.map(scoreOf)); + const after = sdOf( + bunched.map((e) => result.adjusted.get(e.id) as number), + ); + expect( + result.judges.find((j) => j.judgeId === "bunched")?.spread, + ).toBeLessThan(0.7); + expect(after).toBeGreaterThan(before * 1.4); + }); + + it("leaves the normal judge's spread close to where it was", () => { + const normal = of("normal"); + const before = sdOf(normal.map(scoreOf)); + const after = sdOf( + normal.map((e) => result.adjusted.get(e.id) as number), + ); + expect(Math.abs(after - before)).toBeLessThan(before * 0.3); + }); + }); + + describe("a drifting judge", () => { + const { evaluations } = pool({ + projects: 90, + judges: [{ judgeId: "tiring", drift: -0.05 }, { judgeId: "steady" }], + seed: 31, + }); + const result = calibrate(evaluations); + + it("is detrended in grading order", () => { + const mine = evaluations + .filter((e) => e.judgeId === "tiring") + .sort((a, b) => a.submittedAt.getTime() - b.submittedAt.getTime()); + const before = slopeOf(mine.map(scoreOf)); + const after = slopeOf( + mine.map((e) => result.adjusted.get(e.id) as number), + ); + expect(before).toBeLessThan(-0.04); + expect(Math.abs(after)).toBeLessThan(Math.abs(before) * 0.4); + }); + + it("reports a drift that is negative for a judge who got harsher", () => { + const tiring = result.judges.find((j) => j.judgeId === "tiring"); + const steady = result.judges.find((j) => j.judgeId === "steady"); + // Lost about 0.05 a grade over 44 grades, less the shrink. + expect(tiring?.drift).toBeLessThan(-1.2); + expect(Math.abs(steady?.drift ?? 1)).toBeLessThan(0.7); + }); + }); + + it("breaks a tie in submission time by id when ordering a judge's grades", () => { + const climbing = calibrate([ + evaluation("a", "judge", 1), + evaluation("b", "judge", 3), + evaluation("c", "judge", 5), + ]); + const falling = calibrate([ + evaluation("c", "judge", 1), + evaluation("b", "judge", 3), + evaluation("a", "judge", 5), + ]); + expect(climbing.judges[0]?.drift).toBeGreaterThan(0); + expect(falling.judges[0]?.drift).toBeLessThan(0); + }); + + it("barely corrects a judge with a single grade", () => { + const { evaluations } = pool({ + projects: 40, + judges: [{ judgeId: "a" }, { judgeId: "b", offset: 0.3 }], + seed: 41, + }); + const once = evaluation("once", "visitor", 4.9); + const all = [...evaluations, once]; + const result = calibrate(all); + const visitor = result.judges.find((j) => j.judgeId === "visitor"); + expect(visitor?.evaluations).toBe(1); + expect(visitor?.spread).toBe(1); + expect(visitor?.drift).toBe(0); + + // One grade is mostly the project, not the judge: the pull toward the pool + // is 1 / (1 + K_A) of the distance, a quarter. + const mu = all.reduce((sum, e) => sum + scoreOf(e), 0) / all.length; + expect(visitor?.offset).toBeCloseTo((4.9 - mu) / (1 + K_A), 10); + const moved = Math.abs(4.9 - (result.adjusted.get("once") as number)); + expect(moved).toBeLessThan(Math.abs(4.9 - mu) * 0.3); + }); + + it("leaves unbiased judges almost where they were", () => { + const { evaluations } = pool({ + projects: 90, + judges: [{ judgeId: "a" }, { judgeId: "b" }, { judgeId: "c" }], + seed: 53, + }); + const result = calibrate(evaluations); + const moves = evaluations.map((e) => + Math.abs((result.adjusted.get(e.id) as number) - scoreOf(e)), + ); + const average = moves.reduce((sum, move) => sum + move, 0) / moves.length; + // Chance alone gives each judge's thirty projects a slightly different + // mean, so some movement is expected; a biased judge moves several times as far. + expect(average).toBeLessThan(0.2); + expect(Math.max(...moves)).toBeLessThan(0.6); + expect( + spearman( + evaluations.map(scoreOf), + evaluations.map((e) => result.adjusted.get(e.id) as number), + ), + ).toBeGreaterThan(0.95); + for (const judge of result.judges) { + expect(Math.abs(judge.offset)).toBeLessThan(0.2); + expect(judge.spread).toBeGreaterThan(0.8); + expect(judge.spread).toBeLessThan(1.25); + } + }); + + describe("pools", () => { + const table = pool({ + projects: 40, + judges: [{ judgeId: "t1" }, { judgeId: "t2", offset: 0.5 }], + seed: 61, + challengeId: "general", + }).evaluations; + // A sponsor whose guests are harsh, and one guest who also judges at the tables. + const sponsor = pool({ + projects: 20, + judges: [ + { judgeId: "g1", offset: -1 }, + { judgeId: "t1", offset: -1 }, + ], + seed: 67, + challengeId: "sponsor", + }).evaluations; + + it("never mix: a pool calibrates the same alone or beside another", () => { + const together = calibrate([...table, ...sponsor]); + const alone = calibrate(table); + for (const e of table) { + expect(together.adjusted.get(e.id)).toBe(alone.adjusted.get(e.id)); + } + const sponsorAlone = calibrate(sponsor); + for (const e of sponsor) { + expect(together.adjusted.get(e.id)).toBe( + sponsorAlone.adjusted.get(e.id), + ); + } + }); + + it("give a judge in two challenges separate parameters in each", () => { + const together = calibrate([...table, ...sponsor]); + const both = together.judges.filter((j) => j.judgeId === "t1"); + expect(both.map((j) => j.challengeId).sort()).toEqual([ + "general", + "sponsor", + ]); + const [general, other] = [ + both.find((j) => j.challengeId === "general")!, + both.find((j) => j.challengeId === "sponsor")!, + ]; + expect(general.evaluations).toBe(20); + expect(other.evaluations).toBe(10); + // Stricter than the lenient t2 at the tables; as harsh as the sponsor's + // own guest, so near level there (ten grades each leave some chance in it). + expect(general.offset).toBeLessThan(-0.1); + expect(Math.abs(other.offset)).toBeLessThan(0.25); + expect(other.offset - general.offset).toBeGreaterThan(0.2); + }); + }); + + it("keeps every output between 1 and 5", () => { + const random = seeded(71); + const extreme: CalibrationEvaluation[] = []; + for (let i = 0; i < 60; i++) { + // A judge who only ever gives 5s next to one who only gives 1s and 2s, + // and one who swings between the ends. + const judgeId = ["high", "low", "swing"][i % 3]!; + const score = + judgeId === "high" + ? 5 + : judgeId === "low" + ? 1 + Math.floor(random.uniform() * 2) + : random.uniform() < 0.5 + ? 1 + : 5; + extreme.push( + evaluation(`x-${i}`, judgeId, score, { + submittedAt: new Date(DAY + i * 60_000), + }), + ); + } + const result = calibrate(extreme); + expect(result.adjusted.size).toBe(60); + for (const value of result.adjusted.values()) { + expect(value).toBeGreaterThanOrEqual(1); + expect(value).toBeLessThanOrEqual(5); + } + for (const judge of result.judges) { + expect(judge.spread).toBeGreaterThanOrEqual(0.5); + expect(judge.spread).toBeLessThanOrEqual(2); + } + }); + + describe("a judge who gives every project the same score", () => { + const { evaluations } = pool({ + projects: 40, + judges: [{ judgeId: "varied" }, { judgeId: "flat", spread: 0, noise: 0 }], + seed: 83, + }); + const flat = evaluations.filter((e) => e.judgeId === "flat"); + const result = calibrate(evaluations); + + it("is clamped at a spread of 0.5 and given one sane score throughout", () => { + expect(new Set(flat.map(scoreOf)).size).toBe(1); + const judge = result.judges.find((j) => j.judgeId === "flat"); + expect(judge?.spread).toBe(0.5); + expect(judge?.drift).toBeCloseTo(0, 10); + + const values = flat.map((e) => result.adjusted.get(e.id) as number); + for (const value of values) { + expect(Number.isFinite(value)).toBe(true); + expect(value).toBeGreaterThanOrEqual(1); + expect(value).toBeLessThanOrEqual(5); + expect(value).toBeCloseTo(values[0] as number, 10); + } + }); + + it("does not disturb the other judge", () => { + const varied = evaluations.filter((e) => e.judgeId === "varied"); + for (const e of varied) { + const adjusted = result.adjusted.get(e.id) as number; + expect(Number.isFinite(adjusted)).toBe(true); + expect(Math.abs(adjusted - scoreOf(e))).toBeLessThan(0.5); + } + }); + }); + + it("copes with a pool where every score is the same", () => { + const result = calibrate( + ["a", "b", "c", "d"].map((id, i) => + evaluation(id, i % 2 ? "x" : "y", 3.5, { + submittedAt: new Date(DAY + i * 60_000), + }), + ), + ); + for (const value of result.adjusted.values()) + expect(value).toBeCloseTo(3.5, 10); + for (const judge of result.judges) { + expect(judge.spread).toBe(1); + expect(judge.offset).toBeCloseTo(0, 10); + } + }); + + it("handles projects with several evaluations, as older data has", () => { + const { evaluations } = pool({ + projects: 30, + judges: [ + { judgeId: "a", offset: -0.6 }, + { judgeId: "b", offset: 0.6 }, + { judgeId: "c" }, + ], + seed: 97, + }); + // Every judge also graded the first ten projects. + const repeats = evaluations.slice(0, 10).flatMap((first) => + ["a", "b", "c"] + .filter((judgeId) => judgeId !== first.judgeId) + .map((judgeId) => + evaluation(`again-${first.id}-${judgeId}`, judgeId, scoreOf(first), { + entryId: first.entryId, + submittedAt: new Date(DAY + 3_600_000), + }), + ), + ); + const all = [...evaluations, ...repeats]; + const result = calibrate(all); + expect(result.adjusted.size).toBe(all.length); + expect(new Set(all.map((e) => e.entryId)).size).toBe(30); + for (const value of result.adjusted.values()) { + expect(value).toBeGreaterThanOrEqual(1); + expect(value).toBeLessThanOrEqual(5); + } + }); + + it("gives the same answer whatever order the evaluations come in", () => { + const { evaluations } = pool({ + projects: 60, + judges: [ + { judgeId: "strict", offset: -0.7, drift: -0.03 }, + { judgeId: "lenient", offset: 0.7, spread: 0.6 }, + { judgeId: "plain" }, + ], + seed: 101, + }); + const other = pool({ + projects: 20, + judges: [{ judgeId: "plain", offset: 0.4 }, { judgeId: "guest" }], + seed: 103, + challengeId: "sponsor", + }).evaluations; + const all = [...evaluations, ...other]; + + const forward = calibrate(all); + const shuffled = calibrate(seeded(7).shuffle(all)); + const reversed = calibrate([...all].reverse()); + + for (const result of [shuffled, reversed]) { + expect(result.judges).toEqual(forward.judges); + expect([...result.adjusted.entries()].sort()).toEqual( + [...forward.adjusted.entries()].sort(), + ); + } + }); + + it("does not change its input", () => { + const { evaluations } = pool({ + projects: 12, + judges: [{ judgeId: "a" }, { judgeId: "b" }], + seed: 107, + }); + const before = JSON.stringify(evaluations); + calibrate(evaluations); + expect(JSON.stringify(evaluations)).toBe(before); + }); +}); diff --git a/packages/api/src/services/judging/calibration.ts b/packages/api/src/services/judging/calibration.ts new file mode 100644 index 00000000..b8c90298 --- /dev/null +++ b/packages/api/src/services/judging/calibration.ts @@ -0,0 +1,234 @@ +// Judge calibration. Each project is graded once per challenge, by one judge, +// so a strict, lenient, bunched or drifting judge decides the score of every +// project they happened to grade. This takes that judge out of the score. +// +// It is valid only because the judge queue (queue.ts) hands each judge a random +// sample of the projects: a random sample is as good as the pool, so a judge +// whose mean, spread or trend differs from the pool's is showing their own +// habits, not an easier or harder set of projects. Hand a judge picked projects +// and the correction below would punish them for it. +// +// A pool is one root challenge: sponsor judges are compared only with the other +// judges of that sponsor's challenge, table judges with table judges, and a +// judge who grades in two challenges gets separate parameters in each. Within a +// pool, over complete evaluations with a score s (the mean of its ratings): +// +// mu the pool's mean +// drift a judge's grades in order (submittedAt, then id): the slope of s on +// the centred order index, shrunk, removed: s' = s - slope * (i - mean i) +// sd the pool's population standard deviation of s' with each judge's +// own mean taken out (judges with two or more grades) +// offset a = shrink * (mean(s') - mu): stricter (below 0) or more lenient +// spread b = 1 + shrink * (sd_j(s') / sd - 1), in [0.5, 2]: bunched or wide +// +// adjusted = clamp(mu + (s' - mu - a) / b, 1, 5) +// +// where shrink = n / (n + K) for a judge with n grades (K_A, K_B, K_D below). + +import { evaluationScore } from "./scoring"; + +// Pseudo-evaluations: a judge with n grades is trusted n / (n + K) of the way, +// so one with few grades is pulled toward no correction. K_A is for the offset, +// K_B for the spread, K_D for the drift (a trend needs more grades than a mean +// to be believed). +export const K_A = 3; +export const K_B = 6; +export const K_D = 10; + +const MIN_SPREAD = 0.5; +const MAX_SPREAD = 2; +const MIN_RATING = 1; +const MAX_RATING = 5; +// Below this a standard deviation is float noise, not a spread. +const FLAT = 1e-9; + +export type CalibrationEvaluation = { + id: string; + judgeId: string; + /** The root challenge. It is the pool: nothing is compared across it. */ + challengeId: string; + entryId: string; + submittedAt: Date; + isComplete: boolean; + /** The 1-5 rating values it holds. */ + ratings: readonly number[]; +}; + +export type JudgeCalibration = { + judgeId: string; + challengeId: string; + /** Complete, scored evaluations the judge has in this challenge. */ + evaluations: number; + /** Rating points taken off the judge's scores. Positive: more lenient than the pool. */ + offset: number; + /** What the judge's spread was divided by. Below 1: bunched. Above 1: wider than the pool. */ + spread: number; + /** Points gained (positive) or lost (negative) from the judge's first grade to their last. */ + drift: number; +}; + +export type Calibration = { + /** The calibrated score of each complete, scored evaluation, by id. */ + adjusted: Map; + judges: JudgeCalibration[]; +}; + +type Graded = { + id: string; + judgeId: string; + challengeId: string; + at: number; + score: number; +}; + +const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + +const mean = (values: readonly number[]) => { + let sum = 0; + for (const value of values) sum += value; + return sum / values.length; +}; + +const populationSd = (values: readonly number[]) => { + const centre = mean(values); + let sum = 0; + for (const value of values) sum += (value - centre) ** 2; + return Math.sqrt(sum / values.length); +}; + +const clamp = (value: number, low: number, high: number) => + Math.min(high, Math.max(low, value)); + +const shrink = (n: number, k: number) => n / (n + k); + +function groupBy(items: readonly T[], key: (item: T) => string) { + const groups = new Map(); + for (const item of items) { + const list = groups.get(key(item)) ?? []; + list.push(item); + groups.set(key(item), list); + } + return groups; +} + +/** + * Calibrated scores for every complete evaluation, and what was learned about + * each judge. Incomplete evaluations, and complete ones with no ratings, are + * left out of both. The result does not depend on the order of the input. + */ +export function calibrate( + evaluations: readonly CalibrationEvaluation[], +): Calibration { + const graded: Graded[] = []; + for (const evaluation of evaluations) { + if (!evaluation.isComplete) continue; + const score = evaluationScore(evaluation.ratings); + if (score === null) continue; + graded.push({ + id: evaluation.id, + judgeId: evaluation.judgeId, + challengeId: evaluation.challengeId, + at: evaluation.submittedAt.getTime(), + score, + }); + } + + // One canonical order, so the sums below add up the same way whatever order + // the caller listed the evaluations in. + graded.sort( + (a, b) => + compareText(a.challengeId, b.challengeId) || + compareText(a.judgeId, b.judgeId) || + a.at - b.at || + compareText(a.id, b.id), + ); + + const adjusted = new Map(); + const judges: JudgeCalibration[] = []; + + for (const pool of groupBy(graded, (g) => g.challengeId).values()) { + const mu = mean(pool.map((g) => g.score)); + + const byJudge = [...groupBy(pool, (g) => g.judgeId).values()].map( + (grades) => { + const n = grades.length; + const scores = grades.map((g) => g.score); + const centre = (n - 1) / 2; + + // Least-squares slope of score on the centred order index. Three + // grades is the fewest that can show a trend. + let slope = 0; + if (n >= 3) { + const average = mean(scores); + let along = 0; + let across = 0; + scores.forEach((score, index) => { + along += (index - centre) * (score - average); + across += (index - centre) ** 2; + }); + slope = (along / across) * shrink(n, K_D); + } + const detrended = scores.map( + (score, index) => score - slope * (index - centre), + ); + return { grades, slope, detrended }; + }, + ); + + // What a judge's spread is measured against: the pool's spread once each + // judge's own mean is taken out. The plain spread of the pool also holds + // the gaps between judges' means, so beside a lenient judge a strict one + // with an ordinary spread would look bunched, and both would be stretched. + // A judge with one grade has no spread of their own and is left out. + let squares = 0; + let counted = 0; + for (const { detrended } of byJudge) { + if (detrended.length < 2) continue; + const centre = mean(detrended); + for (const value of detrended) squares += (value - centre) ** 2; + counted += detrended.length; + } + const sd = counted > 0 ? Math.sqrt(squares / counted) : 0; + + for (const { grades, slope, detrended } of byJudge) { + const n = grades.length; + const offset = shrink(n, K_A) * (mean(detrended) - mu); + + // A judge who gave every project the same score has a spread of 0, which + // would be a divisor near 0 below. The clamp stops the divisor at 0.5 + // (reached once they have 6 or more grades; with fewer, the shrink keeps + // it higher), so what is left of their score after the offset is at most + // doubled. Every project they graded lands on the same value, inside + // 1 to 5: they said nothing about which project was better. + let spread = 1; + if (n >= 2 && sd > FLAT) { + spread = clamp( + 1 + shrink(n, K_B) * (populationSd(detrended) / sd - 1), + MIN_SPREAD, + MAX_SPREAD, + ); + } + + grades.forEach((grade, index) => { + adjusted.set( + grade.id, + clamp( + mu + ((detrended[index] as number) - mu - offset) / spread, + MIN_RATING, + MAX_RATING, + ), + ); + }); + judges.push({ + judgeId: grades[0]!.judgeId, + challengeId: grades[0]!.challengeId, + evaluations: n, + offset, + spread, + drift: slope * (n - 1), + }); + } + } + + return { adjusted, judges }; +} diff --git a/packages/api/src/services/judging/placements.test.ts b/packages/api/src/services/judging/placements.test.ts index f8161b6a..523f1739 100644 --- a/packages/api/src/services/judging/placements.test.ts +++ b/packages/api/src/services/judging/placements.test.ts @@ -1,13 +1,51 @@ -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { calibrate } from "./calibration"; +import type * as CalibrationModule from "./calibration"; import { computePlacements, formatScore } from "./placements"; import type { PlacementEvaluation } from "./placements"; +// The real calibrate, but spied on so a test can swap it out. The first group +// below is about ranking, scopes and ties, so it runs on scores that are left +// as the raw means; the second runs the real thing. +vi.mock("./calibration", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, calibrate: vi.fn(actual.calibrate) }; +}); +const actualCalibrate = async () => + (await vi.importActual("./calibration")).calibrate; + +const leaveAsRaw: typeof calibrate = (evaluations) => ({ + adjusted: new Map( + evaluations + .filter((e) => e.isComplete && e.ratings.length > 0) + .map((e) => [ + e.id, + e.ratings.reduce((sum, value) => sum + value, 0) / e.ratings.length, + ]), + ), + judges: [], +}); + +let count = 0; const evaluation = ( entryId: string, challengeId: string, ratings: number[], isComplete = true, -): PlacementEvaluation => ({ entryId, challengeId, ratings, isComplete }); + over: Partial = {}, +): PlacementEvaluation => { + count += 1; + return { + id: `evaluation-${count}`, + judgeId: "judge", + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, 0, count)), + entryId, + challengeId, + ratings, + isComplete, + ...over, + }; +}; const challenges = [ { id: "general", label: "General" }, @@ -15,6 +53,12 @@ const challenges = [ ]; describe("computePlacements", () => { + beforeEach(() => { + count = 0; + vi.mocked(calibrate).mockReset(); + vi.mocked(calibrate).mockImplementation(leaveAsRaw); + }); + it("ranks overall and per challenge, ties sharing a placing", () => { const placements = computePlacements({ entryIds: ["a", "b", "c"], @@ -119,6 +163,164 @@ describe("computePlacements", () => { }); }); +describe("computePlacements with judge calibration", () => { + beforeEach(async () => { + count = 0; + vi.mocked(calibrate).mockReset(); + vi.mocked(calibrate).mockImplementation(await actualCalibrate()); + }); + + // A strict judge and a lenient one each graded six projects, dealt out at + // random, so the two sets are alike in quality and the lenient judge's higher + // marks are the judge's, not the projects'. + const strictScores = [3.0, 3.2, 3.4, 3.6, 3.8, 4.0]; + const lenientScores = [3.8, 4.0, 4.2, 4.4, 4.6, 4.8]; + // Marked in a scrambled order, so neither judge looks to be drifting. + const gradingOrder = [2, 5, 0, 4, 1, 3]; + const twoJudges = () => { + const graded = (prefix: string, judgeId: string, scores: number[]) => + gradingOrder.map((i) => + evaluation(`${prefix}${i + 1}`, "general", [scores[i]!], true, { + judgeId, + }), + ); + const evaluations = [ + ...graded("s", "strict", strictScores), + ...graded("l", "lenient", lenientScores), + ]; + const entryIds = evaluations.map((e) => e.entryId); + return computePlacements({ + entryIds, + memberships: entryIds.map((entryId) => ({ + entryId, + challengeId: "general", + })), + challenges, + evaluations, + }); + }; + + it("ranks on the calibrated score, so a strict judge's best is not buried", () => { + const overall = new Map( + twoJudges() + .filter((p) => p.scope === "overall") + .map((p) => [p.entryId, p]), + ); + const place = (entryId: string) => overall.get(entryId)!.placement; + const raw = (entryId: string) => overall.get(entryId)!.rawScore; + + // On raw marks the strict judge's best (4.0) is below the lenient judge's + // third best (4.4). + expect(raw("s6")).toBeLessThan(raw("l4")); + // Calibrated, each of the strict judge's projects beats the lenient + // judge's project whose raw mark is 0.4 above it. + expect(place("s6")).toBeLessThan(place("l4")); + expect(place("s5")).toBeLessThan(place("l3")); + expect(place("s4")).toBeLessThan(place("l2")); + expect(place("s3")).toBeLessThan(place("l1")); + // Within a judge, the order is the judge's own. + expect(place("l6")).toBeLessThan(place("l5")); + expect(place("s2")).toBeLessThan(place("s1")); + }); + + it("keeps the plain mean beside the calibrated score", () => { + const placements = twoJudges(); + const byEntry = new Map( + placements + .filter((p) => p.scope === "overall") + .map((p) => [p.entryId, p]), + ); + expect(byEntry.get("s6")?.rawScore).toBeCloseTo(4.0, 10); + expect(byEntry.get("l1")?.rawScore).toBeCloseTo(3.8, 10); + // The strict judge's score moves up, the lenient judge's down. + expect(byEntry.get("s6")!.score).toBeGreaterThan(4.0); + expect(byEntry.get("l6")!.score).toBeLessThan(4.8); + }); + + it("scores a project the same in a challenge as overall when it has one evaluation", () => { + const placements = twoJudges(); + const pick = (scope: string, entryId: string) => + placements.find((p) => p.scope === scope && p.entryId === entryId); + expect(pick("general", "s3")?.score).toBe(pick("overall", "s3")?.score); + expect(pick("general", "s3")?.rawScore).toBe( + pick("overall", "s3")?.rawScore, + ); + }); + + it("calibrates once, over the live entries' complete evaluations only", () => { + const evaluations = [ + evaluation("a", "general", [4], true, { judgeId: "one" }), + evaluation("b", "sponsor", [3], true, { judgeId: "two" }), + evaluation("a", "general", [2], false, { judgeId: "one" }), + // Pulled from prizes: judges scored it, it must not shift anyone else. + evaluation("pulled", "general", [5], true, { judgeId: "two" }), + ]; + computePlacements({ + entryIds: ["a", "b"], + memberships: [ + { entryId: "a", challengeId: "general" }, + { entryId: "b", challengeId: "sponsor" }, + ], + challenges, + evaluations, + }); + + expect(calibrate).toHaveBeenCalledTimes(1); + const given = vi.mocked(calibrate).mock.calls[0]![0]; + expect(given.map((e) => e.entryId).sort()).toEqual(["a", "a", "b"]); + }); + + it("does not let a pulled project's grades move anyone else's score", () => { + const base = [ + evaluation("a", "general", [4], true, { judgeId: "one" }), + evaluation("b", "general", [3], true, { judgeId: "one" }), + evaluation("c", "general", [3.5], true, { judgeId: "two" }), + ]; + const run = (extra: PlacementEvaluation[]) => + computePlacements({ + entryIds: ["a", "b", "c"], + memberships: ["a", "b", "c"].map((entryId) => ({ + entryId, + challengeId: "general", + })), + challenges, + evaluations: [...base, ...extra], + }).map((p) => [p.scope, p.entryId, p.score]); + + expect( + run([evaluation("pulled", "general", [5], true, { judgeId: "two" })]), + ).toEqual(run([])); + }); + + it("does not compare a sponsor's judges with the table judges", () => { + // The sponsor's two guests both mark high; that must not drag the tables. + const tables = [ + evaluation("a", "general", [3], true, { judgeId: "t1" }), + evaluation("b", "general", [4], true, { judgeId: "t2" }), + ]; + const sponsor = [ + evaluation("c", "sponsor", [5], true, { judgeId: "g1" }), + evaluation("d", "sponsor", [5], true, { judgeId: "g2" }), + ]; + const input = { + entryIds: ["a", "b", "c", "d"], + memberships: [ + { entryId: "a", challengeId: "general" }, + { entryId: "b", challengeId: "general" }, + { entryId: "c", challengeId: "sponsor" }, + { entryId: "d", challengeId: "sponsor" }, + ], + challenges, + }; + const general = (evaluations: PlacementEvaluation[]) => + computePlacements({ ...input, evaluations }) + .filter((p) => p.scope === "general") + .map((p) => [p.entryId, p.score]); + + expect(general([...tables, ...sponsor])).toEqual(general(tables)); + }); +}); + describe("formatScore", () => { it("writes two decimal places for numeric(4,2)", () => { expect(formatScore(4)).toBe("4.00"); diff --git a/packages/api/src/services/judging/placements.ts b/packages/api/src/services/judging/placements.ts index 16fdc4de..57497ddb 100644 --- a/packages/api/src/services/judging/placements.ts +++ b/packages/api/src/services/judging/placements.ts @@ -1,9 +1,11 @@ // Turns evaluations into placings: one ranking per challenge and one overall. -// A project's score is the mean of its complete evaluations (scoring.ts); -// equal scores share a placing. Projects nobody scored are left out rather -// than listed last, since a placing is published and "8th of 8" for a project -// no judge reached would be a claim nobody made. +// A project's score is the mean of its complete evaluations (scoring.ts), each +// calibrated for its judge's habits first (calibration.ts); equal scores share +// a placing. The raw mean is kept beside it. Projects nobody scored are left +// out rather than listed last, since a placing is published and "8th of 8" for +// a project no judge reached would be a claim nobody made. +import { calibrate } from "./calibration"; import { overallScore, rankByScore, scopedScore } from "./scoring"; import type { ScoredEvaluation } from "./scoring"; @@ -11,7 +13,12 @@ import type { ScoredEvaluation } from "./scoring"; export const OVERALL_SCOPE = "overall"; export const OVERALL_LABEL = "Overall"; -export type PlacementEvaluation = ScoredEvaluation & { entryId: string }; +export type PlacementEvaluation = ScoredEvaluation & { + id: string; + entryId: string; + judgeId: string; + submittedAt: Date; +}; export type ComputedPlacement = { /** "overall" or a challenge id. */ @@ -19,7 +26,10 @@ export type ComputedPlacement = { scopeLabel: string; entryId: string; placement: number; + /** Calibrated for judge bias. This is what ranks. */ score: number; + /** The plain mean of the same evaluations. */ + rawScore: number; evaluationCount: number; }; @@ -36,11 +46,18 @@ export function computePlacements(input: { const entryIds = [...new Set(input.entryIds)].sort(); const live = new Set(entryIds); + // Calibrated once, over every complete evaluation of a live entry, so a + // judge's habits are learned from all of their grades and not from the + // slice that fell in one scope. + const liveEvaluations = input.evaluations.filter((evaluation) => + live.has(evaluation.entryId), + ); + const { adjusted } = calibrate(liveEvaluations); + const evaluationsOf = new Map(); - for (const evaluation of input.evaluations) { - if (!live.has(evaluation.entryId)) continue; + for (const evaluation of liveEvaluations) { const list = evaluationsOf.get(evaluation.entryId) ?? []; - list.push(evaluation); + list.push({ ...evaluation, adjustedScore: adjusted.get(evaluation.id) }); evaluationsOf.set(evaluation.entryId, list); } @@ -54,13 +71,29 @@ export function computePlacements(input: { }, ): ComputedPlacement[] => { const scored = candidates - .map((entryId) => ({ - entryId, - ...summarize(evaluationsOf.get(entryId) ?? []), - })) + .map((entryId) => { + const evals = evaluationsOf.get(entryId) ?? []; + return { + entryId, + ...summarize(evals), + // The same evaluations on their plain ratings. + rawScore: summarize( + evals.map((evaluation) => ({ + ...evaluation, + adjustedScore: undefined, + })), + ).score, + }; + }) .filter( - (row): row is { entryId: string; score: number; count: number } => - row.score !== null, + ( + row, + ): row is { + entryId: string; + score: number; + count: number; + rawScore: number; + } => row.score !== null && row.rawScore !== null, ); return rankByScore(scored).map((row) => ({ scope, @@ -68,6 +101,7 @@ export function computePlacements(input: { entryId: row.entryId, placement: row.rank, score: row.score, + rawScore: row.rawScore, evaluationCount: row.count, })); }; diff --git a/packages/api/src/services/judging/scoring.test.ts b/packages/api/src/services/judging/scoring.test.ts index 6b88aab2..287931cb 100644 --- a/packages/api/src/services/judging/scoring.test.ts +++ b/packages/api/src/services/judging/scoring.test.ts @@ -66,6 +66,48 @@ describe("overallScore", () => { }); }); +describe("an adjusted score", () => { + const adjusted: ScoredEvaluation[] = [ + { + challengeId: "general", + isComplete: true, + ratings: [5, 5], + adjustedScore: 4, + }, + { + challengeId: "general", + isComplete: true, + ratings: [3, 3], + adjustedScore: 3.5, + }, + // No adjusted score: the mean of its ratings counts. + { challengeId: "general", isComplete: true, ratings: [2, 4] }, + // Incomplete is still left out, adjusted or not. + { + challengeId: "general", + isComplete: false, + ratings: [1], + adjustedScore: 1, + }, + ]; + + it("counts in place of the mean of the ratings", () => { + // (4 + 3.5 + 3) / 3 + expect(scopedScore(adjusted, "general")).toEqual({ score: 3.5, count: 3 }); + expect(overallScore(adjusted)).toEqual({ score: 3.5, count: 3 }); + }); + + it("leaves evaluations without one on the raw mean", () => { + const raw = adjusted.map(({ challengeId, isComplete, ratings }) => ({ + challengeId, + isComplete, + ratings, + })); + // (5 + 3 + 3) / 3 + expect(overallScore(raw)).toEqual({ score: 11 / 3, count: 3 }); + }); +}); + describe("rankByScore", () => { it("ranks highest first", () => { const ranked = rankByScore([ diff --git a/packages/api/src/services/judging/scoring.ts b/packages/api/src/services/judging/scoring.ts index ec77cc1b..74e9eb8e 100644 --- a/packages/api/src/services/judging/scoring.ts +++ b/packages/api/src/services/judging/scoring.ts @@ -1,7 +1,8 @@ // Pure scoring for room judging. A judge's evaluation scores the mean of its // 1–5 ratings; a project's score in a scope is the mean of those evaluation // scores, counting complete evaluations only. Every evaluation weighs the -// same, whoever wrote it. +// same, whoever wrote it. Placements feed in the calibrated score of each +// evaluation (calibration.ts) in place of the raw mean; the judge desk does not. /** An evaluation as scoring sees it. */ export type ScoredEvaluation = { @@ -9,6 +10,8 @@ export type ScoredEvaluation = { isComplete: boolean; /** The 1–5 rating values it holds. */ ratings: readonly number[]; + /** Counted in place of the mean of the ratings when set: the calibrated score. */ + adjustedScore?: number; }; export type ScoreSummary = { @@ -33,7 +36,10 @@ export function evaluationScore(values: readonly number[]): number | null { function summarize(evals: readonly ScoredEvaluation[]): ScoreSummary { const scores = evals .filter((evaluation) => evaluation.isComplete) - .map((evaluation) => evaluationScore(evaluation.ratings)) + .map( + (evaluation) => + evaluation.adjustedScore ?? evaluationScore(evaluation.ratings), + ) .filter((score): score is number => score !== null); return { score: meanOrNull(scores), count: scores.length }; } diff --git a/packages/db/src/schemas/judging.ts b/packages/db/src/schemas/judging.ts index 35291e40..886915bc 100644 --- a/packages/db/src/schemas/judging.ts +++ b/packages/db/src/schemas/judging.ts @@ -766,7 +766,11 @@ export const judgingPlacements = pgTable( scope: text("scope").notNull().default("overall"), scopeLabel: text("scope_label").notNull().default("Overall"), placement: integer("placement").notNull(), + // Calibrated for judge bias: what the placing is ranked on. score: numeric("score", { precision: 4, scale: 2 }), + // The plain mean of the same evaluations. Null on rows computed before + // calibration existed. + rawScore: numeric("raw_score", { precision: 4, scale: 2 }), evaluationCount: integer("evaluation_count").notNull().default(0), publishedAt: timestamp("published_at"), computedAt: timestamp("computed_at").defaultNow().notNull(), diff --git a/sites/mainweb/components/admin/judging/results/ResultsTab.tsx b/sites/mainweb/components/admin/judging/results/ResultsTab.tsx index ebe3b049..6fac719e 100644 --- a/sites/mainweb/components/admin/judging/results/ResultsTab.tsx +++ b/sites/mainweb/components/admin/judging/results/ResultsTab.tsx @@ -24,6 +24,11 @@ import { eventMoment, } from "@/components/hackathon/judging/event-time"; import { plural, useRefreshJudging } from "@/components/admin/judging/shared"; +import { + describeJudge, + judgeNumbers, +} from "@/components/admin/judging/results/calibration"; +import type { CalibrationChallenge } from "@/components/admin/judging/results/calibration"; // Officer screens poll every 10 seconds while judges are still submitting. const POLL_MS = 10_000; @@ -52,6 +57,9 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { const [error, setError] = useState(null); const [openScopes, setOpenScopes] = useState>(new Set()); + const [openCalibration, setOpenCalibration] = useState>( + new Set(), + ); const [entryFilter, setEntryFilter] = useState<{ id: string; title: string; @@ -128,8 +136,13 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { ); } - const { computedAt, publishedAt, evaluationsSinceCompute, scopes } = - list.data; + const { + computedAt, + publishedAt, + evaluationsSinceCompute, + scopes, + calibration, + } = list.data; const published = !!publishedAt; // Teams, not rows: one team can place in several lists. A team already @@ -290,7 +303,7 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) {

{computedAt ? "No project has a complete evaluation yet, so nothing was ranked. Compute again once judges submit." - : "Compute results once judges have submitted. Each challenge and the overall list rank projects by the mean of their complete evaluations; ties share a placing."} + : "Compute results once judges have submitted. Each challenge and the overall list rank projects by the mean of their complete evaluations, each adjusted for how strict or generous its judge is; ties share a placing."}

) : (
@@ -299,6 +312,11 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { const rows = open ? scope.placements : scope.placements.slice(0, PLACINGS_SHOWN); + // Rows computed before calibration existed have no raw score, + // and their score is the plain mean. + const calibrated = scope.placements.some( + (row) => row.rawScore !== null, + ); return (
@@ -308,6 +326,11 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { {scope.placements.length === 1 ? "project" : "projects"}
+ {calibrated && ( +

+ Adjusted for judge bias +

+ )}
    {rows.map((row) => (
  1. {score(row.score)} {" "} + {row.rawScore !== null && ( + + raw {row.rawScore.toFixed(2)}{" "} + + )} · {row.evaluationCount}{" "} {row.evaluationCount === 1 ? "eval" : "evals"} @@ -426,6 +454,38 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { )} + {calibration.length > 0 && ( +
    +

    Judge calibration

    +

    + Each project is graded once, so a strict or generous judge moves + every project they graded. Scores above are adjusted for that, + comparing judges only within their own challenge. Judges and hackers + see none of this. +

    +
    + {calibration.map((challenge) => ( + + setOpenCalibration((previous) => { + const next = new Set(previous); + if (next.has(challenge.challengeId)) { + next.delete(challenge.challengeId); + } else { + next.add(challenge.challengeId); + } + return next; + }) + } + /> + ))} +
    +
    + )} +
    void; +}) { + return ( +
    + + {open && ( +
      + {challenge.judges.map((judge) => ( +
    • +

      + {judge.judgeName} + {judge.kind === "guest" && ( + (guest) + )} +

      +

      {describeJudge(judge).join(" · ")}

      +

      {judgeNumbers(judge)}

      +
      • + ))} +
    + )} +
    + ); +} + function EvaluationAudit({ hackathonId, entryFilter, diff --git a/sites/mainweb/components/admin/judging/results/calibration.test.ts b/sites/mainweb/components/admin/judging/results/calibration.test.ts new file mode 100644 index 00000000..da491e2d --- /dev/null +++ b/sites/mainweb/components/admin/judging/results/calibration.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { describeJudge, judgeNumbers, signed } from "./calibration"; + +const judge = (over: Partial[0]> = {}) => ({ + grades: 12, + offset: 0, + spread: 1, + drift: 0, + ...over, +}); + +describe("signed", () => { + it("shows the direction of a number", () => { + expect(signed(0.384)).toBe("+0.38"); + expect(signed(-0.451)).toBe("-0.45"); + expect(signed(0)).toBe("0.00"); + }); +}); + +describe("describeJudge", () => { + it("calls a judge in line when nothing stands out", () => { + expect(describeJudge(judge())).toEqual(["In line with the other judges"]); + expect( + describeJudge(judge({ offset: 0.1, spread: 0.9, drift: -0.2 })), + ).toEqual(["In line with the other judges"]); + }); + + it("says which way a judge's marks sit and by how much", () => { + expect(describeJudge(judge({ offset: -0.4 }))).toEqual([ + "Stricter than average by 0.40", + ]); + expect(describeJudge(judge({ offset: 0.55 }))).toEqual([ + "More lenient than average by 0.55", + ]); + }); + + it("says when scores are bunched or spread wide", () => { + expect(describeJudge(judge({ spread: 0.6 }))).toEqual([ + "Scores bunched together", + ]); + expect(describeJudge(judge({ spread: 1.4 }))).toEqual([ + "Scores spread wider than average", + ]); + }); + + it("says when a judge changed over the day", () => { + expect(describeJudge(judge({ drift: -0.8 }))).toEqual([ + "Got harsher through the day", + ]); + expect(describeJudge(judge({ drift: 0.7 }))).toEqual([ + "Got more lenient through the day", + ]); + }); + + it("does not read a habit into what chance alone gives a judge", () => { + expect( + describeJudge(judge({ grades: 30, offset: -0.15, drift: 0.45 })), + ).toEqual(["In line with the other judges"]); + }); + + it("lists everything that applies", () => { + expect( + describeJudge(judge({ offset: -0.3, spread: 0.7, drift: -0.6 })), + ).toEqual([ + "Stricter than average by 0.30", + "Scores bunched together", + "Got harsher through the day", + ]); + }); + + it("does not read much into a judge with few grades", () => { + expect(describeJudge(judge({ grades: 2, offset: -0.4 }))).toEqual([ + "Too few grades to adjust much", + ]); + expect(describeJudge(judge({ grades: 1 }))).toEqual([ + "Too few grades to adjust much", + ]); + }); +}); + +describe("judgeNumbers", () => { + it("prints the grades and the three numbers", () => { + expect( + judgeNumbers(judge({ offset: -0.38, spread: 0.72, drift: -0.45 })), + ).toBe("12 grades · offset -0.38 · spread 0.72 · drift -0.45"); + }); + + it("is singular for one grade", () => { + expect(judgeNumbers(judge({ grades: 1 }))).toBe( + "1 grade · offset 0.00 · spread 1.00 · drift 0.00", + ); + }); +}); diff --git a/sites/mainweb/components/admin/judging/results/calibration.ts b/sites/mainweb/components/admin/judging/results/calibration.ts new file mode 100644 index 00000000..6733782d --- /dev/null +++ b/sites/mainweb/components/admin/judging/results/calibration.ts @@ -0,0 +1,59 @@ +import type { RouterOutputs } from "@query/api"; + +export type CalibrationChallenge = + RouterOutputs["judging"]["results"]["list"]["calibration"][number]; +export type CalibratedJudge = CalibrationChallenge["judges"][number]; + +type Numbers = Pick; + +// How far a number has to sit from "no correction" before it is worth saying +// out loud. Offset and drift are in rating points; spread is a multiple of the +// pool's, so 1 is the same. Chance alone moves them for a judge with no habit +// at all (which projects they drew, and in what order), by about 0.1 for the +// offset and 0.3 for the drift over thirty grades, so these sit about twice +// that above zero. +const OFFSET_NOTICED = 0.2; +const SPREAD_NOTICED = 0.15; +const DRIFT_NOTICED = 0.6; +// With fewer grades than this the correction is mostly shrunk away. +const FEW_GRADES = 3; + +/** +0.38, -0.45, 0.00: the sign shown so a direction reads at a glance. */ +export function signed(value: number) { + const text = Math.abs(value).toFixed(2); + return value > 0 ? `+${text}` : value < 0 ? `-${text}` : text; +} + +/** What a judge's numbers say about how they mark, in plain words. */ +export function describeJudge(judge: Numbers): string[] { + if (judge.grades < FEW_GRADES) return ["Too few grades to adjust much"]; + + const phrases: string[] = []; + const size = Math.abs(judge.offset).toFixed(2); + if (judge.offset <= -OFFSET_NOTICED) { + phrases.push(`Stricter than average by ${size}`); + } else if (judge.offset >= OFFSET_NOTICED) { + phrases.push(`More lenient than average by ${size}`); + } + if (judge.spread <= 1 - SPREAD_NOTICED) { + phrases.push("Scores bunched together"); + } else if (judge.spread >= 1 + SPREAD_NOTICED) { + phrases.push("Scores spread wider than average"); + } + if (judge.drift <= -DRIFT_NOTICED) { + phrases.push("Got harsher through the day"); + } else if (judge.drift >= DRIFT_NOTICED) { + phrases.push("Got more lenient through the day"); + } + return phrases.length > 0 ? phrases : ["In line with the other judges"]; +} + +/** The numbers behind the words: 12 grades · offset -0.38 · spread 0.72 · drift -0.45 */ +export function judgeNumbers(judge: Numbers) { + return [ + `${judge.grades} ${judge.grades === 1 ? "grade" : "grades"}`, + `offset ${signed(judge.offset)}`, + `spread ${judge.spread.toFixed(2)}`, + `drift ${signed(judge.drift)}`, + ].join(" · "); +} From 7d8b63c361e756b53835e3d4ecc507f86a954984 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 20:28:35 -0400 Subject: [PATCH 15/27] feat(judging): keep judges to their queue and sponsors to their rooms A table judge now grades a sponsor's challenge only while seated in a room that serves it. evaluatedChallenge refuses with FORBIDDEN on project, projects, nextProject, saveDraft, submit and startGrading, and desk.context lists only the challenges a member may pick. The schedule's task selection reads the same scope.isSponsorChallenge flag. At an untimed challenge a judge's first claim of a project has to be one of the first QUEUE_WINDOW (3) open projects in their own queue, so each judge's sample stays random for calibration. loadQueue in queue-store.ts is shared by nextProject and the claim check, so the two never disagree, and a project the judge has only opened stays on offer to them. nextProject takes a skip list for a team that is away from its table and reports allSkipped; the desk keeps the skips and shows a Skip link next to Grade next project. A scheduled challenge under a saved schedule is unchanged. The new desk.startGrading mutation holds a project from the moment it is opened with an empty revision 0 draft, so another judge is not sent to the same table, and the editor's first save carries on from it. The deadline auto-submit keeps an empty draft rather than submitting it, and desk.projects no longer reports one as a saved draft. For untimed challenges the project list shows the judge's own projects first, then open ones in queue order. --- .../src/.internal-tests/graded-once.test.ts | 978 +++++++++++++++++- packages/api/src/routers/judging/desk.ts | 372 ++++--- .../services/judging/evaluation-store.test.ts | 128 ++- .../src/services/judging/evaluation-store.ts | 51 +- .../src/services/judging/queue-store.test.ts | 428 ++++++++ .../api/src/services/judging/queue-store.ts | 244 +++++ .../api/src/services/judging/queue.test.ts | 73 +- packages/api/src/services/judging/queue.ts | 41 +- .../src/services/judging/schedule/inputs.ts | 8 +- .../api/src/services/judging/scope.test.ts | 8 + packages/api/src/services/judging/scope.ts | 8 + .../components/judge/EvaluationSheet.tsx | 64 +- sites/mainweb/components/judge/JudgeDesk.tsx | 9 +- .../mainweb/components/judge/ProjectQueue.tsx | 73 +- sites/mainweb/components/judge/desk.test.ts | 29 + sites/mainweb/components/judge/desk.ts | 8 + 16 files changed, 2343 insertions(+), 179 deletions(-) create mode 100644 packages/api/src/services/judging/queue-store.test.ts create mode 100644 packages/api/src/services/judging/queue-store.ts diff --git a/packages/api/src/.internal-tests/graded-once.test.ts b/packages/api/src/.internal-tests/graded-once.test.ts index 26e6d265..0017172a 100644 --- a/packages/api/src/.internal-tests/graded-once.test.ts +++ b/packages/api/src/.internal-tests/graded-once.test.ts @@ -1,8 +1,8 @@ /** * Each project is graded once per challenge, by one judge, at the judge desk: * the refusals on a draft save and a submit, what the project list and sheet - * report, the next project a judge is sent to, and who may grade which - * challenge. + * report, the next project a judge is sent to (its queue window and skips), + * opening a project to hold it, and who may grade which challenge. * * The router runs for real against a stand-in database that answers each select * by the tables it reads and keeps every statement in order, which is how the @@ -13,7 +13,9 @@ import { getTableName } from "drizzle-orm"; import type { SQL } from "drizzle-orm"; import { PgDialect } from "drizzle-orm/pg-core"; import { + hackathons, judgingAppointments, + judgingBuildings, judgingChallenges, judgingConfigs, judgingEntries, @@ -32,7 +34,7 @@ import { } from "@query/db"; import { judgingDeskRouter } from "../routers/judging/desk"; import { GRADING_HOLD_MS } from "../services/judging/grading-hold"; -import { queueOrder } from "../services/judging/queue"; +import { QUEUE_WINDOW, queueOrder } from "../services/judging/queue"; const HACK = "6f1d2c3b-4a5e-4f60-8a7b-9c0d1e2f3a4b"; const GENERAL = "0c4a8e10-2b3d-4c5e-9f60-7a8b9c0d1e2f"; @@ -83,6 +85,7 @@ type Event = { kind: "select" | "execute" | "insert" | "update" | "delete"; table?: string; where?: SQL; + values?: Record; }; /** The nth read of a table gets the nth list. */ @@ -142,16 +145,16 @@ function fakeDb(answers: Answers) { return Promise.resolve(); }, insert: (table: unknown) => ({ - values: () => { - events.push({ kind: "insert", table: via(table) }); + values: (values: Record) => { + events.push({ kind: "insert", table: via(table), values }); const done = Promise.resolve([{ id: "new" }]); return Object.assign(done, { returning: () => done }); }, }), update: (table: unknown) => ({ - set: () => ({ + set: (values: Record) => ({ where: (where: SQL) => { - events.push({ kind: "update", table: via(table), where }); + events.push({ kind: "update", table: via(table), where, values }); return Promise.resolve(); }, }), @@ -212,14 +215,19 @@ const guestWorld = (over: Answers = {}): Answers => ({ ...over, }); -const memberCtx = (db: unknown) => - ({ +// A new user for each context: the desk's flood guard counts calls per user, +// and this file makes more than one user's worth. +let callers = 0; +const memberCtx = (db: unknown) => { + const userId = `judge-user-${(callers += 1)}`; + return { db, - session: { user: { id: "judge-user", name: "Grace Hopper" } }, - userId: "judge-user", + session: { user: { id: userId, name: "Grace Hopper" } }, + userId, clientIp: "127.0.0.1", req: { headers: { get: () => null } }, - }) as never; + } as never; +}; const guestCtx = (db: unknown) => ({ @@ -255,6 +263,11 @@ const gradedElsewhere = const beingGraded = "Another judge is grading this project right now. Pick another one."; +const sponsorRefusal = + "Sponsor judges grade Best Use of AI. Sit in its room to grade it."; +const queueRefusal = + "Grade the projects your queue offers so every judge sees a random mix. Use Grade next project."; + const save = { hackathonId: HACK, entryId: E_OPEN, @@ -533,6 +546,31 @@ describe("desk.projects: each project's grading state", () => { expect(result.judgedCount).toBe(1); }); + it("does not call a project the judge only opened a saved draft", async () => { + const mine = (entryId: string, revision: number) => ({ + entryId, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision, + }); + const result = await list( + memberWorld({ + [via(judgingEntries, judgingEntryChallenges)]: [ + entry(E_OPEN, "Alpha"), + entry(E_HELD, "Bravo"), + ], + [drafts]: [mine(E_OPEN, 0), mine(E_HELD, 2)], + }), + ); + + const hasDraft = Object.fromEntries( + result.projects.map((p) => [p.id, p.hasDraft]), + ); + expect(hasDraft).toEqual({ [E_OPEN]: false, [E_HELD]: true }); + expect(result.projects.map((p) => p.grading)).toEqual(["mine", "mine"]); + }); + it("never says which other judge", async () => { const result = await list(crowded()); @@ -831,10 +869,15 @@ describe("desk.nextProject", () => { expect(result.entryId).toBe(E_STALE); }); + // A table judge can only reach a scheduled challenge that is General's: + // a sponsor's is refused before the room matters (see "who may grade + // which challenge"). + const timedGeneral = { ...general, mode: "scheduled" as const }; + it("asks a judge seated elsewhere to join the room first", async () => { const { result, events } = await next( - scheduled({ [via(judgingPresence, judgingRooms)]: seat(GENERAL) }), - { challengeId: SPONSOR }, + scheduled({ [via(judgingChallenges)]: [timedGeneral, sponsor] }), + { challengeId: GENERAL }, ); expect(result.entryId).toBeNull(); @@ -848,8 +891,11 @@ describe("desk.nextProject", () => { it("asks a judge with no seat to join the room first", async () => { const { result } = await next( - scheduled({ [via(judgingPresence, judgingRooms)]: [] }), - { challengeId: SPONSOR }, + scheduled({ + [via(judgingChallenges)]: [timedGeneral, sponsor], + [via(judgingPresence, judgingRooms)]: [], + }), + { challengeId: GENERAL }, ); expect(result.needsRoom).toBe(true); @@ -948,18 +994,33 @@ describe("who may grade which challenge", () => { judgingDeskRouter .createCaller(memberCtx(db)) .saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), - ).rejects.toMatchObject({ - code: "PRECONDITION_FAILED", - message: - "Join the room or table that judges this challenge to evaluate this project.", - }); + ).rejects.toMatchObject({ code: "FORBIDDEN", message: sponsorRefusal }); expect(wrote(events, judgingEvaluationDrafts)).toBe(false); await expect( judgingDeskRouter .createCaller(memberCtx(fakeDb(sheet(GENERAL)).db)) .submit({ ...save, challengeId: SPONSOR, expectedRevision: 0 }), - ).rejects.toMatchObject({ code: "PRECONDITION_FAILED" }); + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it("still needs the room for a scheduled General", async () => { + const world = memberWorld({ + ...sheet(SPONSOR), + [via(judgingChallenges)]: [{ ...general, mode: "scheduled" }, sponsor], + }); + const { db, events } = fakeDb(world); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .saveDraft({ ...save, challengeId: GENERAL, revision: 0 }), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: + "Join the room or table that judges this challenge to evaluate this project.", + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); }); it("can grade it when seated at the room that judges it", async () => { @@ -974,3 +1035,876 @@ describe("who may grade which challenge", () => { }); }); }); + +/** More projects than the queue window, with ids the desk accepts as input. */ +const eid = (n: number) => + `40e8c254-6f71-4092-9da3-1e2f3a4b5c${String(n).padStart(2, "0")}`; +const POOL = Array.from({ length: 8 }, (_, i) => eid(i + 10)); + +const entriesKey = via(judgingEntries, judgingEntryChallenges); +const presenceKey = via(judgingPresence, judgingRooms); +const seatAt = (challengeId: string) => [ + { roomId: ROOM, roomName: "Table 4", challengeId }, +]; + +describe("sponsor challenges belong to sponsor judges", () => { + const MLH = "0c4a8e12-2b3d-4c5e-9f60-7a8b9c0d1e2f"; + const MONGO = "0c4a8e13-2b3d-4c5e-9f60-7a8b9c0d1e2f"; + const mlh = { + ...challenge(MLH, "MLH Challenges", "unscheduled"), + isGroup: true, + }; + const mongo = { + ...challenge(MONGO, "Best Use of MongoDB", "unscheduled"), + parentId: MLH, + }; + + const world = (seatedAt: string | null, over: Answers = {}) => + memberWorld({ + [via(judgingChallenges)]: [general, sponsor, mlh, mongo], + [presenceKey]: seatedAt ? seatAt(seatedAt) : [], + ...over, + }); + + it("refuses a table judge who is not seated in the sponsor's room, on every call that picks or writes", async () => { + for (const seatedAt of [null, GENERAL, MLH]) { + const { db, events } = fakeDb(world(seatedAt)); + const caller = judgingDeskRouter.createCaller(memberCtx(db)); + const refused = { code: "FORBIDDEN", message: sponsorRefusal }; + + await expect( + caller.projects({ hackathonId: HACK, challengeId: SPONSOR }), + ).rejects.toMatchObject(refused); + await expect( + caller.project({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: SPONSOR, + }), + ).rejects.toMatchObject(refused); + await expect( + caller.nextProject({ hackathonId: HACK, challengeId: SPONSOR }), + ).rejects.toMatchObject(refused); + await expect( + caller.saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).rejects.toMatchObject(refused); + await expect( + caller.submit({ ...save, challengeId: SPONSOR, expectedRevision: 0 }), + ).rejects.toMatchObject(refused); + await expect( + caller.startGrading({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: SPONSOR, + }), + ).rejects.toMatchObject(refused); + + expect(kinds(events)).not.toContain("execute"); + expect(kinds(events)).not.toContain("insert"); + } + }); + + it("names the group when the judge asked for a prize inside it", async () => { + const { db } = fakeDb(world(GENERAL)); + + await expect( + judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK, challengeId: MONGO }), + ).rejects.toMatchObject({ + code: "FORBIDDEN", + message: + "Sponsor judges grade MLH Challenges. Sit in its room to grade it.", + }); + }); + + it("lets a table judge in once they are seated in the room that serves it", async () => { + const { db, events } = fakeDb(world(SPONSOR)); + const caller = judgingDeskRouter.createCaller(memberCtx(db)); + + const listed = await caller.projects({ + hackathonId: HACK, + challengeId: SPONSOR, + inMyRoomOnly: false, + }); + expect(listed.challenge.id).toBe(SPONSOR); + await expect( + caller.nextProject({ hackathonId: HACK, challengeId: SPONSOR }), + ).resolves.toMatchObject({ entryId: E_OPEN }); + await expect( + caller.saveDraft({ ...save, challengeId: SPONSOR, revision: 0 }), + ).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + }); + + it("lets a table judge grade a prize inside a group from a seat at the group's room", async () => { + const { db } = fakeDb(world(MLH)); + + const listed = await judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK, challengeId: MONGO }); + expect(listed.challenge.id).toBe(MLH); + }); + + it("keeps General open to a table judge wherever they sit", async () => { + for (const seatedAt of [null, GENERAL, SPONSOR]) { + const { db } = fakeDb(world(seatedAt)); + const listed = await judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK, challengeId: GENERAL }); + expect(listed.challenge.id).toBe(GENERAL); + } + }); + + it("defaults an unseated table judge to General", async () => { + const { db } = fakeDb(world(null)); + + const listed = await judgingDeskRouter + .createCaller(memberCtx(db)) + .projects({ hackathonId: HACK }); + + expect(listed.challenge.id).toBe(GENERAL); + }); + + describe("the challenges the desk offers", () => { + const offered = async (answers: Answers, ctx = memberCtx) => { + const { db } = fakeDb({ + [via(hackathons)]: [{ id: HACK, name: "Hacklytics" }], + ...answers, + }); + const context = await judgingDeskRouter + .createCaller(ctx(db)) + .context({ hackathonId: HACK }); + return context.challenges.map((c) => c.id); + }; + + it("is General alone for a table judge with no seat or another's", async () => { + expect(await offered(world(null))).toEqual([GENERAL]); + expect(await offered(world(GENERAL))).toEqual([GENERAL]); + }); + + it("adds the sponsor's challenge, and what is below it, from its room", async () => { + expect(await offered(world(SPONSOR))).toEqual([GENERAL, SPONSOR]); + expect(await offered(world(MLH))).toEqual([GENERAL, MLH, MONGO]); + }); + + it("is still only the room's challenge for a guest", async () => { + expect(await offered(guestWorld(), guestCtx)).toEqual([SPONSOR]); + }); + }); +}); + +describe("a judge's first claim is limited to their queue window", () => { + const order = queueOrder(ME, GENERAL, POOL); + const world = (over: Answers = {}) => + memberWorld({ + [entriesKey]: POOL.map((id) => entry(id, "Project")), + ...over, + }); + const firstSave = (entryId: string) => ({ ...save, entryId, revision: 0 }); + const desk = (answers: Answers) => { + const fake = fakeDb(answers); + return { + ...fake, + caller: judgingDeskRouter.createCaller(memberCtx(fake.db)), + }; + }; + + it("saves a draft of any of the first projects the queue offers", async () => { + for (const entryId of order.slice(0, QUEUE_WINDOW)) { + const { caller, events } = desk(world()); + + await expect(caller.saveDraft(firstSave(entryId))).resolves.toMatchObject( + { revision: 1 }, + ); + expect(wrote(events, judgingEvaluationDrafts)).toBe(true); + } + }); + + it("refuses a first save of a project further down, after the grading lock and before any write", async () => { + const { caller, events } = desk(world()); + + await expect( + caller.saveDraft(firstSave(order[QUEUE_WINDOW])), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: queueRefusal, + }); + expect(kinds(events)).toContain("execute"); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a first submission of a project further down", async () => { + const { caller, events } = desk(world()); + + await expect( + caller.submit({ ...firstSave(order[QUEUE_WINDOW]), expectedRevision: 0 }), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: queueRefusal, + }); + expect(wrote(events, judgingEvaluations)).toBe(false); + }); + + it("lets a judge submit a project in the window without ever saving a draft", async () => { + const { caller } = desk(world()); + + await expect( + caller.submit({ ...firstSave(order[0]), expectedRevision: 0 }), + ).resolves.toMatchObject({ isComplete: true }); + }); + + it("slides the window past projects that are graded or held", async () => { + const held = (entryId: string, minutes: number) => ({ + entryId, + judgeId: OTHER, + updatedAt: minutesAgo(minutes), + deadlineAt: null, + }); + const { caller } = desk( + world({ + [evaluations]: byCall([], [], [{ entryId: order[0], judgeId: OTHER }]), + [drafts]: byCall([], [], [held(order[1], 1), held(order[2], 2)]), + }), + ); + + await expect( + caller.saveDraft(firstSave(order[QUEUE_WINDOW])), + ).resolves.toMatchObject({ revision: 1 }); + }); + + it("leaves a project the judge already holds alone, however far down", async () => { + const { caller } = desk( + world({ + [drafts]: byCall([{ id: "mine", revision: 0, deadlineAt: null }]), + }), + ); + + await expect(caller.saveDraft(firstSave(order[7]))).resolves.toMatchObject({ + revision: 1, + }); + }); + + it("leaves a project the judge already graded alone, however far down", async () => { + const { caller, events } = desk( + world({ [evaluations]: byCall([{ appointmentId: null }]) }), + ); + + await expect(caller.saveDraft(firstSave(order[7]))).resolves.toMatchObject({ + revision: 1, + }); + expect(kinds(events)).not.toContain("execute"); + }); + + it("asks nothing of a scheduled challenge: the schedule decides", async () => { + const { caller, events } = desk( + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [via(judgingAppointments)]: [ + { + id: "slot", + roomId: ROOM, + startsAt: minutesAgo(5), + deadlineAt: minutesAhead(5), + endsAt: minutesAhead(10), + }, + ], + [presenceKey]: seatAt(SPONSOR), + [entriesKey]: POOL.map((id) => entry(id, "Project")), + }), + ); + + await expect( + caller.saveDraft({ + ...firstSave(queueOrder(ME, SPONSOR, POOL)[7]), + challengeId: SPONSOR, + }), + ).resolves.toMatchObject({ revision: 1 }); + // Only the check that the project is entered read the projects. + expect(events.filter((e) => e.table === entriesKey)).toHaveLength(1); + }); +}); + +describe("desk.nextProject: the queue window and skips", () => { + const order = queueOrder(ME, GENERAL, POOL); + const next = async ( + input: { challengeId?: string; skip?: string[] } = {}, + over: Answers = {}, + ) => { + const { db } = fakeDb( + memberWorld({ [entriesKey]: POOL.map((id) => ({ id })), ...over }), + ); + return judgingDeskRouter + .createCaller(memberCtx(db)) + .nextProject({ hackathonId: HACK, ...input }); + }; + + it("offers the first project of the judge's queue", async () => { + const result = await next(); + + expect(result.entryId).toBe(order[0]); + expect(result.allSkipped).toBe(false); + }); + + it("passes over skipped projects, one at a time, within the window", async () => { + expect((await next({ skip: [order[0]] })).entryId).toBe(order[1]); + expect((await next({ skip: [order[0], order[1]] })).entryId).toBe(order[2]); + }); + + it("says everything is skipped once the whole window is, and never reaches past it", async () => { + const result = await next({ skip: order.slice(0, QUEUE_WINDOW) }); + + expect(result.entryId).toBeNull(); + expect(result.allSkipped).toBe(true); + expect(result.needsRoom).toBe(false); + }); + + it("ignores a skip outside the window", async () => { + expect((await next({ skip: [order[6]] })).entryId).toBe(order[0]); + }); + + it("does not call a challenge with nothing open skipped", async () => { + const held = POOL.map((entryId) => ({ + entryId, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + })); + + const result = await next({ skip: [order[0]] }, { [drafts]: held }); + + expect(result.entryId).toBeNull(); + expect(result.allSkipped).toBe(false); + }); + + it("slides the window as other judges grade projects", async () => { + const result = await next( + { skip: order.slice(0, QUEUE_WINDOW - 1) }, + { [evaluations]: [{ entryId: order[0], judgeId: OTHER }] }, + ); + + // order[0] is graded, so the window is order[1..3]; skipping order[0] + // and order[1] leaves order[2]. + expect(result.entryId).toBe(order[2]); + const all = await next( + { skip: order.slice(1, QUEUE_WINDOW + 1) }, + { [evaluations]: [{ entryId: order[0], judgeId: OTHER }] }, + ); + expect(all.entryId).toBeNull(); + expect(all.allSkipped).toBe(true); + }); + + it("keeps a project the judge opened and skipped inside the window", async () => { + // Opening a project holds it for the judge (an empty draft); skipping it + // must not let them reach past the window. + const opened = order.slice(0, QUEUE_WINDOW).map((entryId) => ({ + entryId, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision: 0, + })); + + expect((await next({}, { [drafts]: opened.slice(0, 1) })).entryId).toBe( + order[0], + ); + expect( + (await next({ skip: [order[0]] }, { [drafts]: opened.slice(0, 1) })) + .entryId, + ).toBe(order[1]); + const all = await next( + { skip: order.slice(0, QUEUE_WINDOW) }, + { [drafts]: opened }, + ); + expect(all.entryId).toBeNull(); + expect(all.allSkipped).toBe(true); + }); + + it("takes at most 20 skips, and only project ids", async () => { + const many = Array.from({ length: 21 }, (_, i) => eid(i + 30)); + await expect(next({ skip: many })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + await expect(next({ skip: ["not-an-id"] })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + await expect(next({ skip: many.slice(0, 20) })).resolves.toMatchObject({ + entryId: order[0], + }); + }); + + it("offers every running presentation at a scheduled challenge, not just a window", async () => { + const ids = POOL.slice(0, 5); + const sponsorOrder = queueOrder(ME, SPONSOR, ids); + const running = ids.map((entryId) => ({ + id: `slot-${entryId}`, + entryId, + startsAt: minutesAgo(5), + deadlineAt: minutesAhead(5), + })); + const result = await next( + { challengeId: SPONSOR, skip: sponsorOrder.slice(0, 4) }, + { + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [presenceKey]: seatAt(SPONSOR), + [entriesKey]: ids.map((id) => ({ id })), + [via(judgingAppointments)]: running, + }, + ); + + expect(result.entryId).toBe(sponsorOrder[4]); + }); +}); + +describe("desk.projects: open projects in the judge's queue order", () => { + const titled = [ + ["Alpha", eid(40)], + ["Bravo", eid(41)], + ["Charlie", eid(42)], + ["Delta", eid(43)], + ["Echo", eid(44)], + ["Foxtrot", eid(45)], + ["Golf", eid(46)], + ["Hotel", eid(47)], + ].map(([title, id]) => entry(id, title)); + // Alpha is the judge's own, Bravo is held by another judge, Charlie was + // graded by another judge, and the rest are open. + const [MINE, HELD, GRADED, ...OPEN] = titled.map((e) => e.id); + + const list = async ( + over: Answers = {}, + input: { sort?: "scheduled" | "title"; challengeId?: string } = {}, + ) => { + const { db } = fakeDb( + memberWorld({ + [entriesKey]: titled, + [drafts]: [ + { + entryId: MINE, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: HELD, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + ], + [evaluations]: [ + { entryId: GRADED, judgeId: OTHER, revision: 1, isComplete: true }, + ], + ...over, + }), + ); + return judgingDeskRouter.createCaller(memberCtx(db)).projects({ + hackathonId: HACK, + inMyRoomOnly: false, + hideJudged: false, + ...input, + }); + }; + + it("lists the judge's own first, then open projects in their queue order", async () => { + const result = await list(); + const queue = queueOrder(ME, GENERAL, OPEN); + // The test only means something if the queue is not the title order. + expect(queue).not.toEqual(OPEN); + + expect(result.projects.map((p) => p.id)).toEqual([ + MINE, + ...queue, + HELD, + GRADED, + ]); + expect(result.projects.map((p) => p.grading)).toEqual([ + "mine", + ...queue.map(() => "open"), + "in_progress", + "graded", + ]); + }); + + it("puts first the open project that Grade next project goes to", async () => { + const result = await list(); + const { db } = fakeDb( + memberWorld({ + [entriesKey]: titled.map((e) => ({ id: e.id })), + [drafts]: [ + { + entryId: MINE, + judgeId: ME, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: HELD, + judgeId: OTHER, + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + ], + [evaluations]: [{ entryId: GRADED, judgeId: OTHER }], + }), + ); + const next = await judgingDeskRouter + .createCaller(memberCtx(db)) + .nextProject({ hackathonId: HACK }); + + expect(result.projects.find((p) => p.grading === "open")?.id).toBe( + next.entryId, + ); + }); + + it("keeps title order when the judge asks for it", async () => { + const result = await list({}, { sort: "title" }); + + expect( + result.projects.filter((p) => p.grading === "open").map((p) => p.title), + ).toEqual(["Delta", "Echo", "Foxtrot", "Golf", "Hotel"]); + }); + + it("keeps presentation order at a scheduled challenge", async () => { + const slot = (entryId: string, minutes: number) => ({ + id: `slot-${entryId}`, + entryId, + roomId: ROOM, + roomName: "Table 4", + buildingName: null, + startsAt: minutesAhead(minutes), + deadlineAt: minutesAhead(minutes + 10), + endsAt: minutesAhead(minutes + 12), + }); + const [delta, echo, foxtrot] = OPEN; + const result = await list( + { + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [presenceKey]: seatAt(SPONSOR), + [via(judgingAppointments, judgingRooms, judgingBuildings)]: [ + slot(foxtrot, 10), + slot(echo, 20), + slot(delta, 30), + ], + }, + { challengeId: SPONSOR }, + ); + + expect(result.projects.map((p) => p.title)).toEqual([ + "Foxtrot", + "Echo", + "Delta", + "Alpha", + "Golf", + "Hotel", + "Bravo", + "Charlie", + ]); + }); +}); + +describe("desk.startGrading: the hold starts when a project is opened", () => { + const start = (entryId = E_OPEN, challengeId = GENERAL) => ({ + hackathonId: HACK, + entryId, + challengeId, + }); + const desk = (answers: Answers, ctx = memberCtx) => { + const fake = fakeDb(answers); + return { + ...fake, + caller: judgingDeskRouter.createCaller(ctx(fake.db)), + }; + }; + const insertedDraft = (events: Event[]) => + events.find((e) => e.kind === "insert" && e.table === drafts)?.values; + + it("creates an empty draft after taking the grading lock", async () => { + const { caller, events } = desk(memberWorld()); + + const result = await caller.startGrading(start()); + + expect(result).toEqual({ held: true, revision: 0, deadlineAt: null }); + expect(insertedDraft(events)).toMatchObject({ + hackathonId: HACK, + judgeId: ME, + entryId: E_OPEN, + challengeId: GENERAL, + ratings: {}, + responses: {}, + revision: 0, + appointmentId: null, + deadlineAt: null, + }); + expect(kinds(events).indexOf("execute")).toBeGreaterThan(-1); + expect(kinds(events).indexOf("execute")).toBeLessThan( + kinds(events).indexOf("insert"), + ); + }); + + it("picks the challenge a table judge sits at when none is named", async () => { + const { caller, events } = desk( + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [presenceKey]: seatAt(SPONSOR), + }), + ); + + await caller.startGrading({ hackathonId: HACK, entryId: E_OPEN }); + + expect(insertedDraft(events)).toMatchObject({ challengeId: SPONSOR }); + }); + + it("touches a draft the judge already has, keeping its answers and revision", async () => { + const { caller, events } = desk( + memberWorld({ + [drafts]: byCall([{ id: "mine", revision: 3, deadlineAt: null }]), + }), + ); + + const result = await caller.startGrading(start()); + + expect(result).toEqual({ held: true, revision: 3, deadlineAt: null }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + const touched = events.find( + (e) => e.kind === "update" && e.table === drafts, + ); + expect(touched?.values).toHaveProperty("updatedAt"); + expect(touched?.values).not.toHaveProperty("revision"); + expect(touched?.values).not.toHaveProperty("ratings"); + expect(touched?.values).not.toHaveProperty("responses"); + // A draft they hold needs no queue check. + expect(events.filter((e) => e.table === entriesKey)).toHaveLength(1); + }); + + it("does not make the editor's first save stale", async () => { + // The sheet opened with no draft and sends revision 0; the empty draft + // holds revision 0, so that save goes through as an update of it. + const { caller, events } = desk( + memberWorld({ + [drafts]: byCall([{ id: "new", revision: 0, deadlineAt: null }]), + }), + ); + + await expect( + caller.saveDraft({ ...save, revision: 0 }), + ).resolves.toMatchObject({ revision: 1 }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + expect(events.some((e) => e.kind === "update" && e.table === drafts)).toBe( + true, + ); + }); + + it("refuses a project another judge already graded", async () => { + const { caller, events } = desk( + memberWorld({ [evaluations]: byCall([], [{ judgeId: OTHER }]) }), + ); + + await expect(caller.startGrading(start())).rejects.toMatchObject({ + code: "CONFLICT", + message: gradedElsewhere, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a project another judge is grading right now", async () => { + const { caller, events } = desk( + memberWorld({ + [drafts]: byCall( + [], + [{ judgeId: OTHER, updatedAt: minutesAgo(1), deadlineAt: null }], + ), + }), + ); + + await expect(caller.startGrading(start())).rejects.toMatchObject({ + code: "CONFLICT", + message: beingGraded, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("refuses a project outside the judge's queue window", async () => { + const order = queueOrder(ME, GENERAL, POOL); + const { caller, events } = desk( + memberWorld({ [entriesKey]: POOL.map((id) => entry(id, "Project")) }), + ); + + await expect( + caller.startGrading(start(order[QUEUE_WINDOW])), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: queueRefusal, + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + + await expect( + desk( + memberWorld({ [entriesKey]: POOL.map((id) => entry(id, "Project")) }), + ).caller.startGrading(start(order[0])), + ).resolves.toMatchObject({ held: true }); + }); + + it("leaves the judge's own graded project alone", async () => { + const { caller, events } = desk( + memberWorld({ [evaluations]: byCall([{ id: "mine" }]) }), + ); + + await expect(caller.startGrading(start())).resolves.toMatchObject({ + held: false, + }); + expect(kinds(events)).not.toContain("execute"); + expect(kinds(events)).not.toContain("insert"); + expect(kinds(events)).not.toContain("update"); + }); + + it("is refused while judging is not open", async () => { + const { caller, events } = desk( + memberWorld({ + [via(judgingConfigs)]: [ + { state: "draft", showAllScores: false, closedAt: null }, + ], + }), + ); + + await expect(caller.startGrading(start())).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + + it("holds a guest's own room's project the same way", async () => { + const { caller, events } = desk(guestWorld(), guestCtx); + + await caller.startGrading({ hackathonId: HACK, entryId: E_OPEN }); + + expect(insertedDraft(events)).toMatchObject({ + challengeId: SPONSOR, + ratings: {}, + revision: 0, + }); + }); + + describe("for a scheduled presentation", () => { + const presenting = (startsAt: Date, deadlineAt: Date) => + memberWorld({ + [via(judgingChallenges)]: [general, sponsor], + [via(judgingSchedules)]: [{ id: "schedule" }], + [via(judgingAppointments)]: [ + { + id: "slot", + roomId: ROOM, + startsAt, + deadlineAt, + endsAt: minutesAhead(10), + }, + ], + [presenceKey]: seatAt(SPONSOR), + }); + + it("stamps the draft with the presentation and its deadline, as a save does", async () => { + const deadline = minutesAhead(5); + const { caller, events } = desk(presenting(minutesAgo(5), deadline)); + + const result = await caller.startGrading(start(E_OPEN, SPONSOR)); + + expect(result.deadlineAt).toEqual(deadline); + expect(insertedDraft(events)).toMatchObject({ + challengeId: SPONSOR, + appointmentId: "slot", + deadlineAt: deadline, + revision: 0, + }); + }); + + it("keeps the draft's own deadline once the slot's has just passed", async () => { + const kept = minutesAhead(2); + // Within the grace a write still gets, the slot gives no new deadline. + const { caller, events } = desk({ + ...presenting(minutesAgo(10), new Date(Date.now() - 5_000)), + [drafts]: byCall([{ id: "mine", revision: 2, deadlineAt: kept }]), + }); + + const result = await caller.startGrading(start(E_OPEN, SPONSOR)); + + expect(result).toEqual({ held: true, revision: 2, deadlineAt: kept }); + const touched = events.find( + (e) => e.kind === "update" && e.table === drafts, + ); + expect(touched?.values).toMatchObject({ deadlineAt: kept }); + }); + + it("does not start before the presentation does", async () => { + const { caller, events } = desk( + presenting(minutesAhead(10), minutesAhead(20)), + ); + + await expect( + caller.startGrading(start(E_OPEN, SPONSOR)), + ).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: + "This project has not presented yet. You can judge it once its presentation starts.", + }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + }); + + describe("a second judge after the first opens a project", () => { + const secondJudge = (held: Event["values"]) => + memberWorld({ + [via(judgingJudges)]: [{ id: OTHER, displayName: "Alan Turing" }], + [via(judgingEntryMembers)]: [{ name: "Ada Lovelace" }], + [drafts]: [ + { + entryId: held?.entryId, + judgeId: held?.judgeId, + updatedAt: held?.updatedAt, + deadlineAt: held?.deadlineAt, + }, + ], + }); + + it("sees the project in progress, read-only, and cannot take it", async () => { + const first = desk(memberWorld()); + await first.caller.startGrading(start()); + const held = insertedDraft(first.events); + expect(held).toBeDefined(); + + const { db } = fakeDb(secondJudge(held)); + const caller = judgingDeskRouter.createCaller(memberCtx(db)); + + const listed = await caller.projects({ + hackathonId: HACK, + inMyRoomOnly: false, + }); + expect(listed.projects.map((p) => p.grading)).toEqual(["in_progress"]); + + const sheet = await caller.project({ + hackathonId: HACK, + entryId: E_OPEN, + challengeId: GENERAL, + }); + expect(sheet.grading).toBe("in_progress"); + expect(sheet.access).toMatchObject({ + canEdit: false, + reason: beingGraded, + gradedElsewhere: true, + }); + + const { db: db2, events } = fakeDb(secondJudge(held)); + const second = judgingDeskRouter.createCaller(memberCtx(db2)); + await expect(second.startGrading(start())).rejects.toMatchObject({ + code: "CONFLICT", + message: beingGraded, + }); + await expect( + second.saveDraft({ ...save, revision: 0 }), + ).rejects.toMatchObject({ code: "CONFLICT", message: beingGraded }); + expect(wrote(events, judgingEvaluationDrafts)).toBe(false); + }); + }); +}); diff --git a/packages/api/src/routers/judging/desk.ts b/packages/api/src/routers/judging/desk.ts index 5e09a5c7..7112497d 100644 --- a/packages/api/src/routers/judging/desk.ts +++ b/packages/api/src/routers/judging/desk.ts @@ -45,14 +45,12 @@ import { loadChallenges, loadEntryInChallenge, loadRubric, + memberScope, reconcileDueDrafts, seatOf, writeEvaluation, } from "../../services/judging/evaluation-store"; -import { - DEADLINE_GRACE_MS, - evaluationTiming, -} from "../../services/judging/timing"; +import { DEADLINE_GRACE_MS } from "../../services/judging/timing"; import type { ChallengeRow } from "../../services/judging/evaluation-store"; import { gradingRefusal, @@ -60,7 +58,16 @@ import { gradingStates, } from "../../services/judging/grading-hold"; import type { GradingState } from "../../services/judging/grading-hold"; -import { nextToGrade } from "../../services/judging/queue"; +import { + QUEUE_WINDOW, + nextToGrade, + queueOrder, + queueWindow, +} from "../../services/judging/queue"; +import { + assertQueueOffers, + loadQueue, +} from "../../services/judging/queue-store"; import { checkAnswers, responseVisibility, @@ -283,6 +290,20 @@ async function openRoomId(db: DrizzleDB, judgeId: string) { return seat?.roomId ?? null; } +/** + * A member's seat, for the rule that a sponsor's challenge is graded from its + * room. A guest is locked to their room's challenge and needs no seat. + */ +function memberSeat( + db: DrizzleDB, + judge: Judge, + challenges: readonly ChallengeRow[], +) { + return judge.kind === "member" + ? seatOf(db, judge.judgeId, challenges) + : Promise.resolve(null); +} + /** The track a member judge asked for when they applied, or null. */ async function preferredTrackOf( db: DrizzleDB, @@ -706,10 +727,12 @@ export const judgingDeskRouter = createTRPCRouter({ const seat = judge ? await seatOf(db, judge.judgeId, challenges) : null; const roomId = principal.kind === "guest" ? principal.roomId : (seat?.roomId ?? null); + // A guest sees their room's challenge; a table judge sees General's and, + // from a seat in its room, a sponsor's. const visible = principal.kind === "guest" ? guestScope(challenges, principal.roomChallengeId) - : null; + : memberScope(challenges, seat); return { kind: principal.kind, @@ -722,7 +745,7 @@ export const judgingDeskRouter = createTRPCRouter({ showAllScores: principal.kind === "member" && switches.showAllScores, room: roomId ? await roomSummary(db, roomId, challenges) : null, challenges: challenges - .filter((c) => !visible || visible.has(c.id)) + .filter((c) => visible.has(c.id)) .map((c) => ({ id: c.id, label: c.label, @@ -1064,6 +1087,7 @@ export const judgingDeskRouter = createTRPCRouter({ judgeId: judgingEvaluationDrafts.judgeId, updatedAt: judgingEvaluationDrafts.updatedAt, deadlineAt: judgingEvaluationDrafts.deadlineAt, + revision: judgingEvaluationDrafts.revision, }) .from(judgingEvaluationDrafts) .where( @@ -1083,8 +1107,12 @@ export const judgingDeskRouter = createTRPCRouter({ .filter((e) => e.judgeId === judge.judgeId) .map((e) => [e.entryId, e]), ); + // A draft the judge has saved to. Opening a project makes an empty one + // (startGrading), and that is not a draft saved. const drafted = new Set( - drafts.filter((d) => d.judgeId === judge.judgeId).map((d) => d.entryId), + drafts + .filter((d) => d.judgeId === judge.judgeId && d.revision > 0) + .map((d) => d.entryId), ); const grading = gradingStates( judge.judgeId, @@ -1147,10 +1175,30 @@ export const judgingDeskRouter = createTRPCRouter({ state === "graded" ? 2 : state === "in_progress" ? 1 : 0; const byTitle = (a: { title: string }, b: { title: string }) => a.title.localeCompare(b.title); + // Nothing is timed here, so open projects follow the judge's own queue, + // the order Grade next project walks: the natural pick is the fair one. + const queued = + input.sort === "scheduled" && (!schedule || mode !== "scheduled") + ? new Map( + queueOrder( + judge.judgeId, + challenge.id, + projects.filter((p) => p.grading === "open").map((p) => p.id), + ).map((id, place) => [id, place]), + ) + : null; projects.sort((a, b) => { const open = standing(a.grading) - standing(b.grading); if (open !== 0) return open; - if (input.sort === "scheduled") { + if (queued) { + // The judge's own come first, then the open ones in queue order. + const mine = + Number(b.grading === "mine") - Number(a.grading === "mine"); + if (mine !== 0) return mine; + const at = queued.get(a.id); + const bt = queued.get(b.id); + if (at !== undefined && bt !== undefined) return at - bt; + } else if (input.sort === "scheduled") { const at = a.presentation?.startsAt.getTime() ?? Infinity; const bt = b.presentation?.startsAt.getTime() ?? Infinity; if (at !== bt) return at - bt; @@ -1407,10 +1455,18 @@ export const judgingDeskRouter = createTRPCRouter({ // The next project for this judge to grade, in the challenge they pick or // the one they sit at. Each judge walks the projects in an order of their own // (services/judging/queue.ts), so judges at one table rarely reach for the - // same one. Only open projects: not graded, not held by another judge. At a - // scheduled challenge, only presentations running now in the judge's room. + // same one. Only open projects: not graded, not held by another judge. At an + // untimed challenge only the first QUEUE_WINDOW of them are on offer, and + // `skip` passes over projects the judge turned down (a team away from its + // table). At a scheduled challenge, only presentations running now in the + // judge's room. nextProject: judgeProcedure - .input(z.object({ challengeId: uuid.optional() })) + .input( + z.object({ + challengeId: uuid.optional(), + skip: z.array(uuid).max(20).default([]), + }), + ) .query(async ({ ctx, input }) => { const db = ctx.db; const { judge } = ctx; @@ -1425,121 +1481,179 @@ export const judgingDeskRouter = createTRPCRouter({ seat, ); const mode = modeOf(challenges, challenge); - const found = (entryId: string | null, needsRoom = false) => ({ + const found = ( + entryId: string | null, + needsRoom = false, + allSkipped = false, + ) => ({ challenge: { id: challenge.id, label: challenge.label, mode }, entryId, needsRoom, + allSkipped, }); - const [schedule] = await db - .select({ id: judgingSchedules.id }) - .from(judgingSchedules) - .where(eq(judgingSchedules.hackathonId, input.hackathonId)) - .limit(1); - // A scheduled challenge is judged where it presents. - const timed = schedule !== undefined && mode === "scheduled"; - if (timed && seat?.challengeId !== challenge.id) return found(null, true); - - const entries = await db - .select({ id: judgingEntries.id }) - .from(judgingEntries) - .innerJoin( - judgingEntryChallenges, - and( - eq(judgingEntryChallenges.entryId, judgingEntries.id), - eq( - judgingEntryChallenges.challengeId, - input.challengeId ?? challenge.id, - ), - ), - ) - .where( - and( - eq(judgingEntries.hackathonId, input.hackathonId), - isNull(judgingEntries.deletedAt), - ), - ); - if (entries.length === 0) return found(null); - const entryIds = entries.map((entry) => entry.id); + const queue = await loadQueue(db, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + challenge, + seat, + filterId: input.challengeId, + }); + if (queue.needsRoom) return found(null, true); - const evaluations = await db - .select({ - entryId: judgingEvaluations.entryId, - judgeId: judgingEvaluations.judgeId, - }) - .from(judgingEvaluations) - .where( - and( - eq(judgingEvaluations.challengeId, challenge.id), - inArray(judgingEvaluations.entryId, entryIds), - ), - ); - const drafts = await db - .select({ - entryId: judgingEvaluationDrafts.entryId, - judgeId: judgingEvaluationDrafts.judgeId, - updatedAt: judgingEvaluationDrafts.updatedAt, - deadlineAt: judgingEvaluationDrafts.deadlineAt, - }) - .from(judgingEvaluationDrafts) - .where( - and( - eq(judgingEvaluationDrafts.challengeId, challenge.id), - inArray(judgingEvaluationDrafts.entryId, entryIds), - ), - ); - const now = new Date(); - const grading = gradingStates( + // The schedule decides who sees what at a scheduled challenge. + const window = queue.timed ? Infinity : QUEUE_WINDOW; + const entryId = nextToGrade( judge.judgeId, - entryIds, - evaluations, - drafts, - now, + challenge.id, + queue.candidates, + { skip: input.skip, window }, ); - let candidates = entryIds.map((entryId) => ({ - entryId, - grading: grading.get(entryId) ?? ("open" as const), - })); + const allSkipped = + entryId === null && + queueWindow(judge.judgeId, challenge.id, queue.candidates, window) + .length > 0; + return found(entryId, false, allSkipped); + }), - if (timed && seat) { - const slots = await db - .select({ - id: judgingAppointments.id, - entryId: judgingAppointments.entryId, - startsAt: judgingAppointments.startsAt, - deadlineAt: judgingAppointments.deadlineAt, - }) - .from(judgingAppointments) - .where( - and( - eq(judgingAppointments.scheduleId, schedule.id), - eq(judgingAppointments.challengeId, challenge.id), - eq(judgingAppointments.roomId, seat.roomId), - inArray(judgingAppointments.entryId, entryIds), - ), - ); - const running = new Set( - slots - .filter( - (slot) => - evaluationTiming( - { - hasSchedule: true, - mode, - challengeId: challenge.id, - evaluation: null, - appointment: slot, - presenceChallengeId: seat.challengeId, - }, - now, - ).canEdit, + // Opening a project to grade it. Runs the checks a first save would (the + // project is free, the queue offers it) and, when it passes, creates or + // touches the judge's draft with no answers needed, so the hold starts as + // the judge opens the project rather than at their first save. The draft + // keeps saveDraft's deadline and revision, so the editor carries on from it. + startGrading: judgeProcedure + .input(z.object({ entryId: uuid, challengeId: uuid.optional() })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db; + const { judge } = ctx; + await reconcileDueDrafts(db, input.hackathonId); + + const challenges = await loadChallenges(db, input.hackathonId); + const challenge = evaluatedChallenge( + judge, + challenges, + input.challengeId, + await memberSeat(db, judge, challenges), + ); + + try { + return await db.transaction(async (tx) => { + await assertJudgingOpen(tx, input.hackathonId); + await assertGuestSeated(tx, judge); + await loadEntryInChallenge(tx, { + hackathonId: input.hackathonId, + entryId: input.entryId, + challenge, + }); + + const [draft] = await tx + .select({ + id: judgingEvaluationDrafts.id, + revision: judgingEvaluationDrafts.revision, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + }) + .from(judgingEvaluationDrafts) + .where( + taskOf( + judgingEvaluationDrafts, + judge.judgeId, + input.entryId, + challenge.id, + ), ) - .map((slot) => slot.entryId), - ); - candidates = candidates.filter((c) => running.has(c.entryId)); - } + .for("update") + .limit(1); + const [evaluation] = await tx + .select({ id: judgingEvaluations.id }) + .from(judgingEvaluations) + .where( + taskOf( + judgingEvaluations, + judge.judgeId, + input.entryId, + challenge.id, + ), + ) + .limit(1); + // Their own graded project is theirs to edit; there is nothing to hold. + if (evaluation) { + return { + held: false, + revision: draft?.revision ?? 0, + deadlineAt: draft?.deadlineAt ?? null, + }; + } - return found(nextToGrade(judge.judgeId, challenge.id, candidates)); + await claimGrading(tx, { + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + }); + if (!draft) { + await assertQueueOffers(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + }); + } + + const { timing } = await evaluationAccess(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + evaluation: null, + graceMs: DEADLINE_GRACE_MS, + }); + if (!timing.canEdit) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: timing.reason, + }); + } + + // As saveDraft stamps a draft that is not an edit of an evaluation. + const deadlineAt = timing.deadlineAt ?? draft?.deadlineAt ?? null; + const now = new Date(); + if (draft) { + await tx + .update(judgingEvaluationDrafts) + .set({ + appointmentId: timing.appointmentId, + deadlineAt, + updatedAt: now, + }) + .where(eq(judgingEvaluationDrafts.id, draft.id)); + } else { + await tx.insert(judgingEvaluationDrafts).values({ + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challengeId: challenge.id, + ratings: {}, + responses: {}, + appointmentId: timing.appointmentId, + deadlineAt, + revision: 0, + updatedAt: now, + }); + } + + return { held: true, revision: draft?.revision ?? 0, deadlineAt }; + }); + } catch (error) { + if (isUniqueViolation(error)) { + throw new TRPCError({ + code: "CONFLICT", + message: + "This project was just opened from another window. Reload to pick up the latest.", + }); + } + throw error; + } }), // Autosave. `revision` is the draft revision the editor last saw (0 for @@ -1566,7 +1680,7 @@ export const judgingDeskRouter = createTRPCRouter({ judge, challenges, input.challengeId, - null, + await memberSeat(db, judge, challenges), ); const rubric = await loadRubric(db, input.hackathonId); const check = checkAnswers(rubric, input, judge.kind, "strict"); @@ -1622,6 +1736,16 @@ export const judgingDeskRouter = createTRPCRouter({ entryId: input.entryId, challenge, }); + // Starting a project means taking one their queue offers. + if (!draft) { + await assertQueueOffers(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + }); + } } if ((draft?.revision ?? 0) !== input.revision) { @@ -1719,7 +1843,7 @@ export const judgingDeskRouter = createTRPCRouter({ judge, challenges, input.challengeId, - null, + await memberSeat(db, judge, challenges), ); const rubric = await loadRubric(db, input.hackathonId); const check = checkAnswers(rubric, input, judge.kind, "strict"); @@ -1745,7 +1869,7 @@ export const judgingDeskRouter = createTRPCRouter({ // The draft first: a deadline submission holding it finishes // before this reads the evaluation, and then sees the result. - await tx + const [draft] = await tx .select({ id: judgingEvaluationDrafts.id }) .from(judgingEvaluationDrafts) .where( @@ -1784,6 +1908,16 @@ export const judgingDeskRouter = createTRPCRouter({ entryId: input.entryId, challenge, }); + // Starting a project means taking one their queue offers. + if (!draft) { + await assertQueueOffers(tx, { + hackathonId: input.hackathonId, + judgeId: judge.judgeId, + entryId: input.entryId, + challenge, + challenges, + }); + } } if ((existing?.revision ?? 0) !== input.expectedRevision) { diff --git a/packages/api/src/services/judging/evaluation-store.test.ts b/packages/api/src/services/judging/evaluation-store.test.ts index 4a020840..d41bf2bf 100644 --- a/packages/api/src/services/judging/evaluation-store.test.ts +++ b/packages/api/src/services/judging/evaluation-store.test.ts @@ -16,9 +16,11 @@ import { evaluatedChallenge, guestScope, inJudgeScope, + memberScope, reconcileDueDrafts, } from "./evaluation-store"; import type { ChallengeRow, Seat } from "./evaluation-store"; +import type { Answers } from "./evaluation"; import { GRADING_HOLD_MS } from "./grading-hold"; import type { Judge } from "./principal"; import type { DrizzleDB } from "@query/db"; @@ -85,11 +87,34 @@ describe("guest scope", () => { }); }); +describe("member scope", () => { + it("is General's challenges alone for a judge with no seat", () => { + expect([...memberScope(challenges, null)]).toEqual(["general"]); + }); + + it("adds a sponsor's challenge, and what is below it, from its room", () => { + expect([...memberScope(challenges, seat("sponsor"))].sort()).toEqual([ + "general", + "sponsor", + ]); + expect([...memberScope(challenges, seat("mlh"))].sort()).toEqual([ + "domain", + "general", + "mlh", + "mongo", + ]); + }); + + it("does not let a seat at General open a sponsor's challenge", () => { + expect([...memberScope(challenges, seat("general"))]).toEqual(["general"]); + }); +}); + describe("evaluatedChallenge", () => { it("evaluates a child challenge under its root", () => { - expect(evaluatedChallenge(member, challenges, "mongo", null).id).toBe( - "mlh", - ); + expect( + evaluatedChallenge(member, challenges, "mongo", seat("mlh")).id, + ).toBe("mlh"); }); it("defaults a member to their room's challenge, then General", () => { @@ -101,11 +126,64 @@ describe("evaluatedChallenge", () => { ); }); - it("falls back to the first root when there is no General", () => { + it("has nothing for a member to default to when there is only sponsors'", () => { const noGeneral = challenges.filter((c) => c.id !== "general"); - expect(evaluatedChallenge(member, noGeneral, undefined, null).id).toBe( - "sponsor", - ); + expect(() => + evaluatedChallenge(member, noGeneral, undefined, null), + ).toThrowError(expect.objectContaining({ code: "FORBIDDEN" })); + expect( + evaluatedChallenge(member, noGeneral, undefined, seat("sponsor")).id, + ).toBe("sponsor"); + }); + + describe("sponsor challenges", () => { + const refused = (label: string) => + expect.objectContaining({ + code: "FORBIDDEN", + message: `Sponsor judges grade ${label}. Sit in its room to grade it.`, + }); + + it("refuses a table judge with no seat", () => { + expect(() => + evaluatedChallenge(member, challenges, "sponsor", null), + ).toThrowError(refused("sponsor")); + }); + + it("refuses a table judge seated at another challenge's room", () => { + expect(() => + evaluatedChallenge(member, challenges, "sponsor", seat("general")), + ).toThrowError(refused("sponsor")); + expect(() => + evaluatedChallenge(member, challenges, "sponsor", seat("mlh")), + ).toThrowError(refused("sponsor")); + }); + + it("names the root when the judge asked for a prize inside a group", () => { + expect(() => + evaluatedChallenge(member, challenges, "mongo", null), + ).toThrowError(refused("mlh")); + }); + + it("lets a table judge grade it from a seat in its room", () => { + expect( + evaluatedChallenge(member, challenges, "sponsor", seat("sponsor")).id, + ).toBe("sponsor"); + }); + + it("keeps General open whatever the seat", () => { + expect(evaluatedChallenge(member, challenges, "general", null).id).toBe( + "general", + ); + expect( + evaluatedChallenge(member, challenges, "general", seat("sponsor")).id, + ).toBe("general"); + }); + + it("leaves a guest on their room's challenge, seat or none", () => { + expect(evaluatedChallenge(guest, challenges, "mongo", null).id).toBe( + "mlh", + ); + }); }); it("keeps a guest on their room's challenge", () => { @@ -255,19 +333,20 @@ const lockKeys = (events: Event[]) => .map((event) => dialect.sqlToQuery(event.query as SQL).params[0]); const RATING = "10000000-0000-4000-8000-000000000001"; +const RESPONSE = "10000000-0000-4000-8000-000000000002"; const due = ( id: string, judgeId: string, entryId: string, challengeId = "c1", + answers: Answers = { ratings: { [RATING]: 4 }, responses: {} }, ) => ({ id, judgeId, entryId, challengeId, appointmentId: "a1", - ratings: { [RATING]: 4 }, - responses: {}, + ...answers, deadlineAt: new Date("2026-04-18T15:00:00Z"), }); @@ -289,6 +368,14 @@ describe("reconcileDueDrafts and graded once", () => { required: true, guestVisibility: null, }, + { + id: RESPONSE, + kind: "response", + label: "Feedback", + description: "", + required: false, + guestVisibility: null, + }, ], [name(judgingJudges)]: [ ...new Set(drafts.map((draft) => draft.judgeId)), @@ -331,6 +418,29 @@ describe("reconcileDueDrafts and graded once", () => { expect(events.some((e) => e.kind === "update")).toBe(true); }); + it("keeps a due draft with nothing in it, from a judge who only opened the project", async () => { + const opened = due("d1", "j1", "e1", "c1", { ratings: {}, responses: {} }); + const { db, events } = fakeDb(answersFor([opened])); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(0); + expect(events.some((e) => e.kind === "delete")).toBe(false); + expect(events.some((e) => e.kind === "update")).toBe(true); + }); + + it("still submits a due draft with only a written response in it", async () => { + const answered = due("d1", "j1", "e1", "c1", { + ratings: {}, + responses: { [RESPONSE]: { value: "Clear demo.", isPublic: false } }, + }); + const { db, events } = fakeDb(answersFor([answered])); + + await reconcileDueDrafts(db, "h", new Date("2026-04-18T15:05:00Z")); + + expect(written(events, judgingEvaluations)).toHaveLength(1); + }); + it("looks for a grade from any judge, not just the draft's", async () => { const { db, events } = fakeDb(answersFor([due("d1", "j1", "e1")])); diff --git a/packages/api/src/services/judging/evaluation-store.ts b/packages/api/src/services/judging/evaluation-store.ts index d442d6d6..dc585500 100644 --- a/packages/api/src/services/judging/evaluation-store.ts +++ b/packages/api/src/services/judging/evaluation-store.ts @@ -37,7 +37,12 @@ import type { CheckedAnswers } from "./evaluation"; import { gradingLockKey, gradingRefusal, gradingStateOf } from "./grading-hold"; import { GUEST_SESSION_ENDED } from "./principal"; import type { Judge } from "./principal"; -import { descendantsOf, effectiveMode, rootOf } from "./scope"; +import { + descendantsOf, + effectiveMode, + isSponsorChallenge, + rootOf, +} from "./scope"; import type { ChallengeMode } from "./scope"; import { DEADLINE_GRACE_MS, evaluationTiming } from "./timing"; import type { EvaluationTiming } from "./timing"; @@ -138,10 +143,37 @@ export async function seatOf( }; } +/** + * Sponsor challenges belong to the sponsor's own judges: a member (table) + * judge grades one only while seated in a room that serves it. + */ +function memberMayGrade(root: ChallengeRow, seat: Seat | null): boolean { + return !isSponsorChallenge(root) || seat?.challengeId === root.id; +} + +/** + * The challenges a member judge may pick right now: the general ones, and a + * sponsor's (with everything below it) only from a seat in its room. + */ +export function memberScope( + challenges: readonly ChallengeRow[], + seat: Seat | null, +): Set { + return new Set( + challenges + .filter((c) => { + const root = rootOf(challenges, c.id); + return !!root && memberMayGrade(root, seat); + }) + .map((c) => c.id), + ); +} + /** * The root challenge a judge is evaluating. A guest always evaluates their * room's; a filter they pass has to sit inside it. A member evaluates the root - * of what they picked, or else the room they sit in, or else General. + * of what they picked, or else the room they sit in, or else General, and + * only a sponsor's challenge from a seat in the room that serves it. */ export function evaluatedChallenge( judge: Judge, @@ -174,6 +206,12 @@ export function evaluatedChallenge( "This hackathon has no judging challenges yet. Ask an organiser to import projects first.", }); } + if (judge.kind === "member" && !memberMayGrade(root, seat)) { + throw new TRPCError({ + code: "FORBIDDEN", + message: `Sponsor judges grade ${root.label}. Sit in its room to grade it.`, + }); + } return root; } @@ -562,7 +600,8 @@ const RECONCILE_BATCH = 50; * first) wins over this. A draft that can no longer be submitted (the guest's * session was revoked before the deadline, the project was withdrawn, the * schedule was dropped, another judge already graded the project) keeps its - * answers and loses its deadline. + * answers and loses its deadline. So does a draft with no answers in it: the + * judge only opened the project (desk.startGrading), and that is not a grade. * * Each project is graded once, so the batch takes the grading lock of every * project it touches before it writes anything: in key order, and ahead of any @@ -686,7 +725,11 @@ export async function reconcileDueDrafts( } const check = checkAnswers(rubric, draft, kind, "lenient"); - if (!check.ok) { + if ( + !check.ok || + (check.answers.ratings.length === 0 && + check.answers.responses.length === 0) + ) { await retain(draft.id); continue; } diff --git a/packages/api/src/services/judging/queue-store.test.ts b/packages/api/src/services/judging/queue-store.test.ts new file mode 100644 index 00000000..e86dfb2a --- /dev/null +++ b/packages/api/src/services/judging/queue-store.test.ts @@ -0,0 +1,428 @@ +import { describe, expect, it } from "vitest"; +import { getTableName } from "drizzle-orm"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + judgingAppointments, + judgingEntries, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingSchedules, +} from "@query/db"; +import { QUEUE_REFUSAL, assertQueueOffers, loadQueue } from "./queue-store"; +import { QUEUE_WINDOW, queueOrder } from "./queue"; +import type { ChallengeRow, Seat } from "./evaluation-store"; +import { GRADING_HOLD_MS } from "./grading-hold"; + +const HACK = "hackathon"; +const JUDGE = "judge-1"; + +const row = (id: string, over: Partial = {}): ChallengeRow => ({ + id, + parentId: null, + mode: "unscheduled", + label: id, + isGeneral: false, + isGroup: false, + ...over, +}); +const general = row("general", { isGeneral: true }); +const sponsor = row("sponsor", { mode: "scheduled" }); +const mlh = row("mlh", { isGroup: true }); +const mongo = row("mongo", { parentId: "mlh" }); +const challenges = [general, sponsor, mlh, mongo]; + +const seat = (challengeId: string): Seat => ({ + roomId: "room-1", + roomName: "Table 4", + challengeId, +}); + +const entryIds = Array.from({ length: 8 }, (_, i) => `entry-${i}`); + +const name = (table: unknown) => getTableName(table as never); +const via = (...tables: unknown[]) => tables.map(name).join(">"); +const entries = via(judgingEntries, judgingEntryChallenges); +const evaluations = via(judgingEvaluations); +const drafts = via(judgingEvaluationDrafts); +const schedules = via(judgingSchedules); +const appointments = via(judgingAppointments); +const memberships = via(judgingEntryChallenges); + +type Rows = unknown[] | ((call: number) => unknown[]); +type Read = { table: string; where?: SQL; on?: SQL }; + +/** A database whose selects answer by the tables they read, in order. */ +function fakeDb(answers: Record) { + const reads: Read[] = []; + const counts = new Map(); + const select = () => { + const joined: unknown[] = []; + let from: unknown; + let where: SQL | undefined; + let on: SQL | undefined; + const chain: Record = { + from: (table: unknown) => { + from = table; + return chain; + }, + where: (clause: SQL) => { + where = clause; + return chain; + }, + then: ( + ok: (rows: unknown[]) => unknown, + fail: (e: unknown) => unknown, + ) => { + const key = via(from, ...joined); + reads.push({ table: key, where, on }); + const call = counts.get(key) ?? 0; + counts.set(key, call + 1); + const answer = answers[key] ?? []; + return Promise.resolve( + typeof answer === "function" ? answer(call) : answer, + ).then(ok, fail); + }, + }; + for (const join of ["innerJoin", "leftJoin"]) { + chain[join] = (table: unknown, clause: SQL) => { + joined.push(table); + on = clause; + return chain; + }; + } + for (const method of ["orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }; + return { db: { select } as never, reads }; +} + +const ids = (list: readonly string[]) => list.map((id) => ({ id })); +const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000); +const minutesAhead = (minutes: number) => + new Date(Date.now() + minutes * 60_000); + +const input = ( + over: Partial[1]> = {}, +): Parameters[1] => ({ + hackathonId: HACK, + judgeId: JUDGE, + challenge: general, + seat: null, + ...over, +}); + +describe("loadQueue", () => { + it("says where each project stands for the judge", async () => { + const { db } = fakeDb({ + [entries]: ids(entryIds.slice(0, 5)), + [evaluations]: [ + { entryId: "entry-0", judgeId: JUDGE }, + { entryId: "entry-1", judgeId: "other" }, + ], + [drafts]: [ + { + entryId: "entry-2", + judgeId: "other", + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: "entry-3", + judgeId: "other", + updatedAt: minutesAgo(60), + deadlineAt: null, + }, + ], + }); + + const queue = await loadQueue(db, input()); + + expect(queue.timed).toBe(false); + expect(queue.needsRoom).toBe(false); + expect( + Object.fromEntries(queue.candidates.map((c) => [c.entryId, c.grading])), + ).toEqual({ + "entry-0": "mine", + "entry-1": "graded", + "entry-2": "in_progress", + "entry-3": "open", + "entry-4": "open", + }); + }); + + it("keeps a project the judge only opened on offer to them", async () => { + const mine = (entryId: string, revision: number) => ({ + entryId, + judgeId: JUDGE, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision, + }); + const { db } = fakeDb({ + [entries]: ids(entryIds.slice(0, 4)), + [evaluations]: [ + { entryId: "entry-2", judgeId: "other" }, + { entryId: "entry-3", judgeId: JUDGE }, + ], + [drafts]: [ + mine("entry-0", 0), + mine("entry-1", 2), + mine("entry-2", 0), + mine("entry-3", 0), + ], + }); + + const queue = await loadQueue(db, input()); + + expect( + Object.fromEntries(queue.candidates.map((c) => [c.entryId, c.grading])), + ).toEqual({ + // Opened, nothing saved: still on offer. + "entry-0": "open", + // Saved to: the judge is grading it. + "entry-1": "mine", + // Another judge's grade, or the judge's own, wins over an empty draft. + "entry-2": "graded", + "entry-3": "mine", + }); + }); + + it("has no candidates, and reads no grades, for a challenge without projects", async () => { + const { db, reads } = fakeDb({}); + + const queue = await loadQueue(db, input()); + + expect(queue.candidates).toEqual([]); + expect(reads.map((r) => r.table)).toEqual([schedules, entries]); + }); + + it("narrows the projects to a prize when asked", async () => { + const { db, reads } = fakeDb({ [entries]: ids(["entry-0"]) }); + + await loadQueue(db, input({ challenge: mlh, filterId: "mongo" })); + + const params = new PgDialect().sqlToQuery( + reads.find((r) => r.table === entries)?.on as SQL, + ).params; + expect(params).toContain("mongo"); + }); + + describe("at a scheduled challenge under a saved schedule", () => { + const world = (over: Record = {}) => ({ + [schedules]: [{ id: "schedule" }], + [entries]: ids(["entry-0", "entry-1", "entry-2"]), + ...over, + }); + + it("is timed, and asks a judge seated elsewhere to join the room", async () => { + const { db, reads } = fakeDb(world()); + + const queue = await loadQueue(db, input({ challenge: sponsor })); + expect(queue).toEqual({ timed: true, needsRoom: true, candidates: [] }); + + const elsewhere = await loadQueue( + db, + input({ challenge: sponsor, seat: seat("general") }), + ); + expect(elsewhere.needsRoom).toBe(true); + expect(reads.some((r) => r.table === entries)).toBe(false); + }); + + it("offers only the presentations running now in the judge's room", async () => { + const slot = (entryId: string, startsAt: Date, deadlineAt: Date) => ({ + id: `slot-${entryId}`, + entryId, + startsAt, + deadlineAt, + }); + const { db } = fakeDb( + world({ + [appointments]: [ + slot("entry-0", minutesAgo(30), minutesAgo(20)), + slot("entry-1", minutesAgo(5), minutesAhead(5)), + slot("entry-2", minutesAhead(20), minutesAhead(30)), + ], + }), + ); + + const queue = await loadQueue( + db, + input({ challenge: sponsor, seat: seat("sponsor") }), + ); + + expect(queue.timed).toBe(true); + expect(queue.needsRoom).toBe(false); + expect(queue.candidates.map((c) => c.entryId)).toEqual(["entry-1"]); + }); + }); + + it("is untimed for a scheduled challenge nobody has scheduled yet", async () => { + const { db } = fakeDb({ [entries]: ids(["entry-0"]) }); + + const queue = await loadQueue(db, input({ challenge: sponsor })); + + expect(queue.timed).toBe(false); + expect(queue.candidates).toHaveLength(1); + }); +}); + +describe("assertQueueOffers", () => { + const order = queueOrder(JUDGE, "general", entryIds); + const claim = ( + entryId: string, + answers: Record, + over: Partial[1]> = {}, + ) => { + const fake = fakeDb({ [entries]: ids(entryIds), ...answers }); + return { + ...fake, + claim: () => + assertQueueOffers(fake.db, { + hackathonId: HACK, + judgeId: JUDGE, + entryId, + challenge: general, + challenges, + ...over, + }), + }; + }; + + it("lets a judge start any of the first projects their queue offers", async () => { + for (const entryId of order.slice(0, QUEUE_WINDOW)) { + await expect(claim(entryId, {}).claim()).resolves.toBeUndefined(); + } + }); + + it("refuses a project further down their queue", async () => { + for (const entryId of order.slice(QUEUE_WINDOW)) { + await expect(claim(entryId, {}).claim()).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: QUEUE_REFUSAL, + }); + } + }); + + it("says what to do instead", () => { + expect(QUEUE_REFUSAL).toBe( + "Grade the projects your queue offers so every judge sees a random mix. Use Grade next project.", + ); + }); + + it("slides the window past projects that are graded or held", async () => { + const fourth = order[QUEUE_WINDOW]; + const { claim: go } = claim(fourth, { + [evaluations]: [{ entryId: order[0], judgeId: "other" }], + [drafts]: [ + { + entryId: order[1], + judgeId: "other", + updatedAt: minutesAgo(1), + deadlineAt: null, + }, + { + entryId: order[2], + judgeId: "other", + updatedAt: minutesAgo(2), + deadlineAt: null, + }, + ], + }); + + await expect(go()).resolves.toBeUndefined(); + }); + + it("does not slide the window past projects the judge only opened", async () => { + const held = (entryId: string, revision: number) => ({ + entryId, + judgeId: JUDGE, + updatedAt: minutesAgo(1), + deadlineAt: null, + revision, + }); + const opened = order.slice(0, QUEUE_WINDOW).map((id) => held(id, 0)); + await expect( + claim(order[QUEUE_WINDOW], { [drafts]: opened }).claim(), + ).rejects.toMatchObject({ message: QUEUE_REFUSAL }); + + // Once they save to one, they are grading it, and the window moves on. + const started = [held(order[0], 1), ...opened.slice(1)]; + await expect( + claim(order[QUEUE_WINDOW], { [drafts]: started }).claim(), + ).resolves.toBeUndefined(); + }); + + it("does not count a draft left alone past the hold", async () => { + const { claim: go } = claim(order[0], { + [drafts]: [ + { + entryId: order[0], + judgeId: "other", + updatedAt: new Date(Date.now() - GRADING_HOLD_MS - 60_000), + deadlineAt: null, + }, + ], + }); + + await expect(go()).resolves.toBeUndefined(); + }); + + it("asks nothing of a scheduled challenge: the schedule decides", async () => { + const { claim: go, reads } = claim( + order[7], + { [schedules]: [{ id: "schedule" }] }, + { challenge: sponsor }, + ); + + await expect(go()).resolves.toBeUndefined(); + expect(reads.some((r) => r.table === entries)).toBe(false); + }); + + it("still asks it of a scheduled challenge nobody has scheduled yet", async () => { + const worst = queueOrder(JUDGE, "sponsor", entryIds)[7]; + const { claim: go } = claim(worst, {}, { challenge: sponsor }); + + await expect(go()).rejects.toMatchObject({ message: QUEUE_REFUSAL }); + }); + + describe("under a group, with the list narrowed to one prize", () => { + const wholeOrder = queueOrder(JUDGE, "mlh", entryIds); + const prize = (entryId: string, prizeEntries: string[]) => + claim( + entryId, + { + [entries]: (call) => ids(call === 0 ? entryIds : prizeEntries), + [memberships]: [{ challengeId: "mongo" }], + }, + { challenge: mlh }, + ); + + it("offers a project that tops its prize's queue though it is far down the group's", async () => { + const far = wholeOrder[7]; + + await expect(prize(far, [far]).claim()).resolves.toBeUndefined(); + }); + + it("refuses a project that is outside the window of the group and of its prize", async () => { + const prizeEntries = wholeOrder.slice(3, 8); + const prizeOrder = queueOrder(JUDGE, "mlh", prizeEntries); + + await expect( + prize(prizeOrder[QUEUE_WINDOW], prizeEntries).claim(), + ).rejects.toMatchObject({ message: QUEUE_REFUSAL }); + }); + + it("checks the group first and stops there when it offers the project", async () => { + const { claim: go, reads } = prize(wholeOrder[0], []); + + await go(); + + expect(reads.filter((r) => r.table === entries)).toHaveLength(1); + expect(reads.some((r) => r.table === memberships)).toBe(false); + }); + }); +}); diff --git a/packages/api/src/services/judging/queue-store.ts b/packages/api/src/services/judging/queue-store.ts new file mode 100644 index 00000000..6f7c2394 --- /dev/null +++ b/packages/api/src/services/judging/queue-store.ts @@ -0,0 +1,244 @@ +import { TRPCError } from "@trpc/server"; +import { and, eq, inArray, isNull } from "drizzle-orm"; +import { + judgingAppointments, + judgingEntries, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingSchedules, +} from "@query/db"; +import type { JudgingDb } from "./config"; +import type { ChallengeRow, Seat } from "./evaluation-store"; +import { gradingStates } from "./grading-hold"; +import type { GradingState } from "./grading-hold"; +import { queueWindow } from "./queue"; +import { descendantsOf } from "./scope"; +import { evaluationTiming } from "./timing"; + +// The projects a judge is offered at a challenge, read from the database. The +// desk's next-project pick and the check on a judge's first claim of a project +// both go through loadQueue, so they never disagree about what is on offer. + +export type Queue = { + /** A scheduled challenge under a saved schedule: the schedule decides who sees what. */ + timed: boolean; + /** Timed, and the judge is not seated in a room that serves the challenge. */ + needsRoom: boolean; + /** + * The challenge's projects and where each stands for the judge. A project + * the judge has only opened counts as open (see loadQueue). For a timed + * challenge, only the presentations running now in the judge's room. + */ + candidates: { entryId: string; grading: GradingState }[]; +}; + +/** + * `challenge` is the root being graded. `filterId` narrows the projects to one + * prize inside it, as the desk's challenge filter does. + */ +export async function loadQueue( + db: JudgingDb, + input: { + hackathonId: string; + judgeId: string; + challenge: ChallengeRow; + seat: Seat | null; + filterId?: string; + }, + now = new Date(), +): Promise { + const { challenge, seat } = input; + const [schedule] = await db + .select({ id: judgingSchedules.id }) + .from(judgingSchedules) + .where(eq(judgingSchedules.hackathonId, input.hackathonId)) + .limit(1); + // A scheduled challenge is judged where it presents. + const timed = schedule !== undefined && challenge.mode === "scheduled"; + if (timed && seat?.challengeId !== challenge.id) { + return { timed, needsRoom: true, candidates: [] }; + } + const queue = (candidates: Queue["candidates"]): Queue => ({ + timed, + needsRoom: false, + candidates, + }); + + const entries = await db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .innerJoin( + judgingEntryChallenges, + and( + eq(judgingEntryChallenges.entryId, judgingEntries.id), + eq(judgingEntryChallenges.challengeId, input.filterId ?? challenge.id), + ), + ) + .where( + and( + eq(judgingEntries.hackathonId, input.hackathonId), + isNull(judgingEntries.deletedAt), + ), + ); + if (entries.length === 0) return queue([]); + const entryIds = entries.map((entry) => entry.id); + + const evaluations = await db + .select({ + entryId: judgingEvaluations.entryId, + judgeId: judgingEvaluations.judgeId, + }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.challengeId, challenge.id), + inArray(judgingEvaluations.entryId, entryIds), + ), + ); + const drafts = await db + .select({ + entryId: judgingEvaluationDrafts.entryId, + judgeId: judgingEvaluationDrafts.judgeId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, + revision: judgingEvaluationDrafts.revision, + }) + .from(judgingEvaluationDrafts) + .where( + and( + eq(judgingEvaluationDrafts.challengeId, challenge.id), + inArray(judgingEvaluationDrafts.entryId, entryIds), + ), + ); + const grading = gradingStates( + input.judgeId, + entryIds, + evaluations, + drafts, + now, + ); + // A project the judge has only opened (desk.startGrading's draft, never + // saved to) stays on offer to them. Otherwise opening projects and backing + // out of them, or skipping them, would walk the judge down their queue. + const graded = new Set( + evaluations + .filter((evaluation) => evaluation.judgeId === input.judgeId) + .map((evaluation) => evaluation.entryId), + ); + const opened = new Set( + drafts + .filter( + (draft) => + draft.judgeId === input.judgeId && + draft.revision === 0 && + !graded.has(draft.entryId), + ) + .map((draft) => draft.entryId), + ); + let candidates = entryIds.map((entryId) => { + const state = grading.get(entryId) ?? ("open" as const); + return { + entryId, + grading: + state === "mine" && opened.has(entryId) ? ("open" as const) : state, + }; + }); + + if (timed && seat) { + const slots = await db + .select({ + id: judgingAppointments.id, + entryId: judgingAppointments.entryId, + startsAt: judgingAppointments.startsAt, + deadlineAt: judgingAppointments.deadlineAt, + }) + .from(judgingAppointments) + .where( + and( + eq(judgingAppointments.scheduleId, schedule.id), + eq(judgingAppointments.challengeId, challenge.id), + eq(judgingAppointments.roomId, seat.roomId), + inArray(judgingAppointments.entryId, entryIds), + ), + ); + const running = new Set( + slots + .filter( + (slot) => + evaluationTiming( + { + hasSchedule: true, + mode: challenge.mode, + challengeId: challenge.id, + evaluation: null, + appointment: slot, + presenceChallengeId: seat.challengeId, + }, + now, + ).canEdit, + ) + .map((slot) => slot.entryId), + ); + candidates = candidates.filter((c) => running.has(c.entryId)); + } + + return queue(candidates); +} + +export const QUEUE_REFUSAL = + "Grade the projects your queue offers so every judge sees a random mix. Use Grade next project."; + +/** + * For a judge's first claim of a project (they hold no draft of it and have + * not graded it): at an untimed challenge it has to be one of the projects + * their queue offers (see QUEUE_WINDOW). A judge who narrowed the list to one + * prize is offered that prize's own window, so the project counts if it sits + * in the window of any prize it is entered in. Run it after claimGrading, in + * the same transaction. + */ +export async function assertQueueOffers( + tx: JudgingDb, + input: { + hackathonId: string; + judgeId: string; + entryId: string; + challenge: ChallengeRow; + challenges: readonly ChallengeRow[]; + }, + now = new Date(), +) { + const { challenge } = input; + const offers = (queue: Queue) => + queueWindow(input.judgeId, challenge.id, queue.candidates).includes( + input.entryId, + ); + // A timed challenge stops at the schedule check, so the seat is never read. + const base = { hackathonId: input.hackathonId, judgeId: input.judgeId }; + + const whole = await loadQueue(tx, { ...base, challenge, seat: null }, now); + if (whole.timed || offers(whole)) return; + + const below = descendantsOf(input.challenges, challenge.id).map((c) => c.id); + if (below.length > 0) { + const entered = await tx + .select({ challengeId: judgingEntryChallenges.challengeId }) + .from(judgingEntryChallenges) + .where( + and( + eq(judgingEntryChallenges.entryId, input.entryId), + inArray(judgingEntryChallenges.challengeId, below), + ), + ); + for (const { challengeId } of entered) { + const prize = await loadQueue( + tx, + { ...base, challenge, seat: null, filterId: challengeId }, + now, + ); + if (offers(prize)) return; + } + } + + throw new TRPCError({ code: "PRECONDITION_FAILED", message: QUEUE_REFUSAL }); +} diff --git a/packages/api/src/services/judging/queue.test.ts b/packages/api/src/services/judging/queue.test.ts index 5e234ae6..b77c65f5 100644 --- a/packages/api/src/services/judging/queue.test.ts +++ b/packages/api/src/services/judging/queue.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { nextToGrade, queueOrder } from "./queue"; +import { QUEUE_WINDOW, nextToGrade, queueOrder, queueWindow } from "./queue"; import type { GradingState } from "./grading-hold"; const CHALLENGE = "c0000000-0000-4000-8000-000000000001"; @@ -133,3 +133,74 @@ describe("nextToGrade", () => { expect(new Set(picked).size).toBeGreaterThan(1); }); }); + +describe("queueWindow", () => { + it("is the first few open projects in the judge's order", () => { + expect(QUEUE_WINDOW).toBe(3); + expect(queueWindow(judges[0], CHALLENGE, entries.map(open))).toEqual( + queueOrder(judges[0], CHALLENGE, entries).slice(0, QUEUE_WINDOW), + ); + }); + + it("counts only open projects, so held and graded ones do not use it up", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + const candidates = entries.map((entryId) => { + if (entryId === order[0]) return state(entryId, "graded"); + if (entryId === order[1]) return state(entryId, "in_progress"); + if (entryId === order[2]) return state(entryId, "mine"); + return open(entryId); + }); + expect(queueWindow(judges[0], CHALLENGE, candidates)).toEqual( + order.slice(3, 3 + QUEUE_WINDOW), + ); + }); + + it("is shorter when fewer projects are open", () => { + expect( + queueWindow(judges[0], CHALLENGE, entries.slice(0, 2).map(open)), + ).toHaveLength(2); + expect(queueWindow(judges[0], CHALLENGE, [])).toEqual([]); + }); + + it("is every open project when the window is unlimited", () => { + expect( + queueWindow(judges[0], CHALLENGE, entries.map(open), Infinity), + ).toEqual(queueOrder(judges[0], CHALLENGE, entries)); + }); + + it("differs from judge to judge, so each draws their own sample", () => { + const windows = new Set( + judges.map((judge) => + queueWindow(judge, CHALLENGE, entries.map(open)).join(), + ), + ); + expect(windows.size).toBeGreaterThan(30); + }); +}); + +describe("nextToGrade with skips", () => { + const order = queueOrder(judges[0], CHALLENGE, entries); + const next = (skip: string[], window?: number) => + nextToGrade(judges[0], CHALLENGE, entries.map(open), { skip, window }); + + it("passes over a skipped project to the next one in the window", () => { + expect(next([order[0]])).toBe(order[1]); + expect(next([order[0], order[1]])).toBe(order[2]); + }); + + it("is nothing once every project in the window is skipped", () => { + expect(next(order.slice(0, QUEUE_WINDOW))).toBeNull(); + }); + + it("never reaches past the window for a project that was not skipped", () => { + expect(next(order.slice(0, QUEUE_WINDOW))).not.toBe(order[QUEUE_WINDOW]); + }); + + it("ignores a skipped project outside the window", () => { + expect(next([order[10]])).toBe(order[0]); + }); + + it("reaches as far as an unlimited window goes", () => { + expect(next(order.slice(0, 5), Infinity)).toBe(order[5]); + }); +}); diff --git a/packages/api/src/services/judging/queue.ts b/packages/api/src/services/judging/queue.ts index d4cdffc5..73fc85c5 100644 --- a/packages/api/src/services/judging/queue.ts +++ b/packages/api/src/services/judging/queue.ts @@ -35,14 +35,45 @@ export function queueOrder( .map((item) => item.entryId); } -/** The next project for the judge to grade: the first open one in their queue. */ -export function nextToGrade( +type Candidate = { entryId: string; grading: GradingState }; + +// How many open projects at the front of their queue a judge may start at an +// untimed challenge. Calibration depends on each judge's sample being random. +// A small window lets a judge skip a team that is away from its table without +// letting them pick projects. +export const QUEUE_WINDOW = 3; + +/** + * The open projects a judge is offered: the first `window` of them in their + * queue order. A scheduled challenge passes Infinity, since the schedule + * already decides who sees what. + */ +export function queueWindow( judgeId: string, challengeId: string, - candidates: readonly { entryId: string; grading: GradingState }[], -): string | null { + candidates: readonly Candidate[], + window = QUEUE_WINDOW, +): string[] { const open = candidates .filter((candidate) => candidate.grading === "open") .map((candidate) => candidate.entryId); - return queueOrder(judgeId, challengeId, open)[0] ?? null; + return queueOrder(judgeId, challengeId, open).slice(0, window); +} + +/** + * The next project for the judge to grade: the first of the projects their + * queue offers that they have not skipped. + */ +export function nextToGrade( + judgeId: string, + challengeId: string, + candidates: readonly Candidate[], + options: { skip?: readonly string[]; window?: number } = {}, +): string | null { + const skip = new Set(options.skip); + return ( + queueWindow(judgeId, challengeId, candidates, options.window).find( + (entryId) => !skip.has(entryId), + ) ?? null + ); } diff --git a/packages/api/src/services/judging/schedule/inputs.ts b/packages/api/src/services/judging/schedule/inputs.ts index ffb0aae1..b313a523 100644 --- a/packages/api/src/services/judging/schedule/inputs.ts +++ b/packages/api/src/services/judging/schedule/inputs.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { rootOf } from "../scope"; +import { isSponsorChallenge, rootOf } from "../scope"; import type { ChallengeMode } from "../scope"; import { gridOf } from "./slots"; import type { ScheduleTiming } from "./slots"; @@ -86,8 +86,8 @@ export function selectPlan(source: { memberships: readonly SourceMembership[]; }): { rooms: PlanRoom[]; tasks: PlanTask[] } { const scheduled = scheduledChallengeIds(source.challenges); - const general = new Set( - source.challenges.filter((c) => c.isGeneral).map((c) => c.id), + const sponsor = new Set( + source.challenges.filter(isSponsorChallenge).map((c) => c.id), ); const rooms = source.rooms @@ -111,7 +111,7 @@ export function selectPlan(source: { tasks.push({ entryId: row.entryId, challengeId: row.challengeId, - sponsor: !general.has(row.challengeId), + sponsor: sponsor.has(row.challengeId), }); } tasks.sort(byTask); diff --git a/packages/api/src/services/judging/scope.test.ts b/packages/api/src/services/judging/scope.test.ts index dc6737f7..4bad80cd 100644 --- a/packages/api/src/services/judging/scope.test.ts +++ b/packages/api/src/services/judging/scope.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { descendantsOf, effectiveMode, + isSponsorChallenge, normalizeLabel, rootOf, sameLabel, @@ -108,3 +109,10 @@ describe("labels", () => { expect(sameLabel("AI", "A I")).toBe(false); }); }); + +describe("isSponsorChallenge", () => { + it("is every challenge but the general ones", () => { + expect(isSponsorChallenge({ isGeneral: false })).toBe(true); + expect(isSponsorChallenge({ isGeneral: true })).toBe(false); + }); +}); diff --git a/packages/api/src/services/judging/scope.ts b/packages/api/src/services/judging/scope.ts index 7fd40cb4..d8691ed5 100644 --- a/packages/api/src/services/judging/scope.ts +++ b/packages/api/src/services/judging/scope.ts @@ -35,6 +35,14 @@ export function rootOf( return null; } +/** + * Whether a challenge is a sponsor's: every challenge but the general ones. + * The schedule and the judge desk both read the flag this way. + */ +export function isSponsorChallenge(challenge: { isGeneral: boolean }): boolean { + return !challenge.isGeneral; +} + /** How a challenge is judged: always its root's mode. Null when unknown. */ export function effectiveMode( challenges: readonly ChallengeNode[], diff --git a/sites/mainweb/components/judge/EvaluationSheet.tsx b/sites/mainweb/components/judge/EvaluationSheet.tsx index 057a76f3..4a0c854c 100644 --- a/sites/mainweb/components/judge/EvaluationSheet.tsx +++ b/sites/mainweb/components/judge/EvaluationSheet.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react"; import { trpc } from "@/lib/trpc"; import { trpcErrorMessage } from "@/lib/trpc-error"; @@ -62,6 +62,7 @@ export function EvaluationSheet({ entryId, challengeId, onClose, + onGraded, onPickRoom, }: { hackathonId: string; @@ -70,6 +71,8 @@ export function EvaluationSheet({ entryId: string; challengeId: string | undefined; onClose: () => void; + /** The judge submitted their first evaluation of this project. */ + onGraded?: () => void; /** Members only: leaves the project for the room picker. */ onPickRoom?: () => void; }) { @@ -79,6 +82,48 @@ export function EvaluationSheet({ { refetchInterval: 15_000 }, ); const now = useServerNow(sheet.data?.serverNow, sheet.dataUpdatedAt); + const utils = trpc.useUtils(); + + // Opening an open project to grade it takes it: the hold starts now, not at + // the first save, so another judge is not sent to the same table. One call + // for each time it is opened. Another judge getting there first shows as the + // sheet's usual refusal; any other refusal is kept to say so here. + const { refetch } = sheet; + const openKey = + sheet.data && + sheet.data.grading === "open" && + sheet.data.access.canEdit && + !sheet.data.evaluation + ? `${sheet.data.entry.id}:${sheet.data.challenge.id}` + : null; + const challengeToStart = sheet.data?.challenge.id; + const started = useRef(null); + const [refused, setRefused] = useState<{ + entryId: string; + message: string; + } | null>(null); + const { mutate: startGrading } = trpc.judging.desk.startGrading.useMutation({ + onSuccess: () => + void utils.judging.desk.projects.invalidate({ hackathonId }), + onError: (error, variables) => { + if (error.data?.code === "CONFLICT") { + void refetch(); + } else { + setRefused({ + entryId: variables.entryId, + message: trpcErrorMessage( + error, + "This project did not open for you. Try again.", + ), + }); + } + }, + }); + useEffect(() => { + if (!openKey || started.current === openKey) return; + started.current = openKey; + startGrading({ hackathonId, entryId, challengeId: challengeToStart }); + }, [openKey, hackathonId, entryId, challengeToStart, startGrading]); // The project list may have been scrolled far down; the project starts at // its title. @@ -145,6 +190,8 @@ export function EvaluationSheet({ void sheet.refetch().then(() => setReloads((n) => n + 1)) } onNext={onClose} + onGraded={onGraded} + refusal={refused?.entryId === entryId ? refused.message : null} onPickRoom={onPickRoom} /> @@ -160,6 +207,8 @@ function EvaluationForm({ refresh, onReload, onNext, + onGraded, + refusal, onPickRoom, }: { hackathonId: string; @@ -173,6 +222,9 @@ function EvaluationForm({ onReload: () => void; /** Back to the queue, where the next project waits. */ onNext: () => void; + onGraded?: () => void; + /** Why opening the project for grading was refused, if it was. */ + refusal: string | null; onPickRoom?: () => void; }) { const utils = trpc.useUtils(); @@ -236,7 +288,7 @@ function EvaluationForm({ const opensNow = opensIn !== null && opensIn <= 0; const submit = trpc.judging.desk.submit.useMutation(); - const writable = access.canEdit && !timeUp; + const writable = access.canEdit && !timeUp && !refusal; const editable = writable && !sending; const autosave = useDraftAutosave({ @@ -310,6 +362,7 @@ function EvaluationForm({ void utils.judging.desk.project.invalidate({ hackathonId }); void utils.judging.desk.projects.invalidate({ hackathonId }); void utils.judging.desk.myEvaluations.invalidate({ hackathonId }); + if (!evaluation) onGraded?.(); }, onError: (error) => { release(); @@ -430,6 +483,13 @@ function EvaluationForm({ + {refusal && ( +
    +

    Not open to you yet

    +

    {refusal}

    +
    + )} + {entry.flagged && (

    Flagged by organisers

    diff --git a/sites/mainweb/components/judge/JudgeDesk.tsx b/sites/mainweb/components/judge/JudgeDesk.tsx index 952eaeb3..b2ca2d1d 100644 --- a/sites/mainweb/components/judge/JudgeDesk.tsx +++ b/sites/mainweb/components/judge/JudgeDesk.tsx @@ -23,8 +23,8 @@ import { DeskAnnouncements } from "./DeskAnnouncements"; import { EvaluationSheet } from "./EvaluationSheet"; import { GuestNameGate } from "./GuestNameGate"; import { MySubmissions } from "./MySubmissions"; -import { DEFAULT_FILTERS, ProjectQueue } from "./ProjectQueue"; -import type { QueueFilters } from "./ProjectQueue"; +import { DEFAULT_FILTERS, NO_SKIPS, ProjectQueue } from "./ProjectQueue"; +import type { QueueFilters, SkipState } from "./ProjectQueue"; import { RoomSwitcher } from "./RoomSwitcher"; import { Shortlist } from "./Shortlist"; import { ICON_STROKE, deskMeta, deskPage, deskStatus } from "./desk"; @@ -222,6 +222,8 @@ function Desk({ const searchParams = useSearchParams(); const [view, setView] = useState("projects"); const [filters, setFilters] = useState(DEFAULT_FILTERS); + // Kept here, with the filters, because the project sheet replaces the list. + const [skips, setSkips] = useState(NO_SKIPS); const [pickingRoom, setPickingRoom] = useState(false); // Whether the open project was opened from this page, so closing it can // step back in history instead of stacking another entry. @@ -264,6 +266,7 @@ function Desk({ entryId={projectId} challengeId={projectChallenge} onClose={close} + onGraded={() => setSkips(NO_SKIPS)} onPickRoom={ isMember ? () => { @@ -353,6 +356,8 @@ function Desk({ filters={filters} onFilters={setFilters} onOpen={open} + skips={skips} + onSkips={setSkips} onShowSubmitted={() => setView("submitted")} onPickRoom={isMember ? () => setPickingRoom(true) : undefined} /> diff --git a/sites/mainweb/components/judge/ProjectQueue.tsx b/sites/mainweb/components/judge/ProjectQueue.tsx index e38f628c..56a07f24 100644 --- a/sites/mainweb/components/judge/ProjectQueue.tsx +++ b/sites/mainweb/components/judge/ProjectQueue.tsx @@ -18,6 +18,7 @@ import { import type { Tone } from "@/components/portal/ui"; import { ICON_STROKE, + addSkip, clockTime, deskLabel, deskMeta, @@ -45,6 +46,15 @@ export const DEFAULT_FILTERS: QueueFilters = { inMyRoomOnly: true, }; +/** + * What the judge turned down since they last graded a project, and the + * project Grade next project last sent them to. The desk holds it, so it + * outlives the project sheet that replaces this list. + */ +export type SkipState = { skipped: string[]; offered: string | null }; + +export const NO_SKIPS: SkipState = { skipped: [], offered: null }; + function useDebounced(value: T, ms: number) { const [settled, setSettled] = useState(value); useEffect(() => { @@ -101,6 +111,8 @@ export function ProjectQueue({ filters, onFilters, onOpen, + skips, + onSkips, onShowSubmitted, onPickRoom, }: { @@ -109,6 +121,8 @@ export function ProjectQueue({ filters: QueueFilters; onFilters: (next: QueueFilters) => void; onOpen: (entryId: string, challengeId: string) => void; + skips: SkipState; + onSkips: (next: SkipState) => void; /** Switches the desk to the judge's submitted evaluations. */ onShowSubmitted: () => void; /** Members only: opens the room picker. */ @@ -133,6 +147,7 @@ export function ProjectQueue({ challengeId: string; label: string; needsRoom: boolean; + allSkipped: boolean; } | null>(null); const [findError, setFindError] = useState(null); @@ -197,7 +212,7 @@ export function ProjectQueue({ ).length; const mixed = (!filters.hideJudged && judged > 0) || taken > 0; - async function gradeNext() { + async function gradeNext(skipped = skips.skipped) { if (!challenge) return; setFinding(true); setNone(null); @@ -206,7 +221,13 @@ export function ProjectQueue({ const found = await utils.client.judging.desk.nextProject.query({ hackathonId, challengeId: filters.challengeId ?? undefined, + skip: skipped, }); + // With everything on offer skipped, the skips start over, so trying + // again later goes back to the first of them: that team may be back. + onSkips( + found.allSkipped ? NO_SKIPS : { skipped, offered: found.entryId }, + ); if (found.entryId) { onOpen(found.entryId, found.challenge.id); } else { @@ -214,6 +235,7 @@ export function ProjectQueue({ challengeId: found.challenge.id, label: found.challenge.label, needsRoom: found.needsRoom, + allSkipped: found.allSkipped, }); } } catch (error) { @@ -227,6 +249,17 @@ export function ProjectQueue({ } } + // For a team that is away from its table: passes over the project Grade next + // project last sent the judge to, and opens the one after it. + function skipOffered() { + const { offered, skipped } = skips; + if (!offered) return; + void gradeNext(addSkip(skipped, offered)); + } + const offeredTitle = data?.projects.find( + (p) => p.id === skips.offered, + )?.title; + // Above the filters, so it is the first thing on a phone's screen. It only // exists in the judge's own room, which the heading already names. const upNext = @@ -272,14 +305,27 @@ export function ProjectQueue({ {data && challenge && (
    {context.state === "open" && ( - +
    + + {skips.offered && ( + + )} +
    )} {none?.challengeId === challenge.id && (

    @@ -296,6 +342,8 @@ export function ProjectQueue({ )} + ) : none.allSkipped ? ( + "Everything your queue offers right now is skipped. Try again in a few minutes." ) : ( `Nothing left to grade in ${none.label}.` )} @@ -308,6 +356,8 @@ export function ProjectQueue({ )}

    Each project is graded once per challenge, by one judge. + {skips.offered && + " Skip moves on if that team is away from its table."}

    )} @@ -522,8 +572,9 @@ function RoomFilterNote({ if (filter === "untimed") { return (

    - {challengeLabel} has no presentation times, so every project is listed. - Judge them in any order. + {challengeLabel} has no presentation times, so every project is listed, + open ones first in your own queue order. Use Grade next project to take + the next one.

    ); } diff --git a/sites/mainweb/components/judge/desk.test.ts b/sites/mainweb/components/judge/desk.test.ts index de130768..3381dc0b 100644 --- a/sites/mainweb/components/judge/desk.test.ts +++ b/sites/mainweb/components/judge/desk.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; import { + MAX_SKIPS, + addSkip, countdown, countdownTone, formatScore, @@ -156,3 +158,30 @@ describe("stationGroups", () => { expect(stationGroups([])).toEqual([]); }); }); + +describe("addSkip", () => { + it("adds the project last", () => { + expect(addSkip([], "a")).toEqual(["a"]); + expect(addSkip(["a", "b"], "c")).toEqual(["a", "b", "c"]); + }); + + it("holds a project once, moved to the end", () => { + expect(addSkip(["a", "b", "c"], "a")).toEqual(["b", "c", "a"]); + }); + + it("drops the oldest past what the API takes", () => { + const full = Array.from({ length: MAX_SKIPS }, (_, i) => `e${i}`); + + const next = addSkip(full, "new"); + + expect(next).toHaveLength(MAX_SKIPS); + expect(next[0]).toBe("e1"); + expect(next.at(-1)).toBe("new"); + }); + + it("leaves the list it was given alone", () => { + const skipped = ["a"]; + addSkip(skipped, "b"); + expect(skipped).toEqual(["a"]); + }); +}); diff --git a/sites/mainweb/components/judge/desk.ts b/sites/mainweb/components/judge/desk.ts index 0a2e44bd..939b0646 100644 --- a/sites/mainweb/components/judge/desk.ts +++ b/sites/mainweb/components/judge/desk.ts @@ -109,6 +109,14 @@ export function inOrAt(kind: StationKind) { return kind === "table" ? "at" : "in"; } +/** The most skips the desk sends to `nextProject`. */ +export const MAX_SKIPS = 20; + +/** `skipped` with `entryId` last; past MAX_SKIPS the oldest are dropped. */ +export function addSkip(skipped: readonly string[], entryId: string): string[] { + return [...skipped.filter((id) => id !== entryId), entryId].slice(-MAX_SKIPS); +} + /** A link's host, for a label: "github.com". */ export function linkHost(href: string) { try { From 600c78820499aabbf7bb50e67b62c84afe350cf2 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 20:28:36 -0400 Subject: [PATCH 16/27] feat(judging): count auto-submits, reopen grades, retune drift An evaluation now counts toward placings and calibration once it holds a rating for every rubric rating item, so a draft submitted at the deadline with a written response missing still places its project. scoring.hasEveryRating decides it. results.compute, appointmentResults (completeCount becomes countedCount) and evaluationAudit use the same rule, and the results tab says whether an incomplete evaluation counts. judging.admin.reopenGrade deletes one evaluation and every draft for its project and challenge so another judge can grade it. It is admin only, audited as critical, takes the edition lock, then the drafts, then the grading lock, and is refused while results are published. The results tab offers it on each evaluation behind a confirmation. The drift correction was mostly noise: chance alone gave judges with no drift a fitted one of about 0.3 of a point. K_D rises from 10 to 150 and no drift is fitted below MIN_DRIFT_GRADES (20), which keeps that under 0.1 of a point in seeded simulations; a real drift is now only partly removed. The organiser view flags drift from 0.2. judging.results.list caches the per-judge calibration for 60 seconds, keyed on the edition's evaluation count and latest save and on its ranked entries; compute always calibrates fresh. docs/packages/api.md describes these changes and the desk's queue window, hold on open and sponsor room rule. --- docs/packages/api.md | 4 +- .../check-in-flags-results.test.ts | 623 +++++++++++++++++- .../src/.internal-tests/reopen-grade.test.ts | 376 +++++++++++ packages/api/src/middleware/cache.ts | 4 + packages/api/src/routers/judging/admin.ts | 104 +++ packages/api/src/routers/judging/results.ts | 212 ++++-- .../src/services/judging/calibration.test.ts | 156 ++++- .../api/src/services/judging/calibration.ts | 55 +- .../src/services/judging/placements.test.ts | 10 +- .../api/src/services/judging/placements.ts | 6 +- .../api/src/services/judging/scoring.test.ts | 78 ++- packages/api/src/services/judging/scoring.ts | 39 +- .../admin/judging/results/ResultsTab.test.tsx | 177 +++++ .../admin/judging/results/ResultsTab.tsx | 134 +++- .../admin/judging/results/calibration.test.ts | 10 +- .../admin/judging/results/calibration.ts | 7 +- 16 files changed, 1831 insertions(+), 164 deletions(-) create mode 100644 packages/api/src/.internal-tests/reopen-grade.test.ts create mode 100644 sites/mainweb/components/admin/judging/results/ResultsTab.test.tsx diff --git a/docs/packages/api.md b/docs/packages/api.md index 6be97624..51bb0479 100644 --- a/docs/packages/api.md +++ b/docs/packages/api.md @@ -93,7 +93,9 @@ Batch status updates allow up to **2500** ids. The global array cap in `scrubMar Judging itself is the `judging` namespace; its shared helpers are in `services/judging/` (`room-link`, `scope`, `scoring`, `config`). `hackathon.getResults` returns published placings from both the retired `hackathon_result` table and `judging_placement`, in one shape. It leaves out entries pulled from judging (`deleted_at`) or from prizes (`prize_ineligible_at`, set by `judging.admin.setPrizeEligibility`). Check-in rule-break flags (`services/check-in-flags.ts`) reach organisers through `judging.admin.listEntries` and `judging.results.list`, and judges through `judging.desk` as a bare `flagged` boolean; no hacker-facing, public or cached procedure returns them. -Placings rank on scores adjusted for judge bias (`services/judging/calibration.ts`): each evaluation is calibrated within its root challenge for its judge's strictness, spread and drift, on the assumption that `judging.desk.nextProject` (`services/judging/queue.ts`) hands every judge a random sample of projects. `judging_placement.raw_score` keeps the plain mean; only `judging.results.list` (organisers) returns it and the per-judge calibration, and `judging.desk` stays on raw means. +Placings rank on scores adjusted for judge bias (`services/judging/calibration.ts`): each evaluation is calibrated within its root challenge for its judge's strictness, spread and drift, on the assumption that `judging.desk.nextProject` (`services/judging/queue.ts`) hands every judge a random sample of projects. `judging_placement.raw_score` keeps the plain mean; only `judging.results.list` (organisers) returns it and the per-judge calibration, and `judging.desk` stays on raw means. An evaluation counts when it holds a rating for every rating item, whether or not a written response is missing, so a draft submitted at the hard end still places its project. `judging.results.list` keeps the calibration for 60 seconds per edition, keyed on the edition's evaluation count and latest save and on its ranked entries, while `judging.results.compute` always calibrates fresh. `judging.admin.reopenGrade` deletes one evaluation and the drafts for its project and challenge so another judge can grade it; it is refused while results are published. + +At an untimed challenge that random sample is enforced (`services/judging/queue-store.ts`): `judging.desk.nextProject` offers only the first `QUEUE_WINDOW` (3) open projects of the judge's own queue and takes a `skip` list for a team away from its table, and a judge's first `startGrading`, `saveDraft` or `submit` on a project outside that window is refused. `judging.desk.startGrading` holds a project from the moment it is opened, with an empty draft. A table judge grades a sponsor's challenge only from a seat in a room that serves it (`scope.isSponsorChallenge`); `judging.desk.context` lists only the challenges they may pick. ## Club procedures (selected) diff --git a/packages/api/src/.internal-tests/check-in-flags-results.test.ts b/packages/api/src/.internal-tests/check-in-flags-results.test.ts index d7454f2b..bd001ea2 100644 --- a/packages/api/src/.internal-tests/check-in-flags-results.test.ts +++ b/packages/api/src/.internal-tests/check-in-flags-results.test.ts @@ -9,6 +9,10 @@ * both and a per-judge summary, and the public payload carries neither the raw * score nor anything about a judge. * + * It covers which evaluations count (every rating, whether or not a written + * response is missing, as with a draft submitted at the deadline), and the + * calibration the list keeps for a minute, which compute never reads. + * * The schema is the real one; only the database handle is a stand-in, handed * to the router through the context the way judge-edge.test.ts does. Its * selects answer by the table they read from, and the entries query applies @@ -21,6 +25,7 @@ import { hackathonCheckInFlags, hackathonEvents, hackathons, + judgingAppointments, judgingChallenges, judgingConfigs, judgingEntries, @@ -28,9 +33,11 @@ import { judgingEntryMembers, judgingEvaluationDrafts, judgingEvaluationRatings, + judgingEvaluationResponses, judgingEvaluations, judgingJudges, judgingPlacements, + judgingRubricItems, } from "@query/db"; import { appRouter } from "../root"; import { cache } from "../middleware/cache"; @@ -166,7 +173,6 @@ describe("judging.results.compute", () => { challengeId: "ch-1", judgeId, submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), - isComplete: true, }); // One judge marked all three. With the pulled team left out of the // calibration she has two grades, a mean of 3.5 and the pool's own spread, so @@ -212,6 +218,8 @@ describe("judging.results.compute", () => { if (table === judgingChallenges) return []; if (table === judgingEvaluations) return scored; if (table === judgingEvaluationRatings) return ratings; + // One rating item, which every evaluation here rated. + if (table === judgingRubricItems) return [{ n: 1 }]; return []; }); }; @@ -289,7 +297,6 @@ describe("judging.results.compute with judge calibration", () => { challengeId: "ch-1", judgeId, submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), - isComplete: true, }); // A strict judge marked e1 and e2, a lenient one e3 and e4. Raw, e2 and e3 @@ -321,6 +328,7 @@ describe("judging.results.compute with judge calibration", () => { { evaluationId: "v4", value: 5 }, ]; } + if (table === judgingRubricItems) return [{ n: 1 }]; return []; }); @@ -348,8 +356,283 @@ describe("judging.results.compute with judge calibration", () => { }); }); +// --------------------------------------------------------------------------- +describe("judging.results.compute with grades submitted at the deadline", () => { + const entries = ["entry-a", "entry-b", "entry-c"].map((id) => ({ + id, + deletedAt: null, + prizeIneligibleAt: null, + })); + const evaluation = ( + id: string, + entryId: string, + minute: number, + isComplete: boolean, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId: "judge-1", + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + isComplete, + }); + // A rubric of two rating items. entry-a's only grade is an auto-submit: both + // ratings, a required response left empty, so isComplete is false. entry-b's + // stopped after one rating. entry-c's is whole. + const stored = [ + evaluation("eval-a", "entry-a", 0, false), + evaluation("eval-b", "entry-b", 1, false), + evaluation("eval-c", "entry-c", 2, true), + ]; + const ratings = [ + { evaluationId: "eval-a", value: 4 }, + { evaluationId: "eval-a", value: 4 }, + { evaluationId: "eval-b", value: 5 }, + { evaluationId: "eval-c", value: 3 }, + { evaluationId: "eval-c", value: 3 }, + ]; + let evaluationWheres: unknown[] = []; + let ratingWheres: unknown[] = []; + + const wire = () => { + evaluationWheres = []; + ratingWheres = []; + mockSelect.mockImplementation(({ table, wheres }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + if (table === judgingPlacements) return [{ n: 0 }]; + if (table === judgingEntries) return entries; + if (table === judgingRubricItems) return [{ n: 2 }]; + if (table === judgingEvaluations) { + evaluationWheres = wheres; + return stored; + } + if (table === judgingEvaluationRatings) { + ratingWheres = wheres; + return ratings; + } + return []; + }); + }; + + it("places a project whose only grade is such an auto-submit", async () => { + wire(); + + const result = await adminCaller().judging.results.compute({ + hackathonId: HACK, + }); + + const placed = placementsInserted(); + expect( + placed.map((row) => [row.entryId, row.scope, row.placement, row.score]), + ).toEqual([ + ["entry-a", "overall", 1, "4.00"], + ["entry-c", "overall", 2, "3.00"], + ]); + expect(placed.map((row) => row.evaluationCount)).toEqual([1, 1]); + expect(result.placements).toBe(2); + }); + + it("leaves a grade short of a rating out, and does not count it as ranked", async () => { + wire(); + + const result = await adminCaller().judging.results.compute({ + hackathonId: HACK, + }); + + expect(placementsInserted().map((row) => row.entryId)).not.toContain( + "entry-b", + ); + expect(result.evaluations).toBe(2); + }); + + it("does not filter evaluations or ratings on isComplete", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + expect(evaluationWheres.length).toBeGreaterThan(0); + expect(ratingWheres.length).toBeGreaterThan(0); + for (const where of [...evaluationWheres, ...ratingWheres]) { + expect(render(where).sql).not.toContain("is_complete"); + expect(render(where).params).toContain(HACK); + } + }); + + it("asks the rubric how many rating items an evaluation must have", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + const rubric = mockSelect.mock.calls.find( + ([query]) => query.table === judgingRubricItems, + )![0]; + expect(render(rubric.wheres[0]).params).toEqual([HACK, "rating"]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.results.compute beside the list's calibration cache", () => { + const entries = ["e1", "e2", "e3", "e4"].map((id) => ({ + id, + deletedAt: null, + prizeIneligibleAt: null, + })); + const evaluation = ( + id: string, + entryId: string, + judgeId: string, + minute: number, + ) => ({ + id, + entryId, + challengeId: "ch-1", + judgeId, + submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), + }); + const stored = [ + evaluation("v1", "e1", "strict", 0), + evaluation("v2", "e2", "strict", 1), + evaluation("v3", "e3", "lenient", 2), + evaluation("v4", "e4", "lenient", 3), + ]; + let ratings = [ + { evaluationId: "v1", value: 3 }, + { evaluationId: "v2", value: 4 }, + { evaluationId: "v3", value: 4 }, + { evaluationId: "v4", value: 5 }, + ]; + const computedAt = new Date("2027-02-07T09:00:00Z"); + + beforeEach(() => { + ratings = [ + { evaluationId: "v1", value: 3 }, + { evaluationId: "v2", value: 4 }, + { evaluationId: "v3", value: 4 }, + { evaluationId: "v4", value: 5 }, + ]; + }); + + // The same edition answers both procedures. Placements come back as rows for + // the list and as a published count for compute, which asks for `n`. + const wire = () => + mockSelect.mockImplementation(({ table, fields }) => { + if (table === hackathons) { + return [ + { id: HACK, name: "Hacklytics", startDate: null, endDate: null }, + ]; + } + if (table === judgingPlacements) { + return fields && "n" in fields + ? [{ n: 0 }] + : [ + { + id: "p1", + scope: "overall", + scopeLabel: "Overall", + placement: 1, + score: "4.00", + evaluationCount: 1, + publishedAt: null, + computedAt, + entryId: "e1", + title: "Project e1", + deletedAt: null, + prizeIneligibleAt: null, + }, + ]; + } + if (table === judgingEvaluationDrafts) return []; + if (table === judgingConfigs) return [{ hackathonId: HACK }]; + if (table === judgingEntries) { + return fields && "latest" in fields + ? [{ n: 4, latest: new Date("2027-02-06T09:00:00Z") }] + : entries; + } + if (table === judgingEvaluations) { + if (fields && "latest" in fields) { + return [{ n: 4, latest: new Date("2027-02-06T11:00:00Z") }]; + } + return fields && "n" in fields ? [{ n: 0 }] : stored; + } + if (table === judgingRubricItems) return [{ n: 1 }]; + if (table === judgingEvaluationRatings) return ratings; + if (table === judgingJudges) { + return [ + { id: "strict", displayName: "Sam Strict", kind: "member" }, + { id: "lenient", displayName: "Lee Lenient", kind: "member" }, + ]; + } + if (table === judgingChallenges) { + return [{ id: "ch-1", label: "General" }]; + } + return []; + }); + + const ratingReads = () => + mockSelect.mock.calls.filter( + ([query]) => query.table === judgingEvaluationRatings, + ).length; + + it("ranks on the evaluations as they stand, whatever the list has cached", async () => { + wire(); + const getOrSet = vi.spyOn(cache, "getOrSet"); + + // The list fills the cache, and a second poll is answered from it. + await adminCaller().judging.results.list({ hackathonId: HACK }); + await adminCaller().judging.results.list({ hackathonId: HACK }); + expect(ratingReads()).toBe(1); + expect(getOrSet).toHaveBeenCalledTimes(2); + + // A rating changes under it, with nothing the cache key could see. + ratings = ratings.map((rating) => + rating.evaluationId === "v4" ? { ...rating, value: 1 } : rating, + ); + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + // Compute read the ratings itself and did not touch the cache. + expect(ratingReads()).toBe(2); + expect(getOrSet).toHaveBeenCalledTimes(2); + // v4 is the lenient judge's lowest now, so e4 ranks last. + expect(placementsInserted().at(-1)?.entryId).toBe("e4"); + getOrSet.mockRestore(); + }); + + it("reads the ratings again on every compute", async () => { + wire(); + + await adminCaller().judging.results.compute({ hackathonId: HACK }); + await adminCaller().judging.results.compute({ hackathonId: HACK }); + + expect(ratingReads()).toBe(2); + }); +}); + // --------------------------------------------------------------------------- describe("judging.results.list", () => { + // What the calibration cache key is read from: the edition's evaluations and + // its ranked entries, each a count and a latest save. A test changes one + // between two lists to stand for a save, a reopen or a pulled team. + const stamps = { + evaluations: { n: 4, latest: new Date("2027-02-06T11:00:00Z") } as Record< + string, + unknown + >, + entries: { n: 6, latest: new Date("2027-02-06T09:00:00Z") } as Record< + string, + unknown + >, + }; + beforeEach(() => { + stamps.evaluations = { n: 4, latest: new Date("2027-02-06T11:00:00Z") }; + stamps.entries = { n: 6, latest: new Date("2027-02-06T09:00:00Z") }; + }); + const COMPUTED_AT = new Date("2027-02-07T09:00:00Z"); const placement = ( id: string, @@ -400,15 +683,20 @@ describe("judging.results.list", () => { /** Complete evaluations, as the calibration summary reads them. */ evaluations?: unknown[]; ratings?: unknown[]; + /** Rating items on the rubric. The fixtures rate one each unless said. */ + ratingItems?: number; judges?: unknown[]; challenges?: unknown[]; }) => { mockSelect.mockImplementation(({ table, fields }) => { if (table === judgingPlacements) return opts.placements; - // The stale count reads one column; the calibration summary reads many. + // The cache key reads a count and a latest time, the stale count reads + // a count alone, and the calibration summary reads many columns. if (table === judgingEvaluations) { + if (fields && "latest" in fields) return [stamps.evaluations]; return fields && "n" in fields ? [{ n: 0 }] : (opts.evaluations ?? []); } + if (table === judgingRubricItems) return [{ n: opts.ratingItems ?? 1 }]; if (table === judgingEvaluationRatings) return opts.ratings ?? []; if (table === judgingJudges) return opts.judges ?? []; if (table === judgingChallenges) return opts.challenges ?? []; @@ -419,7 +707,10 @@ describe("judging.results.list", () => { { id: "event-2", name: "Sponsor Talk" }, ]; } - if (table === judgingEntries) return opts.portal ?? opts.live ?? []; + if (table === judgingEntries) { + if (fields && "latest" in fields) return [stamps.entries]; + return opts.portal ?? opts.live ?? []; + } if (table === judgingEntryMembers) return []; return []; }); @@ -540,7 +831,6 @@ describe("judging.results.list", () => { challengeId, judgeId, submittedAt: new Date(Date.UTC(2027, 1, 6, 10, minute)), - isComplete: true, }); const challenges = [ { id: "ch-sponsor", label: "Sponsor Prize" }, @@ -646,6 +936,168 @@ describe("judging.results.list", () => { expect(data.calibration).toEqual([]); }); + + it("counts a grade submitted at the deadline with every rating, and not one short of a rating", async () => { + // Two rating items. g1 is an auto-submit: all its ratings, a response + // left empty, so the evaluation is incomplete. g3 stopped after one. + const two = [ + { ...evaluations[0]!, isComplete: false }, + evaluations[1]!, + evaluations[2]!, + ]; + const rated = [ + { evaluationId: "g1", value: 3 }, + { evaluationId: "g1", value: 3 }, + { evaluationId: "g2", value: 3 }, + { evaluationId: "g2", value: 3 }, + { evaluationId: "g3", value: 4 }, + ]; + wire({ + placements: [placement("p1", "entry-0")], + evaluations: two, + ratings: rated, + ratingItems: 2, + judges, + challenges, + live, + }); + + const data = await adminCaller().judging.results.list({ + hackathonId: HACK, + }); + + const general = data.calibration.find((c) => c.challenge === "General"); + expect(general!.judges.map((j) => [j.judgeName, j.grades])).toEqual([ + ["Sam Strict", 2], + ]); + }); + + describe("cache", () => { + const list = () => + adminCaller().judging.results.list({ hackathonId: HACK }); + // The heavy read: every rating of the edition. + const calibrations = () => + mockSelect.mock.calls.filter( + ([query]) => query.table === judgingEvaluationRatings, + ).length; + const setUp = () => + wire({ + placements: [placement("p1", "entry-0")], + evaluations, + ratings, + judges, + challenges, + live, + }); + + it("calibrates once for polls that find nothing changed", async () => { + setUp(); + + const first = await list(); + const second = await list(); + const third = await list(); + + expect(calibrations()).toBe(1); + expect(second.calibration).toEqual(first.calibration); + expect(third.calibration).toEqual(first.calibration); + expect(first.calibration.length).toBeGreaterThan(0); + }); + + it("calibrates again when an evaluation is added or deleted", async () => { + setUp(); + await list(); + + stamps.evaluations = { ...stamps.evaluations, n: 5 }; + await list(); + expect(calibrations()).toBe(2); + + // Two grades reopened, the newest among them: fewer, and an earlier + // latest save. + stamps.evaluations = { + n: 3, + latest: new Date("2027-02-06T10:30:00Z"), + }; + await list(); + expect(calibrations()).toBe(3); + }); + + it("calibrates again when an evaluation is edited", async () => { + setUp(); + await list(); + + // Same count, saved later. + stamps.evaluations = { + ...stamps.evaluations, + latest: new Date("2027-02-06T11:05:00Z"), + }; + await list(); + + expect(calibrations()).toBe(2); + }); + + it("calibrates again when a team is pulled from prizes or put back", async () => { + setUp(); + await list(); + + stamps.entries = { n: 5, latest: new Date("2027-02-07T08:00:00Z") }; + await list(); + expect(calibrations()).toBe(2); + + // Back in, a moment later: the count is as it was, the time is not. + stamps.entries = { n: 6, latest: new Date("2027-02-07T08:01:00Z") }; + await list(); + expect(calibrations()).toBe(3); + }); + + it("keys the result by edition and by what it was computed from", async () => { + setUp(); + const getOrSet = vi.spyOn(cache, "getOrSet"); + + await list(); + + expect(getOrSet).toHaveBeenCalledTimes(1); + const [key, , ttl] = getOrSet.mock.calls[0]!; + const latest = (stamps.evaluations.latest as Date).getTime(); + const entries = (stamps.entries.latest as Date).getTime(); + expect(key).toBe( + `judging:${HACK}:calibration:4:${latest}:6:${entries}`, + ); + expect(ttl).toBe(60); + getOrSet.mockRestore(); + }); + + it("does not share a result between editions", async () => { + setUp(); + await list(); + + await adminCaller().judging.results.list({ + hackathonId: "99999999-9999-4999-8999-999999999999", + }); + + expect(calibrations()).toBe(2); + }); + + it("keeps the result for a minute and no longer", async () => { + setUp(); + const getOrSet = vi.spyOn(cache, "getOrSet"); + await list(); + const key = getOrSet.mock.calls[0]![0]; + getOrSet.mockRestore(); + + // Only the clock the cache reads is moved: the router's own rate + // limiter keeps real time. + const clock = vi.spyOn(Date, "now"); + const now = Date.now(); + try { + clock.mockReturnValue(now + 59_000); + expect(cache.has(key)).toBe(true); + clock.mockReturnValue(now + 61_000); + expect(cache.has(key)).toBe(false); + } finally { + clock.mockRestore(); + } + }); + }); }); it("reads no flags when nothing is placed", async () => { @@ -661,6 +1113,167 @@ describe("judging.results.list", () => { }); }); +// --------------------------------------------------------------------------- +describe("the organiser's evaluation views with a grade short of a response", () => { + const APPOINTMENT = "33333333-3333-4333-8333-333333333333"; + const at = new Date("2027-02-06T11:00:00Z"); + // A rubric of two rating items. Judge A's grade was submitted at the + // deadline with both ratings and a response left empty. B's is whole. C's + // stopped after one rating. + const evaluations = [ + { + id: "eval-a", + entryId: "entry-1", + title: "Project 1", + challengeId: "ch-1", + challenge: "General", + judgeId: "judge-a", + judgeName: "Ada Auto", + kind: "member", + appointmentId: APPOINTMENT, + isComplete: false, + revision: 1, + submittedAt: at, + autoSubmittedAt: at, + updatedAt: at, + }, + { + id: "eval-b", + entryId: "entry-1", + title: "Project 1", + challengeId: "ch-1", + challenge: "General", + judgeId: "judge-b", + judgeName: "Bea Whole", + kind: "member", + appointmentId: APPOINTMENT, + isComplete: true, + revision: 1, + submittedAt: at, + autoSubmittedAt: null, + updatedAt: at, + }, + { + id: "eval-c", + entryId: "entry-1", + title: "Project 1", + challengeId: "ch-1", + challenge: "General", + judgeId: "judge-c", + judgeName: "Cy Short", + kind: "member", + appointmentId: APPOINTMENT, + isComplete: false, + revision: 1, + submittedAt: at, + autoSubmittedAt: at, + updatedAt: at, + }, + ]; + const rating = ( + evaluationId: string, + item: "one" | "two", + value: number, + ) => ({ + evaluationId, + itemId: `item-${item}`, + label: item === "one" ? "Impact" : "Craft", + value, + }); + const ratings = [ + rating("eval-a", "one", 5), + rating("eval-a", "two", 3), + rating("eval-b", "one", 4), + rating("eval-b", "two", 4), + rating("eval-c", "one", 2), + ]; + + const wire = () => + mockSelect.mockImplementation(({ table, fields }) => { + if (table === judgingEvaluationDrafts) return []; + if (table === judgingRubricItems) return [{ n: 2 }]; + if (table === judgingEvaluationRatings) return ratings; + if (table === judgingEvaluationResponses) return []; + if (table === judgingAppointments) { + return [ + { + id: APPOINTMENT, + entryId: "entry-1", + challengeId: "ch-1", + startsAt: at, + endsAt: at, + title: "Project 1", + challenge: "General", + room: "Table 1", + building: null, + }, + ]; + } + if (table === judgingEvaluations) { + // The "elsewhere" check reads the id alone; the rest read the judge. + return fields && "judgeName" in fields ? evaluations : []; + } + return []; + }); + + describe("appointmentResults", () => { + const results = () => + adminCaller().judging.results.appointmentResults({ + hackathonId: HACK, + appointmentId: APPOINTMENT, + }); + + it("counts a grade with every rating, whether or not it is complete", async () => { + wire(); + + const data = await results(); + + const byName = Object.fromEntries( + data.judges.map((judge) => [judge.judgeName, judge]), + ); + expect(byName["Ada Auto"]).toMatchObject({ + status: "incomplete", + autoSubmitted: true, + counted: true, + score: 4, + }); + expect(byName["Ada Auto"]!.ratings.map((r) => r.value)).toEqual([5, 3]); + expect(byName["Bea Whole"]).toMatchObject({ + status: "complete", + counted: true, + score: 4, + }); + }); + + it("leaves a grade short of a rating out of the score and the average", async () => { + wire(); + + const data = await results(); + + const short = data.judges.find((judge) => judge.judgeName === "Cy Short"); + expect(short).toMatchObject({ counted: false, score: null, ratings: [] }); + expect(data.countedCount).toBe(2); + expect(data.average).toBe(4); + }); + }); + + describe("evaluationAudit", () => { + it("says which evaluations the placings count", async () => { + wire(); + + const rows = await adminCaller().judging.results.evaluationAudit({ + hackathonId: HACK, + }); + + expect(rows.map((row) => [row.id, row.isComplete, row.counted])).toEqual([ + ["eval-a", false, true], + ["eval-b", true, true], + ["eval-c", false, false], + ]); + }); + }); +}); + // --------------------------------------------------------------------------- describe("hackathon.getResults", () => { const placementRow = (id: string, over: Record = {}) => ({ diff --git a/packages/api/src/.internal-tests/reopen-grade.test.ts b/packages/api/src/.internal-tests/reopen-grade.test.ts new file mode 100644 index 00000000..87358587 --- /dev/null +++ b/packages/api/src/.internal-tests/reopen-grade.test.ts @@ -0,0 +1,376 @@ +/** + * judging.admin.reopenGrade: an organiser deletes one judge's evaluation, and + * any draft for the same project and challenge, so the project is open for + * another judge. Refused once results are published. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way check-in-flags-organizer.test.ts + * does. It applies the equality conditions the router asks for, so a missing + * edition condition lets another edition's evaluation through and fails the + * test. It keeps every statement in order, which is how the tests see the + * drafts being locked before the project's grading lock and both before a + * delete. + */ +import { beforeEach, describe, expect, it } from "vitest"; +import { getTableConfig } from "drizzle-orm/pg-core"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + auditLogs, + hackathons, + judgingConfigs, + judgingEvaluationDrafts, + judgingEvaluationRatings, + judgingEvaluationResponses, + judgingEvaluationRevisions, + judgingEvaluations, + judgingPlacements, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; + +const HACK_A = "11111111-1111-4111-8111-111111111111"; +const HACK_B = "22222222-2222-4222-8222-222222222222"; +const ENTRY = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1"; +const CHALLENGE = "cccccccc-cccc-4ccc-8ccc-ccccccccccc1"; +const JUDGE = "dddddddd-dddd-4ddd-8ddd-ddddddddddd1"; +const EVAL = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1"; +const EVAL_OTHER_EDITION = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee2"; +const DRAFT_1 = "ffffffff-ffff-4fff-8fff-fffffffffff1"; +const DRAFT_2 = "ffffffff-ffff-4fff-8fff-fffffffffff2"; +const NO_SUCH = "99999999-9999-4999-8999-999999999999"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); + +type Row = Record; +type Event = { + kind: "select" | "execute" | "insert" | "delete" | "update"; + table?: unknown; + where?: unknown; + /** A select that takes row locks. */ + locking?: boolean; +}; + +const camel = (name: string) => + name.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase()); +const TERM = /"[a-z_]+"\."([a-z_]+)" = \$(\d+)/g; + +/** A row filter from a where of `column = value` terms joined by and. */ +function matcher(where: unknown) { + const { sql: text, params } = render(where); + const terms = [...text.matchAll(TERM)].map( + (match) => [camel(match[1]!), params[Number(match[2]) - 1]] as const, + ); + if (text.replace(TERM, "").replace(/[()\s]|and/g, "")) { + throw new Error(`The fake database cannot evaluate: ${text}`); + } + return (row: Row) => terms.every(([key, value]) => row[key] === value); +} + +let adminRow: Row | undefined; +let published = false; +const store = new Map(); +const events: Event[] = []; + +const rowsOf = (table: unknown, where: unknown) => { + const rows = store.get(table) ?? []; + return where ? rows.filter(matcher(where)) : [...rows]; +}; + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: () => { + const event: Event = { kind: "select" }; + events.push(event); + const chain: any = { + then: (onOk: any, onErr: any) => { + // The published check counts placements that carry a publish time. + const rows = + event.table === judgingPlacements + ? [{ value: published ? 3 : 0 }] + : rowsOf(event.table, event.where); + return Promise.resolve(rows).then(onOk, onErr); + }, + }; + chain.from = (table: unknown) => { + event.table = table; + return chain; + }; + chain.where = (where: unknown) => { + event.where = where; + return chain; + }; + chain.for = () => { + event.locking = true; + return chain; + }; + for (const method of ["innerJoin", "leftJoin", "orderBy", "limit"]) { + chain[method] = () => chain; + } + return chain; + }, + execute: async () => { + events.push({ kind: "execute" }); + }, + insert: (table: unknown) => ({ + values: (values: Row) => { + events.push({ kind: "insert", table, where: values }); + return Object.assign(Promise.resolve(undefined), { + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [values], + }); + }, + }), + update: (table: unknown) => ({ + set: () => ({ + where: (where: unknown) => { + events.push({ kind: "update", table, where }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => [], + }); + }, + }), + }), + delete: (table: unknown) => ({ + where: (where: unknown) => { + events.push({ kind: "delete", table, where }); + return Promise.resolve([]); + }, + }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string | undefined) => + appRouter.createCaller({ + db: fakeDb, + session: userId ? { user: { id: userId } } : null, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); + +const admin = () => callerAs("admin_user"); +const reopen = ( + over: Partial<{ hackathonId: string; evaluationId: string }> = {}, +) => + admin().judging.admin.reopenGrade({ + hackathonId: HACK_A, + evaluationId: EVAL, + ...over, + }); + +// Without the audit log's own housekeeping, which prunes old rows. +const deletes = () => + events.filter( + (event) => event.kind === "delete" && event.table !== auditLogs, + ); +const audits = () => + events + .filter((event) => event.kind === "insert" && event.table === auditLogs) + .map((event) => event.where as Row); + +const configRow = (hackathonId: string, state = "open") => ({ + hackathonId, + state, + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: false, + emergencyMode: false, + claimsStartedAt: null, +}); + +beforeEach(() => { + cache.clear(); + events.length = 0; + published = false; + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + store.clear(); + store.set(hackathons, [ + { id: HACK_A, name: "Hacklytics 2027", startDate: null, endDate: null }, + { id: HACK_B, name: "Hacklytics 2026", startDate: null, endDate: null }, + ]); + store.set(judgingConfigs, [configRow(HACK_A), configRow(HACK_B)]); + store.set(judgingEvaluations, [ + { + id: EVAL, + hackathonId: HACK_A, + entryId: ENTRY, + challengeId: CHALLENGE, + judgeId: JUDGE, + revision: 2, + isComplete: false, + }, + { + id: EVAL_OTHER_EDITION, + hackathonId: HACK_B, + entryId: ENTRY, + challengeId: CHALLENGE, + judgeId: JUDGE, + revision: 1, + isComplete: true, + }, + ]); + store.set(judgingEvaluationDrafts, [ + { id: DRAFT_1, entryId: ENTRY, challengeId: CHALLENGE }, + { id: DRAFT_2, entryId: ENTRY, challengeId: CHALLENGE }, + { id: "elsewhere", entryId: NO_SUCH, challengeId: CHALLENGE }, + ]); +}); + +describe("judging.admin.reopenGrade", () => { + it("deletes the evaluation and every draft for the project and challenge", async () => { + await expect(reopen()).resolves.toEqual({ reopened: true, drafts: 2 }); + + const [draftDelete, evaluationDelete] = deletes(); + expect(deletes()).toHaveLength(2); + expect(draftDelete!.table).toBe(judgingEvaluationDrafts); + expect(render(draftDelete!.where).params).toEqual([DRAFT_1, DRAFT_2]); + expect(evaluationDelete!.table).toBe(judgingEvaluations); + expect(render(evaluationDelete!.where).params).toEqual([EVAL]); + }); + + it("deletes an evaluation with no draft beside it", async () => { + store.set(judgingEvaluationDrafts, []); + + await expect(reopen()).resolves.toEqual({ reopened: true, drafts: 0 }); + + expect(deletes().map((event) => event.table)).toEqual([judgingEvaluations]); + }); + + it("locks the drafts, then the project's grading lock, before it deletes", async () => { + await reopen(); + + const draftLock = events.findIndex( + (event) => + event.kind === "select" && + event.table === judgingEvaluationDrafts && + event.locking, + ); + const gradingLock = events.findIndex((event) => event.kind === "execute"); + const firstDelete = events.findIndex((event) => event.kind === "delete"); + const configLock = events.findIndex( + (event) => + event.kind === "select" && + event.table === judgingConfigs && + event.locking, + ); + expect(configLock).toBeGreaterThan(-1); + expect(draftLock).toBeGreaterThan(configLock); + expect(gradingLock).toBeGreaterThan(draftLock); + expect(firstDelete).toBeGreaterThan(gradingLock); + }); + + it("refuses once results are published, and deletes nothing", async () => { + published = true; + + await expect(reopen()).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: + "Results are published. Unpublish them first, then reopen the grade and compute results again.", + }); + + expect(deletes()).toEqual([]); + expect(audits()).toEqual([]); + }); + + it("reads published from the placings of this edition", async () => { + await reopen(); + + const placings = events.find( + (event) => event.kind === "select" && event.table === judgingPlacements, + )!; + const query = render(placings.where); + expect(query.params).toContain(HACK_A); + expect(query.sql).toContain('"published_at" is not null'); + }); + + it("is allowed while judging is closed", async () => { + store.set(judgingConfigs, [configRow(HACK_A, "closed")]); + + await expect(reopen()).resolves.toMatchObject({ reopened: true }); + + expect(deletes()).toHaveLength(2); + }); + + it("does not reach an evaluation of another edition", async () => { + await expect( + reopen({ evaluationId: EVAL_OTHER_EDITION }), + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + + expect(deletes()).toEqual([]); + }); + + it("says so when the evaluation is already gone", async () => { + await expect(reopen({ evaluationId: NO_SUCH })).rejects.toMatchObject({ + code: "NOT_FOUND", + message: "That evaluation is gone. Refresh the list.", + }); + }); + + it("is audited as a critical action, with what was deleted", async () => { + await reopen(); + + expect(audits()).toEqual([ + expect.objectContaining({ + userId: "admin_user", + action: "judging.reopenGrade", + resourceId: EVAL, + severity: "critical", + metadata: { + hackathonId: HACK_A, + entryId: ENTRY, + challengeId: CHALLENGE, + judgeId: JUDGE, + revision: 2, + wasComplete: false, + draftsDeleted: 2, + }, + }), + ]); + }); + + it("is for admins only", async () => { + adminRow = undefined; + await expect(reopen()).rejects.toMatchObject({ code: "FORBIDDEN" }); + + adminRow = { userId: "admin_user", isActive: true, role: "bug_tester" }; + cache.clear(); + await expect(reopen()).rejects.toMatchObject({ code: "FORBIDDEN" }); + + expect(deletes()).toEqual([]); + }); + + it("rejects ids that are not uuids", async () => { + await expect(reopen({ evaluationId: "not-a-uuid" })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + }); +}); + +describe("what goes with an evaluation", () => { + const foreignKeys = (table: Parameters[0]) => + getTableConfig(table).foreignKeys.map((key) => { + const reference = key.reference(); + return { + to: reference.foreignTable, + columns: reference.columns.map((column) => column.name), + onDelete: key.onDelete, + }; + }); + + it.each([ + ["ratings", judgingEvaluationRatings], + ["responses", judgingEvaluationResponses], + ["revisions", judgingEvaluationRevisions], + ])("deletes its %s by cascade", (_, table) => { + expect(foreignKeys(table)).toContainEqual({ + to: judgingEvaluations, + columns: ["evaluation_id"], + onDelete: "cascade", + }); + }); +}); diff --git a/packages/api/src/middleware/cache.ts b/packages/api/src/middleware/cache.ts index 295926a7..8e9326ce 100644 --- a/packages/api/src/middleware/cache.ts +++ b/packages/api/src/middleware/cache.ts @@ -228,6 +228,10 @@ export const CacheKeys = { judge: (userId: string) => `judge:${userId}`, projectLeader: (userId: string) => `project-leader:${userId}`, portalContext: (userId: string) => `user:${userId}:portal`, + // judging.results.list's per-judge calibration; the caller adds what it was + // computed from, so a changed evaluation reads a new key. + judgingCalibration: (hackathonId: string) => + `judging:${hackathonId}:calibration`, } as const; export const invalidatePortalContext = (userId: string) => { diff --git a/packages/api/src/routers/judging/admin.ts b/packages/api/src/routers/judging/admin.ts index d596e6f0..45b089c1 100644 --- a/packages/api/src/routers/judging/admin.ts +++ b/packages/api/src/routers/judging/admin.ts @@ -30,6 +30,7 @@ import { judgingEvaluationDrafts, judgingEvaluations, judgingGuestSessions, + judgingPlacements, judgingPresence, judgingRoomLinks, judgingRooms, @@ -46,6 +47,7 @@ import { ensureJudgingConfig, } from "../../services/judging/config"; import type { JudgingDb } from "../../services/judging/config"; +import { lockGrading } from "../../services/judging/evaluation-store"; import { effectiveMode, sameLabel } from "../../services/judging/scope"; import { assertStructureEditable, @@ -2537,6 +2539,108 @@ export const judgingAdminRouter = createTRPCRouter({ return { cleared: !!cleared }; }), + /** + * Deletes one evaluation and any draft for the same project and challenge, so + * the project is open for another judge to grade (grading-hold.ts). The + * ratings, responses and revisions go with the evaluation by cascade. Allowed + * while judging is closed, as setPrizeEligibility is: closing freezes judges' + * writes, not the organiser's decisions on a project. No judge can grade the + * project until judging is open again. The draft placings still hold the old + * grade until results are computed again. + * + * Refused while results are published: the public list was ranked on it. + */ + reopenGrade: isAdmin + .input(hackathonInput.extend({ evaluationId: uuid })) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const reopened = await db.transaction(async (tx) => { + // Takes the config lock, which holds off every judge write and a + // publish while the grade is deleted. + await lockEdition(tx, input.hackathonId); + + const [published] = await tx + .select({ value: count() }) + .from(judgingPlacements) + .where( + and( + eq(judgingPlacements.hackathonId, input.hackathonId), + isNotNull(judgingPlacements.publishedAt), + ), + ); + if ((published?.value ?? 0) > 0) { + throw refuse( + "Results are published. Unpublish them first, then reopen the grade and compute results again.", + ); + } + + const [evaluation] = await tx + .select({ + entryId: judgingEvaluations.entryId, + challengeId: judgingEvaluations.challengeId, + judgeId: judgingEvaluations.judgeId, + revision: judgingEvaluations.revision, + isComplete: judgingEvaluations.isComplete, + }) + .from(judgingEvaluations) + .where( + and( + eq(judgingEvaluations.id, input.evaluationId), + eq(judgingEvaluations.hackathonId, input.hackathonId), + ), + ) + .limit(1); + if (!evaluation) { + throw notFound("That evaluation is gone. Refresh the list."); + } + + // The drafts first, then the project's grading lock: the order the + // deadline submission takes them in, so the two cannot deadlock. + const drafts = await tx + .select({ id: judgingEvaluationDrafts.id }) + .from(judgingEvaluationDrafts) + .where( + and( + eq(judgingEvaluationDrafts.entryId, evaluation.entryId), + eq(judgingEvaluationDrafts.challengeId, evaluation.challengeId), + ), + ) + .for("update"); + await lockGrading(tx, evaluation.entryId, evaluation.challengeId); + + if (drafts.length > 0) { + await tx.delete(judgingEvaluationDrafts).where( + inArray( + judgingEvaluationDrafts.id, + drafts.map((draft) => draft.id), + ), + ); + } + await tx + .delete(judgingEvaluations) + .where(eq(judgingEvaluations.id, input.evaluationId)); + + return { ...evaluation, drafts: drafts.length }; + }); + + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.reopenGrade", + resourceId: input.evaluationId, + severity: "critical", + metadata: { + hackathonId: input.hackathonId, + entryId: reopened.entryId, + challengeId: reopened.challengeId, + judgeId: reopened.judgeId, + revision: reopened.revision, + wasComplete: reopened.isComplete, + draftsDeleted: reopened.drafts, + }, + }); + return { reopened: true, drafts: reopened.drafts }; + }), + // --------------------------------------------------------------------- // Resets. Each takes the hackathon's name typed out. diff --git a/packages/api/src/routers/judging/results.ts b/packages/api/src/routers/judging/results.ts index 87c925d4..15dabca6 100644 --- a/packages/api/src/routers/judging/results.ts +++ b/packages/api/src/routers/judging/results.ts @@ -10,6 +10,7 @@ import { inArray, isNotNull, isNull, + max, ne, or, } from "drizzle-orm"; @@ -33,6 +34,8 @@ import type { DrizzleDB } from "@query/db"; import { createTRPCRouter } from "../../trpc"; import { isAdmin } from "../../middleware/procedures"; import { recordAdminAction } from "../../middleware/audit"; +import { CacheKeys } from "../../middleware/cache"; +import type { CacheService } from "../../middleware/cache"; import { assertHackathonExists } from "../../services/judging/config"; import { reconcileDueDrafts } from "../../services/judging/evaluation-store"; import type { JudgingDb } from "../../services/judging/config"; @@ -43,7 +46,11 @@ import { notifyResultsPublished, resultsPublishedBefore, } from "../../services/judging/notify"; -import { evaluationScore, meanOrNull } from "../../services/judging/scoring"; +import { + evaluationScore, + hasEveryRating, + meanOrNull, +} from "../../services/judging/scoring"; import { OVERALL_SCOPE, computePlacements, @@ -84,12 +91,29 @@ async function ratingsOf(db: JudgingDb, evaluationIds: string[]) { ); } +/** How many rating items the rubric has: an evaluation counts once it rates them all. */ +async function ratingItemCount(db: JudgingDb, hackathonId: string) { + const [row] = await db + .select({ n: count() }) + .from(judgingRubricItems) + .where( + and( + eq(judgingRubricItems.hackathonId, hackathonId), + eq(judgingRubricItems.kind, "rating"), + ), + ); + return row?.n ?? 0; +} + /** - * Every complete evaluation in the edition with its ratings: what compute ranks - * and the judge calibration summary reads. The ratings are joined rather than - * listed by id, since an edition's evaluations run to thousands. + * Every evaluation in the edition with its ratings, and whether it rated every + * rating item: what compute ranks and the judge calibration summary reads. + * Scoring and calibration count the ones that did, whether or not a written + * response was left empty, which is how a draft submitted at the hard end reads. + * The ratings are joined rather than listed by id, since an edition's + * evaluations run to thousands. */ -async function completeEvaluations(db: JudgingDb, hackathonId: string) { +async function evaluationsForScoring(db: JudgingDb, hackathonId: string) { const evaluations = await db .select({ id: judgingEvaluations.id, @@ -97,15 +121,9 @@ async function completeEvaluations(db: JudgingDb, hackathonId: string) { challengeId: judgingEvaluations.challengeId, judgeId: judgingEvaluations.judgeId, submittedAt: judgingEvaluations.submittedAt, - isComplete: judgingEvaluations.isComplete, }) .from(judgingEvaluations) - .where( - and( - eq(judgingEvaluations.hackathonId, hackathonId), - eq(judgingEvaluations.isComplete, true), - ), - ); + .where(eq(judgingEvaluations.hackathonId, hackathonId)); const ratingRows = await db .select({ evaluationId: judgingEvaluationRatings.evaluationId, @@ -116,43 +134,95 @@ async function completeEvaluations(db: JudgingDb, hackathonId: string) { judgingEvaluations, eq(judgingEvaluations.id, judgingEvaluationRatings.evaluationId), ) - .where( - and( - eq(judgingEvaluations.hackathonId, hackathonId), - eq(judgingEvaluations.isComplete, true), - ), - ); + .where(eq(judgingEvaluations.hackathonId, hackathonId)); + const ratingItems = await ratingItemCount(db, hackathonId); const ratings = groupRatings(ratingRows); - return evaluations.map((evaluation) => ({ - ...evaluation, - ratings: ratings.get(evaluation.id) ?? [], - })); + return evaluations.map((evaluation) => { + const values = ratings.get(evaluation.id) ?? []; + return { + ...evaluation, + ratings: values, + ratingsComplete: hasEveryRating(values, ratingItems), + }; + }); } // Two decimals, and never a negative zero. const round2 = (value: number) => Math.round(value * 100) / 100 || 0; +// The list polls every 10 seconds; calibrating on each poll reads and sorts the +// whole edition's evaluations, so the result is kept for this long. +const CALIBRATION_TTL_SECONDS = 60; + +/** The entries compute ranks: live and still eligible for prizes. */ +const rankedEntries = (hackathonId: string) => + and( + eq(judgingEntries.hackathonId, hackathonId), + isNull(judgingEntries.deletedAt), + isNull(judgingEntries.prizeIneligibleAt), + ); + +/** + * What a calibration was computed from, as a cache key suffix: how many + * evaluations the edition has and when one was last saved, and the same two for + * its ranked entries. A new, edited or deleted evaluation (a reopened grade is + * one), or an entry pulled from prizes, withdrawn or restored, moves one of the + * four, so the next read calibrates afresh. The one pair that cancels out is a + * grade deleted while a draft is submitted as of an earlier deadline, both + * between two polls; the TTL holds that to a minute. Two aggregates; nothing is + * listed. + */ +async function calibrationFingerprint(db: JudgingDb, hackathonId: string) { + const [[evaluations], [entries]] = await Promise.all([ + db + .select({ n: count(), latest: max(judgingEvaluations.updatedAt) }) + .from(judgingEvaluations) + .where(eq(judgingEvaluations.hackathonId, hackathonId)), + db + .select({ n: count(), latest: max(judgingEntries.updatedAt) }) + .from(judgingEntries) + .where(rankedEntries(hackathonId)), + ]); + return [ + evaluations?.n ?? 0, + evaluations?.latest?.getTime() ?? 0, + entries?.n ?? 0, + entries?.latest?.getTime() ?? 0, + ].join(":"); +} + /** * How each judge's scores are being adjusted, grouped by challenge: the * organiser's window onto what compute does. Read from the current evaluations * of live, prize-eligible entries, the same ones compute calibrates. Judges and * hackers never get this; only the organiser's results list returns it. + * + * The calibration is cached per edition (see calibrationFingerprint). compute + * does not read it: a ranking is always calibrated from the evaluations as they + * stand. */ -async function judgeCalibrations(db: JudgingDb, hackathonId: string) { - const evaluations = await completeEvaluations(db, hackathonId); - const entries = await db - .select({ id: judgingEntries.id }) - .from(judgingEntries) - .where( - and( - eq(judgingEntries.hackathonId, hackathonId), - isNull(judgingEntries.deletedAt), - isNull(judgingEntries.prizeIneligibleAt), - ), - ); - const live = new Set(entries.map((entry) => entry.id)); - const { judges: calibrated } = calibrate( - evaluations.filter((evaluation) => live.has(evaluation.entryId)), +async function judgeCalibrations( + db: JudgingDb, + cache: CacheService, + hackathonId: string, +) { + const key = `${CacheKeys.judgingCalibration(hackathonId)}:${await calibrationFingerprint(db, hackathonId)}`; + const calibrated = await cache.getOrSet( + key, + async () => { + const [evaluations, entries] = await Promise.all([ + evaluationsForScoring(db, hackathonId), + db + .select({ id: judgingEntries.id }) + .from(judgingEntries) + .where(rankedEntries(hackathonId)), + ]); + const live = new Set(entries.map((entry) => entry.id)); + return calibrate( + evaluations.filter((evaluation) => live.has(evaluation.entryId)), + ).judges; + }, + CALIBRATION_TTL_SECONDS, ); if (calibrated.length === 0) return []; @@ -204,9 +274,10 @@ async function judgeCalibrations(db: JudgingDb, hackathonId: string) { // Computing and publishing placements, and the organiser's view of what each // judge submitted. export const judgingResultsRouter = createTRPCRouter({ - // Replaces the draft placings with a fresh ranking from every complete - // evaluation. Refused while results are published: recomputing then would - // either change the public list under people's feet or silently pull it. + // Replaces the draft placings with a fresh ranking from every evaluation that + // holds every rating. Refused while results are published: recomputing then + // would either change the public list under people's feet or silently pull + // it. compute: isAdmin.input(hackathonInput).mutation(async ({ ctx, input }) => { const db = ctx.db as DrizzleDB; await assertHackathonExists(db, input.hackathonId); @@ -263,9 +334,11 @@ export const judgingResultsRouter = createTRPCRouter({ asc(judgingChallenges.displayOrder), asc(judgingChallenges.label), ); - const evaluations = await completeEvaluations(tx, input.hackathonId); + const evaluations = await evaluationsForScoring(tx, input.hackathonId); // Each evaluation is calibrated for its judge before it is scored. + // Always from the evaluations as they stand, never from the cache that + // the list keeps for its calibration summary. const placements = computePlacements({ entryIds: entries.map((entry) => entry.id), memberships, @@ -295,7 +368,9 @@ export const judgingResultsRouter = createTRPCRouter({ return { placements: placements.length, scopes: new Set(placements.map((placement) => placement.scope)).size, - evaluations: evaluations.length, + evaluations: evaluations.filter( + (evaluation) => evaluation.ratingsComplete, + ).length, }; }); @@ -502,7 +577,7 @@ export const judgingResultsRouter = createTRPCRouter({ publishedAt, evaluationsSinceCompute, calibration: computedAt - ? await judgeCalibrations(db, input.hackathonId) + ? await judgeCalibrations(db, ctx.cache, input.hackathonId) : [], scopes: [...scopes.values()].sort( (a, b) => @@ -514,7 +589,8 @@ export const judgingResultsRouter = createTRPCRouter({ }), // One presentation's results: each judge who wrote or started an - // evaluation, their ratings and responses once complete, and the mean. + // evaluation, their ratings and responses once every rating is in, and the + // mean. appointmentResults: isAdmin .input(hackathonInput.extend({ appointmentId: z.string().uuid() })) .query(async ({ ctx, input }) => { @@ -604,8 +680,8 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingEvaluationDrafts.appointmentId, appointment.id), ), ), - // Complete evaluations of the same project and challenge outside this - // slot: an untimed visit, or a slot it was moved from. + // Evaluations of the same project and challenge outside this slot: an + // untimed visit, or a slot it was moved from. db .select({ id: judgingEvaluations.id }) .from(judgingEvaluations) @@ -614,7 +690,6 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingEvaluations.hackathonId, input.hackathonId), eq(judgingEvaluations.entryId, appointment.entryId), eq(judgingEvaluations.challengeId, appointment.challengeId), - eq(judgingEvaluations.isComplete, true), or( isNull(judgingEvaluations.appointmentId), ne(judgingEvaluations.appointmentId, appointment.id), @@ -624,10 +699,8 @@ export const judgingResultsRouter = createTRPCRouter({ .limit(1), ]); - const completeIds = evaluations - .filter((evaluation) => evaluation.isComplete) - .map((evaluation) => evaluation.id); - const [ratings, responses] = completeIds.length + const evaluationIds = evaluations.map((evaluation) => evaluation.id); + const [ratings, responses, ratingItems] = evaluationIds.length ? await Promise.all([ db .select({ @@ -642,7 +715,7 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingRubricItems.id, judgingEvaluationRatings.itemId), ) .where( - inArray(judgingEvaluationRatings.evaluationId, completeIds), + inArray(judgingEvaluationRatings.evaluationId, evaluationIds), ) .orderBy(asc(judgingRubricItems.displayOrder)), db @@ -659,18 +732,34 @@ export const judgingResultsRouter = createTRPCRouter({ eq(judgingRubricItems.id, judgingEvaluationResponses.itemId), ) .where( - inArray(judgingEvaluationResponses.evaluationId, completeIds), + inArray(judgingEvaluationResponses.evaluationId, evaluationIds), ) .orderBy(asc(judgingRubricItems.displayOrder)), + ratingItemCount(db, input.hackathonId), ]) - : [[], []]; + : [[], [], 0]; + + // An evaluation counts once it rated every item, whether or not it + // answered every written question. + const counted = new Set( + evaluations + .filter((evaluation) => + hasEveryRating( + ratings + .filter((rating) => rating.evaluationId === evaluation.id) + .map((rating) => rating.value), + ratingItems, + ), + ) + .map((evaluation) => evaluation.id), + ); const judged = new Set( evaluations.map((evaluation) => evaluation.judgeId), ); const judges = [ ...evaluations.map((evaluation) => { - const own = evaluation.isComplete + const own = counted.has(evaluation.id) ? ratings.filter((rating) => rating.evaluationId === evaluation.id) : []; return { @@ -682,6 +771,7 @@ export const judgingResultsRouter = createTRPCRouter({ ? ("complete" as const) : ("incomplete" as const), autoSubmitted: !!evaluation.autoSubmittedAt, + counted: counted.has(evaluation.id), updatedAt: evaluation.updatedAt, score: evaluationScore(own.map((rating) => rating.value)), ratings: own.map(({ itemId, label, value }) => ({ @@ -689,7 +779,7 @@ export const judgingResultsRouter = createTRPCRouter({ label, value, })), - responses: evaluation.isComplete + responses: counted.has(evaluation.id) ? responses .filter((response) => response.evaluationId === evaluation.id) .map(({ itemId, label, value, isPublic }) => ({ @@ -711,6 +801,7 @@ export const judgingResultsRouter = createTRPCRouter({ evaluationId: null, status: "drafting" as const, autoSubmitted: false, + counted: false, updatedAt: draft.updatedAt, score: null, ratings: [], @@ -718,15 +809,15 @@ export const judgingResultsRouter = createTRPCRouter({ })), ].sort((a, b) => a.judgeName.localeCompare(b.judgeName)); - const complete = judges.filter((judge) => judge.status === "complete"); + const counting = judges.filter((judge) => judge.counted); return { appointment, average: meanOrNull( - complete + counting .map((judge) => judge.score) .filter((score): score is number => score !== null), ), - completeCount: complete.length, + countedCount: counting.length, hasResultsElsewhere: elsewhere.length > 0, judges, }; @@ -785,12 +876,15 @@ export const judgingResultsRouter = createTRPCRouter({ db, rows.map((row) => row.id), ); + const ratingItems = await ratingItemCount(db, input.hackathonId); return rows.map(({ autoSubmittedAt, ...row }) => ({ ...row, autoSubmitted: !!autoSubmittedAt, autoSubmittedAt, score: evaluationScore(ratings.get(row.id) ?? []), + // Whether placings count it: it rated every item. + counted: hasEveryRating(ratings.get(row.id) ?? [], ratingItems), })); }), diff --git a/packages/api/src/services/judging/calibration.test.ts b/packages/api/src/services/judging/calibration.test.ts index 2ec47272..9314bff3 100644 --- a/packages/api/src/services/judging/calibration.test.ts +++ b/packages/api/src/services/judging/calibration.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { K_A, K_B, K_D, calibrate } from "./calibration"; +import { K_A, K_B, K_D, MIN_DRIFT_GRADES, calibrate } from "./calibration"; import type { CalibrationEvaluation } from "./calibration"; // A small seeded generator (mulberry32), so every dataset below is the same on @@ -40,7 +40,7 @@ const evaluation = ( challengeId: "general", entryId: `entry-${id}`, submittedAt: new Date(DAY), - isComplete: true, + ratingsComplete: true, ratings: [score], ...over, }); @@ -151,7 +151,8 @@ function slopeOf(values: readonly number[]) { describe("calibrate", () => { it("uses the documented pseudo-evaluation counts", () => { - expect([K_A, K_B, K_D]).toEqual([3, 6, 10]); + expect([K_A, K_B, K_D]).toEqual([3, 6, 150]); + expect(MIN_DRIFT_GRADES).toBe(20); }); it("returns nothing for no evaluations", () => { @@ -182,10 +183,10 @@ describe("calibrate", () => { expect(result.adjusted.get("a")).toBeCloseTo(4, 10); }); - it("ignores incomplete evaluations and ones with no ratings", () => { + it("ignores evaluations short of a rating and ones with no ratings", () => { const result = calibrate([ evaluation("done", "judge", 4), - evaluation("draft", "judge", 1, { isComplete: false }), + evaluation("short", "judge", 1, { ratingsComplete: false }), evaluation("blank", "judge", 0, { ratings: [] }), ]); expect([...result.adjusted.keys()]).toEqual(["done"]); @@ -351,42 +352,151 @@ describe("calibrate", () => { }); const result = calibrate(evaluations); - it("is detrended in grading order", () => { - const mine = evaluations - .filter((e) => e.judgeId === "tiring") - .sort((a, b) => a.submittedAt.getTime() - b.submittedAt.getTime()); + const mine = evaluations + .filter((e) => e.judgeId === "tiring") + .sort((a, b) => a.submittedAt.getTime() - b.submittedAt.getTime()); + const grades = mine.length; + // What shrink leaves of a fitted trend with this many grades. + const trusted = grades / (grades + K_D); + + it("is detrended in grading order, by the shrunk trend", () => { const before = slopeOf(mine.map(scoreOf)); const after = slopeOf( mine.map((e) => result.adjusted.get(e.id) as number), ); expect(before).toBeLessThan(-0.04); - expect(Math.abs(after)).toBeLessThan(Math.abs(before) * 0.4); + expect(Math.abs(after)).toBeLessThan(Math.abs(before)); + expect(after * before).toBeGreaterThan(0); + // Only the trusted share of the trend is taken out (spread and offset + // rescale what is left, so this is not exact). + expect(Math.abs(after)).toBeGreaterThan( + Math.abs(before) * (1 - trusted) * 0.7, + ); }); it("reports a drift that is negative for a judge who got harsher", () => { const tiring = result.judges.find((j) => j.judgeId === "tiring"); const steady = result.judges.find((j) => j.judgeId === "steady"); - // Lost about 0.05 a grade over 44 grades, less the shrink. - expect(tiring?.drift).toBeLessThan(-1.2); - expect(Math.abs(steady?.drift ?? 1)).toBeLessThan(0.7); + // It is the fitted drift over the whole day, times the share trusted. + expect(tiring?.drift).toBeCloseTo( + slopeOf(mine.map(scoreOf)) * (grades - 1) * trusted, + 9, + ); + expect(tiring?.drift).toBeLessThan(-0.3); + expect(Math.abs(steady?.drift ?? 1)).toBeLessThan(0.3); }); }); it("breaks a tie in submission time by id when ordering a judge's grades", () => { - const climbing = calibrate([ - evaluation("a", "judge", 1), - evaluation("b", "judge", 3), - evaluation("c", "judge", 5), - ]); - const falling = calibrate([ - evaluation("c", "judge", 1), - evaluation("b", "judge", 3), - evaluation("a", "judge", 5), - ]); + // Twenty grades at one instant, so the ids alone say which came first. + const day = (idOf: (position: number) => string) => + calibrate( + Array.from({ length: MIN_DRIFT_GRADES }, (_, position) => + evaluation( + idOf(position), + "judge", + 1 + (4 * position) / (MIN_DRIFT_GRADES - 1), + ), + ), + ); + const id = (n: number) => `g${String(n).padStart(2, "0")}`; + const climbing = day((position) => id(position)); + const falling = day((position) => id(MIN_DRIFT_GRADES - 1 - position)); expect(climbing.judges[0]?.drift).toBeGreaterThan(0); expect(falling.judges[0]?.drift).toBeLessThan(0); }); + describe("drift tuning", () => { + // Simulated days: pools of 80 judges who each grade `grades` projects in + // turn, three pools per size, so every number below is over 240 judges and + // the same on every run. Project quality varies by about 0.7 of a point + // and a judge adds 0.2 of noise (pool above). `reported` is the drift + // calibrate removes from each judge; `raw` is the unshrunk least-squares + // drift from first grade to last, taken from the same scores. + const day = (grades: number, drift: number) => { + const reported: number[] = []; + const raw: number[] = []; + for (const seed of [1, 2, 3]) { + const { evaluations } = pool({ + projects: 80 * grades, + judges: Array.from({ length: 80 }, (_, i) => ({ + judgeId: `j${i}`, + drift: drift / (grades - 1), + })), + seed: 1000 * grades + seed, + }); + const byJudge = new Map(); + for (const e of [...evaluations].sort( + (a, b) => a.submittedAt.getTime() - b.submittedAt.getTime(), + )) { + byJudge.set(e.judgeId, [ + ...(byJudge.get(e.judgeId) ?? []), + scoreOf(e), + ]); + } + for (const judge of calibrate(evaluations).judges) { + reported.push(judge.drift); + raw.push( + slopeOf(byJudge.get(judge.judgeId) as number[]) * (grades - 1), + ); + } + } + return { reported, raw }; + }; + const meanOf = (values: readonly number[]) => + values.reduce((sum, value) => sum + value, 0) / values.length; + + // From the shortest day that is fitted to the longest there is. + const sizes = [MIN_DRIFT_GRADES, 30, 45, 60, 100]; + + it.each(sizes)( + "invents under 0.1 of a point of drift for a judge with none, at %i grades", + (grades) => { + const { reported, raw } = day(grades, 0); + expect(sdOf(reported)).toBeLessThan(0.1); + // The same judges under the old K_D = 10, which the review measured at + // about 0.33. + const old = raw.map((d) => (d * grades) / (grades + 10)); + expect(sdOf(old)).toBeGreaterThan(0.2); + }, + ); + + it.each(sizes)( + "takes out the share n / (n + K_D) of a real drift, at %i grades", + (grades) => { + for (const drift of [0.5, 1]) { + const { reported, raw } = day(grades, drift); + // The fit itself is unbiased: the raw drifts average the real one. + expect(Math.abs(meanOf(raw) - drift)).toBeLessThan(0.1); + const removed = meanOf(reported) / drift; + expect(Math.abs(removed - grades / (grades + K_D))).toBeLessThan( + 0.07, + ); + } + }, + ); + + it("does not fit a drift below the fewest grades", () => { + const grades = MIN_DRIFT_GRADES - 1; + const { evaluations } = pool({ + projects: 20 * grades, + judges: Array.from({ length: 20 }, (_, i) => ({ + judgeId: `j${i}`, + drift: -3 / (grades - 1), + })), + seed: 7, + }); + const result = calibrate(evaluations); + expect(result.judges.every((judge) => judge.drift === 0)).toBe(true); + // Nothing detrended: with one habit shared by every judge, the offset and + // spread are the only other corrections, and they are about 0 here. + const moved = evaluations.map((e) => + Math.abs((result.adjusted.get(e.id) as number) - scoreOf(e)), + ); + expect(meanOf(moved)).toBeLessThan(0.25); + }); + }); + it("barely corrects a judge with a single grade", () => { const { evaluations } = pool({ projects: 40, diff --git a/packages/api/src/services/judging/calibration.ts b/packages/api/src/services/judging/calibration.ts index b8c90298..4ac284dc 100644 --- a/packages/api/src/services/judging/calibration.ts +++ b/packages/api/src/services/judging/calibration.ts @@ -11,11 +11,13 @@ // A pool is one root challenge: sponsor judges are compared only with the other // judges of that sponsor's challenge, table judges with table judges, and a // judge who grades in two challenges gets separate parameters in each. Within a -// pool, over complete evaluations with a score s (the mean of its ratings): +// pool, over evaluations that hold every rating, with a score s (the mean of +// its ratings): // // mu the pool's mean -// drift a judge's grades in order (submittedAt, then id): the slope of s on -// the centred order index, shrunk, removed: s' = s - slope * (i - mean i) +// drift a judge's grades in order (submittedAt, then id), from +// MIN_DRIFT_GRADES of them: the slope of s on the centred order +// index, shrunk, removed: s' = s - slope * (i - mean i) // sd the pool's population standard deviation of s' with each judge's // own mean taken out (judges with two or more grades) // offset a = shrink * (mean(s') - mu): stricter (below 0) or more lenient @@ -29,11 +31,31 @@ import { evaluationScore } from "./scoring"; // Pseudo-evaluations: a judge with n grades is trusted n / (n + K) of the way, // so one with few grades is pulled toward no correction. K_A is for the offset, -// K_B for the spread, K_D for the drift (a trend needs more grades than a mean -// to be believed). +// K_B for the spread, K_D for the drift. +// +// K_D is far the largest because a trend is the hardest habit to see. Each +// grade is also the quality of the project it happened to be, which varies by +// about 0.7 of a point, so a judge with no drift at all still has a fitted one +// from first grade to last, off by an sd of 0.5 of a point over 20 grades, 0.4 +// over 30 and 0.3 over 60. K_D = 10 passed most of that on as correction: the +// sd of the drift taken off judges with none was 0.32 of a point at 30 grades. +// K_D = 150 holds it under 0.1 at every size from 20 to 100 grades (0.07 at 30, +// 0.09 at 60). Both are measured on simulated judges in calibration.test.ts, +// "drift tuning". +// +// The price is that a real drift is taken out by the same share, n / (n + K_D): +// 17% of it at 30 grades, 29% at 60. Half a point of real drift across a day is +// mostly left in. Taking most of it out as well was the aim, and no weight does +// both: at that size a real drift and the one chance invents look alike, so a +// smaller K_D gives the invented correction back as fast as the real one (0.15 +// of a point at K_D = 60). Both would take some 150 grades from each judge. export const K_A = 3; export const K_B = 6; -export const K_D = 10; +export const K_D = 150; +// Fewest grades before a drift is fitted. Below this chance alone moves the +// fitted drift by half a point or more (0.9 at 6 grades), as much as a real +// one worth finding, so none is fitted. +export const MIN_DRIFT_GRADES = 20; const MIN_SPREAD = 0.5; const MAX_SPREAD = 2; @@ -49,7 +71,8 @@ export type CalibrationEvaluation = { challengeId: string; entryId: string; submittedAt: Date; - isComplete: boolean; + /** Holds a rating for every rating item of the rubric; only these count. */ + ratingsComplete: boolean; /** The 1-5 rating values it holds. */ ratings: readonly number[]; }; @@ -57,7 +80,7 @@ export type CalibrationEvaluation = { export type JudgeCalibration = { judgeId: string; challengeId: string; - /** Complete, scored evaluations the judge has in this challenge. */ + /** Counted, scored evaluations the judge has in this challenge. */ evaluations: number; /** Rating points taken off the judge's scores. Positive: more lenient than the pool. */ offset: number; @@ -68,7 +91,7 @@ export type JudgeCalibration = { }; export type Calibration = { - /** The calibrated score of each complete, scored evaluation, by id. */ + /** The calibrated score of each counted, scored evaluation, by id. */ adjusted: Map; judges: JudgeCalibration[]; }; @@ -112,16 +135,17 @@ function groupBy(items: readonly T[], key: (item: T) => string) { } /** - * Calibrated scores for every complete evaluation, and what was learned about - * each judge. Incomplete evaluations, and complete ones with no ratings, are - * left out of both. The result does not depend on the order of the input. + * Calibrated scores for every evaluation that holds every rating, and what was + * learned about each judge. Evaluations short of a rating, and ones with no + * ratings at all, are left out of both. The result does not depend on the order + * of the input. */ export function calibrate( evaluations: readonly CalibrationEvaluation[], ): Calibration { const graded: Graded[] = []; for (const evaluation of evaluations) { - if (!evaluation.isComplete) continue; + if (!evaluation.ratingsComplete) continue; const score = evaluationScore(evaluation.ratings); if (score === null) continue; graded.push({ @@ -155,10 +179,9 @@ export function calibrate( const scores = grades.map((g) => g.score); const centre = (n - 1) / 2; - // Least-squares slope of score on the centred order index. Three - // grades is the fewest that can show a trend. + // Least-squares slope of score on the centred order index. let slope = 0; - if (n >= 3) { + if (n >= MIN_DRIFT_GRADES) { const average = mean(scores); let along = 0; let across = 0; diff --git a/packages/api/src/services/judging/placements.test.ts b/packages/api/src/services/judging/placements.test.ts index 523f1739..bc8f401c 100644 --- a/packages/api/src/services/judging/placements.test.ts +++ b/packages/api/src/services/judging/placements.test.ts @@ -17,7 +17,7 @@ const actualCalibrate = async () => const leaveAsRaw: typeof calibrate = (evaluations) => ({ adjusted: new Map( evaluations - .filter((e) => e.isComplete && e.ratings.length > 0) + .filter((e) => e.ratingsComplete && e.ratings.length > 0) .map((e) => [ e.id, e.ratings.reduce((sum, value) => sum + value, 0) / e.ratings.length, @@ -31,7 +31,7 @@ const evaluation = ( entryId: string, challengeId: string, ratings: number[], - isComplete = true, + ratingsComplete = true, over: Partial = {}, ): PlacementEvaluation => { count += 1; @@ -42,7 +42,7 @@ const evaluation = ( entryId, challengeId, ratings, - isComplete, + ratingsComplete, ...over, }; }; @@ -112,7 +112,7 @@ describe("computePlacements", () => { ); }); - it("counts complete evaluations only and leaves unscored projects out", () => { + it("counts evaluations with every rating only and leaves unscored projects out", () => { const placements = computePlacements({ entryIds: ["a", "b"], memberships: [ @@ -247,7 +247,7 @@ describe("computePlacements with judge calibration", () => { ); }); - it("calibrates once, over the live entries' complete evaluations only", () => { + it("calibrates once, over the live entries' evaluations only", () => { const evaluations = [ evaluation("a", "general", [4], true, { judgeId: "one" }), evaluation("b", "sponsor", [3], true, { judgeId: "two" }), diff --git a/packages/api/src/services/judging/placements.ts b/packages/api/src/services/judging/placements.ts index 57497ddb..ec26ffac 100644 --- a/packages/api/src/services/judging/placements.ts +++ b/packages/api/src/services/judging/placements.ts @@ -1,5 +1,5 @@ // Turns evaluations into placings: one ranking per challenge and one overall. -// A project's score is the mean of its complete evaluations (scoring.ts), each +// A project's score is the mean of its counted evaluations (scoring.ts), each // calibrated for its judge's habits first (calibration.ts); equal scores share // a placing. The raw mean is kept beside it. Projects nobody scored are left // out rather than listed last, since a placing is published and "8th of 8" for @@ -14,6 +14,8 @@ export const OVERALL_SCOPE = "overall"; export const OVERALL_LABEL = "Overall"; export type PlacementEvaluation = ScoredEvaluation & { + /** Calibration counts evaluations by this alone. */ + ratingsComplete: boolean; id: string; entryId: string; judgeId: string; @@ -46,7 +48,7 @@ export function computePlacements(input: { const entryIds = [...new Set(input.entryIds)].sort(); const live = new Set(entryIds); - // Calibrated once, over every complete evaluation of a live entry, so a + // Calibrated once, over every counted evaluation of a live entry, so a // judge's habits are learned from all of their grades and not from the // slice that fell in one scope. const liveEvaluations = input.evaluations.filter((evaluation) => diff --git a/packages/api/src/services/judging/scoring.test.ts b/packages/api/src/services/judging/scoring.test.ts index 287931cb..0e0162e6 100644 --- a/packages/api/src/services/judging/scoring.test.ts +++ b/packages/api/src/services/judging/scoring.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { evaluationScore, + hasEveryRating, meanOrNull, overallScore, rankByScore, @@ -19,6 +20,23 @@ describe("meanOrNull", () => { }); }); +describe("hasEveryRating", () => { + it("is whether the evaluation rated every rating item", () => { + expect(hasEveryRating([5, 4, 3], 3)).toBe(true); + expect(hasEveryRating([5, 4], 3)).toBe(false); + expect(hasEveryRating([], 3)).toBe(false); + }); + + it("does not need to be told about written responses", () => { + // Four rating items, however many responses the rubric also asks for. + expect(hasEveryRating([1, 2, 3, 4], 4)).toBe(true); + }); + + it("has nothing to miss on a rubric with no rating items", () => { + expect(hasEveryRating([], 0)).toBe(true); + }); +}); + describe("evaluationScore", () => { it("is the mean of the ratings", () => { expect(evaluationScore([5, 4, 3])).toBe(4); @@ -31,15 +49,15 @@ describe("evaluationScore", () => { }); const evals: ScoredEvaluation[] = [ - { challengeId: "general", isComplete: true, ratings: [5, 5, 5] }, // 5 - { challengeId: "general", isComplete: true, ratings: [3, 3, 3] }, // 3 - { challengeId: "general", isComplete: false, ratings: [1, 1] }, // ignored - { challengeId: "mongo", isComplete: true, ratings: [4, 2] }, // 3 - { challengeId: "mongo", isComplete: true, ratings: [] }, // no score + { challengeId: "general", ratingsComplete: true, ratings: [5, 5, 5] }, // 5 + { challengeId: "general", ratingsComplete: true, ratings: [3, 3, 3] }, // 3 + { challengeId: "general", ratingsComplete: false, ratings: [1, 1] }, // short of a rating: ignored + { challengeId: "mongo", ratingsComplete: true, ratings: [4, 2] }, // 3 + { challengeId: "mongo", ratingsComplete: true, ratings: [] }, // no score ]; describe("scopedScore", () => { - it("averages complete evaluations in that challenge only", () => { + it("averages evaluations with every rating in that challenge only", () => { expect(scopedScore(evals, "general")).toEqual({ score: 4, count: 2 }); expect(scopedScore(evals, "mongo")).toEqual({ score: 3, count: 1 }); }); @@ -48,7 +66,7 @@ describe("scopedScore", () => { expect(scopedScore(evals, "other")).toEqual({ score: null, count: 0 }); expect( scopedScore( - [{ challengeId: "general", isComplete: false, ratings: [5] }], + [{ challengeId: "general", ratingsComplete: false, ratings: [5] }], "general", ), ).toEqual({ score: null, count: 0 }); @@ -56,7 +74,7 @@ describe("scopedScore", () => { }); describe("overallScore", () => { - it("weighs every complete evaluation the same, across challenges", () => { + it("weighs every counted evaluation the same, across challenges", () => { // (5 + 3 + 3) / 3 expect(overallScore(evals)).toEqual({ score: 11 / 3, count: 3 }); }); @@ -66,26 +84,56 @@ describe("overallScore", () => { }); }); +describe("which evaluations count", () => { + const score = (over: Partial) => + overallScore([{ challengeId: "general", ratings: [4, 2], ...over }]); + + it("counts one with every rating, though a written response is missing", () => { + // An auto-submit at the hard end: not complete in the database, but all + // its ratings are there. + expect(score({ ratingsComplete: true, isComplete: false })).toEqual({ + score: 3, + count: 1, + }); + }); + + it("leaves out one short of a rating, though it was submitted whole", () => { + expect(score({ ratingsComplete: false, isComplete: true })).toEqual({ + score: null, + count: 0, + }); + }); + + it("reads isComplete from a caller that gives no ratingsComplete", () => { + expect(score({ isComplete: true })).toEqual({ score: 3, count: 1 }); + expect(score({ isComplete: false })).toEqual({ score: null, count: 0 }); + }); + + it("counts nothing when told nothing", () => { + expect(score({})).toEqual({ score: null, count: 0 }); + }); +}); + describe("an adjusted score", () => { const adjusted: ScoredEvaluation[] = [ { challengeId: "general", - isComplete: true, + ratingsComplete: true, ratings: [5, 5], adjustedScore: 4, }, { challengeId: "general", - isComplete: true, + ratingsComplete: true, ratings: [3, 3], adjustedScore: 3.5, }, // No adjusted score: the mean of its ratings counts. - { challengeId: "general", isComplete: true, ratings: [2, 4] }, - // Incomplete is still left out, adjusted or not. + { challengeId: "general", ratingsComplete: true, ratings: [2, 4] }, + // Short of a rating is still left out, adjusted or not. { challengeId: "general", - isComplete: false, + ratingsComplete: false, ratings: [1], adjustedScore: 1, }, @@ -98,9 +146,9 @@ describe("an adjusted score", () => { }); it("leaves evaluations without one on the raw mean", () => { - const raw = adjusted.map(({ challengeId, isComplete, ratings }) => ({ + const raw = adjusted.map(({ challengeId, ratingsComplete, ratings }) => ({ challengeId, - isComplete, + ratingsComplete, ratings, })); // (5 + 3 + 3) / 3 diff --git a/packages/api/src/services/judging/scoring.ts b/packages/api/src/services/judging/scoring.ts index 74e9eb8e..28d040b7 100644 --- a/packages/api/src/services/judging/scoring.ts +++ b/packages/api/src/services/judging/scoring.ts @@ -1,13 +1,25 @@ // Pure scoring for room judging. A judge's evaluation scores the mean of its // 1–5 ratings; a project's score in a scope is the mean of those evaluation -// scores, counting complete evaluations only. Every evaluation weighs the -// same, whoever wrote it. Placements feed in the calibrated score of each +// scores, counting evaluations that hold every rating. A written response left +// empty does not stop one counting: a draft submitted at the hard end is often +// short of a response and is the project's only grade. Every evaluation weighs +// the same, whoever wrote it. Placements feed in the calibrated score of each // evaluation (calibration.ts) in place of the raw mean; the judge desk does not. /** An evaluation as scoring sees it. */ export type ScoredEvaluation = { challengeId: string; - isComplete: boolean; + /** + * Holds a rating for every rating item of the rubric (hasEveryRating). It + * counts when this is true, whatever was left unwritten. + */ + ratingsComplete?: boolean; + /** + * For a caller that only knows the evaluation was submitted whole, as the + * judge desk does, which implies every rating is there. Read only when + * ratingsComplete is not given. + */ + isComplete?: boolean; /** The 1–5 rating values it holds. */ ratings: readonly number[]; /** Counted in place of the mean of the ratings when set: the calibrated score. */ @@ -15,7 +27,7 @@ export type ScoredEvaluation = { }; export type ScoreSummary = { - /** Null when no complete evaluation has a rating. */ + /** Null when no counted evaluation has a rating. */ score: number | null; /** How many evaluations the score is the mean of. */ count: number; @@ -33,9 +45,24 @@ export function evaluationScore(values: readonly number[]): number | null { return meanOrNull(values); } +/** + * Whether an evaluation rated every rating item. The rubric is fixed once an + * evaluation exists, and a rating is stored once per item, so counting them + * against the rubric's rating items is enough. Responses are not looked at. + */ +export function hasEveryRating( + ratings: readonly number[], + ratingItems: number, +): boolean { + return ratings.length >= ratingItems; +} + +const counts = (evaluation: ScoredEvaluation) => + evaluation.ratingsComplete ?? evaluation.isComplete ?? false; + function summarize(evals: readonly ScoredEvaluation[]): ScoreSummary { const scores = evals - .filter((evaluation) => evaluation.isComplete) + .filter(counts) .map( (evaluation) => evaluation.adjustedScore ?? evaluationScore(evaluation.ratings), @@ -52,7 +79,7 @@ export function scopedScore( return summarize(evals.filter((e) => e.challengeId === challengeId)); } -/** A project's overall score: every complete evaluation, in any challenge. */ +/** A project's overall score: every counted evaluation, in any challenge. */ export function overallScore(evals: readonly ScoredEvaluation[]): ScoreSummary { return summarize(evals); } diff --git a/sites/mainweb/components/admin/judging/results/ResultsTab.test.tsx b/sites/mainweb/components/admin/judging/results/ResultsTab.test.tsx new file mode 100644 index 00000000..c24f9d2e --- /dev/null +++ b/sites/mainweb/components/admin/judging/results/ResultsTab.test.tsx @@ -0,0 +1,177 @@ +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { ResultsTab } from "./ResultsTab"; + +// Rendered to markup with the tRPC hooks stubbed: the web app has no DOM test +// environment, so these check what each state shows and disables, not clicks. + +// The root vitest config compiles JSX to React.createElement, and components +// written for Next's automatic runtime do not import React. +Object.assign(globalThis, { React }); + +const mocks = vi.hoisted(() => { + const at = new Date("2027-02-06T11:00:00Z"); + const audit = ( + id: string, + judgeName: string, + over: Record, + ) => ({ + id, + entryId: "entry-1", + title: "Tidewater", + challengeId: "ch-1", + challenge: "General", + judgeId: `judge-${id}`, + judgeName, + kind: "member", + appointmentId: null, + isComplete: true, + revision: 1, + submittedAt: at, + autoSubmitted: false, + autoSubmittedAt: null, + updatedAt: at, + score: 4, + counted: true, + ...over, + }); + return { + at, + readOnly: false, + publishedAt: null as Date | null, + // One whole evaluation, one auto-submitted with every rating and a + // response missing, one auto-submitted short of a rating. + rows: [ + audit("whole", "Bea Whole", {}), + audit("auto", "Ada Auto", { + isComplete: false, + autoSubmitted: true, + autoSubmittedAt: at, + }), + audit("short", "Cy Short", { + isComplete: false, + autoSubmitted: true, + autoSubmittedAt: at, + counted: false, + }), + ], + }; +}); + +vi.mock("@/lib/use-portal-context", () => ({ + READ_ONLY_TITLE: "Read-only", + useReadOnly: () => mocks.readOnly, +})); + +vi.mock("@/lib/trpc", () => { + const idle = () => ({ + isPending: false, + variables: undefined, + error: null, + mutate: () => undefined, + }); + const query = (data: unknown) => () => ({ + data, + isLoading: false, + isError: false, + error: null, + refetch: () => undefined, + }); + return { + trpc: { + useUtils: () => ({}), + hackathon: { + getById: { useQuery: query({ status: "active" }) }, + update: { useMutation: idle }, + }, + judging: { + admin: { + setPrizeEligibility: { useMutation: idle }, + reopenGrade: { useMutation: idle }, + }, + results: { + compute: { useMutation: idle }, + publish: { useMutation: idle }, + unpublish: { useMutation: idle }, + list: { + useQuery: () => ({ + data: { + computedAt: mocks.at, + publishedAt: mocks.publishedAt, + evaluationsSinceCompute: 0, + calibration: [], + scopes: [], + }, + isLoading: false, + isError: false, + error: null, + refetch: () => undefined, + }), + }, + evaluationAudit: { useQuery: query(mocks.rows) }, + }, + }, + }, + }; +}); + +const render = () => + renderToStaticMarkup( + , + ); + +// The attribute, not the `disabled:` variants in the class list. +const isDisabled = (tag: string) => /\sdisabled=""/.test(tag); + +/** The opening tag of every Reopen button. */ +const reopenButtons = (markup: string) => + [...markup.matchAll(/]*>(?=Reopen for grading)/g)].map( + (match) => match[0], + ); + +beforeEach(() => { + mocks.readOnly = false; + mocks.publishedAt = null; +}); + +describe("ResultsTab evaluations", () => { + it("offers to reopen each evaluation for grading", () => { + const buttons = reopenButtons(render()); + + expect(buttons).toHaveLength(3); + for (const button of buttons) expect(isDisabled(button)).toBe(false); + }); + + it("says which incomplete evaluations count toward the placings", () => { + const markup = render(); + + expect(markup).toContain("Complete"); + expect(markup).toContain("Counts, response missing"); + expect(markup).toContain("Incomplete, not counted"); + }); + + it("holds the button back while results are published, and says why", () => { + mocks.publishedAt = mocks.at; + + const buttons = reopenButtons(render()); + + expect(buttons).toHaveLength(3); + for (const button of buttons) { + expect(isDisabled(button)).toBe(true); + expect(button).toContain("Unpublish first to reopen a grade."); + } + }); + + it("holds the button back for a read-only tester", () => { + mocks.readOnly = true; + + const buttons = reopenButtons(render()); + + expect(buttons).toHaveLength(3); + for (const button of buttons) { + expect(isDisabled(button)).toBe(true); + expect(button).toContain("Read-only"); + } + }); +}); diff --git a/sites/mainweb/components/admin/judging/results/ResultsTab.tsx b/sites/mainweb/components/admin/judging/results/ResultsTab.tsx index 6fac719e..e7f99d3e 100644 --- a/sites/mainweb/components/admin/judging/results/ResultsTab.tsx +++ b/sites/mainweb/components/admin/judging/results/ResultsTab.tsx @@ -23,7 +23,12 @@ import { eventClock, eventMoment, } from "@/components/hackathon/judging/event-time"; -import { plural, useRefreshJudging } from "@/components/admin/judging/shared"; +import { + ErrorLine, + NoticeLine, + plural, + useRefreshJudging, +} from "@/components/admin/judging/shared"; import { describeJudge, judgeNumbers, @@ -44,10 +49,24 @@ type AuditRow = RouterOutputs["judging"]["results"]["evaluationAudit"][number]; const score = (value: number | null) => value === null ? "No score" : value.toFixed(2); +// An evaluation counts toward the placings once every rating is in, so one +// submitted at the deadline with a response left empty still does. +const gradeBadge = ( + state: "complete" | "incomplete" | "drafting", + counted: boolean, +) => + state === "complete" + ? { tone: "success" as const, text: "Complete" } + : state === "drafting" + ? { tone: "neutral" as const, text: "Still writing" } + : counted + ? { tone: "warning" as const, text: "Counts, response missing" } + : { tone: "warning" as const, text: "Incomplete, not counted" }; + /** - * Ranking and releasing results: compute draft placings from complete - * evaluations, check them, publish them to the hackathon's Results tab, and - * look behind any number at who scored what. + * Ranking and releasing results: compute draft placings from the evaluations, + * check them, publish them to the hackathon's Results tab, look behind any + * number at who scored what, and reopen a grade for another judge. */ export function ResultsTab({ hackathonId }: { hackathonId: string }) { const utils = trpc.useUtils(); @@ -302,8 +321,8 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) { {scopes.length === 0 ? (

    {computedAt - ? "No project has a complete evaluation yet, so nothing was ranked. Compute again once judges submit." - : "Compute results once judges have submitted. Each challenge and the overall list rank projects by the mean of their complete evaluations, each adjusted for how strict or generous its judge is; ties share a placing."} + ? "No project has an evaluation with every rating yet, so nothing was ranked. Compute again once judges submit." + : "Compute results once judges have submitted. Each challenge and the overall list rank projects by the mean of their evaluations that have every rating, each adjusted for how strict or generous its judge is; ties share a placing."}

    ) : (
    @@ -489,6 +508,7 @@ export function ResultsTab({ hackathonId }: { hackathonId: string }) {
    setEntryFilter(null)} /> @@ -543,13 +563,19 @@ function CalibrationChallengeRow({ function EvaluationAudit({ hackathonId, + published, entryFilter, onClearFilter, }: { hackathonId: string; + published: boolean; entryFilter: { id: string; title: string } | null; onClearFilter: () => void; }) { + const readOnly = useReadOnly(); + const refreshJudging = useRefreshJudging(); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); const [shown, setShown] = useState(AUDIT_PAGE); const [expanded, setExpanded] = useState<{ id: string; @@ -561,6 +587,22 @@ function EvaluationAudit({ { refetchInterval: POLL_MS, placeholderData: (previous) => previous }, ); + // Deleting the grade changes the project list and the judges' queues as + // well as this one, so refresh all of judging. + const reopen = trpc.judging.admin.reopenGrade.useMutation({ + onSuccess: () => { + setError(null); + setNotice( + "The grade is deleted and another judge can grade the project. Compute results again to update the placings.", + ); + void refreshJudging(); + }, + onError: (e) => { + setNotice(null); + setError(trpcErrorMessage(e, "That did not go through. Try again.")); + }, + }); + const toggle = (id: string, view: "appointment" | "history") => setExpanded((current) => current?.id === id && current.view === view ? null : { id, view }, @@ -588,6 +630,17 @@ function EvaluationAudit({

    )} + {error && ( +
    + {error} +
    + )} + {notice && ( +
    + {notice} +
    + )} + {audit.isLoading ? (

    Loading evaluations…

    ) : audit.isError ? ( @@ -618,6 +671,28 @@ function EvaluationAudit({ row={row} expanded={expanded?.id === row.id ? expanded.view : null} onToggle={(view) => toggle(row.id, view)} + reopenDisabled={readOnly || published || reopen.isPending} + reopenTitle={ + readOnly + ? READ_ONLY_TITLE + : published + ? "Unpublish first to reopen a grade." + : undefined + } + reopening={ + reopen.isPending && reopen.variables?.evaluationId === row.id + } + onReopen={() => { + if ( + !window.confirm( + `Reopen ${row.title} for grading? ${row.judgeName}'s grade is deleted, with its ratings, responses and history, and another judge can grade the project. This cannot be undone.`, + ) + ) + return; + setError(null); + setNotice(null); + reopen.mutate({ hackathonId, evaluationId: row.id }); + }} /> ))} @@ -641,12 +716,24 @@ function AuditItem({ row, expanded, onToggle, + reopenDisabled, + reopenTitle, + reopening, + onReopen, }: { hackathonId: string; row: AuditRow; expanded: "appointment" | "history" | null; onToggle: (view: "appointment" | "history") => void; + reopenDisabled: boolean; + reopenTitle: string | undefined; + reopening: boolean; + onReopen: () => void; }) { + const badge = gradeBadge( + row.isComplete ? "complete" : "incomplete", + row.counted, + ); return (
  2. @@ -666,9 +753,7 @@ function AuditItem({ {score(row.score)} - - {row.isComplete ? "Complete" : "Incomplete"} - + {badge.text} {row.appointmentId && ( +
    3. {expanded === "appointment" && row.appointmentId && ( @@ -725,7 +819,7 @@ function AppointmentDetail({ ); } - const { appointment, average, completeCount, hasResultsElsewhere, judges } = + const { appointment, average, countedCount, hasResultsElsewhere, judges } = detail.data; const where = [appointment.building, appointment.room] .filter(Boolean) @@ -742,13 +836,13 @@ function AppointmentDetail({ {score(average)} {" "} - from {completeCount} complete{" "} - {completeCount === 1 ? "evaluation" : "evaluations"}. + from {countedCount} counted{" "} + {countedCount === 1 ? "evaluation" : "evaluations"}.

    {hasResultsElsewhere && (

    - This project also has complete evaluations for this challenge outside - this slot. + This project also has evaluations for this challenge outside this + slot.

    )} {judges.length === 0 ? ( @@ -766,18 +860,10 @@ function AppointmentDetail({

    - {judge.status === "complete" - ? "Complete" - : judge.status === "incomplete" - ? "Incomplete" - : "Still writing"} + {gradeBadge(judge.status, judge.counted).text} {judge.score !== null && ( diff --git a/sites/mainweb/components/admin/judging/results/calibration.test.ts b/sites/mainweb/components/admin/judging/results/calibration.test.ts index da491e2d..0e921cc0 100644 --- a/sites/mainweb/components/admin/judging/results/calibration.test.ts +++ b/sites/mainweb/components/admin/judging/results/calibration.test.ts @@ -21,7 +21,7 @@ describe("describeJudge", () => { it("calls a judge in line when nothing stands out", () => { expect(describeJudge(judge())).toEqual(["In line with the other judges"]); expect( - describeJudge(judge({ offset: 0.1, spread: 0.9, drift: -0.2 })), + describeJudge(judge({ offset: 0.1, spread: 0.9, drift: -0.15 })), ).toEqual(["In line with the other judges"]); }); @@ -44,23 +44,23 @@ describe("describeJudge", () => { }); it("says when a judge changed over the day", () => { - expect(describeJudge(judge({ drift: -0.8 }))).toEqual([ + expect(describeJudge(judge({ drift: -0.4 }))).toEqual([ "Got harsher through the day", ]); - expect(describeJudge(judge({ drift: 0.7 }))).toEqual([ + expect(describeJudge(judge({ drift: 0.3 }))).toEqual([ "Got more lenient through the day", ]); }); it("does not read a habit into what chance alone gives a judge", () => { expect( - describeJudge(judge({ grades: 30, offset: -0.15, drift: 0.45 })), + describeJudge(judge({ grades: 30, offset: -0.15, drift: 0.15 })), ).toEqual(["In line with the other judges"]); }); it("lists everything that applies", () => { expect( - describeJudge(judge({ offset: -0.3, spread: 0.7, drift: -0.6 })), + describeJudge(judge({ offset: -0.3, spread: 0.7, drift: -0.25 })), ).toEqual([ "Stricter than average by 0.30", "Scores bunched together", diff --git a/sites/mainweb/components/admin/judging/results/calibration.ts b/sites/mainweb/components/admin/judging/results/calibration.ts index 6733782d..97c00739 100644 --- a/sites/mainweb/components/admin/judging/results/calibration.ts +++ b/sites/mainweb/components/admin/judging/results/calibration.ts @@ -10,11 +10,12 @@ type Numbers = Pick; // out loud. Offset and drift are in rating points; spread is a multiple of the // pool's, so 1 is the same. Chance alone moves them for a judge with no habit // at all (which projects they drew, and in what order), by about 0.1 for the -// offset and 0.3 for the drift over thirty grades, so these sit about twice -// that above zero. +// offset over thirty grades and under 0.1 for the drift (the API shrinks it to +// that, and fits none below twenty grades), so these sit about twice that +// above zero. const OFFSET_NOTICED = 0.2; const SPREAD_NOTICED = 0.15; -const DRIFT_NOTICED = 0.6; +const DRIFT_NOTICED = 0.2; // With fewer grades than this the correction is mostly shrunk away. const FEW_GRADES = 3; From 964f93e05dcc430b590bc7135fb075b637427fee Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 18:56:14 -0400 Subject: [PATCH 17/27] feat(judging): assign one judge to every presentation Add a second scheduling stage. Once every presentation has a time and a room, each one gets exactly one judge, so every project has a time, a place and a grader. Schema: judging_appointment.assigned_judge_id (nullable, FK to judging_judge, on delete set null) and an index on it. Model (services/judging/schedule/judges.ts): a judge grades only the challenges they may grade, a guest only in their own room, one presentation per slot, the schedule's breaks to walk to another room or building (staying in a room needs none), and at most MAX_CONSECUTIVE slots in a row. Goals in order: every presentation has a judge, sponsor presentations go to the sponsor's own judges rather than table judges, even load (busiest judge, then spread inside each pool), then mix (few repeats of a room, as much of the day as possible). validateJudgeAssignment and scoreJudgeAssignment check and measure an assignment; assignJudges is the deterministic fallback for servers without OR-Tools: three greedy starts, then a bounded local search. cpsat.py: with a judges array it assigns judges as a second CP-SAT model after the placement, writes assignments into the solution, and keeps a challenge's presentations in a slot within its judges as the first placement goal. A problem with placements only assigns judges, with locked assignments kept and an assignmentHint to start from. Version-1 files without judges are unchanged. cpsat-bridge and cpsat-runner carry the judges, validate what comes back with validateJudgeAssignment and fall back to assignJudges, and allow the extra time limit. Router: saving a schedule stores each presentation's judge; the board read returns each appointment's judge, per-judge loads and warnings for slots with too few judges; assignJudges re-assigns what has not started and has no evaluation or draft, keeping the rest (CP-SAT when the server has it); setAssignedJudge and judgeChoices change one appointment's judge by hand; moving an appointment clears a judge who can no longer grade it. Organiser UI: judge on each appointment, an Assign judges button with warnings, a per-judge load list, and a dialog to change one judge. --- docs/operations/environment.md | 2 +- .../api/scripts/judging-schedule/cpsat.py | 548 +++++++++- .../scripts/judging-schedule/test_cpsat.py | 206 +++- .../.internal-tests/schedule-judges.test.ts | 940 ++++++++++++++++++ packages/api/src/routers/judging/schedule.ts | 641 +++++++++++- .../judging/schedule/cpsat-bridge.test.ts | 245 ++++- .../services/judging/schedule/cpsat-bridge.ts | 197 +++- .../judging/schedule/cpsat-runner.test.ts | 107 +- .../services/judging/schedule/cpsat-runner.ts | 25 +- .../judging/schedule/judge-plan.test.ts | 251 +++++ .../services/judging/schedule/judge-plan.ts | 217 ++++ .../services/judging/schedule/judges.test.ts | 532 ++++++++++ .../src/services/judging/schedule/judges.ts | 784 +++++++++++++++ packages/db/src/schemas/judging.ts | 7 + .../judging/schedule/CpsatPanel.test.tsx | 20 + .../admin/judging/schedule/CpsatPanel.tsx | 20 +- .../admin/judging/schedule/JudgeDialog.tsx | 180 ++++ .../judging/schedule/JudgesPanel.test.tsx | 280 ++++++ .../admin/judging/schedule/JudgesPanel.tsx | 173 ++++ .../admin/judging/schedule/LiveBoard.tsx | 40 +- 20 files changed, 5356 insertions(+), 59 deletions(-) create mode 100644 packages/api/src/.internal-tests/schedule-judges.test.ts create mode 100644 packages/api/src/services/judging/schedule/judge-plan.test.ts create mode 100644 packages/api/src/services/judging/schedule/judge-plan.ts create mode 100644 packages/api/src/services/judging/schedule/judges.test.ts create mode 100644 packages/api/src/services/judging/schedule/judges.ts create mode 100644 sites/mainweb/components/admin/judging/schedule/JudgeDialog.tsx create mode 100644 sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx create mode 100644 sites/mainweb/components/admin/judging/schedule/JudgesPanel.tsx diff --git a/docs/operations/environment.md b/docs/operations/environment.md index 3eadf40f..84a42174 100644 --- a/docs/operations/environment.md +++ b/docs/operations/environment.md @@ -62,7 +62,7 @@ Without `DATABASE_URL`, `db` is null, sessions fall back to JWT, and tRPC proced | Variable | Default / notes | | --- | --- | -| `JUDGING_CPSAT_PYTHON` | A Python with OR-Tools (`python3` or a path). Set, the schedule section can run `packages/api/scripts/judging-schedule/cpsat.py` on the server (`judging.schedule.solveCpsat`). Unset means CP-SAT is download, run on a laptop, upload | +| `JUDGING_CPSAT_PYTHON` | A Python with OR-Tools (`python3` or a path). Set, the schedule section can run `packages/api/scripts/judging-schedule/cpsat.py` on the server (`judging.schedule.solveCpsat`), and `judging.schedule.assignJudges` uses it to pick judges. Unset means CP-SAT is download, run on a laptop, upload, and the app assigns judges itself | | `JUDGING_CPSAT_SCRIPT` | Path to `cpsat.py`. Default: found by walking up from the working directory | ## Security / proxy diff --git a/packages/api/scripts/judging-schedule/cpsat.py b/packages/api/scripts/judging-schedule/cpsat.py index f81afaad..080f0ad8 100644 --- a/packages/api/scripts/judging-schedule/cpsat.py +++ b/packages/api/scripts/judging-schedule/cpsat.py @@ -35,6 +35,36 @@ cross-building walks only the same-building break and marks the solution `relaxed`; the portal makes an organiser confirm that before saving. +Judges +------ +When the problem lists `judges`, the script goes on to the second stage: one +judge for every presentation, solved as its own CP-SAT model once the +placement is known. + + hard a judge grades only challenges they may grade, and a guest only in + their own room; + one presentation per judge per slot; + moving to another room in the same building needs `sameGap` empty + slots, to another building `crossGap`; staying in one room needs none; + at most `maxConsecutive` slots in a row, then a free slot. + goals in this order, each one never traded for a later one: + 1. every presentation has a judge (the rest is reported, not failed) + 2. sponsor presentations go to the sponsor's own judges, not to table + judges (members) + 3. the busiest judge grades as little as possible + 4. judges of one pool (same challenges, same room) grade evenly + 5. a judge repeats a room as little as possible + 6. a judge's presentations cover as much of the day as possible, as + judge-bias calibration assumes every judge sees a random sample + +The placement stage then also keeps, in every slot, the presentations of a +challenge within the number of judges who may grade it, as far as it can +(goal 0, `judgeShortfall`, ahead of the five above). + +A problem with `placements` is a schedule already made: the placement stage is +skipped and only the judges are assigned. `locked` lists assignments that must +stay as they are. Files without `judges` work as before. + Exit codes: 0 solved, 2 no schedule exists, 3 the time limit ran out before any schedule was found, 1 bad input. """ @@ -46,12 +76,16 @@ import sys import time from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field PROBLEM_FORMAT = "query-judging-schedule-problem" SOLUTION_FORMAT = "query-judging-schedule-solution" VERSION = 1 +# Same as MAX_CONSECUTIVE and DAY_PARTS in judges.ts. +MAX_CONSECUTIVE = 4 +DAY_PARTS = 4 + class InputError(Exception): pass @@ -72,6 +106,15 @@ class Task: sponsor: bool +@dataclass(frozen=True) +class Judge: + id: str + kind: str # "member" or "guest" + challenge_ids: frozenset[str] + room_id: str | None + max_consecutive: int + + @dataclass class Problem: fingerprint: str @@ -81,6 +124,13 @@ class Problem: rooms: list[Room] tasks: list[Task] hint: dict[tuple[str, str], tuple[str, int]] + judges: list[Judge] = field(default_factory=list) + # A finished schedule to assign judges to, instead of one to look for. + fixed: dict[tuple[str, str], tuple[str, int]] | None = None + # Assignments to keep: (entry id, challenge id) -> judge id. + locked: dict[tuple[str, str], str] = field(default_factory=dict) + # Assignments to start from, as `hint` is for the placement. + assignment_hint: dict[tuple[str, str], str] = field(default_factory=dict) def _require(obj: dict, key: str, kind: type, where: str): @@ -133,6 +183,63 @@ def load_problem(raw: dict) -> Problem: for p in raw.get("hint") or []: hint[(p["entryId"], p["challengeId"])] = (p["roomId"], p["slot"]) + judges = [] + for i, j in enumerate(_require(raw, "judges", list, "problem") if "judges" in raw else []): + where = f"judges[{i}]" + if not isinstance(j, dict): + raise InputError(f"{where} must be an object") + kind = _require(j, "kind", str, where) + if kind not in ("member", "guest"): + raise InputError(f"{where}.kind must be member or guest") + challenge_ids = _require(j, "challengeIds", list, where) + if not all(isinstance(c, str) for c in challenge_ids): + raise InputError(f"{where}.challengeIds must be strings") + room_id = j.get("roomId") + if room_id is not None and not isinstance(room_id, str): + raise InputError(f"{where}.roomId must be a string or null") + max_consecutive = j.get("maxConsecutive", MAX_CONSECUTIVE) + if not isinstance(max_consecutive, int) or isinstance(max_consecutive, bool) or max_consecutive < 1: + raise InputError(f"{where}.maxConsecutive must be a positive integer") + judges.append( + Judge(_require(j, "id", str, where), kind, frozenset(challenge_ids), room_id, max_consecutive) + ) + if len({j.id for j in judges}) != len(judges): + raise InputError("judge ids repeat") + + fixed = None + if raw.get("placements") is not None: + room_ids = {r.id for r in rooms} + known = {(t.entry_id, t.challenge_id) for t in tasks} + fixed = {} + for i, p in enumerate(_require(raw, "placements", list, "problem")): + where = f"placements[{i}]" + key = (_require(p, "entryId", str, where), _require(p, "challengeId", str, where)) + room_id = _require(p, "roomId", str, where) + slot = _require(p, "slot", int, where) + if key not in known or room_id not in room_ids or key in fixed: + raise InputError(f"{where} does not match a presentation and room of this problem") + if not 0 <= slot < slot_count: + raise InputError(f"{where}.slot is outside the window") + fixed[key] = (room_id, slot) + if set(fixed) != known: + raise InputError("placements must place every presentation") + if not judges: + raise InputError("placements without judges leave nothing to solve") + + locked = {} + for i, row in enumerate(_require(raw, "locked", list, "problem") if "locked" in raw else []): + where = f"locked[{i}]" + locked[(_require(row, "entryId", str, where), _require(row, "challengeId", str, where))] = _require( + row, "judgeId", str, where + ) + + assignment_hint = {} + for i, row in enumerate(_require(raw, "assignmentHint", list, "problem") if "assignmentHint" in raw else []): + where = f"assignmentHint[{i}]" + assignment_hint[(_require(row, "entryId", str, where), _require(row, "challengeId", str, where))] = _require( + row, "judgeId", str, where + ) + return Problem( fingerprint=_require(raw, "fingerprint", str, "problem"), slot_count=slot_count, @@ -141,6 +248,10 @@ def load_problem(raw: dict) -> Problem: rooms=rooms, tasks=tasks, hint=hint, + judges=judges, + fixed=fixed, + locked=locked, + assignment_hint=assignment_hint, ) @@ -205,13 +316,40 @@ def score(problem: Problem, placement: Placement) -> dict[str, int]: for (s1, b1), (s2, b2) in zip(seq, seq[1:]): changes += b1 != b2 breaks += s2 - s1 - 1 - return { + goals = { "sponsorFinish": sponsor_finish, "sponsorIdle": sponsor_idle, "finish": finish, "buildingChanges": changes, "breakSlots": breaks, } + if problem.judges: + # Ahead of the rest, as in solve(). + goals = {"judgeShortfall": shortfall(problem, placement), **goals} + return goals + + +def judge_capacity(problem: Problem) -> dict[str, int]: + """How many judges may grade each challenge that has presentations and any judge.""" + counts = defaultdict(int) + for judge in problem.judges: + for challenge in judge.challenge_ids: + counts[challenge] += 1 + return {c: counts[c] for c in sorted({t.challenge_id for t in problem.tasks}) if counts[c] > 0} + + +def shortfall(problem: Problem, placement: Placement) -> int: + """Presentations of a challenge in one slot beyond its judges, for the worst + slot of each challenge, summed. 0 when judges could cover every slot.""" + capacity = judge_capacity(problem) + per_slot = defaultdict(int) + for i, task in enumerate(problem.tasks): + per_slot[(task.challenge_id, placement[i][1])] += 1 + worst = defaultdict(int) + for (challenge, _slot), count in per_slot.items(): + if challenge in capacity: + worst[challenge] = max(worst[challenge], count - capacity[challenge]) + return sum(worst.values()) def violations(problem: Problem, placement: Placement, cross_gap: int) -> list[str]: @@ -398,7 +536,25 @@ def busiest(selected) -> int: model.add(idle == 0).only_enforce_if(used.Not()) idle_terms.append(idle) + # Judges: in any slot, a challenge's presentations should not outnumber the + # judges who may grade it. Kept as a goal, the first, rather than a rule: too + # few judges must not make a schedule impossible. Each challenge's capacity + # is its judges plus an extra, the most it is over in any slot. + overloads = [] + for challenge, judges_for in judge_capacity(problem).items(): + members = [i for i, task in enumerate(tasks) if task.challenge_id == challenge] + if len(members) <= judges_for: + continue + extra = int_var(0, len(members) - judges_for, f"judge_extra_{challenge}") + model.add_cumulative( + [model.new_fixed_size_interval_var(start[i], 1, f"at_{i}") for i in members], + [1] * len(members), + judges_for + extra, + ) + overloads.append(extra) + goals = { + "judgeShortfall": sum(overloads), "sponsorFinish": sponsor_finish, "sponsorIdle": sum(idle_terms), "finish": finish, @@ -408,6 +564,8 @@ def busiest(selected) -> int: stages = [g for g in GOALS if not (g == "sponsorIdle" and not idle_terms) and not (g == "sponsorFinish" and not any(t.sponsor for t in tasks)) and not (g in ("buildingChanges", "breakSlots") and not break_terms)] + if overloads: + stages.insert(0, "judgeShortfall") def hint_placement(placement: Placement): model.clear_hints() @@ -497,6 +655,304 @@ def model_value(goal: str, placement: Placement) -> int: return ("optimal" if all_optimal else "feasible", best) +# Assignment: task index -> judge index, or None when nobody grades it. +Assignment = dict[int, "int | None"] + +# The second stage's goals, in order. Lower is better except the last. +JUDGE_GOALS = ["unassigned", "memberOnSponsor", "maxLoad", "loadSpread", "repeats", "dayPartsCovered"] + + +def may_grade(judge: Judge, task: Task, room: Room) -> bool: + if task.challenge_id not in judge.challenge_ids: + return False + # A guest grades only in their own room, and with no room grades nowhere. + if judge.room_id is not None: + return judge.room_id == room.id + return judge.kind != "guest" + + +def day_part(problem: Problem, slot: int) -> int: + return min(DAY_PARTS - 1, slot * DAY_PARTS // problem.slot_count) + + +def judge_pools(problem: Problem) -> list[list[int]]: + """Judges who are interchangeable (same challenges, same room); only within + one of these can the load be evened out.""" + grouped = defaultdict(list) + for k, judge in enumerate(problem.judges): + grouped[(tuple(sorted(judge.challenge_ids)), judge.room_id or "")].append(k) + return list(grouped.values()) + + +def locked_tasks(problem: Problem) -> dict[int, int]: + """Task index -> judge index for the assignments to keep that name a known + presentation and judge.""" + task_index = {(t.entry_id, t.challenge_id): i for i, t in enumerate(problem.tasks)} + judge_index = {j.id: k for k, j in enumerate(problem.judges)} + return { + task_index[key]: judge_index[judge_id] + for key, judge_id in problem.locked.items() + if key in task_index and judge_id in judge_index + } + + +def judges_clash(problem: Problem, placement: Placement, i: int, q: int) -> bool: + """Whether one judge cannot grade both: at the same time, or too soon to walk + between their rooms. Staying in one room needs no walk.""" + (room_i, slot_i), (room_q, slot_q) = placement[i], placement[q] + apart = abs(slot_i - slot_q) + if apart == 0: + return True + if room_i == room_q: + return False + same = problem.rooms[room_i].building_id == problem.rooms[room_q].building_id + return apart <= (problem.same_gap if same else problem.cross_gap) + + +def assignment_score(problem: Problem, placement: Placement, assignment: Assignment) -> dict[str, float]: + """The second stage's goals for a finished assignment, from the assignment alone.""" + judges, tasks = problem.judges, problem.tasks + load = [0] * len(judges) + rooms_seen = [defaultdict(int) for _ in judges] + parts_seen = [set() for _ in judges] + unassigned = member_on_sponsor = 0 + for i, task in enumerate(tasks): + k = assignment.get(i) + if k is None: + unassigned += 1 + continue + load[k] += 1 + rooms_seen[k][placement[i][0]] += 1 + parts_seen[k].add(day_part(problem, placement[i][1])) + member_on_sponsor += task.sponsor and judges[k].kind == "member" + spread = sum( + max(load[k] for k in pool) - min(load[k] for k in pool) + for pool in judge_pools(problem) + if len(pool) > 1 + ) + graders = [k for k in range(len(judges)) if load[k]] + return { + "unassigned": unassigned, + "memberOnSponsor": member_on_sponsor, + "maxLoad": max(load, default=0), + "loadSpread": spread, + "repeats": sum(count - 1 for seen in rooms_seen for count in seen.values()), + "dayPartsCovered": sum(len(parts) for parts in parts_seen), + "avgDistinctRooms": round(sum(len(rooms_seen[k]) for k in graders) / len(graders), 2) if graders else 0, + } + + +def assignment_violations(problem: Problem, placement: Placement, assignment: Assignment) -> list[str]: + """Independent re-check of the second stage's hard rules.""" + judges, tasks, rooms = problem.judges, problem.tasks, problem.rooms + kept = locked_tasks(problem) + found = [] + mine = defaultdict(list) + for i, k in assignment.items(): + if k is None: + continue + mine[k].append(i) + if i not in kept and not may_grade(judges[k], tasks[i], rooms[placement[i][0]]): + found.append(f"judge {judges[k].id} may not grade {tasks[i].entry_id}/{tasks[i].challenge_id}") + max_gap = max(problem.same_gap, problem.cross_gap) + for k, members in mine.items(): + members.sort(key=lambda i: placement[i][1]) + for a, i in enumerate(members): + for q in members[a + 1:]: + if placement[q][1] - placement[i][1] > max_gap: + break + if judges_clash(problem, placement, i, q): + found.append(f"judge {judges[k].id} cannot be at both {tasks[i].entry_id} and {tasks[q].entry_id}") + run, previous = 0, -2 + for slot in sorted({placement[i][1] for i in members}): + run = run + 1 if slot == previous + 1 else 1 + previous = slot + if run == judges[k].max_consecutive + 1: + found.append(f"judge {judges[k].id} grades more than {judges[k].max_consecutive} slots in a row") + return found + + +def assign(problem: Problem, placement: Placement, seconds: float, workers: int, log): + """Lexicographic CP-SAT solve of who grades what, once the schedule is fixed. + Returns (status, assignment or None). A presentation nobody may grade, or + that no judge has time for, is left out rather than failing the solve.""" + from ortools.sat.python import cp_model + + judges, tasks, rooms = problem.judges, problem.tasks, problem.rooms + n = len(tasks) + slot_of = [placement[i][1] for i in range(n)] + room_of = [placement[i][0] for i in range(n)] + kept = locked_tasks(problem) + + model = cp_model.CpModel() + tracked = [] + + def int_var(lo: int, hi: int, name: str): + v = model.new_int_var(lo, hi, name) + tracked.append(v) + return v + + def flag(name: str): + v = model.new_bool_var(name) + tracked.append(v) + return v + + # A kept presentation may go to its judge alone; the rest to whoever may grade it. + eligible = [ + [kept[i]] if i in kept + else [k for k, judge in enumerate(judges) if may_grade(judge, tasks[i], rooms[room_of[i]])] + for i in range(n) + ] + pick = {(i, k): flag(f"task_{i}_judge_{k}") for i in range(n) for k in eligible[i]} + left_out = [flag(f"left_out_{i}") for i in range(n)] + for i in range(n): + model.add_exactly_one([pick[i, k] for k in eligible[i]] + [left_out[i]]) + if i in kept: + model.add(pick[i, kept[i]] == 1) + + max_gap = max(problem.same_gap, problem.cross_gap) + by_slot = defaultdict(list) + for i in range(n): + by_slot[slot_of[i]].append(i) + + mine = [] # judge -> {task: literal} + for k, judge in enumerate(judges): + grades = {i: pick[i, k] for i in range(n) if (i, k) in pick} + mine.append(grades) + # One place at a time, and time to walk between rooms. + for i, lit in grades.items(): + for slot in range(slot_of[i], slot_of[i] + max_gap + 1): + for q in by_slot.get(slot, ()): + if q not in grades or (slot == slot_of[i] and q <= i): + continue + if judges_clash(problem, placement, i, q): + model.add_bool_or([lit.Not(), grades[q].Not()]) + # No more than max_consecutive slots in a row: any window one longer + # holds at most max_consecutive. + run = judge.max_consecutive + in_slot = defaultdict(list) + for i, lit in grades.items(): + in_slot[slot_of[i]].append(lit) + for first in range(problem.slot_count - run): + window = [lit for slot in range(first, first + run + 1) for lit in in_slot.get(slot, ())] + if len(window) > run: + model.add(sum(window) <= run) + + load = [] + for k, grades in enumerate(mine): + v = int_var(0, len(grades), f"load_{k}") + model.add(v == sum(grades.values())) + load.append(v) + + # Goals, each (name, expression to minimise, whether it has anything to decide). + member_on_sponsor = [ + pick[i, k] for (i, k) in pick if tasks[i].sponsor and judges[k].kind == "member" + ] + busiest = int_var(0, n, "busiest") + if load: + model.add_max_equality(busiest, load) + spreads = [] + for pool in judge_pools(problem): + if len(pool) < 2: + continue + high, low = int_var(0, n, "pool_high"), int_var(0, n, "pool_low") + for k in pool: + model.add(high >= load[k]) + model.add(low <= load[k]) + spreads.append(high - low) + rooms_used, parts_used = [], [] + for k, grades in enumerate(mine): + in_room, in_part = defaultdict(list), defaultdict(list) + for i, lit in grades.items(): + in_room[room_of[i]].append(lit) + in_part[day_part(problem, slot_of[i])].append(lit) + for lits, used in ((in_room, rooms_used), (in_part, parts_used)): + for group in lits.values(): + seen = flag(f"seen_{k}_{len(used)}") + model.add(seen <= sum(group)) + used.append(seen) + + stages = [ + ("unassigned", sum(left_out), n > 0), + ("memberOnSponsor", sum(member_on_sponsor), bool(member_on_sponsor)), + ("maxLoad", busiest, bool(load)), + ("loadSpread", sum(spreads), bool(spreads)), + ("repeats", -sum(rooms_used), bool(rooms_used)), + ("dayPartsCovered", -sum(parts_used), bool(parts_used)), + ] + stages = [(name, expression) for name, expression, decides in stages if decides] + + def new_solver(limit: float): + solver = cp_model.CpSolver() + solver.parameters.max_time_in_seconds = max(1.0, limit) + solver.parameters.num_workers = workers + solver.parameters.random_seed = 7 + # Presolve and the symmetry search cost seconds on a big event, longer + # than a stage's share, and every stage after the first starts from a + # complete assignment, so skip both. + solver.parameters.cp_model_presolve = False + solver.parameters.symmetry_level = 0 + return solver + + def take(solver) -> Assignment: + model.clear_hints() + for v in tracked: + model.add_hint(v, solver.value(v)) + return { + i: next((k for k in eligible[i] if solver.boolean_value(pick[i, k])), None) + for i in range(n) + } + + # Start from the hinted assignment where it names a pair the model has. One + # that keeps every rule is already an answer, as a good `hint` is for the + # placement, so running out of time can only fail to improve it. + judge_index = {j.id: k for k, j in enumerate(judges)} + task_index = {(t.entry_id, t.challenge_id): i for i, t in enumerate(tasks)} + hinted: Assignment = {i: kept.get(i) for i in range(n)} + for key, judge_id in problem.assignment_hint.items(): + i, k = task_index.get(key), judge_index.get(judge_id) + if i is not None and k is not None and (i, k) in pick and i not in kept: + hinted[i] = k + deadline = time.monotonic() + seconds + best: Assignment | None = None + if problem.assignment_hint: + model.clear_hints() + for (i, k), lit in pick.items(): + model.add_hint(lit, hinted[i] == k) + for i in range(n): + model.add_hint(left_out[i], hinted[i] is None) + if not assignment_violations(problem, placement, hinted): + best = hinted + all_optimal = True + for position, (name, expression) in enumerate(stages): + remaining = deadline - time.monotonic() + share = remaining / (len(stages) - position) + if position == 0: + share = max(share, remaining / 2) + solver = new_solver(share) + model.minimize(expression) + status = solver.solve(model) + if status not in (cp_model.OPTIMAL, cp_model.FEASIBLE): + if status == cp_model.INFEASIBLE and position == 0: + return ("infeasible", None) + if best is None: + return ("unknown", None) + all_optimal = False + log(f" {name}: no improvement within its time share, {solver.status_name(status)}") + continue + all_optimal = all_optimal and status == cp_model.OPTIMAL + best = take(solver) + value = int(solver.objective_value) + shown = assignment_score(problem, placement, best)[name] + log(f" {name}: {shown}{'' if status == cp_model.OPTIMAL else ' (time limit)'}") + model.add(expression <= value) + + if best is None: + # Nothing to decide: every presentation is left out or settled by its kept judge. + best = {i: kept.get(i) for i in range(n)} + return ("optimal" if all_optimal else "feasible", best) + + def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("problem", help="problem.json exported from the judging console, " @@ -534,35 +990,62 @@ def log(message: str): f"{problem.slot_count} slots (OR-Tools {ortools.__version__})") began = time.monotonic() - attempts = [(False, problem.cross_gap)] - if problem.cross_gap > problem.same_gap: - attempts.append((True, problem.same_gap)) - status, placement, relaxed = "infeasible", None, False - reasons = capacity_problems(problem, problem.same_gap) - for relaxed, cross_gap in [] if reasons else attempts: - log("relaxed: cross-building walks get the same-building break" if relaxed - else "strict: full cross-building breaks") - status, placement = solve(problem, cross_gap, args.time_limit, args.workers, log) - if placement is not None: - broken = violations(problem, placement, cross_gap) + reasons: list[str] = [] + if problem.fixed is not None: + # The schedule is decided; only the judges remain. + room_index = {r.id: i for i, r in enumerate(problem.rooms)} + placement = { + i: (room_index[problem.fixed[t.entry_id, t.challenge_id][0]], + problem.fixed[t.entry_id, t.challenge_id][1]) + for i, t in enumerate(problem.tasks) + } + status = "optimal" + else: + attempts = [(False, problem.cross_gap)] + if problem.cross_gap > problem.same_gap: + attempts.append((True, problem.same_gap)) + + reasons = capacity_problems(problem, problem.same_gap) + for relaxed, cross_gap in [] if reasons else attempts: + log("relaxed: cross-building walks get the same-building break" if relaxed + else "strict: full cross-building breaks") + status, placement = solve(problem, cross_gap, args.time_limit, args.workers, log) + if placement is not None: + broken = violations(problem, placement, cross_gap) + if broken: + # Would be a bug in the model above; never hand the portal a bad file. + log("internal check failed: " + "; ".join(broken[:5])) + return 1 + break + if status == "unknown": + # Running out of time proves nothing, so the breaks are only + # shortened once the full ones are shown to be impossible. + break + + assignment: Assignment | None = None + assignment_status = None + if placement is not None and problem.judges: + log(f"{len(problem.judges)} judges") + assignment_status, assignment = assign(problem, placement, args.time_limit, args.workers, log) + if assignment is not None: + broken = assignment_violations(problem, placement, assignment) if broken: - # Would be a bug in the model above; never hand the portal a bad file. log("internal check failed: " + "; ".join(broken[:5])) return 1 - break - if status == "unknown": - # Running out of time proves nothing, so the breaks are only - # shortened once the full ones are shown to be impossible. - break + elif problem.fixed is not None: + # Nothing else to hand back. + status = assignment_status + reasons = ["the assignments to keep break the judges' rules"] if status == "infeasible" else [] + solved = placement is not None and (problem.fixed is None or assignment is not None) solution = { "format": SOLUTION_FORMAT, "version": VERSION, "fingerprint": problem.fingerprint, - "status": status, - "relaxed": relaxed if placement is not None else False, - "objective": score(problem, placement) if placement is not None else None, + "status": status if not (solved and problem.fixed is not None) else assignment_status, + "relaxed": relaxed if solved else False, + "objective": score(problem, placement) if solved else None, "placements": [ { "entryId": problem.tasks[i].entry_id, @@ -571,7 +1054,7 @@ def log(message: str): "slot": slot, } for i, (r, slot) in sorted(placement.items()) - ] if placement is not None else [], + ] if solved else [], "reasons": reasons, "solver": { "name": "cp-sat", @@ -579,6 +1062,18 @@ def log(message: str): "seconds": round(time.monotonic() - began, 2), }, } + if assignment is not None: + solution["assignmentStatus"] = assignment_status + solution["judging"] = assignment_score(problem, placement, assignment) + solution["assignments"] = [ + { + "entryId": problem.tasks[i].entry_id, + "challengeId": problem.tasks[i].challenge_id, + "judgeId": problem.judges[k].id, + } + for i, k in sorted(assignment.items()) + if k is not None + ] text = json.dumps(solution, indent=2) + "\n" if args.output: with open(args.output, "w", encoding="utf-8") as handle: @@ -586,9 +1081,12 @@ def log(message: str): else: sys.stdout.write(text) - if placement is not None: - log(f"{status}{', relaxed' if solution['relaxed'] else ''}: " + if solved: + log(f"{solution['status']}{', relaxed' if solution['relaxed'] else ''}: " + ", ".join(f"{k} {v}" for k, v in solution["objective"].items())) + if assignment is not None: + log(f"judges ({assignment_status}): " + + ", ".join(f"{k} {v}" for k, v in solution["judging"].items())) return 0 if status == "unknown": log("no schedule found within the time limit; try a larger --time-limit") diff --git a/packages/api/scripts/judging-schedule/test_cpsat.py b/packages/api/scripts/judging-schedule/test_cpsat.py index 84ca5ae2..8c23cfba 100644 --- a/packages/api/scripts/judging-schedule/test_cpsat.py +++ b/packages/api/scripts/judging-schedule/test_cpsat.py @@ -10,6 +10,7 @@ import contextlib import io +import itertools import json import random import subprocess @@ -188,16 +189,20 @@ def test_a_valid_hint_survives_running_out_of_time(self): self.assertEqual(placement, hinted) +def run_cli(problem: dict | str, *args: str): + text = problem if isinstance(problem, str) else json.dumps(problem) + return subprocess.run( + [sys.executable, "-I", str(HERE / "cpsat.py"), "-", *args], + input=text.encode("utf-8"), + capture_output=True, + timeout=120, + ) + + @unittest.skipUnless(HAVE_ORTOOLS, "OR-Tools is not installed") class CommandLineTest(unittest.TestCase): def run_script(self, problem: dict | str, *args: str): - text = problem if isinstance(problem, str) else json.dumps(problem) - return subprocess.run( - [sys.executable, "-I", str(HERE / "cpsat.py"), "-", *args], - input=text.encode("utf-8"), - capture_output=True, - timeout=120, - ) + return run_cli(problem, *args) def test_reads_the_problem_from_stdin(self): raw = small_problem(1) @@ -259,5 +264,192 @@ def test_exit_codes(self): self.assertTrue(json.loads(done.stdout)["reasons"]) +JUDGE_KEYS = cpsat.JUDGE_GOALS + + +def judge_problem(seed: int) -> dict: + """small_problem with a finished schedule and judges: two table judges (one + with a short run limit) and a guest in one sponsor room.""" + rng = random.Random(seed * 7 + 1) + raw = small_problem(seed) + free = [(r["id"], s) for r in raw["rooms"] for s in range(raw["slotCount"])] + rng.shuffle(free) + placements, taken = [], set() + for task in raw["tasks"]: + room, slot = next( + (r, s) for r, s in free + if (r, s) not in taken + and next(x for x in raw["rooms"] if x["id"] == r)["challengeId"] == task["challengeId"] + ) + taken.add((room, slot)) + placements.append( + {"entryId": task["entryId"], "challengeId": task["challengeId"], "roomId": room, "slot": slot} + ) + everything = ["general", "acme", "globex"] + raw["placements"] = placements + raw["judges"] = [ + {"id": "m1", "kind": "member", "challengeIds": everything, "roomId": None}, + {"id": "m2", "kind": "member", "challengeIds": everything, "roomId": None, + "maxConsecutive": rng.choice([1, 2, 4])}, + {"id": "gs", "kind": "guest", "challengeIds": ["acme"], "roomId": "s1"}, + ] + return raw + + +def judge_key(score: dict) -> tuple[int, ...]: + """Lower is better, in goal order; the last goal counts what is covered.""" + return tuple(-score[k] if k == "dayPartsCovered" else score[k] for k in JUDGE_KEYS) + + +def brute_force_assign(problem: cpsat.Problem, placement: cpsat.Placement): + """Best goal vector over every assignment that keeps the judges' rules.""" + rooms, tasks, judges = problem.rooms, problem.tasks, problem.judges + options = [ + [None] + [k for k, judge in enumerate(judges) + if cpsat.may_grade(judge, tasks[i], rooms[placement[i][0]])] + for i in range(len(tasks)) + ] + best = None + for combo in itertools.product(*options): + assignment = dict(enumerate(combo)) + if cpsat.assignment_violations(problem, placement, assignment): + continue + key = judge_key(cpsat.assignment_score(problem, placement, assignment)) + if best is None or key < best: + best = key + return best + + +@unittest.skipUnless(HAVE_ORTOOLS, "OR-Tools is not installed") +class JudgeTest(unittest.TestCase): + def fixed(self, raw: dict) -> tuple[cpsat.Problem, cpsat.Placement]: + problem = cpsat.load_problem(raw) + index = {r.id: i for i, r in enumerate(problem.rooms)} + placement = { + i: (index[problem.fixed[t.entry_id, t.challenge_id][0]], + problem.fixed[t.entry_id, t.challenge_id][1]) + for i, t in enumerate(problem.tasks) + } + return problem, placement + + def test_assignment_matches_brute_force(self): + for seed in range(12): + problem, placement = self.fixed(judge_problem(seed)) + with self.subTest(seed=seed): + status, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + self.assertEqual(status, "optimal") + self.assertEqual(cpsat.assignment_violations(problem, placement, assignment), []) + self.assertEqual( + judge_key(cpsat.assignment_score(problem, placement, assignment)), + brute_force_assign(problem, placement), + ) + + def test_judges_are_assigned_from_the_command_line(self): + raw = judge_problem(2) + done = run_cli(raw, "--time-limit", "10", "--workers", "1") + self.assertEqual(done.returncode, 0, done.stderr.decode()) + solution = json.loads(done.stdout) + self.assertEqual(solution["status"], solution["assignmentStatus"]) + self.assertEqual(solution["placements"], raw["placements"]) + self.assertEqual(list(solution["judging"])[:6], JUDGE_KEYS) + # One judge at most per presentation, and the rest counted as left out. + done_for = {(a["entryId"], a["challengeId"]) for a in solution["assignments"]} + self.assertEqual(len(done_for), len(solution["assignments"])) + self.assertEqual( + len(solution["assignments"]) + solution["judging"]["unassigned"], len(raw["tasks"]) + ) + + def test_locked_assignments_stay(self): + raw = judge_problem(4) + first = raw["tasks"][0] + raw["locked"] = [{"entryId": first["entryId"], "challengeId": first["challengeId"], "judgeId": "m2"}] + problem, placement = self.fixed(raw) + _, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + self.assertEqual(problem.judges[assignment[0]].id, "m2") + + def test_a_valid_hint_survives_running_out_of_time(self): + from ortools.sat.python import cp_model + + raw = judge_problem(1) + problem, placement = self.fixed(raw) + _, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + raw["assignmentHint"] = [ + { + "entryId": problem.tasks[i].entry_id, + "challengeId": problem.tasks[i].challenge_id, + "judgeId": problem.judges[k].id, + } + for i, k in assignment.items() + if k is not None + ] + problem, placement = self.fixed(raw) + # Every solve runs out of time without finding anything. + with mock.patch.object(cp_model.CpSolver, "solve", return_value=cp_model.UNKNOWN): + status, kept = cpsat.assign(problem, placement, 1, 1, quiet) + self.assertEqual(status, "feasible") + self.assertEqual(kept, assignment) + + def test_a_presentation_nobody_may_grade_is_left_out_not_failed(self): + raw = judge_problem(1) + for judge in raw["judges"]: + judge["challengeIds"] = ["general"] + problem, placement = self.fixed(raw) + status, assignment = cpsat.assign(problem, placement, 20, 1, quiet) + self.assertEqual(status, "optimal") + sponsor = [i for i, t in enumerate(problem.tasks) if t.challenge_id != "general"] + self.assertTrue(sponsor) + self.assertTrue(all(assignment[i] is None for i in sponsor)) + + def test_placement_keeps_a_challenge_within_its_judges(self): + # Three teams, each in general once, three tables, one judge for general. + # Without judges all three go at once; with one they must take turns. + raw = { + "format": cpsat.PROBLEM_FORMAT, "version": cpsat.VERSION, "fingerprint": "capacity", + "slotCount": 4, "sameGap": 0, "crossGap": 0, + "rooms": [{"id": f"g{n}", "challengeId": "general", "buildingId": "a", "sponsor": False} + for n in range(3)], + "tasks": [{"entryId": f"p{n}", "challengeId": "general", "sponsor": False} + for n in range(3)], + } + problem = cpsat.load_problem(raw) + _, together = cpsat.solve(problem, 0, 10, 1, quiet) + self.assertEqual({slot for _, slot in together.values()}, {0}) + self.assertEqual(list(cpsat.score(problem, together)), KEYS) + + raw["judges"] = [{"id": "m1", "kind": "member", "challengeIds": ["general"], "roomId": None}] + problem = cpsat.load_problem(raw) + _, turns = cpsat.solve(problem, 0, 10, 1, quiet) + self.assertEqual(sorted(slot for _, slot in turns.values()), [0, 1, 2]) + self.assertEqual(cpsat.score(problem, turns)["judgeShortfall"], 0) + + def test_too_few_judges_do_not_make_a_schedule_impossible(self): + raw = { + "format": cpsat.PROBLEM_FORMAT, "version": cpsat.VERSION, "fingerprint": "short", + "slotCount": 2, "sameGap": 0, "crossGap": 0, + "rooms": [{"id": f"g{n}", "challengeId": "general", "buildingId": "a", "sponsor": False} + for n in range(3)], + "tasks": [{"entryId": f"p{n}", "challengeId": "general", "sponsor": False} + for n in range(5)], + "judges": [{"id": "m1", "kind": "member", "challengeIds": ["general"], "roomId": None}], + } + problem = cpsat.load_problem(raw) + status, placement = cpsat.solve(problem, 0, 10, 1, quiet) + self.assertIn(status, ("optimal", "feasible")) + # Five presentations in two slots: at best three and two at a time, so + # the worst slot has two more than the one judge. + self.assertEqual(cpsat.score(problem, placement)["judgeShortfall"], 2) + + def test_rejects_bad_judges_and_placements(self): + raw = judge_problem(0) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({**raw, "judges": [{**raw["judges"][0], "kind": "robot"}]}) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({**raw, "judges": [raw["judges"][0], raw["judges"][0]]}) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({**raw, "placements": raw["placements"][:-1]}) + with self.assertRaises(cpsat.InputError): + cpsat.load_problem({k: v for k, v in raw.items() if k != "judges"}) + + if __name__ == "__main__": unittest.main() diff --git a/packages/api/src/.internal-tests/schedule-judges.test.ts b/packages/api/src/.internal-tests/schedule-judges.test.ts new file mode 100644 index 00000000..d20aebc4 --- /dev/null +++ b/packages/api/src/.internal-tests/schedule-judges.test.ts @@ -0,0 +1,940 @@ +/** + * Judges on the saved schedule: saving gives every presentation a judge as far + * as there are judges, the board shows who grades what and where judges run + * short, "assign judges" picks again for what has not started, and an + * organiser can set one appointment's judge by hand until it starts. + * + * The schema is the real one and so is the scheduling code; only the database + * handle is a stand-in, handed to the router through the context the way + * check-in-flags-organizer.test.ts does. It keeps the rows of the tables the + * router reads and records what it writes. The CP-SAT runner is a stub, so a + * test decides what the script "printed" and sees exactly what it was asked. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + auditLogs, + hackathons, + judgingAppointments, + judgingChallenges, + judgingConfigs, + judgingEntryChallenges, + judgingEvaluationDrafts, + judgingEvaluations, + judgingGuestSessions, + judgingJudges, + judgingPresence, + judgingRooms, + judgingSchedules, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; +import { assignJudges } from "../services/judging/schedule/judges"; +import type * as Store from "../services/judging/evaluation-store"; +import type * as Runner from "../services/judging/schedule/cpsat-runner"; + +const mockAvailable = vi.fn(); +const mockRun = vi.fn(); + +// Nothing is due on any board read here, so there is nothing to reconcile. +vi.mock("../services/judging/evaluation-store", async (importActual) => ({ + ...(await importActual()), + reconcileDueDrafts: async () => undefined, +})); +vi.mock("../services/judging/schedule/cpsat-runner", async (importActual) => ({ + ...(await importActual()), + cpsatAvailable: () => mockAvailable(), + runCpsat: (...args: unknown[]) => mockRun(...args), +})); + +const id = (n: number) => + `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const HACK = id(1); +const GENERAL = id(10); +const ACME = id(11); +const G1 = id(20); +const G2 = id(21); +const A1 = id(22); +const A2 = id(23); +const E1 = id(30); +const E2 = id(31); +const E3 = id(32); +const APPT = [id(40), id(41), id(42), id(43)] as const; +const [P1, P2, P3, P4] = APPT; +const M1 = id(50); +const M2 = id(51); +const GS = id(52); +const NOROOM = id(53); +const SCHEDULE = id(60); + +// Slots are 10 minutes from 14:00. It is 15:00, so P1 (14:00) is over and the +// rest (15:30 and 16:00) are still to come. +const NOW = new Date("2027-02-06T15:00:00Z"); +const at = (time: string) => new Date(`2027-02-06T${time}:00Z`); + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); + +type Row = Record; +type Write = { table: unknown; values: Row | Row[]; where?: unknown }; + +const camel = (name: string) => + name.replace(/_([a-z])/g, (_, letter: string) => letter.toUpperCase()); +const TERM = /"[a-z_]+"\."([a-z_]+)" = \$(\d+)/g; + +/** A row filter from a where of `column = value` terms, or null for anything else. */ +function matcher(where: unknown) { + const { sql: text, params } = render(where); + const terms = [...text.matchAll(TERM)].map( + (match) => [camel(match[1]!), params[Number(match[2]) - 1]] as const, + ); + if (text.replace(TERM, "").replace(/[()\s]|and/g, "")) return null; + return (row: Row) => terms.every(([key, value]) => row[key] === value); +} + +let adminRow: Row | undefined; +const store = new Map(); +const inserts: Write[] = []; +const updates: Write[] = []; +// Counts reads of a table, so a test can change what a later read returns. +const reads = new Map(); +let onRead: ((table: unknown, count: number) => Row[] | undefined) | null = + null; + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: () => { + let table: unknown = null; + let where: unknown = null; + const chain: any = { + then: (ok: any, err: any) => { + const count = (reads.get(table) ?? 0) + 1; + reads.set(table, count); + const rows = onRead?.(table, count) ?? store.get(table) ?? []; + const keep = where ? matcher(where) : null; + return Promise.resolve(keep ? rows.filter(keep) : [...rows]).then( + ok, + err, + ); + }, + }; + chain.from = (from: unknown) => { + table = from; + return chain; + }; + chain.where = (condition: unknown) => { + where = condition; + return chain; + }; + for (const method of ["innerJoin", "leftJoin", "orderBy", "limit", "for"]) { + chain[method] = () => chain; + } + return chain; + }, + insert: (table: unknown) => ({ + values: (values: Row | Row[]) => { + inserts.push({ table, values }); + return Object.assign(Promise.resolve(undefined), { + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [values], + }); + }, + }), + update: (table: unknown) => ({ + set: (values: Row) => ({ + where: (where: unknown) => { + const keep = matcher(where); + const hit = keep ? (store.get(table) ?? []).filter(keep) : []; + for (const row of hit) Object.assign(row, values); + updates.push({ table, values, where }); + return Object.assign(Promise.resolve(undefined), { + returning: async () => hit.map((row) => ({ ...row })), + }); + }, + }), + }), + delete: () => ({ where: async () => [] }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string | undefined) => + appRouter.createCaller({ + db: fakeDb, + session: userId ? { user: { id: userId } } : null, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); +const admin = () => callerAs("admin_user"); + +const audits = () => + inserts + .filter((write) => write.table === auditLogs) + .map((write) => write.values as Row); + +/** The appointments an update touched, by the ids in its where. */ +const touched = (write: Write) => + render(write.where).params.filter((p) => APPT.includes(p as never)); + +/** Which judge each appointment was left with by the judge updates. */ +function judgeAfter() { + const result = new Map(); + for (const write of updates) { + if (write.table !== judgingAppointments) continue; + if (!("assignedJudgeId" in (write.values as Row))) continue; + for (const apptId of touched(write)) { + result.set(apptId as string, (write.values as Row).assignedJudgeId); + } + } + return result; +} + +const appointment = (apptId: string, over: Row): Row => ({ + id: apptId, + scheduleId: SCHEDULE, + hackathonId: HACK, + revision: 0, + assignedJudgeId: null, + entryDeletedAt: null, + deadlineAt: null, + ...over, +}); +const times = (start: string) => ({ + startsAt: at(start), + deadlineAt: at(start), + endsAt: new Date(at(start).getTime() + 10 * 60_000), +}); + +// Over (P1), then P2 and P3 at once, then P4 for the sponsor. +const appointments = (): Row[] => [ + appointment(P1, { + entryId: E1, + challengeId: GENERAL, + roomId: G1, + entryTitle: "Project 1", + ...times("14:00"), + }), + appointment(P2, { + entryId: E2, + challengeId: GENERAL, + roomId: G1, + entryTitle: "Project 2", + ...times("15:30"), + }), + appointment(P3, { + entryId: E3, + challengeId: GENERAL, + roomId: G2, + entryTitle: "Project 3", + ...times("15:30"), + }), + appointment(P4, { + entryId: E1, + challengeId: ACME, + roomId: A1, + entryTitle: "Project 1", + ...times("16:00"), + }), +]; + +const judge = (judgeId: string, kind: string, displayName: string) => ({ + id: judgeId, + kind, + userId: kind === "member" ? `user-${displayName}` : null, + displayName, +}); + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + vi.clearAllMocks(); + cache.clear(); + inserts.length = 0; + updates.length = 0; + reads.clear(); + onRead = null; + mockAvailable.mockReturnValue(false); + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + + store.clear(); + // Every table the router reads by edition holds that edition's rows. + const set = store.set.bind(store); + store.set = (table, rows) => + set( + table, + table === hackathons || + table === judgingSchedules || + table === judgingConfigs + ? rows + : rows.map((row) => ({ hackathonId: HACK, ...row })), + ); + store.set(hackathons, [ + { id: HACK, name: "Hacklytics 2027", startDate: null, endDate: null }, + ]); + store.set(judgingConfigs, [{ hackathonId: HACK, state: "open" }]); + store.set(judgingSchedules, [ + { + id: SCHEDULE, + hackathonId: HACK, + windowStart: at("14:00"), + windowEnd: at("17:00"), + setupMinutes: 2, + judgingMinutes: 6, + teardownMinutes: 2, + sameBuildingBreakMinutes: 10, + crossBuildingBreakMinutes: 20, + reducedBreaksAcknowledgedAt: null, + firstResultAt: null, + createdById: "admin_user", + createdAt: NOW, + }, + ]); + const challenge = (cid: string, label: string, general: boolean) => ({ + id: cid, + label, + parentId: null, + mode: "scheduled", + isGeneral: general, + isGroup: false, + }); + store.set(judgingChallenges, [ + challenge(GENERAL, "General", true), + challenge(ACME, "Acme", false), + ]); + const room = ( + rid: string, + name: string, + challengeId: string, + building: string, + ) => ({ + id: rid, + name, + kind: "table", + challengeId, + buildingId: building, + buildingName: building, + archivedAt: null, + }); + store.set(judgingRooms, [ + room(G1, "Table G1", GENERAL, "ENG"), + room(G2, "Table G2", GENERAL, "ENG"), + room(A1, "Acme 1", ACME, "HEC"), + room(A2, "Acme 2", ACME, "HEC"), + ]); + store.set(judgingEntryChallenges, [ + { entryId: E1, challengeId: GENERAL, title: "Project 1" }, + { entryId: E2, challengeId: GENERAL, title: "Project 2" }, + { entryId: E3, challengeId: GENERAL, title: "Project 3" }, + { entryId: E1, challengeId: ACME, title: "Project 1" }, + ]); + store.set(judgingAppointments, appointments()); + store.set(judgingEvaluations, []); + store.set(judgingEvaluationDrafts, []); + store.set(judgingPresence, []); + store.set(judgingJudges, [ + judge(M1, "member", "Ada"), + judge(M2, "member", "Bea"), + judge(GS, "guest", "Grace"), + judge(NOROOM, "guest", "Nobody"), + ]); + store.set(judgingGuestSessions, [ + { judgeId: GS, roomId: A1, createdAt: NOW }, + ]); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +/** Gives an appointment a judge in the stored rows. */ +const give = (apptId: string, judgeId: string) => { + store + .get(judgingAppointments)! + .find((a) => a.id === apptId)!.assignedJudgeId = judgeId; +}; + +// --------------------------------------------------------------------------- +describe("judging.schedule.get", () => { + it("shows each appointment's judge, each judge's load and where judges run short", async () => { + // Only Ada and Grace have joined: two general presentations at 15:30 need two. + store.set(judgingJudges, [ + judge(M1, "member", "Ada"), + judge(GS, "guest", "Grace"), + ]); + give(P2, M1); + give(P4, GS); + + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + + expect( + board.appointments.map((a) => [ + a.id, + a.assignedJudgeName, + a.canAssignJudge, + ]), + ).toEqual([ + [P1, null, false], + [P2, "Ada", true], + [P3, null, true], + [P4, "Grace", true], + ]); + expect(board.judging).toMatchObject({ + judges: 2, + assigned: 2, + unassigned: 1, + loads: [ + { judgeId: M1, name: "Ada", kind: "member", load: 1, available: true }, + { + judgeId: GS, + name: "Grace", + kind: "guest", + roomName: "Acme 1", + load: 1, + }, + ], + warnings: [ + { + slot: 9, + challengeId: GENERAL, + challengeLabel: "General", + presentations: 2, + judgesFree: 1, + message: "Slot 10: 2 presentations in General, only 1 judge free", + }, + ], + }); + expect(board.judging!.warnings[0]!.startsAt).toEqual(at("15:30")); + }); + + it("lists a judge who holds appointments but can no longer take any", async () => { + store.set(judgingJudges, [ + judge(M1, "member", "Ada"), + judge(GS, "guest", "Grace"), + ]); + // Grace's session was revoked: she has no room, but P4 is still hers. + store.set(judgingGuestSessions, []); + give(P4, GS); + + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(board.judging!.loads).toEqual([ + expect.objectContaining({ judgeId: M1, load: 0, available: true }), + expect.objectContaining({ + judgeId: GS, + name: "Grace", + load: 1, + available: false, + }), + ]); + }); + + it("says nothing of judges before a schedule is saved", async () => { + store.set(judgingSchedules, []); + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(board.judging).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.assignJudges", () => { + const run = () => + admin().judging.schedule.assignJudges({ hackathonId: HACK }); + + it("gives every appointment that has not started a judge who may grade it", async () => { + const result = await run(); + + expect(result).toMatchObject({ + solver: "in-app", + changed: 3, + assigned: 3, + unassigned: 0, + warnings: [], + }); + const after = judgeAfter(); + // The sponsor presentation goes to the sponsor's own judge. + expect(after.get(P4)).toBe(GS); + // The two at 15:30 are in different rooms at once: two judges. + expect(new Set([after.get(P2), after.get(P3)])).toEqual(new Set([M1, M2])); + // The one that is over is left as it is. + expect(after.has(P1)).toBe(false); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.schedule.assignJudges", + resourceId: HACK, + metadata: expect.objectContaining({ + solver: "in-app", + changed: 3, + unassigned: 0, + }), + }), + ]); + }); + + it("keeps the judge of an appointment a judge has begun, and plans around it", async () => { + give(P2, M1); + // Ada has a draft on P2: it stays hers, so P3 goes to Bea. + store.set(judgingEvaluationDrafts, [ + { entryId: E2, challengeId: GENERAL, appointmentId: null }, + ]); + + const result = await run(); + + expect(result).toMatchObject({ kept: 1, changed: 2 }); + const after = judgeAfter(); + expect(after.has(P2)).toBe(false); + expect(after.get(P3)).toBe(M2); + }); + + it("leaves appointments alone that already have the judge it would choose", async () => { + await run(); + const first = judgeAfter(); + updates.length = 0; + for (const [apptId, judgeId] of first) give(apptId, judgeId!); + + const again = await run(); + expect(again.changed).toBe(0); + expect(updates.filter((u) => u.table === judgingAppointments)).toEqual([]); + }); + + it("runs CP-SAT when the server has it and takes its assignments", async () => { + mockAvailable.mockReturnValue(true); + mockRun.mockImplementation(async (problem: Row) => ({ + status: "solved", + log: [], + solution: JSON.stringify({ + format: "query-judging-schedule-solution", + version: 1, + fingerprint: problem.fingerprint, + status: "optimal", + relaxed: false, + placements: problem.placements, + // Bea at 15:30 in G1 and Ada in G2: the in-app order is the other way. + assignments: [ + { entryId: E2, challengeId: GENERAL, judgeId: M2 }, + { entryId: E3, challengeId: GENERAL, judgeId: M1 }, + { entryId: E1, challengeId: ACME, judgeId: GS }, + ], + }), + })); + + const result = await admin().judging.schedule.assignJudges({ + hackathonId: HACK, + seconds: 30, + }); + + expect(result.solver).toBe("cp-sat"); + expect(judgeAfter().get(P2)).toBe(M2); + expect(judgeAfter().get(P3)).toBe(M1); + // It was asked for the schedule as it stands, with the judges. + const [problem, options] = mockRun.mock.calls[0]!; + expect(options).toMatchObject({ seconds: 30 }); + expect(problem.placements).toHaveLength(3); + expect(problem.judges.map((j: Row) => j.id)).toEqual([M1, M2, GS]); + }); + + it("falls back to the in-app assigner when CP-SAT fails or answers against the rules", async () => { + mockAvailable.mockReturnValue(true); + mockRun.mockResolvedValueOnce({ + status: "failed", + reason: "no python", + log: [], + }); + expect((await run()).solver).toBe("in-app"); + + // Ada in two rooms at once. + mockRun.mockImplementationOnce(async (problem: Row) => ({ + status: "solved", + log: [], + solution: JSON.stringify({ + format: "query-judging-schedule-solution", + version: 1, + fingerprint: problem.fingerprint, + status: "optimal", + relaxed: false, + placements: problem.placements, + assignments: [ + { entryId: E2, challengeId: GENERAL, judgeId: M1 }, + { entryId: E3, challengeId: GENERAL, judgeId: M1 }, + ], + }), + })); + expect((await run()).solver).toBe("in-app"); + }); + + it("warns of each slot where judges run short", async () => { + store.set(judgingJudges, [judge(M1, "member", "Ada")]); + const result = await run(); + expect(result.unassigned).toBe(1); + expect(result.warnings.map((w) => w.message)).toEqual([ + "Slot 10: 2 presentations in General, only 1 judge free", + ]); + }); + + it("refuses when no judge can be assigned yet", async () => { + store.set(judgingJudges, [judge(NOROOM, "guest", "Nobody")]); + await expect(run()).rejects.toMatchObject({ code: "PRECONDITION_FAILED" }); + }); + + it("refuses without a saved schedule", async () => { + store.set(judgingSchedules, []); + await expect(run()).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("refuses anyone but an admin", async () => { + adminRow = undefined; + await expect( + callerAs("a_hacker").judging.schedule.assignJudges({ hackathonId: HACK }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(updates).toEqual([]); + }); + + it("refuses to write when the judges changed while it worked", async () => { + // The first read of judges sees three; the read inside the transaction two. + onRead = (table, count) => + table === judgingJudges && count > 1 + ? [judge(M1, "member", "Ada"), judge(GS, "guest", "Grace")] + : undefined; + await expect(run()).rejects.toMatchObject({ code: "CONFLICT" }); + expect(updates.filter((u) => u.table === judgingAppointments)).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.setAssignedJudge", () => { + const set = (appointmentId: string, judgeId: string | null) => + admin().judging.schedule.setAssignedJudge({ + hackathonId: HACK, + appointmentId, + judgeId, + }); + + it("sets the judge of an appointment that has not started, and audits it", async () => { + const result = await set(P2, M1); + + expect(result).toEqual({ + appointmentId: P2, + fromJudgeId: null, + toJudgeId: M1, + }); + expect(judgeAfter().get(P2)).toBe(M1); + expect(audits()).toEqual([ + expect.objectContaining({ + action: "judging.schedule.setJudge", + resourceId: P2, + metadata: expect.objectContaining({ toJudgeId: M1 }), + }), + ]); + }); + + it("clears a judge with null", async () => { + give(P2, M1); + await set(P2, null); + expect(judgeAfter().get(P2)).toBeNull(); + }); + + it("refuses a judge who may not grade it: a guest outside their room", async () => { + await expect(set(P2, GS)).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: "Grace cannot grade General at Table G1.", + }); + // Their own room is fine. + await expect(set(P4, GS)).resolves.toMatchObject({ toJudgeId: GS }); + }); + + it("refuses a judge who is already grading something else then", async () => { + give(P3, M1); + await expect(set(P2, M1)).rejects.toMatchObject({ + code: "CONFLICT", + message: "Ada is already grading Project 3 at that time.", + }); + await expect(set(P2, M2)).resolves.toMatchObject({ toJudgeId: M2 }); + }); + + it("refuses a judge who is not one of the edition's, or a guest with no room", async () => { + await expect(set(P2, id(99))).rejects.toMatchObject({ code: "NOT_FOUND" }); + await expect(set(P2, NOROOM)).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("refuses once the presentation has started or a judge has begun on it", async () => { + await expect(set(P1, M1)).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + }); + + store.set(judgingEvaluationDrafts, [ + { entryId: E2, challengeId: GENERAL, appointmentId: P2 }, + ]); + await expect(set(P2, M1)).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + }); + expect(judgeAfter().size).toBe(0); + }); + + it("refuses an appointment that does not exist", async () => { + await expect(set(id(98), M1)).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); + + it("refuses anyone but an admin", async () => { + adminRow = undefined; + await expect( + callerAs("a_hacker").judging.schedule.setAssignedJudge({ + hackathonId: HACK, + appointmentId: P2, + judgeId: M1, + }), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(updates).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.judgeChoices", () => { + it("lists the judges who may grade it, free ones first, with those booked marked", async () => { + give(P3, M1); + give(P4, M2); + + const result = await admin().judging.schedule.judgeChoices({ + hackathonId: HACK, + appointmentId: P2, + }); + + expect(result.appointment).toMatchObject({ + id: P2, + challengeLabel: "General", + roomName: "Table G1", + canAssignJudge: true, + }); + expect(result.choices).toEqual([ + expect.objectContaining({ + judgeId: M2, + name: "Bea", + load: 1, + booked: false, + }), + expect.objectContaining({ + judgeId: M1, + name: "Ada", + load: 1, + booked: true, + bookedWith: "Project 3", + }), + ]); + }); + + it("offers a sponsor presentation to the sponsor's judge and the members", async () => { + const result = await admin().judging.schedule.judgeChoices({ + hackathonId: HACK, + appointmentId: P4, + }); + expect(result.choices.map((c) => c.judgeId).sort()).toEqual( + [GS, M1, M2].sort(), + ); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.schedule.move", () => { + const staffed = () => + store.set( + judgingPresence, + [G1, G2, A1, A2].map((roomId) => ({ + judgeId: M1, + roomId, + joinedAt: at("14:00"), + lastSeenAt: NOW, + leftAt: null, + })), + ); + const move = (appointmentId: string, roomId: string, startsAt: Date) => + admin().judging.schedule.move({ + hackathonId: HACK, + appointmentId, + revision: 0, + roomId, + startsAt, + }); + const written = () => updates.filter((u) => u.table === judgingAppointments); + + it("clears a guest's assignment when the presentation moves to another room", async () => { + staffed(); + give(P4, GS); + + const moved = await move(P4, A2, at("16:00")); + + expect(moved.clearedJudgeId).toBe(GS); + expect(written()[0]!.values).toMatchObject({ + roomId: A2, + assignedJudgeId: null, + }); + }); + + it("clears a member's assignment when they would be in two places at the new time", async () => { + staffed(); + give(P2, M1); + give(P4, M1); + + // P4 is at 16:00; moving P2 there puts Ada in two rooms at once. + const moved = await move(P2, G2, at("16:00")); + + expect(moved.clearedJudgeId).toBe(M1); + expect(written()[0]!.values).toMatchObject({ assignedJudgeId: null }); + }); + + it("keeps the judge when they can still grade it at the new time", async () => { + staffed(); + give(P2, M1); + + const moved = await move(P2, G2, at("15:40")); + + expect(moved.clearedJudgeId).toBeNull(); + expect("assignedJudgeId" in (written()[0]!.values as Row)).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +describe("saving a schedule", () => { + const timing = { + windowStart: at("16:00"), + windowEnd: at("19:00"), + setupMinutes: 2, + judgingMinutes: 6, + teardownMinutes: 2, + sameBuildingBreakMinutes: 10, + crossBuildingBreakMinutes: 20, + }; + const rowsWritten = () => + inserts + .filter((write) => write.table === judgingAppointments) + .flatMap((write) => write.values as Row[]); + + beforeEach(() => { + // Nothing saved yet. + store.set(judgingSchedules, []); + store.set(judgingAppointments, []); + }); + + it("gives every presentation a judge as far as there are judges", async () => { + const preview = await admin().judging.schedule.preview({ + hackathonId: HACK, + timing, + }); + if (!preview.ok) throw new Error("the inventory fits the window"); + + const saved = await admin().judging.schedule.save({ + hackathonId: HACK, + timing, + candidateKey: preview.candidateKey, + }); + + expect(saved).toMatchObject({ appointments: 4, judged: 4 }); + const rows = rowsWritten(); + expect(rows).toHaveLength(4); + // Grace judges Acme's presentation; the three general ones are Ada's and Bea's. + const judgeOf = (challengeId: string) => + rows + .filter((r) => r.challengeId === challengeId) + .map((r) => r.assignedJudgeId); + expect(judgeOf(ACME)).toEqual([GS]); + expect(judgeOf(GENERAL).every((j) => j === M1 || j === M2)).toBe(true); + expect(audits()[0]).toMatchObject({ + action: "judging.schedule.save", + metadata: expect.objectContaining({ judged: 4 }), + }); + }); + + it("still saves when no judge has joined: the presentations have none yet", async () => { + store.set(judgingJudges, []); + const preview = await admin().judging.schedule.preview({ + hackathonId: HACK, + timing, + }); + if (!preview.ok) throw new Error("the inventory fits the window"); + + const saved = await admin().judging.schedule.save({ + hackathonId: HACK, + timing, + candidateKey: preview.candidateKey, + }); + + expect(saved).toMatchObject({ appointments: 4, judged: 0 }); + expect(rowsWritten().every((r) => r.assignedJudgeId === null)).toBe(true); + }); + + it("puts the judges in the CP-SAT problem, and saves the assignments CP-SAT chose", async () => { + const problem = await admin().judging.schedule.cpsatProblem({ + hackathonId: HACK, + timing, + }); + const file = JSON.parse(problem.contents); + expect(problem.judges).toBe(3); + expect(file.judges.map((j: Row) => j.id)).toEqual([M1, M2, GS]); + expect(file.judges.find((j: Row) => j.id === GS)).toMatchObject({ + kind: "guest", + roomId: A1, + challengeIds: [ACME], + }); + + // The in-app schedule with its judges, the two general judges swapped. + const preview = await admin().judging.schedule.preview({ + hackathonId: HACK, + timing, + }); + if (!preview.ok) throw new Error("the inventory fits the window"); + const placements = preview.appointments.map((a) => ({ + entryId: a.entryId, + challengeId: a.challengeId, + roomId: a.roomId, + slot: (a.startsAt.getTime() - timing.windowStart.getTime()) / 600_000, + })); + const inputs = { + rooms: file.rooms, + tasks: file.tasks, + slots: file.slotCount, + sameGap: file.sameGap, + crossGap: file.crossGap, + }; + const inApp = assignJudges(inputs, placements, file.judges).assignment; + const swapped = inApp.map((a) => ({ + ...a, + judgeId: a.judgeId === M1 ? M2 : a.judgeId === M2 ? M1 : a.judgeId, + })); + const solution = JSON.stringify({ + format: "query-judging-schedule-solution", + version: 1, + fingerprint: file.fingerprint, + status: "feasible", + relaxed: false, + placements, + assignments: swapped, + }); + + const checked = await admin().judging.schedule.previewCpsat({ + hackathonId: HACK, + timing, + solution, + }); + expect(checked.judging).toEqual({ + judges: 3, + presentations: 4, + unassigned: 0, + source: "cp-sat", + }); + + await admin().judging.schedule.saveCpsat({ + hackathonId: HACK, + timing, + solution, + candidateKey: checked.candidateKey, + }); + const chosen = new Map( + swapped.map((a) => [`${a.entryId}:${a.challengeId}`, a.judgeId]), + ); + for (const row of rowsWritten()) { + expect(row.assignedJudgeId).toBe( + chosen.get(`${row.entryId}:${row.challengeId}`), + ); + } + expect(audits()[0]).toMatchObject({ + metadata: expect.objectContaining({ solver: "cp-sat", judges: "cp-sat" }), + }); + }); +}); diff --git a/packages/api/src/routers/judging/schedule.ts b/packages/api/src/routers/judging/schedule.ts index 1b34a501..3018b004 100644 --- a/packages/api/src/routers/judging/schedule.ts +++ b/packages/api/src/routers/judging/schedule.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { TRPCError } from "@trpc/server"; -import { and, asc, eq, isNull, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import { judgingAppointments, judgingBuildings, @@ -8,7 +8,10 @@ import { judgingConfigs, judgingEntries, judgingEntryChallenges, + judgingEvaluationDrafts, judgingEvaluations, + judgingGuestSessions, + judgingJudges, judgingPresence, judgingRooms, judgingSchedules, @@ -37,9 +40,31 @@ import type { import { aamoghCPSolver } from "../../services/judging/schedule/aamogh-cp-solver"; import { CPSAT_COMMAND, + cpsatAssignmentProblem, cpsatProblem, + readCpsatAssignments, readCpsatSolution, } from "../../services/judging/schedule/cpsat-bridge"; +import { + assignJudges, + describeShortfall, + judgeShortfalls, + mayGrade, +} from "../../services/judging/schedule/judges"; +import type { + JudgeAssignment, + JudgeInput, +} from "../../services/judging/schedule/judges"; +import { + judgeChanges, + judgeChoices, + judgePlanOf, + rosterOf, +} from "../../services/judging/schedule/judge-plan"; +import type { + Roster, + SavedAppointment, +} from "../../services/judging/schedule/judge-plan"; import { CPSAT_SECONDS, CPSAT_UNAVAILABLE, @@ -131,6 +156,7 @@ async function readInventory(db: JudgingDb, hackathonId: string) { parentId: judgingChallenges.parentId, mode: judgingChallenges.mode, isGeneral: judgingChallenges.isGeneral, + isGroup: judgingChallenges.isGroup, }) .from(judgingChallenges) .where(eq(judgingChallenges.hackathonId, hackathonId)), @@ -171,6 +197,49 @@ async function readInventory(db: JudgingDb, hackathonId: string) { type Inventory = Awaited>; +/** The judges an edition can give presentations to, with every judge's name. */ +async function readRoster( + db: JudgingDb, + hackathonId: string, + inventory: Inventory, +): Promise { + const [judges, sessions] = await Promise.all([ + db + .select({ + id: judgingJudges.id, + kind: judgingJudges.kind, + userId: judgingJudges.userId, + displayName: judgingJudges.displayName, + }) + .from(judgingJudges) + .where(eq(judgingJudges.hackathonId, hackathonId)), + db + .select({ + judgeId: judgingGuestSessions.judgeId, + roomId: judgingGuestSessions.roomId, + createdAt: judgingGuestSessions.createdAt, + }) + .from(judgingGuestSessions) + .where( + and( + eq(judgingGuestSessions.hackathonId, hackathonId), + isNull(judgingGuestSessions.revokedAt), + ), + ), + ]); + return rosterOf({ + hackathonId, + judges, + seats: sessions.flatMap((s) => + s.judgeId + ? [{ judgeId: s.judgeId, roomId: s.roomId, createdAt: s.createdAt }] + : [], + ), + challenges: inventory.challenges, + rooms: selectPlan(inventory).rooms, + }); +} + function namesOf(inventory: Inventory) { const labels = new Map(inventory.challenges.map((c) => [c.id, c.label])); const titles = new Map(inventory.memberships.map((m) => [m.entryId, m.title])); @@ -319,7 +388,13 @@ async function writeSchedule( db: DrizzleDB, userId: string, input: { hackathonId: string; timing: ScheduleTiming }, - schedule: { placements: Placement[]; relaxed: boolean; fingerprint: string }, + schedule: { + placements: Placement[]; + relaxed: boolean; + fingerprint: string; + /** The judge of each presentation, as far as there are judges. */ + assignments: readonly JudgeAssignment[]; + }, ) { return db.transaction(async (tx) => { await ensureJudgingConfig(tx, input.hackathonId); @@ -351,12 +426,28 @@ async function writeSchedule( }) .returning({ id: judgingSchedules.id }); + // A judge removed since the assignment was made has no appointments left. + const judges = new Set( + ( + await tx + .select({ id: judgingJudges.id }) + .from(judgingJudges) + .where(eq(judgingJudges.hackathonId, input.hackathonId)) + ).map((judge) => judge.id), + ); + const judgeOf = new Map( + schedule.assignments + .filter((a) => judges.has(a.judgeId)) + .map((a) => [taskKey(a.entryId, a.challengeId), a.judgeId]), + ); + const rows = schedule.placements.map((p) => ({ scheduleId: row!.id, hackathonId: input.hackathonId, entryId: p.entryId, challengeId: p.challengeId, roomId: p.roomId, + assignedJudgeId: judgeOf.get(taskKey(p.entryId, p.challengeId)) ?? null, ...appointmentTimes(input.timing, p.slot), })); // Postgres takes at most 65535 parameters per statement. @@ -364,14 +455,26 @@ async function writeSchedule( await tx.insert(judgingAppointments).values(rows.slice(i, i + 1000)); } - return { scheduleId: row!.id, appointments: rows.length }; + return { + scheduleId: row!.id, + appointments: rows.length, + judged: rows.filter((r) => r.assignedJudgeId).length, + }; }); } -/** The uploaded CP-SAT solution, checked against the edition as it stands. */ -function readUpload(inventory: Inventory, timing: ScheduleTiming, solution: string) { +/** + * The uploaded CP-SAT solution, checked against the edition as it stands, + * with its judges checked against `judges` as they are now. + */ +function readUpload( + inventory: Inventory, + timing: ScheduleTiming, + solution: string, + judges: readonly JudgeInput[], +) { const { inputs, fingerprint } = planOf(inventory, timing); - const read = readCpsatSolution(inputs, fingerprint, solution); + const read = readCpsatSolution(inputs, fingerprint, solution, judges); if (!read.ok) { throw new TRPCError({ code: "BAD_REQUEST", message: read.reason }); } @@ -383,6 +486,21 @@ function readUpload(inventory: Inventory, timing: ScheduleTiming, solution: stri }; } +/** What a CP-SAT preview says about the judges. */ +function describeJudging( + read: { judging: { source: "cp-sat" | "in-app"; metrics: { unassigned: number } } }, + judges: readonly JudgeInput[], + presentations: number, +) { + return { + judges: judges.length, + presentations, + unassigned: read.judging.metrics.unassigned, + // Who worked the assignment out. + source: read.judging.source, + }; +} + /** The saved schedule with everything the board and the move pickers need. */ async function loadBoard( db: JudgingDb, @@ -404,6 +522,7 @@ async function loadBoard( deadlineAt: judgingAppointments.deadlineAt, endsAt: judgingAppointments.endsAt, revision: judgingAppointments.revision, + assignedJudgeId: judgingAppointments.assignedJudgeId, entryTitle: judgingEntries.title, entryDeletedAt: judgingEntries.deletedAt, }) @@ -464,6 +583,175 @@ async function loadBoard( type Board = Awaited>; type SavedSchedule = NonNullable; +/** + * The board's judges: who can be given presentations, and the saved + * appointments the way the assigner takes them. An appointment is open to a + * new judge while it has not started and nobody has begun an evaluation. + */ +async function loadJudging(db: JudgingDb, board: Board, schedule: SavedSchedule) { + const hackathonId = schedule.hackathonId; + const [roster, drafts] = await Promise.all([ + readRoster(db, hackathonId, board.inventory), + db + .select({ + entryId: judgingEvaluationDrafts.entryId, + challengeId: judgingEvaluationDrafts.challengeId, + appointmentId: judgingEvaluationDrafts.appointmentId, + }) + .from(judgingEvaluationDrafts) + .where(eq(judgingEvaluationDrafts.hackathonId, hackathonId)), + ]); + + const timing = timingOf(schedule); + const planned = new Set( + board.plan.tasks.map((t) => taskKey(t.entryId, t.challengeId)), + ); + const rootOfChallenge = (id: string) => + rootOf(board.inventory.challenges, id)?.id ?? null; + const started = (a: Board["appointments"][number]) => + drafts.some( + (d) => + d.entryId === a.entryId && + (d.appointmentId + ? d.appointmentId === a.id + : rootOfChallenge(d.challengeId) === a.challengeId), + ); + + const saved: SavedAppointment[] = board.appointments.flatMap((a) => { + // Saved appointments sit on the grid; one that does not has no slot to compare. + const slot = slotAt(timing, a.startsAt); + if (slot === null) return []; + return [ + { + id: a.id, + entryId: a.entryId, + challengeId: a.challengeId, + roomId: a.roomId, + slot, + assignedJudgeId: a.assignedJudgeId, + removed: + !!a.entryDeletedAt || !planned.has(taskKey(a.entryId, a.challengeId)), + open: a.status === "future" && !started(a), + }, + ]; + }); + + return { roster, saved, plan: planOfSaved(board, timing, saved), timing }; +} + +/** The saved appointments as the judge assigner takes them. */ +function planOfSaved( + board: Board, + timing: ScheduleTiming, + saved: readonly SavedAppointment[], +) { + return judgePlanOf({ + rooms: board.inventory.rooms, + sponsorChallengeIds: new Set( + board.inventory.challenges.filter((c) => !c.isGeneral).map((c) => c.id), + ), + grid: gridOf(timing), + appointments: saved, + }); +} + +type Judging = Awaited>; + +/** What the board shows about judges: loads, and where there are too few. */ +function describeBoardJudging(judging: Judging, board: Board, now: Date) { + const { roster, saved, plan, timing } = judging; + const live = saved.filter((a) => !a.removed); + const rooms = new Map(board.inventory.rooms.map((r) => [r.id, r.name])); + + const loads = new Map(); + for (const a of live) { + if (a.assignedJudgeId) { + loads.set(a.assignedJudgeId, (loads.get(a.assignedJudgeId) ?? 0) + 1); + } + } + const available = new Map(roster.judges.map((j) => [j.id, j])); + const listed = new Set([...available.keys(), ...loads.keys()]); + + const from = firstSlotFrom(timing, now); + return { + judges: roster.judges.length, + assigned: live.filter((a) => a.assignedJudgeId).length, + // Still waiting for a judge: ones that have started or are over cannot + // be given one. + unassigned: live.filter((a) => a.open && !a.assignedJudgeId).length, + loads: [...listed] + .map((id) => { + const judge = available.get(id); + return { + judgeId: id, + name: roster.names.get(id) ?? "Removed judge", + kind: judge?.kind ?? null, + roomName: judge?.roomId ? (rooms.get(judge.roomId) ?? null) : null, + // Cannot take presentations: a guest with no room or table. + available: !!judge, + load: loads.get(id) ?? 0, + }; + }) + .sort( + (a, b) => + (a.kind === "guest" ? 1 : 0) - (b.kind === "guest" ? 1 : 0) || + a.name.localeCompare(b.name), + ), + warnings: judgeShortfalls( + plan.inputs, + plan.placements, + roster.judges, + plan.current, + ) + .filter((w) => w.slot >= from) + .map((w) => { + const challengeLabel = board.names.label(w.challengeId); + return { + slot: w.slot, + startsAt: slotStartsAt(timing, w.slot), + challengeId: w.challengeId, + challengeLabel, + presentations: w.presentations, + judgesFree: w.judgesFree, + message: describeShortfall(w, challengeLabel), + }; + }), + }; +} + +/** + * Who should grade the open presentations: CP-SAT when this server runs it + * and its answer passes the checks, the in-app assigner otherwise. + */ +async function makeAssignments( + judging: Judging, + options: { seconds: number; signal?: AbortSignal }, +) { + const { plan, roster } = judging; + if (cpsatAvailable()) { + const run = await runCpsat( + cpsatAssignmentProblem(plan.inputs, plan.placements, roster.judges, plan.keep), + options, + ); + if (run.status === "solved") { + const read = readCpsatAssignments( + plan.inputs, + plan.placements, + roster.judges, + plan.keep, + run.solution, + ); + if (read.ok) { + return { assignments: read.assignments, solver: "cp-sat" as const }; + } + } + } + const made = assignJudges(plan.inputs, plan.placements, roster.judges, { + keep: plan.keep, + }); + return { assignments: made.assignment, solver: "in-app" as const }; +} + /** * Every free future slot `task` can take, best first. Only rooms with a * judge seen in the last 15 minutes are offered: a slot in an empty room is @@ -562,6 +850,12 @@ const noSchedule = () => message: "No schedule is saved for this hackathon. Preview one and save it first.", }); +const noJudgeNeeded = () => + new TRPCError({ + code: "PRECONDITION_FAILED", + message: "That project is no longer entered, so it needs no judge.", + }); + const scheduleExists = () => new TRPCError({ code: "CONFLICT", @@ -638,7 +932,7 @@ export const judgingScheduleRouter = createTRPCRouter({ // Solved before the transaction so no lock is held while it computes. const inventory = await readInventory(db, input.hackathonId); - const { result, key, fingerprint } = solveInventory( + const { inputs, result, key, fingerprint } = solveInventory( inventory, input.timing, ); @@ -662,10 +956,15 @@ export const judgingScheduleRouter = createTRPCRouter({ }); } + // Judges first come with the schedule, as far as there are any yet; + // assignJudges picks up the rest once more have joined. + const roster = await readRoster(db, input.hackathonId, inventory); const written = await writeSchedule(db, ctx.userId, input, { placements: result.placements, relaxed: result.relaxed, fingerprint, + assignments: assignJudges(inputs, result.placements, roster.judges) + .assignment, }); const saved = { ...written, @@ -736,10 +1035,12 @@ export const judgingScheduleRouter = createTRPCRouter({ const inventory = await readInventory(db, input.hackathonId); const { inputs, result, fingerprint } = solveInventory(inventory, input.timing); + const roster = await readRoster(db, input.hackathonId, inventory); const problem = cpsatProblem( inputs, fingerprint, result.ok ? result.placements : undefined, + roster.judges, ); return { fileName: "problem.json", @@ -747,6 +1048,7 @@ export const judgingScheduleRouter = createTRPCRouter({ command: CPSAT_COMMAND, presentations: inputs.tasks.length, hinted: result.ok, + judges: roster.judges.length, }; }), @@ -766,12 +1068,19 @@ export const judgingScheduleRouter = createTRPCRouter({ if (await findSchedule(db, input.hackathonId)) throw scheduleExists(); const inventory = await readInventory(db, input.hackathonId); - const { inputs, read, key } = readUpload(inventory, input.timing, input.solution); + const roster = await readRoster(db, input.hackathonId, inventory); + const { inputs, read, key } = readUpload( + inventory, + input.timing, + input.solution, + roster.judges, + ); return { ...describePlan(read, inputs, namesOf(inventory), input.timing), candidateKey: key, // CP-SAT's own word: "optimal" means it proved nothing beats this. solverStatus: read.status, + judging: describeJudging(read, roster.judges, inputs.tasks.length), }; }), @@ -792,10 +1101,12 @@ export const judgingScheduleRouter = createTRPCRouter({ assertWindowAhead(input.timing); const inventory = await readInventory(db, input.hackathonId); + const roster = await readRoster(db, input.hackathonId, inventory); const { read, key, fingerprint } = readUpload( inventory, input.timing, input.solution, + roster.judges, ); if (key !== input.candidateKey) { throw new TRPCError({ @@ -815,12 +1126,14 @@ export const judgingScheduleRouter = createTRPCRouter({ placements: read.placements, relaxed: read.relaxed, fingerprint, + assignments: read.judging.assignments, }); const saved = { ...written, relaxed: read.relaxed, reducedBreaks: read.reducedBreaks.length, solver: "cp-sat", + judges: read.judging.source, }; await recordAdminAction(db, { @@ -873,8 +1186,14 @@ export const judgingScheduleRouter = createTRPCRouter({ const inventory = await readInventory(db, input.hackathonId); const { inputs, result, fingerprint } = solveInventory(inventory, input.timing); + const roster = await readRoster(db, input.hackathonId, inventory); const run = await runCpsat( - cpsatProblem(inputs, fingerprint, result.ok ? result.placements : undefined), + cpsatProblem( + inputs, + fingerprint, + result.ok ? result.placements : undefined, + roster.judges, + ), { seconds: input.seconds, signal }, ); if (run.status === "failed") { @@ -889,11 +1208,17 @@ export const judgingScheduleRouter = createTRPCRouter({ } // A proof of infeasibility is refused here with its reasons; a schedule // has to pass the same checks as an upload. - const { read, key } = readUpload(inventory, input.timing, run.solution); + const { read, key } = readUpload( + inventory, + input.timing, + run.solution, + roster.judges, + ); return { ...describePlan(read, inputs, namesOf(inventory), input.timing), candidateKey: key, solverStatus: read.status, + judging: describeJudging(read, roster.judges, inputs.tasks.length), solution: run.solution, }; }), @@ -914,6 +1239,7 @@ export const judgingScheduleRouter = createTRPCRouter({ await reconcileDueDrafts(db, input.hackathonId, now); const board = await loadBoard(db, input.hackathonId, now); const { schedule, names } = board; + const judging = schedule ? await loadJudging(db, board, schedule) : null; const planned = new Set(board.plan.tasks.map((t) => taskKey(t.entryId, t.challengeId))); const booked = new Set( @@ -966,7 +1292,17 @@ export const judgingScheduleRouter = createTRPCRouter({ status: a.status, canMove: a.canMove, evaluationCount: a.evaluationCount, + assignedJudgeId: a.assignedJudgeId, + assignedJudgeName: a.assignedJudgeId + ? (judging?.roster.names.get(a.assignedJudgeId) ?? "Removed judge") + : null, + // Its judge can be chosen anew: it has not started and nobody has + // begun on it. + canAssignJudge: judging?.saved.some( + (s) => s.id === a.id && s.open && !s.removed, + ) ?? false, })), + judging: judging ? describeBoardJudging(judging, board, now) : null, unscheduled: schedule ? board.plan.tasks .filter((t) => !booked.has(taskKey(t.entryId, t.challengeId))) @@ -1074,6 +1410,31 @@ export const judgingScheduleRouter = createTRPCRouter({ input, timingOf(board.schedule), ); + + // Its judge stays only if they may grade in the new room and are free + // at the new time; otherwise the organiser assigns judges again. + let clearsJudge = false; + if (appointment.assignedJudgeId) { + const judging = await loadJudging(tx, board, board.schedule); + const judge = judging.roster.judges.find( + (j) => j.id === appointment.assignedJudgeId, + ); + const slot = slotAt(judging.timing, choice.startsAt); + clearsJudge = + !judge || + slot === null || + !mayGrade(judge, { + challengeId: appointment.challengeId, + roomId: choice.roomId, + }) || + judging.saved.some( + (other) => + other.id !== appointment.id && + !other.removed && + other.assignedJudgeId === judge.id && + other.slot === slot, + ); + } const [updated] = await tx .update(judgingAppointments) .set({ @@ -1082,6 +1443,7 @@ export const judgingScheduleRouter = createTRPCRouter({ deadlineAt: choice.deadlineAt, endsAt: choice.endsAt, revision: sql`${judgingAppointments.revision} + 1`, + ...(clearsJudge ? { assignedJudgeId: null } : {}), }) .where( and( @@ -1100,6 +1462,7 @@ export const judgingScheduleRouter = createTRPCRouter({ toRoomId: choice.roomId, toStartsAt: choice.startsAt, reducedBreaks: choice.reducedBreaks, + clearedJudgeId: clearsJudge ? appointment.assignedJudgeId : null, }; }); @@ -1113,6 +1476,264 @@ export const judgingScheduleRouter = createTRPCRouter({ return moved; }), + /** + * Who could grade an appointment instead: the judges who may grade its + * challenge in its room, least loaded first, with those already grading + * something else at that time marked. + */ + judgeChoices: isAdmin + .input(z.object({ hackathonId: uuid, appointmentId: uuid })) + .query(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + const now = new Date(); + const board = await loadBoard(db, input.hackathonId, now); + const appointment = board.appointments.find( + (a) => a.id === input.appointmentId, + ); + if (!board.schedule || !appointment) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "That appointment no longer exists. Reload the board.", + }); + } + const judging = await loadJudging(db, board, board.schedule); + const saved = judging.saved.find((s) => s.id === appointment.id); + if (!saved || saved.removed) throw noJudgeNeeded(); + + const titles = new Map(board.appointments.map((a) => [a.id, a.entryTitle])); + const loads = new Map(); + for (const s of judging.saved) { + if (s.assignedJudgeId && !s.removed) { + loads.set(s.assignedJudgeId, (loads.get(s.assignedJudgeId) ?? 0) + 1); + } + } + const kinds = new Map(judging.roster.judges.map((j) => [j.id, j.kind])); + return { + appointment: { + id: appointment.id, + entryTitle: appointment.entryTitle, + challengeLabel: board.names.label(appointment.challengeId), + ...board.names.room(appointment.roomId), + startsAt: appointment.startsAt, + assignedJudgeId: appointment.assignedJudgeId, + canAssignJudge: saved.open, + }, + choices: judgeChoices(judging.roster, saved, judging.saved) + .map((choice) => ({ + judgeId: choice.judgeId, + name: judging.roster.names.get(choice.judgeId) ?? "Removed judge", + kind: kinds.get(choice.judgeId)!, + load: loads.get(choice.judgeId) ?? 0, + bookedWith: choice.bookedWith ? (titles.get(choice.bookedWith) ?? null) : null, + booked: choice.bookedWith !== null, + })) + .sort( + (a, b) => + Number(a.booked) - Number(b.booked) || + a.load - b.load || + a.name.localeCompare(b.name), + ), + }; + }), + + /** + * Sets, or with null clears, the judge of one appointment, until it starts. + * The judge has to be one who may grade it there and is not already + * grading something else at that time. + */ + setAssignedJudge: isAdmin + .input( + z.object({ + hackathonId: uuid, + appointmentId: uuid, + judgeId: uuid.nullable(), + }), + ) + .mutation(async ({ ctx, input }) => { + const db = ctx.db as DrizzleDB; + + const changed = await db.transaction(async (tx) => { + const now = new Date(); + const board = await loadBoard(tx, input.hackathonId, now, true); + const appointment = board.appointments.find( + (a) => a.id === input.appointmentId, + ); + if (!board.schedule || !appointment) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "That appointment no longer exists. Reload the board.", + }); + } + const judging = await loadJudging(tx, board, board.schedule); + const saved = judging.saved.find((s) => s.id === appointment.id); + if (!saved || saved.removed) throw noJudgeNeeded(); + if (!saved.open) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: + "This presentation has started, or a judge has begun on it, so its judge stays.", + }); + } + + if (input.judgeId) { + const judge = judging.roster.judges.find((j) => j.id === input.judgeId); + if (!judge) { + throw new TRPCError({ + code: "NOT_FOUND", + message: + "That judge cannot be assigned. A guest judge needs a room or table of their own.", + }); + } + const name = judging.roster.names.get(judge.id) ?? "That judge"; + if (!mayGrade(judge, saved)) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `${name} cannot grade ${board.names.label(saved.challengeId)} at ${board.names.room(saved.roomId).roomName}.`, + }); + } + const booked = judgeChoices(judging.roster, saved, judging.saved).find( + (c) => c.judgeId === judge.id, + )?.bookedWith; + if (booked) { + throw new TRPCError({ + code: "CONFLICT", + message: `${name} is already grading ${board.appointments.find((a) => a.id === booked)?.entryTitle ?? "another project"} at that time.`, + }); + } + } + + await tx + .update(judgingAppointments) + .set({ assignedJudgeId: input.judgeId }) + .where(eq(judgingAppointments.id, appointment.id)); + return { + appointmentId: appointment.id, + fromJudgeId: appointment.assignedJudgeId, + toJudgeId: input.judgeId, + }; + }); + + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.schedule.setJudge", + resourceId: changed.appointmentId, + metadata: { hackathonId: input.hackathonId, ...changed }, + }); + + return changed; + }), + + /** + * Gives every appointment that has not started, and that no judge has + * begun on, a judge again. The others keep theirs. CP-SAT does it when + * this server runs it, otherwise the in-app assigner. Warns of each slot + * where there are too few judges. + */ + assignJudges: isAdmin + .input( + z.object({ + hackathonId: uuid, + seconds: z + .number() + .int() + .min(CPSAT_SECONDS.min) + .max(CPSAT_SECONDS.max) + .default(CPSAT_SECONDS.default), + }), + ) + .mutation(async ({ ctx, input, signal }) => { + const db = ctx.db as DrizzleDB; + await assertHackathonExists(db, input.hackathonId); + + // Worked out before the transaction, so no lock is held while CP-SAT + // runs, and checked against the board again inside it. + const first = await loadBoard(db, input.hackathonId, new Date()); + if (!first.schedule) throw noSchedule(); + const before = await loadJudging(db, first, first.schedule); + if (before.roster.judges.length === 0) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: + "No judge can be assigned yet. Judges appear here once they have joined a room or table.", + }); + } + const made = await makeAssignments(before, { + seconds: input.seconds, + signal, + }); + + const result = await db.transaction(async (tx) => { + const now = new Date(); + const board = await loadBoard(tx, input.hackathonId, now, true); + if (!board.schedule) throw noSchedule(); + const judging = await loadJudging(tx, board, board.schedule); + const stale = (a: Judging) => + JSON.stringify([a.saved, a.roster.judges]); + if (stale(judging) !== stale(before)) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Appointments or judges changed while judges were being assigned. Assign judges again.", + }); + } + + const changes = judgeChanges(judging.saved, made.assignments); + // One update per judge, however many appointments they gain. + const byJudge = new Map(); + for (const change of changes) { + const list = byJudge.get(change.assignedJudgeId); + if (list) list.push(change.id); + else byJudge.set(change.assignedJudgeId, [change.id]); + } + for (const [judgeId, ids] of byJudge) { + for (let i = 0; i < ids.length; i += 1000) { + await tx + .update(judgingAppointments) + .set({ assignedJudgeId: judgeId }) + .where(inArray(judgingAppointments.id, ids.slice(i, i + 1000))); + } + } + + const chosen = new Map(changes.map((c) => [c.id, c.assignedJudgeId])); + const saved = judging.saved.map((s) => + chosen.has(s.id) ? { ...s, assignedJudgeId: chosen.get(s.id)! } : s, + ); + const summary = describeBoardJudging( + { + ...judging, + saved, + plan: planOfSaved(board, judging.timing, saved), + }, + board, + now, + ); + return { + solver: made.solver, + changed: changes.length, + kept: judging.plan.keep.length, + assigned: summary.assigned, + unassigned: summary.unassigned, + warnings: summary.warnings, + }; + }); + + await recordAdminAction(db, { + userId: ctx.userId, + action: "judging.schedule.assignJudges", + resourceId: input.hackathonId, + metadata: { + solver: result.solver, + changed: result.changed, + kept: result.kept, + assigned: result.assigned, + unassigned: result.unassigned, + warnings: result.warnings.length, + }, + }); + + return result; + }), + /** Where a presentation with no appointment yet could go, best first. */ assignChoices: isAdmin .input(z.object({ hackathonId: uuid, entryId: uuid, challengeId: uuid })) diff --git a/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts b/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts index 620fae94..8e28cf3a 100644 --- a/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts +++ b/packages/api/src/services/judging/schedule/cpsat-bridge.test.ts @@ -1,7 +1,13 @@ import { describe, expect, it } from "vitest"; import { aamoghCPSolver } from "./aamogh-cp-solver"; -import { cpsatProblem, readCpsatSolution } from "./cpsat-bridge"; -import type { PlanInputs } from "./inputs"; +import { + cpsatAssignmentProblem, + cpsatProblem, + readCpsatAssignments, + readCpsatSolution, +} from "./cpsat-bridge"; +import type { Placement, PlanInputs } from "./inputs"; +import type { JudgeAssignment, JudgeInput } from "./judges"; const inputs: PlanInputs = { rooms: [ @@ -58,6 +64,9 @@ describe("cpsatProblem", () => { expect(problem.rooms.map((r) => r.sponsor)).toEqual([false, false, true]); expect(problem.tasks).toHaveLength(5); expect("hint" in problem).toBe(false); + // A file without judges is still the version-1 file it always was. + expect("judges" in problem).toBe(false); + expect("assignmentHint" in problem).toBe(false); }); it("passes a starting schedule along as the hint", () => { @@ -126,4 +135,236 @@ describe("readCpsatSolution", () => { expect(result).toMatchObject({ ok: true, relaxed: true }); if (result.ok) expect(result.reducedBreaks).toHaveLength(1); }); + + it("still reads a version-1 file written before judges", () => { + expect(read(solution)).toMatchObject({ + ok: true, + judging: { source: "in-app", assignments: [] }, + }); + }); +}); + +const judges: JudgeInput[] = [ + { id: "m1", kind: "member", challengeIds: ["general", "sp1"], roomId: null }, + { id: "m2", kind: "member", challengeIds: ["general", "sp1"], roomId: null }, + { id: "gs", kind: "guest", challengeIds: ["sp1"], roomId: "s1" }, +]; + +// What `uv run cpsat.py problem.json --workers 1` wrote for cpsatProblem(inputs, +// "fp-test", undefined, judges), verbatim apart from the timing. +const judged = { + format: "query-judging-schedule-solution", + version: 1, + fingerprint: "fp-test", + status: "optimal", + relaxed: false, + objective: { + judgeShortfall: 0, + sponsorFinish: 2, + sponsorIdle: 0, + finish: 4, + buildingChanges: 0, + breakSlots: 2, + }, + placements: [ + { entryId: "a", challengeId: "general", roomId: "g2", slot: 2 }, + { entryId: "a", challengeId: "sp1", roomId: "s1", slot: 0 }, + { entryId: "b", challengeId: "general", roomId: "g2", slot: 3 }, + { entryId: "b", challengeId: "sp1", roomId: "s1", slot: 1 }, + { entryId: "c", challengeId: "general", roomId: "g2", slot: 1 }, + ], + reasons: [], + solver: { name: "cp-sat", ortools: "9.15.6755", seconds: 0 }, + assignmentStatus: "optimal", + judging: { + unassigned: 0, + memberOnSponsor: 0, + maxLoad: 2, + loadSpread: 1, + repeats: 2, + dayPartsCovered: 5, + avgDistinctRooms: 1.0, + }, + assignments: [ + { entryId: "a", challengeId: "general", judgeId: "m1" }, + { entryId: "a", challengeId: "sp1", judgeId: "gs" }, + { entryId: "b", challengeId: "general", judgeId: "m2" }, + { entryId: "b", challengeId: "sp1", judgeId: "gs" }, + { entryId: "c", challengeId: "general", judgeId: "m2" }, + ], +}; + +const readJudged = (value: unknown, now: JudgeInput[] = judges) => + readCpsatSolution(inputs, "fp-test", JSON.stringify(value), now); + +describe("cpsatProblem with judges", () => { + it("lists the judges with their run limit", () => { + const problem = cpsatProblem(inputs, "fp-test", undefined, judges); + expect(problem.judges).toEqual([ + { id: "m1", kind: "member", challengeIds: ["general", "sp1"], roomId: null, maxConsecutive: 4 }, + { id: "m2", kind: "member", challengeIds: ["general", "sp1"], roomId: null, maxConsecutive: 4 }, + { id: "gs", kind: "guest", challengeIds: ["sp1"], roomId: "s1", maxConsecutive: 4 }, + ]); + expect("assignmentHint" in problem).toBe(false); + expect("judges" in cpsatProblem(inputs, "fp-test", undefined, [])).toBe(false); + }); + + it("sends where the in-app assignment would put the judges for a hint", () => { + const problem = cpsatProblem(inputs, "fp-test", judged.placements, [ + { ...judges[0]!, maxConsecutive: 2 }, + ...judges.slice(1), + ]); + expect(problem.judges![0]!.maxConsecutive).toBe(2); + const hint = (problem as { assignmentHint?: JudgeAssignment[] }).assignmentHint!; + expect(hint).toHaveLength(5); + expect(new Set(hint.map((h) => h.judgeId))).toEqual(new Set(["m1", "m2", "gs"])); + }); +}); + +describe("readCpsatSolution with judges", () => { + it("takes the judges the script chose, once they pass the rules", () => { + const result = readJudged(judged); + expect(result).toMatchObject({ + ok: true, + judging: { + source: "cp-sat", + assignments: judged.assignments, + metrics: { unassigned: 0, maxLoad: 2, loadSpread: 1, repeats: 2 }, + }, + }); + }); + + it("assigns the judges itself when the file has none", () => { + const { assignments: _dropped, ...without } = judged; + const result = readJudged(without); + if (!result.ok) throw new Error("the schedule is fine"); + expect(result.judging.source).toBe("in-app"); + expect(result.judging.metrics.unassigned).toBe(0); + expect(result.judging.assignments).toHaveLength(5); + }); + + it("assigns the judges itself when the file's break the rules for the judges now", () => { + // The guest has since gone: gs is not among the judges any more. + const result = readJudged(judged, judges.slice(0, 2)); + if (!result.ok) throw new Error("the placements are still fine"); + expect(result.judging.source).toBe("in-app"); + expect(result.judging.assignments.every((a) => a.judgeId !== "gs")).toBe(true); + }); + + it("assigns the judges itself when the file puts one judge in two places", () => { + const clashing = { + ...judged, + assignments: judged.assignments.map((a) => + a.entryId === "c" ? { ...a, judgeId: "gs" } : a, + ), + }; + const result = readJudged(clashing); + if (!result.ok) throw new Error("the placements are fine"); + expect(result.judging.source).toBe("in-app"); + }); + + it("leaves presentations without a judge rather than refuse the schedule", () => { + const result = readJudged(judged, []); + if (!result.ok) throw new Error("the placements are fine"); + expect(result.judging.assignments).toEqual([]); + expect(result.judging.metrics.unassigned).toBe(5); + }); +}); + +describe("assigning judges to a schedule already made", () => { + const placements: Placement[] = judged.placements; + const keep: JudgeAssignment[] = [{ entryId: "c", challengeId: "general", judgeId: "m1" }]; + const problem = cpsatAssignmentProblem(inputs, placements, judges, keep); + const answer = (over: Record = {}) => + JSON.stringify({ + ...judged, + fingerprint: problem.fingerprint, + assignments: [ + { entryId: "c", challengeId: "general", judgeId: "m1" }, + { entryId: "a", challengeId: "general", judgeId: "m2" }, + { entryId: "a", challengeId: "sp1", judgeId: "gs" }, + { entryId: "b", challengeId: "general", judgeId: "m2" }, + { entryId: "b", challengeId: "sp1", judgeId: "gs" }, + ], + ...over, + }); + const readAnswer = (text: string, kept = keep) => + readCpsatAssignments(inputs, placements, judges, kept, text); + + it("writes the schedule, the judges and what to keep into the problem", () => { + expect(problem.placements).toEqual(placements); + expect(problem.locked).toEqual(keep); + expect(problem.judges).toHaveLength(3); + expect(problem.assignmentHint).toHaveLength(5); + expect("locked" in cpsatAssignmentProblem(inputs, placements, judges)).toBe(false); + }); + + it("reads the assignments, scored", () => { + expect(readAnswer(answer())).toMatchObject({ + ok: true, + status: "optimal", + metrics: { unassigned: 0, memberOnSponsor: 0 }, + }); + }); + + it("refuses an answer for another schedule", () => { + expect(readAnswer(answer({ fingerprint: "fp-test" }))).toMatchObject({ ok: false }); + }); + + it("refuses a proof that nothing fits and a timeout", () => { + expect(readAnswer(answer({ status: "infeasible" }))).toMatchObject({ ok: false }); + expect(readAnswer(answer({ status: "unknown" }))).toMatchObject({ ok: false }); + }); + + it("refuses an answer with no assignments or no JSON", () => { + const { assignments: _dropped, ...without } = JSON.parse(answer()); + expect(readAnswer(JSON.stringify(without))).toMatchObject({ ok: false }); + expect(readAnswer("nope")).toMatchObject({ ok: false }); + expect(readAnswer("{}")).toMatchObject({ ok: false }); + }); + + it("refuses assignments that break a rule", () => { + // m2 at a:sp1 (slot 0) and c:general (slot 1) in different buildings. + const result = readAnswer( + answer({ + assignments: [ + { entryId: "c", challengeId: "general", judgeId: "m1" }, + { entryId: "a", challengeId: "sp1", judgeId: "m1" }, + ], + }), + ); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) expect(result.reason).toContain("against the rules"); + }); + + it("refuses an answer that moves what it was told to keep", () => { + const result = readAnswer( + answer({ + assignments: [ + { entryId: "c", challengeId: "general", judgeId: "m2" }, + { entryId: "a", challengeId: "general", judgeId: "m1" }, + ], + }), + ); + expect(result).toMatchObject({ ok: false }); + if (!result.ok) expect(result.reason).toContain("keep"); + }); + + it("does not blame the script for a rule the kept assignments already broke", () => { + // m1 kept at b:sp1 (slot 1, HEC) and c:general (slot 1, HEC): two places + // at once, so before CP-SAT ran. + const hasty: JudgeAssignment[] = [ + { entryId: "b", challengeId: "sp1", judgeId: "m1" }, + { entryId: "c", challengeId: "general", judgeId: "m1" }, + ]; + const text = answer({ + assignments: [ + ...hasty, + { entryId: "a", challengeId: "general", judgeId: "m2" }, + { entryId: "a", challengeId: "sp1", judgeId: "gs" }, + { entryId: "b", challengeId: "general", judgeId: "m2" }, + ], + }); + expect(readAnswer(text, hasty)).toMatchObject({ ok: true }); + }); }); diff --git a/packages/api/src/services/judging/schedule/cpsat-bridge.ts b/packages/api/src/services/judging/schedule/cpsat-bridge.ts index 7e8e9f0c..696f35ab 100644 --- a/packages/api/src/services/judging/schedule/cpsat-bridge.ts +++ b/packages/api/src/services/judging/schedule/cpsat-bridge.ts @@ -1,5 +1,13 @@ import { z } from "zod"; +import { candidateKey, taskKey } from "./inputs"; import type { Placement, PlanInputs } from "./inputs"; +import { + MAX_CONSECUTIVE, + assignJudges, + scoreJudgeAssignment, + validateJudgeAssignment, +} from "./judges"; +import type { JudgeAssignment, JudgeInput, JudgeMetrics } from "./judges"; import { scoreSchedule, validateSchedule } from "./rules"; import type { PlanScore, ReducedBreak } from "./rules"; @@ -18,14 +26,33 @@ export const CPSAT_SOLUTION_FORMAT = "query-judging-schedule-solution"; export const CPSAT_COMMAND = "uv run packages/api/scripts/judging-schedule/cpsat.py problem.json -o solution.json"; +const judgesOf = (judges: readonly JudgeInput[]) => + judges.map((judge) => ({ + id: judge.id, + kind: judge.kind, + challengeIds: [...judge.challengeIds], + roomId: judge.roomId, + maxConsecutive: judge.maxConsecutive ?? MAX_CONSECUTIVE, + })); + +const assignmentsOf = (rows: readonly JudgeAssignment[]) => + rows.map((row) => ({ + entryId: row.entryId, + challengeId: row.challengeId, + judgeId: row.judgeId, + })); + /** * The problem file cpsat.py reads. `hint` is a schedule to start from, such - * as the best partial one the in-app solver found. + * as the best partial one the in-app solver found. `judges`, when there are + * any, asks for a judge for every presentation as well, and also keeps each + * challenge's presentations in any slot within its judges. */ export function cpsatProblem( inputs: PlanInputs, fingerprint: string, hint?: readonly Placement[], + judges: readonly JudgeInput[] = [], ) { const sponsorChallenges = new Set( inputs.tasks.filter((t) => t.sponsor).map((t) => t.challengeId), @@ -58,6 +85,51 @@ export function cpsatProblem( })), } : {}), + ...(judges.length > 0 + ? { + judges: judgesOf(judges), + // Where the in-app assignment would put the judges for that hint, + // for CP-SAT to improve on. + ...(hint && hint.length > 0 + ? { + assignmentHint: assignmentsOf( + assignJudges(inputs, hint, judges).assignment, + ), + } + : {}), + } + : {}), + }; +} + +/** Binds an assignment-only run to the placements it assigns judges to. */ +const assignmentFingerprint = (placements: readonly Placement[]) => + candidateKey("judges", false, placements); + +/** + * A problem that only assigns judges: the schedule is given and CP-SAT leaves + * it alone. `keep` stays as it is, such as the presentations already under + * way. `inputs` hold just the presentations in `placements`. + */ +export function cpsatAssignmentProblem( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + keep: readonly JudgeAssignment[] = [], +) { + const base = cpsatProblem(inputs, assignmentFingerprint(placements), undefined, judges); + return { + ...base, + placements: placements.map((p) => ({ + entryId: p.entryId, + challengeId: p.challengeId, + roomId: p.roomId, + slot: p.slot, + })), + ...(keep.length > 0 ? { locked: assignmentsOf(keep) } : {}), + assignmentHint: assignmentsOf( + assignJudges(inputs, placements, judges, { keep }).assignment, + ), }; } @@ -76,8 +148,25 @@ const solutionSchema = z.object({ }), ), reasons: z.array(z.string()).optional(), + assignments: z + .array( + z.object({ + entryId: z.string(), + challengeId: z.string(), + judgeId: z.string(), + }), + ) + .optional(), }); +/** Who grades what for a schedule, and who worked it out. */ +export type JudgeOutcome = { + assignments: JudgeAssignment[]; + metrics: JudgeMetrics; + /** "cp-sat" when the file's own assignments passed the checks. */ + source: "cp-sat" | "in-app"; +}; + export type CpsatImport = | { ok: true; @@ -87,14 +176,44 @@ export type CpsatImport = placements: Placement[]; score: PlanScore; reducedBreaks: ReducedBreak[]; + judging: JudgeOutcome; } | { ok: false; reason: string }; -/** Reads and checks an uploaded solution.json against the current problem. */ +/** + * The file's assignments when it has some and they keep the rules for the + * judges as they are now; otherwise the in-app ones. + */ +function judgeOutcome( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + given: readonly JudgeAssignment[] | undefined, +): JudgeOutcome { + if ( + given && + given.length > 0 && + validateJudgeAssignment(inputs, placements, judges, given).valid + ) { + return { + assignments: assignmentsOf(given), + metrics: scoreJudgeAssignment(inputs, placements, judges, given), + source: "cp-sat", + }; + } + const made = assignJudges(inputs, placements, judges); + return { assignments: made.assignment, metrics: made.metrics, source: "in-app" }; +} + +/** + * Reads and checks an uploaded solution.json against the current problem. + * `judges` are the judges as they are now, whatever the problem listed. + */ export function readCpsatSolution( inputs: PlanInputs, fingerprint: string, text: string, + judges: readonly JudgeInput[] = [], ): CpsatImport { let raw: unknown; try { @@ -154,5 +273,79 @@ export function readCpsatSolution( placements, score: scoreSchedule(inputs, placements), reducedBreaks: check.reducedBreaks, + judging: judgeOutcome(inputs, placements, judges, solution.assignments), + }; +} + +export type CpsatAssignmentImport = + | { + ok: true; + status: "optimal" | "feasible"; + assignments: JudgeAssignment[]; + metrics: JudgeMetrics; + } + | { ok: false; reason: string }; + +/** + * Reads what cpsat.py printed for a cpsatAssignmentProblem. Unlike a new + * schedule, nothing here is optional: the assignments must keep the rules + * for the judges, leave every one in `keep` as it was, and break no rule that + * `keep` did not already break. + */ +export function readCpsatAssignments( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + keep: readonly JudgeAssignment[], + text: string, +): CpsatAssignmentImport { + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { ok: false, reason: "CP-SAT did not print a solution." }; + } + const parsed = solutionSchema.safeParse(raw); + if (!parsed.success) { + return { ok: false, reason: "CP-SAT did not print a solution from cpsat.py." }; + } + const solution = parsed.data; + if (solution.fingerprint !== assignmentFingerprint(placements)) { + return { ok: false, reason: "CP-SAT answered a different schedule." }; + } + if (solution.status === "infeasible" || solution.status === "unknown") { + return { ok: false, reason: "CP-SAT found no assignment of judges in the time it had." }; + } + if (!solution.assignments) { + return { ok: false, reason: "CP-SAT did not assign judges." }; + } + + const assignments = assignmentsOf(solution.assignments); + const before = new Set( + validateJudgeAssignment(inputs, placements, judges, keep).problems, + ); + const broken = validateJudgeAssignment( + inputs, + placements, + judges, + assignments, + ).problems.filter((problem) => !before.has(problem)); + if (broken.length > 0) { + return { + ok: false, + reason: `CP-SAT assigned judges against the rules: ${broken.slice(0, 3).join(" ")}`, + }; + } + const given = new Map( + assignments.map((a) => [taskKey(a.entryId, a.challengeId), a.judgeId]), + ); + if (keep.some((k) => given.get(taskKey(k.entryId, k.challengeId)) !== k.judgeId)) { + return { ok: false, reason: "CP-SAT changed an assignment it was told to keep." }; + } + return { + ok: true, + status: solution.status, + assignments, + metrics: scoreJudgeAssignment(inputs, placements, judges, assignments), }; } diff --git a/packages/api/src/services/judging/schedule/cpsat-runner.test.ts b/packages/api/src/services/judging/schedule/cpsat-runner.test.ts index 46a834c5..9d7c611a 100644 --- a/packages/api/src/services/judging/schedule/cpsat-runner.test.ts +++ b/packages/api/src/services/judging/schedule/cpsat-runner.test.ts @@ -4,7 +4,12 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import type * as ChildProcess from "node:child_process"; import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { cpsatProblem, readCpsatSolution } from "./cpsat-bridge"; +import { + cpsatAssignmentProblem, + cpsatProblem, + readCpsatAssignments, + readCpsatSolution, +} from "./cpsat-bridge"; import { CPSAT_UNAVAILABLE, cpsatAvailable, runCpsat } from "./cpsat-runner"; import type { PlanInputs } from "./inputs"; @@ -212,6 +217,44 @@ process.exitCode = 1; if (run.status === "failed") expect(run.reason).toContain("after 70 seconds"); }); + it("allows one more time limit when the script also assigns judges", async () => { + fakeScript(`setInterval(() => undefined, 1000);`); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const judged = cpsatProblem(inputs, "fp-test", undefined, [ + { id: "m1", kind: "member", challengeIds: ["general", "sp1"], roomId: null }, + ]); + const running = runCpsat(judged, { seconds: 5 }); + while (spawned.length === 0) await new Promise((next) => setImmediate(next)); + // 5 seconds for each of two attempts and for the judges, plus 60. + await vi.advanceTimersByTimeAsync(74_000); + expect(spawned[0]!.child.killed).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + expect(spawned[0]!.child.killed).toBe(true); + + const run = await running; + if (run.status === "failed") expect(run.reason).toContain("after 75 seconds"); + }); + + it("allows a single time limit when the schedule is given and only judges are assigned", async () => { + fakeScript(`setInterval(() => undefined, 1000);`); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const placements = [{ entryId: "a", challengeId: "general", roomId: "g1", slot: 0 }]; + const judged = cpsatAssignmentProblem( + { ...inputs, tasks: [inputs.tasks[0]!] }, + placements, + [{ id: "m1", kind: "member", challengeIds: ["general"], roomId: null }], + ); + const running = runCpsat(judged, { seconds: 5 }); + while (spawned.length === 0) await new Promise((next) => setImmediate(next)); + await vi.advanceTimersByTimeAsync(64_000); + expect(spawned[0]!.child.killed).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + expect(spawned[0]!.child.killed).toBe(true); + + const run = await running; + if (run.status === "failed") expect(run.reason).toContain("after 65 seconds"); + }); + it("kills a run when its signal fires, and removes the problem file", async () => { // Would run for the full 70 seconds if nothing stopped it. fakeScript(`console.error("loading"); setInterval(() => undefined, 1000);`); @@ -288,4 +331,66 @@ describe.runIf(realPython)("runCpsat with cpsat.py", () => { ok: true, }); }); + + it("assigns judges to the bundled example", { timeout: 120_000 }, async () => { + vi.stubEnv("JUDGING_CPSAT_PYTHON", realPython!); + vi.stubEnv("JUDGING_CPSAT_SCRIPT", ""); + const example = JSON.parse( + readFileSync( + fileURLToPath( + new URL("../../../../scripts/judging-schedule/example-problem.json", import.meta.url), + ), + "utf8", + ), + ); + const exampleInputs: PlanInputs = { + rooms: example.rooms.map((r: PlanInputs["rooms"][number]) => ({ + id: r.id, + challengeId: r.challengeId, + buildingId: r.buildingId, + })), + tasks: example.tasks, + slots: example.slotCount, + sameGap: example.sameGap, + crossGap: example.crossGap, + }; + const everything = [...new Set(example.tasks.map((t: { challengeId: string }) => t.challengeId))]; + const judges = Array.from({ length: 8 }, (_, n) => ({ + id: `judge-${n}`, + kind: "member" as const, + challengeIds: everything as string[], + roomId: null, + })); + + const run = await runCpsat( + cpsatProblem(exampleInputs, example.fingerprint, undefined, judges), + { seconds: 5 }, + ); + expect(run).toMatchObject({ status: "solved" }); + if (run.status !== "solved") return; + const read = readCpsatSolution(exampleInputs, example.fingerprint, run.solution, judges); + expect(read).toMatchObject({ ok: true, judging: { source: "cp-sat" } }); + if (!read.ok) return; + + // Then the same schedule again, only the judges: one assignment is kept. + const keep = read.judging.assignments.slice(0, 1); + const again = await runCpsat( + cpsatAssignmentProblem(exampleInputs, read.placements, judges, keep), + { seconds: 5 }, + ); + expect(again).toMatchObject({ status: "solved" }); + if (again.status !== "solved") return; + const assigned = readCpsatAssignments( + exampleInputs, + read.placements, + judges, + keep, + again.solution, + ); + expect(assigned).toMatchObject({ ok: true }); + if (assigned.ok) { + expect(assigned.assignments).toEqual(expect.arrayContaining(keep)); + expect(assigned.assignments).toHaveLength(read.placements.length); + } + }); }); diff --git a/packages/api/src/services/judging/schedule/cpsat-runner.ts b/packages/api/src/services/judging/schedule/cpsat-runner.ts index b8ff9fea..01bbebe4 100644 --- a/packages/api/src/services/judging/schedule/cpsat-runner.ts +++ b/packages/api/src/services/judging/schedule/cpsat-runner.ts @@ -7,7 +7,8 @@ import { dirname, join, resolve } from "node:path"; // Runs the CP-SAT script on this server, for hosts that have a Python with // OR-Tools, so an organiser does not have to download the problem and run it // on a laptop. It is the same script fed the same problem file, and what it -// prints is read exactly like an upload (readCpsatSolution): this file only +// prints is read exactly like an upload (readCpsatSolution, or +// readCpsatAssignments when only the judges are assigned): this file only // starts the process and reports how it ended, it never decides whether a // schedule is good. // @@ -29,6 +30,17 @@ export const CPSAT_SECONDS = { min: 5, max: 60, default: 20 } as const; /** Lines of the script's log kept for error messages. */ const LOG_LINES = 20; +/** + * How many times the script may use its whole time limit on `problem`: twice + * to place the presentations (strict, then relaxed) unless the schedule is + * given, and once more to assign judges when there are any. + */ +function timeLimits(problem: unknown): number { + const shape = problem as { judges?: unknown; placements?: unknown } | null; + const judged = Array.isArray(shape?.judges) && shape.judges.length > 0; + return (Array.isArray(shape?.placements) ? 0 : 2) + (judged ? 1 : 0); +} + type Command = { python: string; script: string }; let probe: { env: string; command: Command | null } | undefined; @@ -108,11 +120,12 @@ export async function runCpsat( Math.max(CPSAT_SECONDS.min, Math.round(options.seconds ?? CPSAT_SECONDS.default)), ); const workers = Math.min(4, availableParallelism()); - // Up to two attempts of `seconds` each. Starting Python, loading OR-Tools - // and building the model for each attempt come on top and do not depend - // on `seconds`: on a busy machine they took close to 30 seconds for a - // 2,000-presentation edition, so the margin leaves room for twice that. - const killAfter = seconds * 2 + 60; + // Up to two attempts of `seconds` each, and one more for the judges. + // Starting Python, loading OR-Tools and building the model for each attempt + // come on top and do not depend on `seconds`: on a busy machine they took + // close to 30 seconds for a 2,000-presentation edition, so the margin leaves + // room for twice that. + const killAfter = seconds * timeLimits(problem) + 60; // A directory of its own, so two runs at once never share a file. const dir = await mkdtemp(join(tmpdir(), "cpsat-")); diff --git a/packages/api/src/services/judging/schedule/judge-plan.test.ts b/packages/api/src/services/judging/schedule/judge-plan.test.ts new file mode 100644 index 00000000..9efe5a45 --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-plan.test.ts @@ -0,0 +1,251 @@ +import { describe, expect, it } from "vitest"; +import { + judgeChanges, + judgeChoices, + judgePlanOf, + rosterOf, +} from "./judge-plan"; +import type { JudgeRow, SavedAppointment } from "./judge-plan"; +import type { ChallengeRow } from "../evaluation-store"; + +const challenge = ( + id: string, + over: Partial = {}, +): ChallengeRow => ({ + id, + parentId: null, + mode: "scheduled", + label: id, + isGeneral: false, + isGroup: false, + ...over, +}); + +// General and one sponsor are scheduled; MLH is a group of two unscheduled prizes. +const challenges: ChallengeRow[] = [ + challenge("general", { isGeneral: true }), + challenge("acme"), + challenge("mlh", { isGroup: true, mode: "unscheduled" }), + challenge("mongo", { parentId: "mlh", mode: "unscheduled" }), +]; + +const rooms = [ + { id: "g1", challengeId: "general", buildingId: "ENG" }, + { id: "a1", challengeId: "acme", buildingId: "HEC" }, +]; + +const judgeRows: JudgeRow[] = [ + { id: "m1", kind: "member", userId: "u1", displayName: "Ada" }, + { id: "gs", kind: "guest", userId: null, displayName: "Grace" }, + { id: "lost", kind: "guest", userId: null, displayName: "Nobody" }, +]; + +describe("rosterOf", () => { + const roster = (over: Partial[0]> = {}) => + rosterOf({ + hackathonId: "h", + judges: judgeRows, + seats: [ + { + judgeId: "gs", + roomId: "a1", + createdAt: new Date("2027-02-06T12:00:00Z"), + }, + ], + challenges, + rooms, + ...over, + }); + + it("lets a member grade every scheduled challenge and a guest only their room's", () => { + const { judges } = roster(); + expect(judges).toEqual([ + { id: "gs", kind: "guest", challengeIds: ["acme"], roomId: "a1" }, + { + id: "m1", + kind: "member", + challengeIds: ["acme", "general"], + roomId: null, + }, + ]); + }); + + it("leaves out a guest with no session, or whose room takes no appointments", () => { + expect(roster({ seats: [] }).judges.map((j) => j.id)).toEqual(["m1"]); + expect( + roster({ + seats: [ + { judgeId: "gs", roomId: "unscheduled-room", createdAt: new Date() }, + ], + }).judges.map((j) => j.id), + ).toEqual(["m1"]); + }); + + it("seats a guest in the room of their latest session", () => { + const { judges } = roster({ + seats: [ + { + judgeId: "gs", + roomId: "g1", + createdAt: new Date("2027-02-06T09:00:00Z"), + }, + { + judgeId: "gs", + roomId: "a1", + createdAt: new Date("2027-02-06T13:00:00Z"), + }, + { + judgeId: "gs", + roomId: "g1", + createdAt: new Date("2027-02-06T11:00:00Z"), + }, + ], + }); + expect(judges.find((j) => j.id === "gs")).toMatchObject({ + roomId: "a1", + challengeIds: ["acme"], + }); + }); + + it("names every judge, including those it cannot assign", () => { + expect(roster().names.get("lost")).toBe("Nobody"); + }); +}); + +const appointment = ( + id: string, + over: Partial = {}, +): SavedAppointment => ({ + id, + entryId: `e-${id}`, + challengeId: "general", + roomId: "g1", + slot: 0, + assignedJudgeId: null, + removed: false, + open: true, + ...over, +}); + +describe("judgePlanOf", () => { + const plan = (appointments: SavedAppointment[]) => + judgePlanOf({ + rooms: [ + ...rooms, + { id: "old", challengeId: "general", buildingId: null }, + ], + sponsorChallengeIds: new Set(["acme"]), + grid: { slots: 10, sameGap: 1, crossGap: 2 }, + appointments, + }); + + it("plans the appointments that are open or already have a judge", () => { + const result = plan([ + appointment("open"), + appointment("kept", { open: false, assignedJudgeId: "m1", slot: 3 }), + appointment("under-way", { open: false, slot: 5 }), + appointment("gone", { removed: true, assignedJudgeId: "m1", slot: 6 }), + ]); + expect(result.placements.map((p) => p.entryId)).toEqual([ + "e-open", + "e-kept", + ]); + expect(result.inputs.tasks).toHaveLength(2); + expect(result.keep).toEqual([ + { entryId: "e-kept", challengeId: "general", judgeId: "m1" }, + ]); + expect(result.current).toEqual(result.keep); + }); + + it("marks sponsor challenges and gives a room without a building its own", () => { + const result = plan([ + appointment("s", { challengeId: "acme", roomId: "a1" }), + ]); + expect(result.inputs.tasks[0]).toMatchObject({ sponsor: true }); + expect(result.inputs.rooms.find((r) => r.id === "old")!.buildingId).toBe( + "room:old", + ); + expect(result.inputs).toMatchObject({ slots: 10, sameGap: 1, crossGap: 2 }); + }); + + it("counts a judge on an open appointment as current, not kept", () => { + const result = plan([appointment("o", { assignedJudgeId: "m1" })]); + expect(result.keep).toEqual([]); + expect(result.current).toHaveLength(1); + }); +}); + +describe("judgeChanges", () => { + it("lists the open appointments whose judge differs, null where nobody is left", () => { + const appointments = [ + appointment("same", { assignedJudgeId: "m1" }), + appointment("new"), + appointment("moved", { assignedJudgeId: "m1" }), + appointment("dropped", { assignedJudgeId: "m1" }), + appointment("closed", { open: false, assignedJudgeId: "m1" }), + ]; + const changes = judgeChanges(appointments, [ + { entryId: "e-same", challengeId: "general", judgeId: "m1" }, + { entryId: "e-new", challengeId: "general", judgeId: "m2" }, + { entryId: "e-moved", challengeId: "general", judgeId: "m2" }, + { entryId: "e-closed", challengeId: "general", judgeId: "m2" }, + ]); + expect(changes).toEqual([ + { id: "new", assignedJudgeId: "m2" }, + { id: "moved", assignedJudgeId: "m2" }, + { id: "dropped", assignedJudgeId: null }, + ]); + }); +}); + +describe("judgeChoices", () => { + const roster = rosterOf({ + hackathonId: "h", + judges: judgeRows, + seats: [{ judgeId: "gs", roomId: "a1", createdAt: new Date() }], + challenges, + rooms, + }); + + it("offers the judges who may grade the appointment there, and marks the booked", () => { + const target = appointment("t", { slot: 2 }); + const all = [ + target, + appointment("other", { + roomId: "a1", + challengeId: "acme", + slot: 2, + assignedJudgeId: "m1", + }), + ]; + expect(judgeChoices(roster, target, all)).toEqual([ + { judgeId: "m1", bookedWith: "other" }, + ]); + // Free once their other appointment is at another time. + all[1] = { ...all[1]!, slot: 4 }; + expect(judgeChoices(roster, target, all)).toEqual([ + { judgeId: "m1", bookedWith: null }, + ]); + }); + + it("offers a guest only their own room", () => { + const sponsor = appointment("s", { challengeId: "acme", roomId: "a1" }); + expect( + judgeChoices(roster, sponsor, [sponsor]).map((c) => c.judgeId), + ).toEqual(["gs", "m1"]); + }); + + it("ignores a booking for a project no longer entered", () => { + const target = appointment("t", { slot: 2 }); + const gone = appointment("g", { + removed: true, + slot: 2, + assignedJudgeId: "m1", + roomId: "a1", + challengeId: "acme", + }); + expect(judgeChoices(roster, target, [target, gone])[0]).toMatchObject({ + bookedWith: null, + }); + }); +}); diff --git a/packages/api/src/services/judging/schedule/judge-plan.ts b/packages/api/src/services/judging/schedule/judge-plan.ts new file mode 100644 index 00000000..b499bfee --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-plan.ts @@ -0,0 +1,217 @@ +import { inJudgeScope } from "../evaluation-store"; +import type { ChallengeRow } from "../evaluation-store"; +import type { Judge } from "../principal"; +import { scheduledChallengeIds, taskKey } from "./inputs"; +import type { Placement, PlanInputs, PlanRoom } from "./inputs"; +import { mayGrade } from "./judges"; +import type { JudgeAssignment, JudgeInput } from "./judges"; + +// What the router hands the judge assigner and reads back: who can judge, +// which saved appointments are up for assignment, and the numbers the board +// shows. No database in here; the router reads the rows and writes the result. + +export type JudgeRow = { + id: string; + kind: "member" | "guest"; + userId: string | null; + displayName: string; +}; + +/** A guest's session: the room they scanned in at. */ +export type GuestSeat = { judgeId: string; roomId: string; createdAt: Date }; + +export type Roster = { + /** Judges who can be given presentations: members, and guests with a room. */ + judges: JudgeInput[]; + /** Every judge's name, those who cannot be given presentations too. */ + names: Map; +}; + +/** + * The judges an edition can assign. A member may grade whatever the judge + * desk lets them see; a guest only their room's challenge (inJudgeScope), + * and only in that room, which is the room of their latest session. A guest + * with no session, or whose room takes no appointments, has nothing to grade. + */ +export function rosterOf(source: { + hackathonId: string; + judges: readonly JudgeRow[]; + /** Sessions that have not been revoked. */ + seats: readonly GuestSeat[]; + challenges: readonly ChallengeRow[]; + /** The rooms that take appointments. */ + rooms: readonly PlanRoom[]; +}): Roster { + const scheduled = [...scheduledChallengeIds(source.challenges)].sort(); + const rooms = new Map(source.rooms.map((room) => [room.id, room])); + + const seatOf = new Map(); + for (const seat of source.seats) { + const current = seatOf.get(seat.judgeId); + if (!current || seat.createdAt > current.createdAt) + seatOf.set(seat.judgeId, seat); + } + + const judges: JudgeInput[] = []; + for (const row of source.judges) { + if (row.kind === "member") { + const principal: Judge = { + kind: "member", + hackathonId: source.hackathonId, + userId: row.userId ?? "", + judgeId: row.id, + displayName: row.displayName, + }; + judges.push({ + id: row.id, + kind: "member", + challengeIds: scheduled.filter((id) => + inJudgeScope(principal, source.challenges, id), + ), + roomId: null, + }); + continue; + } + const room = rooms.get(seatOf.get(row.id)?.roomId ?? ""); + if (!room) continue; + const principal: Judge = { + kind: "guest", + hackathonId: source.hackathonId, + sessionId: "", + linkId: "", + roomId: room.id, + roomChallengeId: room.challengeId, + judgeId: row.id, + displayName: row.displayName, + }; + judges.push({ + id: row.id, + kind: "guest", + challengeIds: scheduled.filter((id) => + inJudgeScope(principal, source.challenges, id), + ), + roomId: room.id, + }); + } + judges.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + + return { + judges, + names: new Map(source.judges.map((row) => [row.id, row.displayName])), + }; +} + +export type SavedAppointment = { + id: string; + entryId: string; + challengeId: string; + roomId: string; + slot: number; + assignedJudgeId: string | null; + /** Entered no more: needs no judge. */ + removed: boolean; + /** Not started, nothing evaluated or drafted: its judge may be chosen anew. */ + open: boolean; +}; + +/** + * The saved schedule as the judge assigner takes it: the appointments still + * entered that are open or already have a judge. `keep` is the judges of those + * that are not open, which stay; `current` is every judge there is now. + */ +export function judgePlanOf(source: { + /** Every room, archived ones too; a room with no building counts as its own. */ + rooms: readonly { + id: string; + challengeId: string; + buildingId: string | null; + }[]; + sponsorChallengeIds: ReadonlySet; + grid: Pick; + appointments: readonly SavedAppointment[]; +}) { + // One that is over or under way with nobody assigned has no judge to find. + const live = source.appointments.filter( + (a) => !a.removed && (a.open || a.assignedJudgeId), + ); + const inputs: PlanInputs = { + ...source.grid, + rooms: source.rooms.map((room) => ({ + id: room.id, + challengeId: room.challengeId, + buildingId: room.buildingId ?? `room:${room.id}`, + })), + tasks: live.map((a) => ({ + entryId: a.entryId, + challengeId: a.challengeId, + sponsor: source.sponsorChallengeIds.has(a.challengeId), + })), + }; + const placements: Placement[] = live.map((a) => ({ + entryId: a.entryId, + challengeId: a.challengeId, + roomId: a.roomId, + slot: a.slot, + })); + const given = (a: SavedAppointment): JudgeAssignment => ({ + entryId: a.entryId, + challengeId: a.challengeId, + judgeId: a.assignedJudgeId!, + }); + return { + inputs, + placements, + keep: live.filter((a) => !a.open && a.assignedJudgeId).map(given), + current: live.filter((a) => a.assignedJudgeId).map(given), + }; +} + +/** + * What `assignment` changes on the open appointments: the new judge of every + * one whose judge differs, null where it now has none. + */ +export function judgeChanges( + appointments: readonly SavedAppointment[], + assignment: readonly JudgeAssignment[], +): { id: string; assignedJudgeId: string | null }[] { + const chosen = new Map( + assignment.map((a) => [taskKey(a.entryId, a.challengeId), a.judgeId]), + ); + return appointments + .filter((a) => a.open && !a.removed) + .map((a) => ({ + id: a.id, + assignedJudgeId: chosen.get(taskKey(a.entryId, a.challengeId)) ?? null, + })) + .filter((change) => { + const was = appointments.find((a) => a.id === change.id)!.assignedJudgeId; + return was !== change.assignedJudgeId; + }); +} + +export type JudgeChoice = { + judgeId: string; + /** The appointment of theirs in the same slot, if they are already booked. */ + bookedWith: string | null; +}; + +/** Judges who may grade `appointment`, and whether each is free in its slot. */ +export function judgeChoices( + roster: Roster, + appointment: Pick, + all: readonly SavedAppointment[], +): JudgeChoice[] { + return roster.judges + .filter((judge) => mayGrade(judge, appointment)) + .map((judge) => ({ + judgeId: judge.id, + bookedWith: + all.find( + (other) => + other.id !== appointment.id && + !other.removed && + other.assignedJudgeId === judge.id && + other.slot === appointment.slot, + )?.id ?? null, + })); +} diff --git a/packages/api/src/services/judging/schedule/judges.test.ts b/packages/api/src/services/judging/schedule/judges.test.ts new file mode 100644 index 00000000..dc66a53c --- /dev/null +++ b/packages/api/src/services/judging/schedule/judges.test.ts @@ -0,0 +1,532 @@ +import { describe, expect, it } from "vitest"; +import { + MAX_CONSECUTIVE, + assignJudges, + compareJudgeMetrics, + describeShortfall, + judgeShortfalls, + mayGrade, + scoreJudgeAssignment, + validateJudgeAssignment, +} from "./judges"; +import type { JudgeAssignment, JudgeInput } from "./judges"; +import type { Placement, PlanInputs } from "./inputs"; + +const inputs: PlanInputs = { + rooms: [ + { id: "g1", challengeId: "general", buildingId: "ENG" }, + { id: "g2", challengeId: "general", buildingId: "ENG" }, + { id: "g3", challengeId: "general", buildingId: "HEC" }, + { id: "s1", challengeId: "sponsor", buildingId: "HEC" }, + ], + tasks: [ + { entryId: "a", challengeId: "general", sponsor: false }, + { entryId: "b", challengeId: "general", sponsor: false }, + { entryId: "c", challengeId: "general", sponsor: false }, + { entryId: "d", challengeId: "general", sponsor: false }, + { entryId: "a", challengeId: "sponsor", sponsor: true }, + { entryId: "b", challengeId: "sponsor", sponsor: true }, + ], + slots: 12, + sameGap: 1, + crossGap: 2, +}; + +const placed = ( + entryId: string, + challengeId: string, + roomId: string, + slot: number, +): Placement => ({ entryId, challengeId, roomId, slot }); + +const member = (id: string, extra: Partial = {}): JudgeInput => ({ + id, + kind: "member", + challengeIds: ["general", "sponsor"], + roomId: null, + ...extra, +}); + +const guest = ( + id: string, + roomId: string | null, + challengeIds = ["sponsor"], +): JudgeInput => ({ id, kind: "guest", challengeIds, roomId }); + +const row = ( + entryId: string, + challengeId: string, + judgeId: string, +): JudgeAssignment => ({ entryId, challengeId, judgeId }); + +const placements: Placement[] = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g2", 0), + placed("c", "general", "g1", 3), + placed("d", "general", "g3", 3), + placed("a", "sponsor", "s1", 6), + placed("b", "sponsor", "s1", 7), +]; + +describe("mayGrade", () => { + it("lets a member grade the challenges they are given, in any room", () => { + const judge = member("m", { challengeIds: ["general"] }); + expect(mayGrade(judge, { challengeId: "general", roomId: "g2" })).toBe( + true, + ); + expect(mayGrade(judge, { challengeId: "sponsor", roomId: "s1" })).toBe( + false, + ); + }); + + it("keeps a guest to their own room, and a guest with no room out", () => { + expect( + mayGrade(guest("x", "s1"), { challengeId: "sponsor", roomId: "s1" }), + ).toBe(true); + expect( + mayGrade(guest("x", "s1"), { challengeId: "sponsor", roomId: "s2" }), + ).toBe(false); + expect( + mayGrade(guest("x", null), { challengeId: "sponsor", roomId: "s1" }), + ).toBe(false); + }); +}); + +describe("validateJudgeAssignment", () => { + const check = ( + judges: JudgeInput[], + assignment: JudgeAssignment[], + spots = placements, + ) => validateJudgeAssignment(inputs, spots, judges, assignment); + + it("accepts a judge who keeps every rule, and does not mind presentations left out", () => { + const judges = [member("m1"), guest("g", "s1")]; + const result = check(judges, [ + row("a", "general", "m1"), + row("c", "general", "m1"), + row("a", "sponsor", "g"), + ]); + expect(result).toEqual({ valid: true, problems: [] }); + }); + + it("refuses a challenge or a room the judge may not grade", () => { + const judges = [ + member("m1", { challengeIds: ["general"] }), + guest("g", "s1"), + ]; + const problems = check(judges, [ + row("a", "sponsor", "m1"), + row("a", "general", "g"), + ]).problems.join(" "); + expect(problems).toContain( + "Judge m1 may not grade a:sponsor: wrong challenge.", + ); + expect(problems).toContain( + "Judge g may not grade a:general: wrong challenge.", + ); + const wrongRoom = check( + [guest("g", "s1", ["general", "sponsor"])], + [row("a", "general", "g")], + ); + expect(wrongRoom.problems).toEqual([ + "Judge g may not grade a:general: wrong room.", + ]); + }); + + it("refuses unknown presentations and judges, and two judges for one presentation", () => { + const { problems } = check( + [member("m1"), member("m2")], + [ + row("zz", "general", "m1"), + row("a", "general", "nobody"), + row("c", "general", "m1"), + row("c", "general", "m2"), + ], + ); + expect(problems).toContain("No such presentation: zz:general."); + expect(problems).toContain("No such judge: nobody."); + expect(problems).toContain("c:general has more than one judge."); + }); + + it("refuses a judge in two rooms at once", () => { + const { problems } = check( + [member("m1")], + [row("a", "general", "m1"), row("b", "general", "m1")], + ); + expect(problems).toEqual(["Judge m1 is booked twice in slot 1."]); + }); + + it("needs the same-building break to change rooms and the longer one to change buildings", () => { + const both = (second: Placement) => + validateJudgeAssignment( + inputs, + [placed("a", "general", "g1", 0), second], + [member("m1")], + [ + row("a", "general", "m1"), + row(second.entryId, second.challengeId, "m1"), + ], + ); + + // g1 to g2 is one building: one free slot between (slots 0 and 2). + expect(both(placed("b", "general", "g2", 1)).problems).toHaveLength(1); + expect(both(placed("b", "general", "g2", 2)).valid).toBe(true); + // g1 to g3 is another building: two free slots between. + expect(both(placed("b", "general", "g3", 2)).problems).toHaveLength(1); + expect(both(placed("b", "general", "g3", 3)).valid).toBe(true); + }); + + it("lets a judge stay in one room without a break, up to the run limit", () => { + const wide: PlanInputs = { + ...inputs, + tasks: ["a", "b", "c", "d", "e", "f"].map((entryId) => ({ + entryId, + challengeId: "general", + sponsor: false, + })), + }; + const run = (count: number, judge: JudgeInput) => { + const spots = ["a", "b", "c", "d", "e", "f"] + .slice(0, count) + .map((entryId, slot) => placed(entryId, "general", "g1", slot)); + return validateJudgeAssignment( + wide, + spots, + [judge], + spots.map((p) => row(p.entryId, "general", judge.id)), + ); + }; + expect(MAX_CONSECUTIVE).toBe(4); + expect(run(4, member("m1")).valid).toBe(true); + expect(run(5, member("m1")).problems).toEqual([ + "Judge m1 grades more than 4 slots in a row.", + ]); + expect(run(5, member("m1", { maxConsecutive: 5 })).valid).toBe(true); + expect(run(6, member("m1", { maxConsecutive: 2 })).valid).toBe(false); + }); + + it("allows a free slot to end a run", () => { + const spots = [0, 1, 2, 3, 5].map((slot, i) => + placed(["a", "b", "c", "d", "e"][i]!, "general", "g1", slot), + ); + const wide: PlanInputs = { + ...inputs, + tasks: spots.map((p) => ({ + entryId: p.entryId, + challengeId: "general", + sponsor: false, + })), + }; + const result = validateJudgeAssignment( + wide, + spots, + [member("m1")], + spots.map((p) => row(p.entryId, "general", "m1")), + ); + expect(result.valid).toBe(true); + }); +}); + +describe("scoreJudgeAssignment", () => { + it("counts what is left out, the loads, the rooms and the parts of the day", () => { + const judges = [member("m1"), member("m2"), guest("g", "s1")]; + const metrics = scoreJudgeAssignment(inputs, placements, judges, [ + row("a", "general", "m1"), + row("c", "general", "m1"), + row("b", "general", "m2"), + row("a", "sponsor", "g"), + row("b", "sponsor", "m2"), + ]); + expect(metrics).toMatchObject({ + unassigned: 1, + memberOnSponsor: 1, + maxLoad: 2, + // Members: 2 against 2; the guest is a pool of one. + loadSpread: 0, + // m1 is in g1 twice. + repeats: 1, + loads: { m1: 2, m2: 2, g: 1 }, + }); + // m1: g1; m2: g2, s1; g: s1. + expect(metrics.avgDistinctRooms).toBeCloseTo(4 / 3); + }); + + it("ranks fewer left out ahead of everything else", () => { + const judges = [member("m1"), member("m2")]; + const some = scoreJudgeAssignment(inputs, placements, judges, [ + row("a", "general", "m1"), + ]); + const more = scoreJudgeAssignment(inputs, placements, judges, [ + row("a", "general", "m1"), + row("c", "general", "m1"), + ]); + expect(compareJudgeMetrics(more, some)).toBeLessThan(0); + expect(compareJudgeMetrics(some, some)).toBe(0); + }); +}); + +describe("assignJudges", () => { + it("gives every presentation a judge who may grade it, keeping the rules", () => { + const judges = [member("m1"), member("m2"), guest("g", "s1")]; + const { assignment, metrics } = assignJudges(inputs, placements, judges); + expect(assignment).toHaveLength(placements.length); + expect(metrics.unassigned).toBe(0); + expect( + validateJudgeAssignment(inputs, placements, judges, assignment), + ).toEqual({ + valid: true, + problems: [], + }); + }); + + it("leaves sponsor presentations to the sponsor's judge while that judge can take them", () => { + const judges = [member("m1"), member("m2"), guest("g", "s1")]; + const { assignment, metrics } = assignJudges(inputs, placements, judges); + const sponsor = assignment.filter((r) => r.challengeId === "sponsor"); + expect(sponsor.map((r) => r.judgeId)).toEqual(["g", "g"]); + expect(metrics.memberOnSponsor).toBe(0); + }); + + it("asks a table judge to cover the sponsor when no sponsor judge can", () => { + // Both sponsor presentations at once: the one guest can take only one. + const clash = [ + placed("a", "sponsor", "s1", 6), + placed("b", "sponsor", "s1", 7), + placed("a", "general", "g1", 0), + ]; + const tight = { ...inputs, sameGap: 1 }; + const judges = [member("m1"), guest("g", "s1")]; + const { assignment, metrics } = assignJudges(tight, clash, judges); + // Back to back in one room needs no break: the guest takes both. + expect(assignment.filter((r) => r.judgeId === "g")).toHaveLength(2); + expect(metrics.memberOnSponsor).toBe(0); + + const together = [ + placed("a", "sponsor", "s1", 6), + placed("b", "sponsor", "s2", 6), + ]; + const withTwoRooms: PlanInputs = { + ...tight, + rooms: [ + ...tight.rooms, + { id: "s2", challengeId: "sponsor", buildingId: "HEC" }, + ], + }; + const result = assignJudges(withTwoRooms, together, [ + member("m1"), + guest("g", "s1"), + ]); + expect(result.metrics.unassigned).toBe(0); + expect(result.metrics.memberOnSponsor).toBe(1); + expect(result.assignment.find((r) => r.entryId === "b")!.judgeId).toBe( + "m1", + ); + }); + + it("evens the load between judges of one pool", () => { + const only = placements.filter((p) => p.challengeId === "general"); + const judges = [member("m1"), member("m2")]; + const { metrics } = assignJudges(inputs, only, judges); + expect(metrics.loads).toEqual({ m1: 2, m2: 2 }); + expect(metrics.maxLoad).toBe(2); + expect(metrics.loadSpread).toBe(0); + }); + + it("spreads one judge over rooms and over the day", () => { + const wide: PlanInputs = { + ...inputs, + sameGap: 0, + crossGap: 0, + slots: 8, + tasks: ["a", "b", "c", "d"].map((entryId) => ({ + entryId, + challengeId: "general", + sponsor: false, + })), + }; + // Two presentations in each of two rooms, early then late. + const spots = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g2", 1), + placed("c", "general", "g1", 6), + placed("d", "general", "g2", 7), + ]; + const { metrics, assignment } = assignJudges(wide, spots, [ + member("m1"), + member("m2"), + ]); + expect(metrics.loads).toEqual({ m1: 2, m2: 2 }); + // Each judge sees both rooms and both ends of the day. + expect(metrics.repeats).toBe(0); + expect(metrics.avgDistinctRooms).toBe(2); + expect(metrics.dayPartsCovered).toBe(4); + expect(new Set(assignment.map((r) => r.judgeId)).size).toBe(2); + }); + + it("reports the slot where judges run out", () => { + const rush = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g2", 0), + placed("c", "general", "g3", 0), + ]; + const judges = [member("m1"), member("m2")]; + const { assignment, metrics } = assignJudges(inputs, rush, judges); + expect(metrics.unassigned).toBe(1); + const shortfalls = judgeShortfalls(inputs, rush, judges, assignment); + expect(shortfalls).toEqual([ + { slot: 0, challengeId: "general", presentations: 3, judgesFree: 2 }, + ]); + expect(describeShortfall(shortfalls[0]!, "General")).toBe( + "Slot 1: 3 presentations in General, only 2 judges free", + ); + expect( + describeShortfall( + { slot: 4, challengeId: "x", presentations: 1, judgesFree: 1 }, + "Acme", + ), + ).toBe("Slot 5: 1 presentation in Acme, only 1 judge free"); + }); + + it("does not count a judge who cannot walk there as free", () => { + // m1 grades in ENG at slot 0; slot 1 is HEC and needs two free slots. + const spots = [ + placed("a", "general", "g1", 0), + placed("b", "general", "g3", 1), + ]; + const judges = [member("m1")]; + const { assignment } = assignJudges(inputs, spots, judges); + expect(assignment).toHaveLength(1); + const shortfalls = judgeShortfalls(inputs, spots, judges, assignment); + expect(shortfalls).toEqual([ + { + slot: assignment[0]!.entryId === "a" ? 1 : 0, + challengeId: "general", + presentations: 1, + judgesFree: 0, + }, + ]); + }); + + it("keeps assignments it is told to keep, and plans around them", () => { + const only = placements.filter((p) => p.challengeId === "general"); + const judges = [member("m1"), member("m2")]; + const keep = [row("a", "general", "m1"), row("c", "general", "m1")]; + const { assignment } = assignJudges(inputs, only, judges, { keep }); + expect(assignment).toHaveLength(4); + expect(assignment).toEqual(expect.arrayContaining(keep)); + // b and d share a slot with the kept ones, so only m2 is free for them. + const rest = assignment.filter( + (r) => r.entryId === "b" || r.entryId === "d", + ); + expect(rest.map((r) => r.judgeId)).toEqual(["m2", "m2"]); + }); + + it("gives the same answer however its inputs are ordered", () => { + const judges = [member("m1"), member("m2"), member("m3"), guest("g", "s1")]; + const first = assignJudges(inputs, placements, judges); + const second = assignJudges( + inputs, + [...placements].reverse(), + [...judges].reverse(), + ); + expect(second.assignment).toEqual(first.assignment); + expect(second.metrics).toEqual(first.metrics); + }); + + it("assigns nothing without judges", () => { + const result = assignJudges(inputs, placements, []); + expect(result.assignment).toEqual([]); + expect(result.metrics.unassigned).toBe(placements.length); + expect(result.metrics.avgDistinctRooms).toBe(0); + }); +}); + +// A small deterministic generator, so these cases never change between runs. +function lcg(seed: number) { + let state = seed >>> 0; + return (n: number) => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state % n; + }; +} + +function randomCase(seed: number) { + const next = lcg(seed); + const rooms = [ + { id: "r1", challengeId: "c1", buildingId: "A" }, + { id: "r2", challengeId: "c1", buildingId: "A" }, + { id: "r3", challengeId: "c1", buildingId: "B" }, + { id: "r4", challengeId: "c2", buildingId: "B" }, + ]; + const slots = 8; + const taken = new Set(); + const spots: Placement[] = []; + for (let n = 0; n < 7; n += 1) { + const room = rooms[next(rooms.length)]!; + const slot = next(slots); + if (taken.has(`${room.id}@${slot}`)) continue; + taken.add(`${room.id}@${slot}`); + spots.push(placed(`e${n}`, room.challengeId, room.id, slot)); + } + const judges: JudgeInput[] = [ + { id: "m1", kind: "member", challengeIds: ["c1", "c2"], roomId: null }, + { id: "m2", kind: "member", challengeIds: ["c1", "c2"], roomId: null }, + { id: "g4", kind: "guest", challengeIds: ["c2"], roomId: "r4" }, + ]; + const plan: PlanInputs = { + rooms, + tasks: spots.map((p) => ({ + entryId: p.entryId, + challengeId: p.challengeId, + sponsor: p.challengeId === "c2", + })), + slots, + sameGap: next(2), + crossGap: 1 + next(2), + }; + plan.crossGap = Math.max(plan.crossGap, plan.sameGap); + return { plan, spots, judges }; +} + +/** The fewest presentations any valid assignment leaves without a judge. */ +function fewestLeftOut( + plan: PlanInputs, + spots: Placement[], + judges: JudgeInput[], +): number { + let best = spots.length; + const rows: JudgeAssignment[] = []; + const go = (i: number) => { + if (i === spots.length) { + best = Math.min(best, spots.length - rows.length); + return; + } + go(i + 1); + const p = spots[i]!; + for (const judge of judges) { + if (!mayGrade(judge, p)) continue; + rows.push(row(p.entryId, p.challengeId, judge.id)); + if (validateJudgeAssignment(plan, spots, judges, rows).valid) go(i + 1); + rows.pop(); + } + }; + go(0); + return best; +} + +describe("assignJudges against every possibility", () => { + it("keeps the rules and leaves out no more than it must", () => { + for (let seed = 1; seed <= 40; seed += 1) { + const { plan, spots, judges } = randomCase(seed); + const result = assignJudges(plan, spots, judges); + expect( + validateJudgeAssignment(plan, spots, judges, result.assignment) + .problems, + `seed ${seed}`, + ).toEqual([]); + expect(result.metrics.unassigned, `seed ${seed}`).toBe( + fewestLeftOut(plan, spots, judges), + ); + } + }); +}); diff --git a/packages/api/src/services/judging/schedule/judges.ts b/packages/api/src/services/judging/schedule/judges.ts new file mode 100644 index 00000000..6a5f0310 --- /dev/null +++ b/packages/api/src/services/judging/schedule/judges.ts @@ -0,0 +1,784 @@ +import { taskKey } from "./inputs"; +import type { Placement, PlanInputs } from "./inputs"; + +// The second scheduling stage. Once every presentation has a time and a room, +// each one gets exactly one judge, so every project has a time, a place and a +// grader. A presentation is a project in one root challenge, and that +// challenge is graded once, by one judge. +// +// Table judges (members) handle general judging; a sponsor's judges are other +// people, tied to the sponsor's room, who grade only that sponsor's challenge. +// A judge is in one place per slot, needs the schedule's breaks to walk to +// another room, and takes a rest after a long run of presentations. Judge bias +// is calibrated on the assumption that every judge sees a random sample of +// projects, so after everything is graded the aim is an even load, then each +// judge spread over as many rooms and as much of the day as possible. +// +// Pure and deterministic, written apart from the CP-SAT script on purpose: the +// script's answer is re-checked here (validateJudgeAssignment), and a server +// without OR-Tools builds its assignment here (assignJudges). + +/** Most slots in a row a judge grades before at least one free slot. */ +export const MAX_CONSECUTIVE = 4; + +/** + * The day is cut into this many equal parts when asking whether a judge's + * presentations are spread across it. cpsat.py cuts it the same way. + */ +export const DAY_PARTS = 4; + +export type JudgeInput = { + id: string; + kind: "member" | "guest"; + /** The root challenges this judge may grade. */ + challengeIds: readonly string[]; + /** The one room a guest grades in; null for a member, who moves about. */ + roomId: string | null; + /** Defaults to MAX_CONSECUTIVE. */ + maxConsecutive?: number; +}; + +/** The judge of one presentation. */ +export type JudgeAssignment = { + entryId: string; + challengeId: string; + judgeId: string; +}; + +/** Lower is better, except the last, which counts what the judges cover. */ +export type JudgeMetrics = { + /** Presentations with no judge. */ + unassigned: number; + /** Sponsor presentations graded by a table judge rather than a sponsor's. */ + memberOnSponsor: number; + /** Presentations of the busiest judge. */ + maxLoad: number; + /** How far apart the judges of one pool are, summed over the pools. */ + loadSpread: number; + /** Times a judge grades in a room they have already graded in. */ + repeats: number; + /** Rooms a judge grades in, averaged over the judges who grade at all. */ + avgDistinctRooms: number; + /** Parts of the day a judge grades in, summed over the judges. Higher is better. */ + dayPartsCovered: number; + /** Presentations per judge, for every judge given. */ + loads: Record; +}; + +/** + * Judges with the same challenges and room share a pool: they are + * interchangeable, so only within a pool can the load be evened out. + */ +const poolKey = (judge: JudgeInput) => + `${[...judge.challengeIds].sort().join(",")}|${judge.roomId ?? ""}`; + +/** Whether `judge` may grade a presentation of `challengeId` held in `roomId`. */ +export function mayGrade( + judge: JudgeInput, + presentation: { challengeId: string; roomId: string }, +): boolean { + if (!judge.challengeIds.includes(presentation.challengeId)) return false; + // A guest grades only in their own room, and with no room grades nowhere. + if (judge.roomId !== null) return judge.roomId === presentation.roomId; + return judge.kind !== "guest"; +} + +// Same hash for every run, so ties break the same way each time. +function stableHash(text: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < text.length; i += 1) { + hash ^= text.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash; +} + +const byHash = (a: string, b: string) => + stableHash(a) - stableHash(b) || (a < b ? -1 : a > b ? 1 : 0); + +type Item = { + index: number; + key: string; + entryId: string; + challengeId: string; + roomId: string; + buildingId: string; + slot: number; + part: number; + sponsor: boolean; +}; + +type Prepared = { + input: JudgeInput; + max: number; + /** Judges are numbered in id order, the same for any input order. */ + pool: number; +}; + +type Context = { + items: Item[]; + judges: Prepared[]; + sameGap: number; + crossGap: number; + maxGap: number; + itemIndex: Map; + judgeIndex: Map; + /** Judges who may grade each item, in judge order. */ + eligible: number[][]; + pools: number[][]; +}; + +function contextOf( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], +): Context { + const buildings = new Map(inputs.rooms.map((r) => [r.id, r.buildingId])); + const sponsor = new Map( + inputs.tasks.map((t) => [taskKey(t.entryId, t.challengeId), t.sponsor]), + ); + + const items: Item[] = []; + const seen = new Set(); + for (const p of placements) { + const key = taskKey(p.entryId, p.challengeId); + const buildingId = buildings.get(p.roomId); + if (seen.has(key) || buildingId === undefined) continue; + seen.add(key); + items.push({ + index: 0, + key, + entryId: p.entryId, + challengeId: p.challengeId, + roomId: p.roomId, + buildingId, + slot: p.slot, + part: Math.min( + DAY_PARTS - 1, + Math.max( + 0, + Math.floor((p.slot * DAY_PARTS) / Math.max(1, inputs.slots)), + ), + ), + sponsor: sponsor.get(key) ?? false, + }); + } + items.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + items.forEach((item, index) => { + item.index = index; + }); + + const unique = [...new Map(judges.map((j) => [j.id, j])).values()].sort( + (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0), + ); + const poolIds = new Map(); + const prepared = unique.map((input) => { + const key = poolKey(input); + if (!poolIds.has(key)) poolIds.set(key, poolIds.size); + return { + input, + max: Math.max(1, input.maxConsecutive ?? MAX_CONSECUTIVE), + pool: poolIds.get(key)!, + }; + }); + const pools: number[][] = [...poolIds.keys()].map(() => []); + prepared.forEach((judge, j) => pools[judge.pool]!.push(j)); + + return { + items, + judges: prepared, + sameGap: inputs.sameGap, + crossGap: inputs.crossGap, + maxGap: Math.max(inputs.sameGap, inputs.crossGap), + itemIndex: new Map(items.map((item) => [item.key, item.index])), + judgeIndex: new Map(prepared.map((judge, j) => [judge.input.id, j])), + eligible: items.map((item) => + prepared.flatMap((judge, j) => (mayGrade(judge.input, item) ? [j] : [])), + ), + pools, + }; +} + +/** + * Whether one judge cannot do both presentations: they are at the same time, + * or the walk between two rooms needs more free slots than lie between them. + * Staying in one room needs no walk. + */ +function clash(context: Context, a: Item, b: Item): boolean { + const apart = Math.abs(a.slot - b.slot); + if (apart === 0) return true; + if (a.roomId === b.roomId) return false; + const gap = + a.buildingId === b.buildingId ? context.sameGap : context.crossGap; + return apart <= gap; +} + +/** Where the search stands: who grades what, and the counts the score needs. */ +class State { + readonly owner: number[]; + /** Per judge, the item they grade in each slot. */ + readonly slots: Map[]; + readonly load: number[]; + readonly rooms: Map[]; + readonly parts: number[][]; + unassigned: number; + memberOnSponsor = 0; + repeats = 0; + covered = 0; + + constructor(private readonly context: Context) { + this.owner = context.items.map(() => -1); + this.slots = context.judges.map(() => new Map()); + this.load = context.judges.map(() => 0); + this.rooms = context.judges.map(() => new Map()); + this.parts = context.judges.map(() => Array(DAY_PARTS).fill(0)); + this.unassigned = context.items.length; + } + + add(i: number, j: number) { + const item = this.context.items[i]!; + this.owner[i] = j; + this.slots[j]!.set(item.slot, i); + this.load[j]! += 1; + this.unassigned -= 1; + if (item.sponsor && this.context.judges[j]!.input.kind === "member") { + this.memberOnSponsor += 1; + } + const rooms = this.rooms[j]!; + const seen = rooms.get(item.roomId) ?? 0; + if (seen > 0) this.repeats += 1; + rooms.set(item.roomId, seen + 1); + if (this.parts[j]![item.part]! === 0) this.covered += 1; + this.parts[j]![item.part]! += 1; + } + + remove(i: number) { + const j = this.owner[i]!; + const item = this.context.items[i]!; + this.owner[i] = -1; + this.slots[j]!.delete(item.slot); + this.load[j]! -= 1; + this.unassigned += 1; + if (item.sponsor && this.context.judges[j]!.input.kind === "member") { + this.memberOnSponsor -= 1; + } + const rooms = this.rooms[j]!; + const left = rooms.get(item.roomId)! - 1; + if (left > 0) { + this.repeats -= 1; + rooms.set(item.roomId, left); + } else { + rooms.delete(item.roomId); + } + this.parts[j]![item.part]! -= 1; + if (this.parts[j]![item.part]! === 0) this.covered -= 1; + } + + /** + * Whether judge `j` could also take `item`: free in its slot, far enough + * from their other rooms, and not past their run limit. `ignore` is an + * item of theirs to leave out, as when it is about to be swapped away. + */ + fits(j: number, item: Item, ignore = -1): boolean { + const mine = this.slots[j]!; + const taken = (slot: number) => { + const other = mine.get(slot); + return other === undefined || other === ignore ? undefined : other; + }; + if (taken(item.slot) !== undefined) return false; + for (let d = 1; d <= this.context.maxGap; d += 1) { + for (const slot of [item.slot - d, item.slot + d]) { + const other = taken(slot); + if ( + other !== undefined && + clash(this.context, this.context.items[other]!, item) + ) { + return false; + } + } + } + let run = 1; + for (let s = item.slot - 1; taken(s) !== undefined; s -= 1) run += 1; + for (let s = item.slot + 1; taken(s) !== undefined; s += 1) run += 1; + return run <= this.context.judges[j]!.max; + } + + /** The score as a vector, compared left to right; lower is better. */ + vector(): number[] { + let maxLoad = 0; + for (const load of this.load) maxLoad = Math.max(maxLoad, load); + return [ + this.unassigned, + this.memberOnSponsor, + maxLoad, + this.spread(), + this.repeats, + -this.covered, + ]; + } + + spread(): number { + let total = 0; + for (const pool of this.context.pools) { + if (pool.length < 2) continue; + let low = Infinity; + let high = 0; + for (const j of pool) { + low = Math.min(low, this.load[j]!); + high = Math.max(high, this.load[j]!); + } + total += high - low; + } + return total; + } + + metrics(): JudgeMetrics { + const [unassigned, memberOnSponsor, maxLoad, loadSpread, repeats] = + this.vector() as [number, number, number, number, number]; + let graders = 0; + let distinct = 0; + const loads: Record = {}; + this.context.judges.forEach((judge, j) => { + loads[judge.input.id] = this.load[j]!; + if (this.load[j]! === 0) return; + graders += 1; + distinct += this.rooms[j]!.size; + }); + return { + unassigned, + memberOnSponsor, + maxLoad, + loadSpread, + repeats, + avgDistinctRooms: graders === 0 ? 0 : distinct / graders, + dayPartsCovered: this.covered, + loads, + }; + } +} + +function compareVectors(a: readonly number[], b: readonly number[]): number { + for (let i = 0; i < a.length; i += 1) { + if (a[i] !== b[i]) return a[i]! - b[i]!; + } + return 0; +} + +/** Lower is better, compared as the fields are listed in JudgeMetrics. */ +export function compareJudgeMetrics(a: JudgeMetrics, b: JudgeMetrics): number { + return compareVectors( + [ + a.unassigned, + a.memberOnSponsor, + a.maxLoad, + a.loadSpread, + a.repeats, + -a.dayPartsCovered, + ], + [ + b.unassigned, + b.memberOnSponsor, + b.maxLoad, + b.loadSpread, + b.repeats, + -b.dayPartsCovered, + ], + ); +} + +/** A state holding `assignment`, skipping rows that name nothing known. */ +function stateOf(context: Context, assignment: readonly JudgeAssignment[]) { + const state = new State(context); + for (const row of assignment) { + const i = context.itemIndex.get(taskKey(row.entryId, row.challengeId)); + const j = context.judgeIndex.get(row.judgeId); + if (i === undefined || j === undefined || state.owner[i]! !== -1) continue; + state.add(i, j); + } + return state; +} + +/** + * Checks an assignment against the hard rules. A presentation with no judge + * is not an error here (see judgeShortfalls); one with the wrong judge, or + * two, is. + */ +export function validateJudgeAssignment( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + assignment: readonly JudgeAssignment[], +) { + const problems = new Set(); + const context = contextOf(inputs, placements, judges); + const given = new Set(); + const mine = context.judges.map((): number[] => []); + + for (const row of assignment) { + const key = taskKey(row.entryId, row.challengeId); + const i = context.itemIndex.get(key); + const j = context.judgeIndex.get(row.judgeId); + if (i === undefined) { + problems.add(`No such presentation: ${key}.`); + continue; + } + if (j === undefined) { + problems.add(`No such judge: ${row.judgeId}.`); + continue; + } + if (given.has(key)) problems.add(`${key} has more than one judge.`); + given.add(key); + const judge = context.judges[j]!.input; + const item = context.items[i]!; + if (!judge.challengeIds.includes(item.challengeId)) { + problems.add(`Judge ${judge.id} may not grade ${key}: wrong challenge.`); + } else if (!mayGrade(judge, item)) { + problems.add(`Judge ${judge.id} may not grade ${key}: wrong room.`); + } + mine[j]!.push(i); + } + + context.judges.forEach((judge, j) => { + const items = mine[j]!.map((i) => context.items[i]!).sort( + (a, b) => a.slot - b.slot, + ); + for (let a = 0; a < items.length; a += 1) { + for (let b = a + 1; b < items.length; b += 1) { + if (items[b]!.slot - items[a]!.slot > context.maxGap) break; + if (clash(context, items[a]!, items[b]!)) { + problems.add( + items[a]!.slot === items[b]!.slot + ? `Judge ${judge.input.id} is booked twice in slot ${items[a]!.slot + 1}.` + : `Judge ${judge.input.id} has too little time between ${items[a]!.key} and ${items[b]!.key}.`, + ); + } + } + } + let run = 0; + let previous = -2; + for (const item of items) { + if (item.slot === previous) continue; + run = item.slot === previous + 1 ? run + 1 : 1; + previous = item.slot; + if (run === judge.max + 1) { + problems.add( + `Judge ${judge.input.id} grades more than ${judge.max} slots in a row.`, + ); + } + } + }); + + return { valid: problems.size === 0, problems: [...problems] }; +} + +/** How even and how mixed an assignment is, and how many presentations it leaves out. */ +export function scoreJudgeAssignment( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + assignment: readonly JudgeAssignment[], +): JudgeMetrics { + const context = contextOf(inputs, placements, judges); + return stateOf(context, assignment).metrics(); +} + +/** A slot where some presentations of one challenge have no judge. */ +export type JudgeShortfall = { + slot: number; + challengeId: string; + presentations: number; + /** How many of them judges could cover at that time. */ + judgesFree: number; +}; + +const plural = (n: number, one: string) => `${n} ${n === 1 ? one : `${one}s`}`; + +/** The shortfall as a sentence; `challenge` is the challenge's label. */ +export function describeShortfall( + shortfall: JudgeShortfall, + challenge: string, +): string { + return `Slot ${shortfall.slot + 1}: ${plural(shortfall.presentations, "presentation")} in ${challenge}, only ${plural(shortfall.judgesFree, "judge")} free`; +} + +/** + * Each slot and challenge with a presentation that has no judge, earliest + * first. A judge counts as free for a challenge when they may grade it there, + * are not grading another challenge in that slot, and could walk there. + */ +export function judgeShortfalls( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + assignment: readonly JudgeAssignment[], +): JudgeShortfall[] { + const context = contextOf(inputs, placements, judges); + const state = stateOf(context, assignment); + + const groups = new Map(); + for (const item of context.items) { + const key = `${item.slot}|${item.challengeId}`; + const list = groups.get(key); + if (list) list.push(item.index); + else groups.set(key, [item.index]); + } + + const found: JudgeShortfall[] = []; + for (const members of groups.values()) { + if (!members.some((i) => state.owner[i]! === -1)) continue; + const first = context.items[members[0]!]!; + // Presentations of this group each need a different free judge: match them. + const options = members.map((i) => + context.eligible[i]!.filter((j) => { + const held = state.slots[j]!.get(first.slot); + const own = held !== undefined && members.includes(held) ? held : -1; + return ( + (held === undefined || own !== -1) && + state.fits(j, context.items[i]!, own) + ); + }), + ); + const taker = new Map(); + const place = (m: number, tried: Set): boolean => { + for (const j of options[m]!) { + if (tried.has(j)) continue; + tried.add(j); + const holder = taker.get(j); + if (holder === undefined || place(holder, tried)) { + taker.set(j, m); + return true; + } + } + return false; + }; + let covered = 0; + for (let m = 0; m < members.length; m += 1) { + if (place(m, new Set())) covered += 1; + } + found.push({ + slot: first.slot, + challengeId: first.challengeId, + presentations: members.length, + judgesFree: covered, + }); + } + return found.sort( + (a, b) => a.slot - b.slot || (a.challengeId < b.challengeId ? -1 : 1), + ); +} + +export type JudgeAssignmentResult = { + /** Every presentation that has a judge, the kept ones included. */ + assignment: JudgeAssignment[]; + metrics: JudgeMetrics; +}; + +/** + * Moves and swaps tried, and blocking items moved out of the way, before the + * search stops, however large the event. + */ +const MAX_EVALUATIONS = 50_000; +const MAX_DISPLACEMENTS = 100_000; +const MAX_PASSES = 12; + +/** + * Assigns a judge to every presentation it can: a greedy pass, each + * presentation to the judge that leaves the score best, from three orders of + * going through them, then a bounded local search on the best of the three + * that moves, swaps and displaces. The same inputs give the same answer, + * whatever order they come in: ties break on a hash of the ids. `keep` is + * assignments that stay as they are, and count towards loads and breaks. + */ +export function assignJudges( + inputs: PlanInputs, + placements: readonly Placement[], + judges: readonly JudgeInput[], + options: { keep?: readonly JudgeAssignment[] } = {}, +): JudgeAssignmentResult { + const context = contextOf(inputs, placements, judges); + const start = stateOf(context, options.keep ?? []); + const locked = new Set( + context.items + .filter((item) => start.owner[item.index]! !== -1) + .map((i) => i.index), + ); + const rank = (j: number) => stableHash(context.judges[j]!.input.id); + const free = context.items.filter((item) => !locked.has(item.index)); + + // Judges to try for an item, in a fixed order that does not favour the + // first by id. + const order = (i: number) => + [...context.eligible[i]!].sort((a, b) => rank(a) - rank(b) || a - b); + + /** + * The judge for `item` that leaves the score best, or -1. With `stay`, one + * who was in this room the slot before comes first: staying costs a judge + * no break, where moving on spends one. + */ + const bestJudge = (state: State, item: Item, skip = -1, stay = false) => { + let best = -1; + let bestVector: number[] = []; + for (const j of order(item.index)) { + if (j === skip || !state.fits(j, item)) continue; + const before = state.slots[j]!.get(item.slot - 1); + const staying = + before !== undefined && context.items[before]!.roomId === item.roomId; + state.add(item.index, j); + const vector = [stay && !staying ? 1 : 0, ...state.vector()]; + state.remove(item.index); + if (best === -1 || compareVectors(vector, bestVector) < 0) { + best = j; + bestVector = vector; + } + } + return best; + }; + + // Three starting points, the best of them is improved below: most + // constrained first, which is what a short supply of judges needs; and + // through the day in order, with or without judges staying where they are, + // which is what a long run of crowded slots needs. + const construct = (sort: (a: Item, b: Item) => number, stay: boolean) => { + const state = stateOf(context, options.keep ?? []); + for (const item of [...free].sort(sort)) { + const j = bestJudge(state, item, -1, stay); + if (j !== -1) state.add(item.index, j); + } + return state; + }; + const tightFirst = (a: Item, b: Item) => + context.eligible[a.index]!.length - context.eligible[b.index]!.length || + a.slot - b.slot || + byHash(a.key, b.key); + const inOrder = (a: Item, b: Item) => a.slot - b.slot || byHash(a.key, b.key); + const state = [ + construct(tightFirst, false), + construct(inOrder, true), + construct(inOrder, false), + ].reduce((best, next) => + compareVectors(next.vector(), best.vector()) < 0 ? next : best, + ); + + let evaluations = 0; + let displacements = 0; + + /** Moves `i` to judge `to` if that scores better than `current`, and says so. */ + const tryMove = (i: number, to: number, current: number[]) => { + const from = state.owner[i]!; + evaluations += 1; + state.remove(i); + state.add(i, to); + if (compareVectors(state.vector(), current) < 0) return true; + state.remove(i); + state.add(i, from); + return false; + }; + + /** Swaps the judges of `i` and `q` if that scores better than `current`, and says so. */ + const trySwap = (i: number, q: number, current: number[]) => { + const a = state.owner[i]!; + const b = state.owner[q]!; + evaluations += 1; + state.remove(i); + state.remove(q); + if (state.fits(b, context.items[i]!) && state.fits(a, context.items[q]!)) { + state.add(i, b); + state.add(q, a); + if (compareVectors(state.vector(), current) < 0) return true; + state.remove(i); + state.remove(q); + } + state.add(i, a); + state.add(q, b); + return false; + }; + + /** + * Gives `item` a judge, moving blocking items of that judge to other + * judges, `depth` moves deep. Changes nothing if it cannot. + */ + const displace = (item: Item, depth: number): boolean => { + const direct = bestJudge(state, item); + if (direct !== -1) { + state.add(item.index, direct); + return true; + } + if (depth === 0) return false; + for (const j of order(item.index)) { + const reach = Math.max(context.maxGap, context.judges[j]!.max); + for (const q of [...state.slots[j]!.values()]) { + if ( + locked.has(q) || + Math.abs(context.items[q]!.slot - item.slot) > reach + ) { + continue; + } + if (displacements >= MAX_DISPLACEMENTS) return false; + displacements += 1; + state.remove(q); + if (state.fits(j, item)) { + state.add(item.index, j); + if (displace(context.items[q]!, depth - 1)) return true; + state.remove(item.index); + } + state.add(q, j); + } + } + return false; + }; + + for ( + let pass = 0; + pass < MAX_PASSES && evaluations < MAX_EVALUATIONS; + pass += 1 + ) { + let improved = false; + + // A presentation with no judge: clear its way by sending an item that + // blocks a judge to somebody else, who may in turn send one on. + for (const item of free) { + if (state.owner[item.index]! !== -1) continue; + if (displacements >= MAX_DISPLACEMENTS) break; + if (displace(item, 4)) improved = true; + } + + // A presentation that scores better with another judge, alone or swapped + // with one of theirs. + for (const item of free) { + const from = state.owner[item.index]!; + if (from === -1) continue; + const current = state.vector(); + for (const to of order(item.index)) { + if (evaluations >= MAX_EVALUATIONS) break; + if (to === from) continue; + if (!state.slots[to]!.has(item.slot) && state.fits(to, item)) { + if (tryMove(item.index, to, current)) { + improved = true; + break; + } + } + let swapped = false; + for (const q of [...state.slots[to]!.values()]) { + if (evaluations >= MAX_EVALUATIONS) break; + if (locked.has(q) || !context.eligible[q]!.includes(from)) continue; + if (trySwap(item.index, q, current)) { + improved = true; + swapped = true; + break; + } + } + if (swapped) break; + } + } + if (!improved) break; + } + + const assignment = context.items + .filter((item) => state.owner[item.index]! !== -1) + .map((item) => ({ + entryId: item.entryId, + challengeId: item.challengeId, + judgeId: context.judges[state.owner[item.index]!]!.input.id, + })); + return { assignment, metrics: state.metrics() }; +} diff --git a/packages/db/src/schemas/judging.ts b/packages/db/src/schemas/judging.ts index 886915bc..0ab2b1a6 100644 --- a/packages/db/src/schemas/judging.ts +++ b/packages/db/src/schemas/judging.ts @@ -463,6 +463,12 @@ export const judgingAppointments = pgTable( endsAt: timestamp("ends_at").notNull(), // Compare-and-swap for moves made from two organiser screens at once. revision: integer("revision").notNull().default(0), + // The one judge who grades this presentation. Null until judges are + // assigned, and again if that judge is removed. + assignedJudgeId: uuid("assigned_judge_id").references( + () => judgingJudges.id, + { onDelete: "set null" }, + ), }, (table) => [ uniqueIndex("judging_appointment_task_idx").on( @@ -476,6 +482,7 @@ export const judgingAppointments = pgTable( table.startsAt, ), index("judging_appointment_entry_idx").on(table.entryId), + index("judging_appointment_judge_idx").on(table.assignedJudgeId), foreignKey({ name: "judging_appointment_challenge_fk", columns: [table.challengeId], diff --git a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx index b5016dda..92009679 100644 --- a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx +++ b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.test.tsx @@ -120,6 +120,7 @@ function solution(overrides: Record = {}) { reducedBreaks: [], candidateKey: "a".repeat(64), solverStatus: "optimal", + judging: { judges: 3, presentations: 2, unassigned: 0, source: "cp-sat" }, solution: "{}", ...overrides, }; @@ -226,6 +227,25 @@ describe("CpsatPanel", () => { expect(mixed).toContain("Rooms or tables"); }); + it("says who assigned the judges, or that there are none yet", () => { + mocks.solved = solution(); + expect(render()).toContain("CP-SAT gave 2 of 2 presentations a judge, from 3 judges."); + + mocks.reported = false; + mocks.solved = solution({ + judging: { judges: 1, presentations: 2, unassigned: 1, source: "in-app" }, + }); + expect(render()).toContain( + "The in-app assigner gave 1 of 2 presentations a judge, from 1 judge.", + ); + + mocks.reported = false; + mocks.solved = solution({ + judging: { judges: 0, presentations: 2, unassigned: 2, source: "in-app" }, + }); + expect(render()).toContain("No judge has joined yet."); + }); + it("says when the time limit ended the search", () => { mocks.solved = solution({ solverStatus: "feasible" }); const html = render(); diff --git a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx index bc80df91..c69fb376 100644 --- a/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx +++ b/sites/mainweb/components/admin/judging/schedule/CpsatPanel.tsx @@ -16,7 +16,7 @@ import { sectionRule, status, } from "@/components/portal/ui"; -import { ErrorLine } from "@/components/admin/judging/shared"; +import { ErrorLine, plural } from "@/components/admin/judging/shared"; import type { CpsatPreview, ScheduleBoard, Timing } from "./format"; import { PreviewFigures, @@ -264,7 +264,8 @@ export function CpsatPanel({

    CP-SAT is searching. It can take up to {seconds} seconds of solving, or twice that if no schedule keeps the full break between buildings, - plus up to a minute of setup on large events. + and as long again to pick judges once some have joined, plus up to a + minute of setup on large events.

    ) : check.isPending ? (

    @@ -294,6 +295,19 @@ export function CpsatPanel({ ); } +/** What a CP-SAT schedule says about its judges. */ +function judgesLine({ + judges, + presentations, + unassigned, + source, +}: CpsatPreview["judging"]) { + if (judges === 0) { + return "No judge has joined yet. Assign judges from the schedule once they have."; + } + return `${source === "cp-sat" ? "CP-SAT" : "The in-app assigner"} gave ${presentations - unassigned} of ${plural(presentations, "presentation")} a judge, from ${plural(judges, "judge")}.`; +} + /** One CP-SAT schedule, shown like the preview, and saving it. */ function CpsatResult({ hackathonId, @@ -342,6 +356,8 @@ function CpsatResult({ +

    {judgesLine(result.judging)}

    + {result.relaxed && ( void; +}) { + const utils = trpc.useUtils(); + const readOnly = useReadOnly(); + const [picked, setPicked] = useState(null); + + const query = trpc.judging.schedule.judgeChoices.useQuery( + { hackathonId, appointmentId }, + { retry: false }, + ); + const refresh = () => utils.judging.schedule.get.invalidate({ hackathonId }); + const save = trpc.judging.schedule.setAssignedJudge.useMutation({ + onSuccess: async () => { + await refresh(); + onClose(); + }, + // Someone else took the judge, or the presentation started: both the + // board and the choices are out of date. + onError: () => { + void refresh(); + void query.refetch(); + }, + }); + + const data = query.data; + const current = data?.appointment.assignedJudgeId ?? ""; + const selected = picked ?? current; + const choices = data?.choices ?? []; + + return ( + { + if (!save.isPending) onClose(); + }} + maxWidth="lg" + label="Choose a judge" + > +
    +
    +

    Choose a judge

    +

    + {data?.appointment.entryTitle ?? "Appointment"} +

    + {data && ( +

    + {[ + data.appointment.challengeLabel, + data.appointment.roomName, + slotTime(data.appointment.startsAt, windowStart), + ].join(" · ")} +

    + )} +
    + + {query.isLoading ? ( +

    + Finding judges… +

    + ) : query.error ? ( + + {trpcErrorMessage(query.error, "Could not load the judges.")} + + ) : ( +
    + + + {data && !data.appointment.canAssignJudge ? ( +

    + This presentation has started, or a judge has begun on it, so + its judge stays. +

    + ) : choices.length === 0 ? ( +

    + No judge can grade this yet. A guest grades only at their own + room or table, and a table judge appears once they have joined. +

    + ) : null} +
    + )} + + {save.error && ( + + {trpcErrorMessage(save.error, "Could not set the judge.")} + + )} + +
    + + +
    +
    +     + ); +} diff --git a/sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx b/sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx new file mode 100644 index 00000000..c4b2bc30 --- /dev/null +++ b/sites/mainweb/components/admin/judging/schedule/JudgesPanel.test.tsx @@ -0,0 +1,280 @@ +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { JudgeDialog } from "./JudgeDialog"; +import { JudgesPanel } from "./JudgesPanel"; + +// Rendered to markup with the tRPC hooks stubbed, as CpsatPanel.test.tsx does: +// these check what each state shows and disables, not clicks. +Object.assign(globalThis, { React }); + +const mocks = vi.hoisted(() => ({ + readOnly: false, + pending: false, + assigned: null as Record | null, + choices: null as Record | null, +})); + +vi.mock("@/lib/use-portal-context", () => ({ + READ_ONLY_TITLE: "Read-only", + useReadOnly: () => mocks.readOnly, +})); + +vi.mock("@/lib/trpc", () => { + const idle = () => ({ + isPending: false, + error: null, + mutate: () => undefined, + reset: () => undefined, + }); + return { + trpc: { + useUtils: () => ({}), + judging: { + schedule: { + assignJudges: { + // Static rendering cannot click, so a finished run is simply + // there as the mutation's data. + useMutation: () => ({ + ...idle(), + data: mocks.assigned ?? undefined, + isPending: mocks.pending, + }), + }, + setAssignedJudge: { useMutation: idle }, + judgeChoices: { + useQuery: () => ({ + isLoading: mocks.choices === null, + error: null, + data: mocks.choices ?? undefined, + refetch: () => undefined, + }), + }, + }, + }, + }, + }; +}); + +const windowStart = new Date(2026, 10, 1, 10, 0); + +const judging = (overrides: Record = {}) => ({ + judges: 3, + assigned: 5, + unassigned: 0, + loads: [ + { + judgeId: "m1", + name: "Ada", + kind: "member", + roomName: null, + available: true, + load: 3, + }, + { + judgeId: "gs", + name: "Grace", + kind: "guest", + roomName: "Acme 1", + available: true, + load: 2, + }, + { + judgeId: "old", + name: "Linus", + kind: "guest", + roomName: null, + available: false, + load: 0, + }, + ], + warnings: [], + ...overrides, +}); + +function panel(over: Record = {}) { + return renderToStaticMarkup( + , + ); +} + +const button = (html: string, text: string) => + html.match(new RegExp(`]*>${text}`))?.[0] ?? ""; +const DISABLED = ' disabled=""'; + +beforeEach(() => { + mocks.readOnly = false; + mocks.pending = false; + mocks.assigned = null; + mocks.choices = null; +}); + +describe("JudgesPanel", () => { + it("states the rule and what is assigned, with a button to assign again", () => { + const html = panel(); + expect(html).toContain("graded once, by one judge"); + expect(html).toContain( + "5 presentations with a judge. Every upcoming one has its judge.", + ); + expect(button(html, "Assign judges")).not.toContain(DISABLED); + }); + + it("counts the presentations still waiting for a judge", () => { + const html = panel({ unassigned: 2 }); + expect(html).toContain( + "5 presentations with a judge, 2 still waiting for one.", + ); + }); + + it("lists each judge with their load, and says who has no room or table", () => { + const html = panel(); + expect(html).toContain("Presentations per judge"); + expect(html).toContain("Ada"); + expect(html).toContain("Table judge"); + expect(html).toContain("Guest · Acme 1"); + expect(html).toContain("Linus"); + expect(html).toContain("has no room or table now"); + }); + + it("warns of the slots where judges run short, with their times", () => { + const html = panel({ + warnings: [ + { + slot: 9, + startsAt: new Date(2026, 10, 1, 11, 30), + challengeId: "general", + challengeLabel: "General", + presentations: 2, + judgesFree: 1, + message: "Slot 10: 2 presentations in General, only 1 judge free", + }, + ], + }); + expect(html).toContain("1 slot has too few judges"); + expect(html).toContain( + "Slot 10: 2 presentations in General, only 1 judge free, at 11:30", + ); + }); + + it("cannot assign before any judge has joined", () => { + const html = panel({ judges: 0, assigned: 0, unassigned: 4, loads: [] }); + expect(html).toContain("No judge has joined a room or table yet"); + expect(button(html, "Assign judges")).toContain(DISABLED); + }); + + it("is read-only for a read-only organiser, and busy while it works", () => { + mocks.readOnly = true; + expect(button(panel(), "Assign judges")).toContain(DISABLED); + + mocks.readOnly = false; + mocks.pending = true; + expect(button(panel(), "Assigning…")).toContain(DISABLED); + }); + + it("reports what an assignment changed and who worked it out", () => { + mocks.assigned = { changed: 4, solver: "cp-sat" }; + expect(panel()).toContain( + "4 presentations given a judge, worked out by CP-SAT.", + ); + + mocks.assigned = { changed: 0, solver: "in-app" }; + expect(panel()).toContain("Nothing changed"); + }); +}); + +const dialog = () => + renderToStaticMarkup( + undefined} + />, + ); + +const choices = (over: Record = {}) => ({ + appointment: { + id: "appointment-1", + entryTitle: "Pathfinder", + challengeLabel: "General", + roomName: "Table G1", + startsAt: new Date(2026, 10, 1, 11, 30), + assignedJudgeId: "m1", + canAssignJudge: true, + }, + choices: [ + { + judgeId: "m2", + name: "Bea", + kind: "member", + load: 2, + bookedWith: null, + booked: false, + }, + { + judgeId: "m1", + name: "Ada", + kind: "member", + load: 3, + bookedWith: null, + booked: false, + }, + { + judgeId: "gs", + name: "Grace", + kind: "guest", + load: 1, + bookedWith: "Pathfinder", + booked: true, + }, + ], + ...over, +}); + +describe("JudgeDialog", () => { + it("waits for the judges", () => { + expect(dialog()).toContain("Finding judges…"); + }); + + it("offers only the judges the server says may grade it, and no judge", () => { + mocks.choices = choices(); + const html = dialog(); + expect(html).toContain("Pathfinder"); + expect(html).toContain("General · Table G1 · 11:30"); + expect(html).toContain(''); + expect(html).toContain("Bea · 2 presentations"); + expect(html).toContain( + "Grace (guest) · 1 presentation · grading Pathfinder then", + ); + }); + + it("starts on the current judge, and cannot save without a change", () => { + mocks.choices = choices(); + const html = dialog(); + expect(html).toMatch(/
    ); diff --git a/sites/mainweb/components/judge/Itinerary.tsx b/sites/mainweb/components/judge/Itinerary.tsx new file mode 100644 index 00000000..2cbd7939 --- /dev/null +++ b/sites/mainweb/components/judge/Itinerary.tsx @@ -0,0 +1,114 @@ +"use client"; + +import { trpc } from "@/lib/trpc"; +import { itemTitle, object, textLink } from "@/components/portal/ui"; +import { + clockTime, + deskLabel, + deskMeta, + deskStatus, + inOrAt, + nextAssignment, + roomPlace, + startsIn, +} from "./desk"; +import type { Assignment } from "./desk"; +import { useServerNow } from "./use-server-now"; + +/** "1116 · Klaus": the room or table, then its building. */ +function placeName(assignment: Assignment) { + return roomPlace(assignment.roomName, assignment.buildingName); +} + +/** + * The presentations organisers assigned to this judge, soonest first: the one + * to go to next, and the rest in brief. Nothing shows for a judge with none. + */ +export function Itinerary({ + hackathonId, + onPickRoom, +}: { + hackathonId: string; + /** Members only: opens the room picker. */ + onPickRoom?: () => void; +}) { + const itinerary = trpc.judging.desk.itinerary.useQuery( + { hackathonId }, + { refetchInterval: 15_000 }, + ); + const now = useServerNow(itinerary.data?.serverNow, itinerary.dataUpdatedAt); + + const assignments = itinerary.data?.presentations ?? []; + if (assignments.length === 0) return null; + + const next = nextAssignment(assignments); + const rest = assignments.filter((a) => a !== next); + const startsAt = next?.startsAt.getTime() ?? 0; + + return ( +
    +

    Assigned to you

    + + {next ? ( +
    +

    + Your next project: {next.title} {inOrAt(next.roomKind)}{" "} + {placeName(next)}, {clockTime(next.startsAt)} +

    +

    + {next.challengeLabel} + {now !== null && + (startsAt <= now + ? " · Presenting now" + : ` · Starts ${startsIn(startsAt - now)}`)} +

    + {!next.inRoom && ( +

    + You are not seated there. Go to {placeName(next)} to judge it.{" "} + {onPickRoom && ( + + )} +

    + )} +
    + ) : ( +

    + Nothing left to judge from what was assigned to you. +

    + )} + + {rest.length > 0 && ( +
      + {rest.map((assignment) => ( +
    • + + {clockTime(assignment.startsAt)} + + + {assignment.title} + + {placeName(assignment)} + {assignment.graded ? ( + Graded + ) : assignment.takenOver ? ( + Being graded + ) : null} +
      • + ))} +
    + )} +
    + ); +} diff --git a/sites/mainweb/components/judge/JudgeDesk.tsx b/sites/mainweb/components/judge/JudgeDesk.tsx index b2ca2d1d..a97ef02b 100644 --- a/sites/mainweb/components/judge/JudgeDesk.tsx +++ b/sites/mainweb/components/judge/JudgeDesk.tsx @@ -22,6 +22,7 @@ import { import { DeskAnnouncements } from "./DeskAnnouncements"; import { EvaluationSheet } from "./EvaluationSheet"; import { GuestNameGate } from "./GuestNameGate"; +import { Itinerary } from "./Itinerary"; import { MySubmissions } from "./MySubmissions"; import { DEFAULT_FILTERS, NO_SKIPS, ProjectQueue } from "./ProjectQueue"; import type { QueueFilters, SkipState } from "./ProjectQueue"; @@ -336,6 +337,13 @@ function Desk({ {!pickingRoom && } + {!pickingRoom && ( + setPickingRoom(true) : undefined} + /> + )} + {/* Picking a room is a mode: the list waits until the judge is done. */} {!pickingRoom && (context.challenges.length === 0 ? ( diff --git a/sites/mainweb/components/judge/MySubmissions.tsx b/sites/mainweb/components/judge/MySubmissions.tsx index 27c8559d..67c799e1 100644 --- a/sites/mainweb/components/judge/MySubmissions.tsx +++ b/sites/mainweb/components/judge/MySubmissions.tsx @@ -16,7 +16,10 @@ import type { Submission } from "./desk"; function standing(row: Submission) { if (row.withdrawn) return { tone: "neutral" as const, text: "Withdrawn" }; - if (!row.isComplete) return { tone: "warning" as const, text: "Incomplete" }; + // The score shows once placings count the evaluation: every rating is in. + if (row.score === null) { + return { tone: "warning" as const, text: "Incomplete" }; + } return { tone: "success" as const, text: `Score ${formatScore(row.score)}` }; } diff --git a/sites/mainweb/components/judge/ProjectQueue.tsx b/sites/mainweb/components/judge/ProjectQueue.tsx index 56a07f24..c0f673b5 100644 --- a/sites/mainweb/components/judge/ProjectQueue.tsx +++ b/sites/mainweb/components/judge/ProjectQueue.tsx @@ -68,14 +68,22 @@ function progress( project: ListedProject, now: number | null, ): { tone: Tone; text: string } | null { - if (project.myEvaluation?.isComplete) - return { tone: "success", text: "Judged" }; + if (project.myEvaluation?.counted) return { tone: "success", text: "Judged" }; if (project.myEvaluation) return { tone: "warning", text: "Incomplete" }; // Another judge's, so the judge's own half-written draft is not the news. if (project.grading === "graded") return { tone: "neutral", text: "Graded" }; if (project.grading === "in_progress") { return { tone: "neutral", text: "Being graded" }; } + // Never says which judge. + if (project.grading === "reserved") { + return { + tone: "neutral", + text: project.reservedUntil + ? `Reserved for another judge until ${clockTime(project.reservedUntil)}` + : "Reserved for another judge", + }; + } if (project.hasDraft) return { tone: "warning", text: "Draft saved" }; const p = project.presentation; if ( @@ -141,6 +149,12 @@ export function ProjectQueue({ ); const now = useServerNow(list.data?.serverNow, list.dataUpdatedAt); const utils = trpc.useUtils(); + // Skipping a project the judge only opened gives it back. A failure changes + // nothing the judge needs to hear: the hold lapses by itself. + const { mutate: release } = trpc.judging.desk.releaseGrading.useMutation({ + onSettled: () => + void utils.judging.desk.projects.invalidate({ hackathonId }), + }); const [finding, setFinding] = useState(false); // Where the last "Grade next project" found nothing, and why. const [none, setNone] = useState<{ @@ -206,9 +220,12 @@ export function ProjectQueue({ // With judged projects shown, a count of all of them hides how many are left. // The same goes for projects with other judges. - const judged = rest.filter((p) => p.myEvaluation?.isComplete).length; + const judged = rest.filter((p) => p.myEvaluation?.counted).length; const taken = rest.filter( - (p) => p.grading === "graded" || p.grading === "in_progress", + (p) => + p.grading === "graded" || + p.grading === "in_progress" || + p.grading === "reserved", ).length; const mixed = (!filters.hideJudged && judged > 0) || taken > 0; @@ -254,6 +271,7 @@ export function ProjectQueue({ function skipOffered() { const { offered, skipped } = skips; if (!offered) return; + release({ hackathonId, entryId: offered, challengeId: challenge?.id }); void gradeNext(addSkip(skipped, offered)); } const offeredTitle = data?.projects.find( @@ -277,6 +295,9 @@ export function ProjectQueue({

    {next.title}

    {where(next, false)}

    + {next.assignedToYou && ( + Assigned to you + )} {next.flagged && ( {where(project, data.roomFilter !== "applied")} + {project.assignedToYou && ( + + Assigned to you + + )} {state && ( {state.text} diff --git a/sites/mainweb/components/judge/RoomSwitcher.tsx b/sites/mainweb/components/judge/RoomSwitcher.tsx index 6bbb5378..d14cb002 100644 --- a/sites/mainweb/components/judge/RoomSwitcher.tsx +++ b/sites/mainweb/components/judge/RoomSwitcher.tsx @@ -38,6 +38,7 @@ export function RoomSwitcher({ utils.judging.desk.context.invalidate({ hackathonId }), utils.judging.desk.rooms.invalidate({ hackathonId }), utils.judging.desk.projects.invalidate({ hackathonId }), + utils.judging.desk.itinerary.invalidate({ hackathonId }), utils.judging.desk.announcements.invalidate({ hackathonId }), ]); onDone(true); diff --git a/sites/mainweb/components/judge/Shortlist.tsx b/sites/mainweb/components/judge/Shortlist.tsx index 2b9f242e..3a631898 100644 --- a/sites/mainweb/components/judge/Shortlist.tsx +++ b/sites/mainweb/components/judge/Shortlist.tsx @@ -105,10 +105,13 @@ export function Shortlist({ ].some((m) => m.isPending); const sections = list.data ?? []; - // A project judged for two challenges is still one project to rank. + // A project judged for two challenges is still one project to rank. One + // with a score counts, as placings count it, so it can go on the shortlist. const finished = new Map(); for (const row of mine.data ?? []) { - if (row.isComplete && !row.withdrawn) finished.set(row.entryId, row.title); + if (row.score !== null && !row.withdrawn) { + finished.set(row.entryId, row.title); + } } const actions: Actions = { diff --git a/sites/mainweb/components/judge/desk.test.ts b/sites/mainweb/components/judge/desk.test.ts index 3381dc0b..ad4d3b97 100644 --- a/sites/mainweb/components/judge/desk.test.ts +++ b/sites/mainweb/components/judge/desk.test.ts @@ -6,6 +6,7 @@ import { countdownTone, formatScore, linkHost, + nextAssignment, nextPresentation, roomPlace, startsIn, @@ -108,6 +109,60 @@ describe("nextPresentation", () => { expect(nextPresentation([finished], now)).toBeNull(); expect(nextPresentation([], now)).toBeNull(); }); + + it("puts a presentation assigned to the judge that is running now first", () => { + const earlier = { id: "a", presentation: slot(8), myEvaluation: null }; + const mine = { + id: "b", + presentation: slot(12), + myEvaluation: null, + assignedToYou: true, + }; + expect(nextPresentation([earlier, mine], now)?.id).toBe("b"); + expect(nextPresentation([mine, earlier], now)?.id).toBe("b"); + }); + + it("leaves an assigned one that has not started to its place in time", () => { + const running = { id: "a", presentation: slot(10), myEvaluation: null }; + const mine = { + id: "b", + presentation: slot(20), + myEvaluation: null, + assignedToYou: true, + }; + expect(nextPresentation([running, mine], now)?.id).toBe("a"); + }); + + it("skips an assigned one the judge has judged", () => { + const judged = { + id: "a", + presentation: slot(10), + myEvaluation: { revision: 1 }, + assignedToYou: true, + }; + const open = { id: "b", presentation: slot(12), myEvaluation: null }; + expect(nextPresentation([judged, open], now)?.id).toBe("b"); + }); +}); + +describe("nextAssignment", () => { + const item = (id: string, graded = false, takenOver = false) => ({ + id, + graded, + takenOver, + }); + + it("is the first presentation nobody has graded or taken over", () => { + expect( + nextAssignment([item("a", true), item("b", false, true), item("c")])?.id, + ).toBe("c"); + expect(nextAssignment([item("a"), item("b")])?.id).toBe("a"); + }); + + it("is null when nothing is left", () => { + expect(nextAssignment([item("a", true)])).toBeNull(); + expect(nextAssignment([])).toBeNull(); + }); }); describe("stationGroups", () => { diff --git a/sites/mainweb/components/judge/desk.ts b/sites/mainweb/components/judge/desk.ts index 939b0646..11b390a4 100644 --- a/sites/mainweb/components/judge/desk.ts +++ b/sites/mainweb/components/judge/desk.ts @@ -9,6 +9,7 @@ export type ListedProject = ProjectList["projects"][number]; export type ProjectSheet = Desk["project"]; export type Submission = Desk["myEvaluations"][number]; export type ShortlistSection = Desk["deliberation"]["list"][number]; +export type Assignment = Desk["itinerary"]["presentations"][number]; /** * The desk's column. Narrower than the member pages: it is read on a phone @@ -78,27 +79,43 @@ export function formatScore(score: number | null) { /** * The presentation a seated judge should do now, or next: the earliest one - * still running or yet to start that they have not evaluated. + * still running or yet to start that they have not evaluated. One assigned to + * them that is running now comes before the rest. */ export function nextPresentation< P extends { presentation: { startsAt: Date; endsAt: Date } | null; myEvaluation: unknown; + assignedToYou?: boolean; }, >(projects: readonly P[], now: number): P | null { let next: P | null = null; + let nextRank = Infinity; let nextStart = Infinity; for (const project of projects) { const slot = project.presentation; if (!slot || project.myEvaluation || slot.endsAt.getTime() <= now) continue; - if (slot.startsAt.getTime() < nextStart) { + const start = slot.startsAt.getTime(); + const rank = project.assignedToYou && start <= now ? 0 : 1; + if (rank < nextRank || (rank === nextRank && start < nextStart)) { next = project; - nextStart = slot.startsAt.getTime(); + nextRank = rank; + nextStart = start; } } return next; } +/** + * The assigned presentation to go to next: the first, in the order the server + * sends them (soonest first), that nobody has graded or taken over. + */ +export function nextAssignment< + A extends { graded: boolean; takenOver: boolean }, +>(assignments: readonly A[]): A | null { + return assignments.find((a) => !a.graded && !a.takenOver) ?? null; +} + /** Room, then building: "1116 · Klaus". */ export function roomPlace(room: string, building: string | null) { return building ? `${room} · ${building}` : room; From 51aaca19606a1db8bef15a7c28e0b4ef078896d0 Mon Sep 17 00:00:00 2001 From: aamoghS Date: Thu, 15 Oct 2026 22:08:36 -0400 Subject: [PATCH 20/27] feat(judging): show assigned judges' progress and clear stale placings The live board now shows, next to each started presentation's assigned judge, whether they are grading it, have graded it, were taken over by another judge, or have not started once the takeover time has passed. It never says who took over. The rule is in schedule/judge-progress.ts. It uses the desk's takeover time and the board's existing rule for matching evaluations and drafts to appointments, and adds no new query to judging.schedule.get. reopenGrade now deletes the edition's unpublished placings in the same transaction, under the config lock that compute and publish take. Publishing then needs a fresh compute, and the organiser is warned before confirming. Organiser counts, the board and hacker itinerary status, and hacker feedback now count an evaluation as placings do: every rating in, whether or not a written response was left empty. The judging API docs describe assigned judges, the new desk procedures and this counting rule. --- docs/packages/api.md | 4 +- .../.internal-tests/evaluation-counts.test.ts | 363 ++++++++++++++++++ .../src/.internal-tests/reopen-grade.test.ts | 105 ++++- .../.internal-tests/schedule-judges.test.ts | 154 ++++++++ packages/api/src/routers/judging/admin.ts | 37 +- packages/api/src/routers/judging/hacker.ts | 11 +- packages/api/src/routers/judging/results.ts | 2 +- packages/api/src/routers/judging/schedule.ts | 38 +- .../judging/appointment-status.test.ts | 16 +- .../services/judging/appointment-status.ts | 11 +- .../src/services/judging/itinerary.test.ts | 44 ++- .../api/src/services/judging/itinerary.ts | 11 +- .../services/judging/schedule/board.test.ts | 2 +- .../src/services/judging/schedule/board.ts | 22 +- .../judging/schedule/judge-progress.test.ts | 221 +++++++++++ .../judging/schedule/judge-progress.ts | 130 +++++++ .../admin/judging/ChecklistSection.tsx | 2 +- .../components/admin/judging/EntriesList.tsx | 2 +- .../admin/judging/results/ResultsTab.tsx | 8 +- .../admin/judging/schedule/LiveBoard.tsx | 8 + .../admin/judging/schedule/format.ts | 11 + 21 files changed, 1135 insertions(+), 67 deletions(-) create mode 100644 packages/api/src/.internal-tests/evaluation-counts.test.ts create mode 100644 packages/api/src/services/judging/schedule/judge-progress.test.ts create mode 100644 packages/api/src/services/judging/schedule/judge-progress.ts diff --git a/docs/packages/api.md b/docs/packages/api.md index 51bb0479..554a0b46 100644 --- a/docs/packages/api.md +++ b/docs/packages/api.md @@ -93,10 +93,12 @@ Batch status updates allow up to **2500** ids. The global array cap in `scrubMar Judging itself is the `judging` namespace; its shared helpers are in `services/judging/` (`room-link`, `scope`, `scoring`, `config`). `hackathon.getResults` returns published placings from both the retired `hackathon_result` table and `judging_placement`, in one shape. It leaves out entries pulled from judging (`deleted_at`) or from prizes (`prize_ineligible_at`, set by `judging.admin.setPrizeEligibility`). Check-in rule-break flags (`services/check-in-flags.ts`) reach organisers through `judging.admin.listEntries` and `judging.results.list`, and judges through `judging.desk` as a bare `flagged` boolean; no hacker-facing, public or cached procedure returns them. -Placings rank on scores adjusted for judge bias (`services/judging/calibration.ts`): each evaluation is calibrated within its root challenge for its judge's strictness, spread and drift, on the assumption that `judging.desk.nextProject` (`services/judging/queue.ts`) hands every judge a random sample of projects. `judging_placement.raw_score` keeps the plain mean; only `judging.results.list` (organisers) returns it and the per-judge calibration, and `judging.desk` stays on raw means. An evaluation counts when it holds a rating for every rating item, whether or not a written response is missing, so a draft submitted at the hard end still places its project. `judging.results.list` keeps the calibration for 60 seconds per edition, keyed on the edition's evaluation count and latest save and on its ranked entries, while `judging.results.compute` always calibrates fresh. `judging.admin.reopenGrade` deletes one evaluation and the drafts for its project and challenge so another judge can grade it; it is refused while results are published. +Placings rank on scores adjusted for judge bias (`services/judging/calibration.ts`): each evaluation is calibrated within its root challenge for its judge's strictness, spread and drift, on the assumption that `judging.desk.nextProject` (`services/judging/queue.ts`) hands every judge a random sample of projects. `judging_placement.raw_score` keeps the plain mean; only `judging.results.list` (organisers) returns it and the per-judge calibration, and `judging.desk` stays on raw means. An evaluation counts when it holds a rating for every rating item (`scoring.hasEveryRating`, and `scoring-sql.holdsEveryRating` where the database counts), whether or not a written response is missing, so a draft submitted at the hard end still places its project. Organiser counts, the board and hacker itinerary status, hacker feedback, and the desk's scores, judged count and shortlist all use that rule. `judging.results.list` keeps the calibration for 60 seconds per edition, keyed on the edition's evaluation count and latest save and on its ranked entries, while `judging.results.compute` always calibrates fresh. `judging.admin.reopenGrade` deletes one evaluation and the drafts for its project and challenge so another judge can grade it, and clears the edition's unpublished placings, so publishing needs a fresh compute; it is refused while results are published. At an untimed challenge that random sample is enforced (`services/judging/queue-store.ts`): `judging.desk.nextProject` offers only the first `QUEUE_WINDOW` (3) open projects of the judge's own queue and takes a `skip` list for a team away from its table, and a judge's first `startGrading`, `saveDraft` or `submit` on a project outside that window is refused. `judging.desk.startGrading` holds a project from the moment it is opened, with an empty draft. A table judge grades a sponsor's challenge only from a seat in a room that serves it (`scope.isSponsorChallenge`); `judging.desk.context` lists only the challenges they may pick. +A scheduled presentation can have one assigned judge (`judging_appointment.assigned_judge_id`, set by `judging.schedule`; `services/judging/schedule/judges.ts` decides who may be assigned it). The desk never writes it and never tells a judge which other judge is assigned to, holds or graded a project. From the presentation's start until `TAKEOVER_AFTER_MS` (5 minutes) later only the assigned judge may open it (`services/judging/timing.ts`), and other judges see it as `reserved` with the time it opens. After that any judge who may grade it there can take it over, unless the assigned judge has started: their draft holds it (`grading-hold.ts`). Every claim checks this under the grading lock. `judging.desk.nextProject` offers the judge's own running assignments first; the queue window does not apply to scheduled challenges. `judging.desk.itinerary` lists the caller's assigned presentations that have not ended, soonest first, with the room and whether they are seated there. `judging.desk.releaseGrading` deletes the caller's own draft while it is still the empty one `startGrading` made, so a skipped or accidentally opened project is free again. `judging.schedule.get` gives organisers each started assignment's `judgeProgress` (grading, graded, taken over, not started). + ## Club procedures (selected) **Members:** `me`, `register`, `update`, `history`, `myPass`, `rotatePass`, `checkStatus`; admin `list`, `getById`, `adminSearch`, `adminHistory`, `adminGrant`, `adminRevoke`. diff --git a/packages/api/src/.internal-tests/evaluation-counts.test.ts b/packages/api/src/.internal-tests/evaluation-counts.test.ts new file mode 100644 index 00000000..44773b28 --- /dev/null +++ b/packages/api/src/.internal-tests/evaluation-counts.test.ts @@ -0,0 +1,363 @@ +/** + * Counts of evaluations that organisers and hackers read agree with placings: + * an evaluation counts once it holds a rating for every rating item, whether + * or not a written response was left empty, which is what an auto-submit at + * the deadline often leaves. The organiser's overview and project list, the + * hacker's feedback and the status the itinerary reports all read it that way, + * and none reads the judge's own "submitted whole" flag. + * + * The schema is the real one; only the database handle is a stand-in, handed + * to the router through the context the way check-in-flags-organizer.test.ts + * does. It answers each select by the table it reads from, so what these tests + * look at is the SQL each router asks the database to apply (the rule is + * checked against scoring.hasEveryRating in services/judging/scoring-sql.test.ts) + * and what the router makes of the rows it is given. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { + hackathonParticipants, + hackathons, + judgingAppointments, + judgingChallenges, + judgingConfigs, + judgingEntries, + judgingEntryChallenges, + judgingEntryMembers, + judgingEvaluationResponses, + judgingEvaluations, + judgingPresence, + judgingRooms, +} from "@query/db"; +import { appRouter } from "../root"; +import { cache } from "../middleware/cache"; +import type * as Store from "../services/judging/evaluation-store"; + +// Nothing is due on any read here, so there is nothing to reconcile. +vi.mock("../services/judging/evaluation-store", async (importActual) => ({ + ...(await importActual()), + reconcileDueDrafts: async () => undefined, +})); + +const HACK = "11111111-1111-4111-8111-111111111111"; +const ENTRY = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaa1"; +const GENERAL = "cccccccc-cccc-4ccc-8ccc-ccccccccccc1"; +const ROOM = "dddddddd-dddd-4ddd-8ddd-ddddddddddd1"; +const SLOT = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1"; +const EVAL_1 = "ffffffff-ffff-4fff-8fff-fffffffffff1"; +const EVAL_2 = "ffffffff-ffff-4fff-8fff-fffffffffff2"; + +const dialect = new PgDialect(); +const render = (condition: unknown) => dialect.sqlToQuery(condition as never); +/** The SQL with its spacing and parameter numbers evened out. */ +const flat = (condition: unknown) => + render(condition) + .sql.replace(/\s+/g, " ") + .replace(/\( /g, "(") + .replace(/ \)/g, ")") + .replace(/\$\d+/g, "$?"); + +// scoring.hasEveryRating, as SQL: the evaluation's ratings are at least as +// many as the rubric's rating items. +const HOLDS_EVERY_RATING = + '(select count(*) from "judging_evaluation_rating" as r where r."evaluation_id" = "judging_evaluation"."id") >= (select count(*) from "judging_rubric_item" as i where i."hackathon_id" = $? and i."kind" = \'rating\')'; + +type Row = Record; +type SelectCall = { + fields: Record | undefined; + table: unknown; + where: unknown; +}; + +let adminRow: Row | undefined; +const selects: SelectCall[] = []; +// The rows each table answers with, whatever condition it was asked for. +const rows = new Map(); + +const fakeDb: any = { + query: { admins: { findFirst: async () => adminRow } }, + select: (fields?: Record) => { + const call: SelectCall = { fields, table: null, where: null }; + selects.push(call); + const chain: any = { + then: (onOk: any, onErr: any) => + Promise.resolve(rows.get(call.table) ?? []).then(onOk, onErr), + }; + chain.from = (table: unknown) => { + call.table = table; + return chain; + }; + chain.where = (where: unknown) => { + call.where = where; + return chain; + }; + for (const method of [ + "innerJoin", + "leftJoin", + "groupBy", + "orderBy", + "limit", + "offset", + "for", + ]) { + chain[method] = () => chain; + } + return chain; + }, + insert: () => ({ + values: () => + Object.assign(Promise.resolve(undefined), { + onConflictDoNothing: () => ({ returning: async () => [] }), + returning: async () => [{ id: "row" }], + }), + }), + delete: () => ({ where: async () => [] }), + transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), +}; + +const callerAs = (userId: string) => + appRouter.createCaller({ + db: fakeDb, + session: { user: { id: userId } }, + userId, + cache, + clientIp: "127.0.0.1", + req: { headers: { get: () => null } }, + } as any); +const admin = () => callerAs("admin_user"); +const hacker = () => callerAs("hacker_user"); + +const callsOn = (table: unknown) => + selects.filter((call) => call.table === table); + +const at = (time: string) => new Date(`2027-02-06T${time}:00Z`); + +const config = (over: Row = {}) => ({ + hackathonId: HACK, + state: "open", + openedAt: null, + closedAt: null, + showAllScores: false, + inventoryLockedAt: null, + groupsInitializedAt: null, + itineraryPublished: true, + emergencyMode: false, + claimsStartedAt: null, + ...over, +}); + +beforeEach(() => { + cache.clear(); + selects.length = 0; + rows.clear(); + adminRow = { userId: "admin_user", isActive: true, role: "admin" }; + rows.set(hackathons, [{ id: HACK, name: "Hacklytics 2027" }]); + rows.set(judgingConfigs, [config()]); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.overview", () => { + it("counts the evaluations that hold every rating, not the ones submitted whole", async () => { + rows.set(judgingEvaluations, [{ total: 5, complete: 3 }]); + + const { counts } = await admin().judging.admin.overview({ + hackathonId: HACK, + }); + + expect(counts.evaluations).toMatchObject({ total: 5, complete: 3 }); + const aggregate = callsOn(judgingEvaluations).find( + (call) => call.fields && "complete" in call.fields, + )!; + const text = flat(aggregate.fields!.complete); + expect(text).toContain(`filter (where ${HOLDS_EVERY_RATING})`); + expect(text).not.toContain("is_complete"); + // For this edition's rubric. + expect(render(aggregate.fields!.complete).params).toEqual([HACK]); + }); +}); + +// --------------------------------------------------------------------------- +describe("judging.admin.listEntries", () => { + it("counts each project's evaluations that hold every rating", async () => { + rows.set(judgingEntries, [ + { + id: ENTRY, + title: "Tidewater", + value: 1, + deletedAt: null, + prizeIneligible: false, + }, + ]); + rows.set(judgingEvaluations, [{ entryId: ENTRY, value: 2 }]); + + const page = await admin().judging.admin.listEntries({ hackathonId: HACK }); + + expect(page.entries[0]).toMatchObject({ + id: ENTRY, + completeEvaluations: 2, + }); + const [counted] = callsOn(judgingEvaluations); + const text = flat(counted!.where); + expect(text).toContain(HOLDS_EVERY_RATING); + expect(text).not.toContain("is_complete"); + expect(render(counted!.where).params).toEqual([ENTRY, HACK]); + }); +}); + +// --------------------------------------------------------------------------- +describe("hackathon participant reads", () => { + const ratedResponse = ( + evaluationId: string, + label: string, + value: string, + ) => ({ + evaluationId, + label, + displayOrder: 1, + value, + }); + + beforeEach(() => { + rows.set(hackathonParticipants, [{ status: "checked_in" }]); + rows.set(judgingEntryMembers, [ + { + memberId: "m1", + entryId: ENTRY, + title: "Tidewater", + participantCount: 2, + deletedAt: null, + }, + ]); + rows.set(judgingChallenges, [ + { + id: GENERAL, + parentId: null, + mode: "scheduled", + label: "General", + displayOrder: 0, + isGeneral: true, + isGroup: false, + }, + ]); + }); + + describe("judging.hacker.feedback", () => { + it("asks only for member evaluations that hold every rating", async () => { + rows.set(judgingEvaluations, []); + + await hacker().judging.hacker.feedback({ hackathonId: HACK }); + + const [read] = callsOn(judgingEvaluations); + const text = flat(read!.where); + expect(text).toContain(HOLDS_EVERY_RATING); + expect(text).not.toContain("is_complete"); + // Of this project, in this edition, by a member judge. + expect(render(read!.where).params).toEqual( + expect.arrayContaining([HACK, ENTRY, "member"]), + ); + }); + + it("shows the written responses that exist and leaves out the ones left empty", async () => { + // The first auto-submitted at the deadline with every rating and one + // response; the second has none written at all. + rows.set(judgingEvaluations, [ + { id: EVAL_1, challengeId: GENERAL, submittedAt: at("15:00") }, + { id: EVAL_2, challengeId: GENERAL, submittedAt: at("15:10") }, + ]); + rows.set(judgingEvaluationResponses, [ + ratedResponse(EVAL_1, "Strengths", " Clear demo "), + { ...ratedResponse(EVAL_1, "Next steps", ""), displayOrder: 2 }, + ]); + + const feedback = await hacker().judging.hacker.feedback({ + hackathonId: HACK, + }); + + expect(feedback).toEqual({ + available: true, + items: [ + { + challenge: "General", + responses: [{ label: "Strengths", value: "Clear demo" }], + }, + ], + }); + }); + + it("asks for public responses only", async () => { + rows.set(judgingEvaluations, [ + { id: EVAL_1, challengeId: GENERAL, submittedAt: at("15:00") }, + ]); + + await hacker().judging.hacker.feedback({ hackathonId: HACK }); + + const [read] = callsOn(judgingEvaluationResponses); + expect(flat(read!.where)).toContain( + '"judging_evaluation_response"."is_public" = $?', + ); + expect(render(read!.where).params).toContain(true); + }); + }); + + describe("judging.hacker.itinerary", () => { + beforeEach(() => { + rows.set(judgingEntries, [{ id: ENTRY, title: "Tidewater" }]); + rows.set(judgingEntryChallenges, [{ challengeId: GENERAL }]); + rows.set(judgingRooms, [ + { + id: ROOM, + challengeId: GENERAL, + name: "Table 1", + kind: "table", + building: null, + displayOrder: 0, + archivedAt: null, + }, + ]); + rows.set(judgingAppointments, [ + { + id: SLOT, + challengeId: GENERAL, + roomId: ROOM, + startsAt: at("10:00"), + endsAt: at("10:10"), + }, + ]); + rows.set(judgingPresence, []); + }); + + const evaluation = (ratingsComplete: boolean) => ({ + judgeId: "judge-1", + challengeId: GENERAL, + appointmentId: SLOT, + ratingsComplete, + }); + const statusOfSlot = async () => { + const itinerary = await hacker().judging.hacker.itinerary({ + hackathonId: HACK, + }); + return itinerary.appointments.map((a) => a.status); + }; + + it("reads whether an evaluation holds every rating, not whether it was submitted whole", async () => { + rows.set(judgingEvaluations, [evaluation(true)]); + + await statusOfSlot(); + + const [read] = callsOn(judgingEvaluations); + expect(flat(read!.fields!.ratingsComplete)).toBe(HOLDS_EVERY_RATING); + expect(read!.fields).not.toHaveProperty("isComplete"); + expect(render(read!.fields!.ratingsComplete).params).toEqual([HACK]); + }); + + it("calls a presentation judged once its evaluation holds every rating", async () => { + rows.set(judgingEvaluations, [evaluation(true)]); + await expect(statusOfSlot()).resolves.toEqual(["complete"]); + }); + + it("calls it presented while an evaluation is short of a rating", async () => { + rows.set(judgingEvaluations, [evaluation(false)]); + await expect(statusOfSlot()).resolves.toEqual(["incomplete"]); + }); + }); +}); diff --git a/packages/api/src/.internal-tests/reopen-grade.test.ts b/packages/api/src/.internal-tests/reopen-grade.test.ts index 87358587..218ec8bc 100644 --- a/packages/api/src/.internal-tests/reopen-grade.test.ts +++ b/packages/api/src/.internal-tests/reopen-grade.test.ts @@ -1,7 +1,9 @@ /** * judging.admin.reopenGrade: an organiser deletes one judge's evaluation, and * any draft for the same project and challenge, so the project is open for - * another judge. Refused once results are published. + * another judge. The edition's draft placings were ranked with that grade, so + * they go in the same transaction and results have to be computed again + * before they can be published. Refused once results are published. * * The schema is the real one; only the database handle is a stand-in, handed * to the router through the context the way check-in-flags-organizer.test.ts @@ -84,10 +86,22 @@ const fakeDb: any = { events.push(event); const chain: any = { then: (onOk: any, onErr: any) => { - // The published check counts placements that carry a publish time. + // Counts of placements: those that carry a publish time (the published + // check), or every one of the edition's (what publish finds to publish). + const placings = () => { + const { sql: text, params } = render(event.where); + const count = text.includes('"published_at" is not null') + ? published + ? 3 + : 0 + : (store.get(judgingPlacements) ?? []).filter((row) => + params.includes(row.hackathonId), + ).length; + return [{ value: count, n: count }]; + }; const rows = event.table === judgingPlacements - ? [{ value: published ? 3 : 0 }] + ? placings() : rowsOf(event.table, event.where); return Promise.resolve(rows).then(onOk, onErr); }, @@ -134,7 +148,25 @@ const fakeDb: any = { delete: (table: unknown) => ({ where: (where: unknown) => { events.push({ kind: "delete", table, where }); - return Promise.resolve([]); + // The placings the delete takes: this edition's that carry no publish + // time, which the fake reads off the condition's parameters. + const hit = + table === judgingPlacements + ? (store.get(table) ?? []).filter( + (row) => + render(where).params.includes(row.hackathonId) && + row.publishedAt === null, + ) + : []; + if (table === judgingPlacements) { + store.set( + table, + (store.get(table) ?? []).filter((row) => !hit.includes(row)), + ); + } + return Object.assign(Promise.resolve([]), { + returning: async () => hit.map((row) => ({ id: row.id })), + }); }, }), transaction: async (callback: (tx: unknown) => unknown) => callback(fakeDb), @@ -214,6 +246,11 @@ beforeEach(() => { isComplete: true, }, ]); + store.set(judgingPlacements, [ + { id: "p1", hackathonId: HACK_A, scope: "overall", publishedAt: null }, + { id: "p2", hackathonId: HACK_A, scope: CHALLENGE, publishedAt: null }, + { id: "p3", hackathonId: HACK_B, scope: "overall", publishedAt: null }, + ]); store.set(judgingEvaluationDrafts, [ { id: DRAFT_1, entryId: ENTRY, challengeId: CHALLENGE }, { id: DRAFT_2, entryId: ENTRY, challengeId: CHALLENGE }, @@ -223,10 +260,14 @@ beforeEach(() => { describe("judging.admin.reopenGrade", () => { it("deletes the evaluation and every draft for the project and challenge", async () => { - await expect(reopen()).resolves.toEqual({ reopened: true, drafts: 2 }); + await expect(reopen()).resolves.toEqual({ + reopened: true, + drafts: 2, + placings: 2, + }); const [draftDelete, evaluationDelete] = deletes(); - expect(deletes()).toHaveLength(2); + expect(deletes()).toHaveLength(3); expect(draftDelete!.table).toBe(judgingEvaluationDrafts); expect(render(draftDelete!.where).params).toEqual([DRAFT_1, DRAFT_2]); expect(evaluationDelete!.table).toBe(judgingEvaluations); @@ -236,9 +277,15 @@ describe("judging.admin.reopenGrade", () => { it("deletes an evaluation with no draft beside it", async () => { store.set(judgingEvaluationDrafts, []); - await expect(reopen()).resolves.toEqual({ reopened: true, drafts: 0 }); + await expect(reopen()).resolves.toMatchObject({ + reopened: true, + drafts: 0, + }); - expect(deletes().map((event) => event.table)).toEqual([judgingEvaluations]); + expect(deletes().map((event) => event.table)).toEqual([ + judgingEvaluations, + judgingPlacements, + ]); }); it("locks the drafts, then the project's grading lock, before it deletes", async () => { @@ -264,6 +311,45 @@ describe("judging.admin.reopenGrade", () => { expect(firstDelete).toBeGreaterThan(gradingLock); }); + it("deletes the edition's draft placings once the grade is gone, and no other edition's", async () => { + await expect(reopen()).resolves.toMatchObject({ placings: 2 }); + + const [, evaluationDelete, placingDelete] = deletes(); + expect(evaluationDelete!.table).toBe(judgingEvaluations); + expect(placingDelete!.table).toBe(judgingPlacements); + const query = render(placingDelete!.where); + expect(query.params).toEqual([HACK_A]); + expect(query.sql).toContain('"judging_placement"."hackathon_id" = $1'); + expect(query.sql).toContain('"published_at" is null'); + expect(store.get(judgingPlacements)!.map((row) => row.id)).toEqual(["p3"]); + }); + + it("leaves nothing to publish until results are computed again", async () => { + const publish = () => + admin().judging.results.publish({ hackathonId: HACK_A }); + + // The draft placings as computed, with the grade in them. + await expect(publish()).resolves.toMatchObject({ publishedAt: expect.any(Date) }); + + await reopen(); + + await expect(publish()).rejects.toMatchObject({ + code: "PRECONDITION_FAILED", + message: expect.stringContaining( + "Reopening a grade clears the draft placings, so compute again after one.", + ), + }); + }); + + it("reports none cleared when results were never computed", async () => { + store.set(judgingPlacements, []); + + await expect(reopen()).resolves.toMatchObject({ + reopened: true, + placings: 0, + }); + }); + it("refuses once results are published, and deletes nothing", async () => { published = true; @@ -293,7 +379,7 @@ describe("judging.admin.reopenGrade", () => { await expect(reopen()).resolves.toMatchObject({ reopened: true }); - expect(deletes()).toHaveLength(2); + expect(deletes()).toHaveLength(3); }); it("does not reach an evaluation of another edition", async () => { @@ -328,6 +414,7 @@ describe("judging.admin.reopenGrade", () => { revision: 2, wasComplete: false, draftsDeleted: 2, + placingsDeleted: 2, }, }), ]); diff --git a/packages/api/src/.internal-tests/schedule-judges.test.ts b/packages/api/src/.internal-tests/schedule-judges.test.ts index d20aebc4..94fcd334 100644 --- a/packages/api/src/.internal-tests/schedule-judges.test.ts +++ b/packages/api/src/.internal-tests/schedule-judges.test.ts @@ -435,6 +435,160 @@ describe("judging.schedule.get", () => { }); }); +// --------------------------------------------------------------------------- +describe("judging.schedule.get: where each assigned judge has got", () => { + const evaluation = ( + judgeId: string, + entryId: string, + challengeId: string, + appointmentId: string | null, + ratingsComplete = true, + ) => ({ judgeId, entryId, challengeId, appointmentId, ratingsComplete }); + const draft = ( + judgeId: string, + entryId: string, + challengeId: string, + appointmentId: string | null, + updatedAt = NOW, + ) => ({ judgeId, entryId, challengeId, appointmentId, updatedAt }); + + // The clock stays at 15:00: the rate limiter reads it, and a clock that + // runs backwards between tests starves its bucket. Presentations start + // earlier instead. + const startAt = (apptId: string, time: string) => { + store.get(judgingAppointments)!.find((a) => a.id === apptId)!.startsAt = + at(time); + }; + + const progress = async () => { + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + return Object.fromEntries( + board.appointments.map((a) => [a.id, a.judgeProgress]), + ); + }; + + it("tells graded, grading, taken over and not started apart, and leaves the rest blank", async () => { + // It is 15:00: P1 is over, P2 and P3 began long enough ago for the + // takeover time to have passed and P4 is still to come. + startAt(P2, "14:10"); + startAt(P3, "14:20"); + give(P1, M1); + give(P2, M1); + give(P3, M2); + give(P4, GS); + store.set(judgingEvaluations, [ + // Ada graded P1. + evaluation(M1, E1, GENERAL, P1), + // Bea graded P2, which was Ada's. + evaluation(M2, E2, GENERAL, P2), + ]); + + await expect(progress()).resolves.toEqual({ + [P1]: "graded", + [P2]: "taken_over", + // Nobody has begun it and Bea had 5 minutes. + [P3]: "not_started", + // Not started yet. + [P4]: null, + }); + + // A draft is a start. + store.set(judgingEvaluations, []); + store.set(judgingEvaluationDrafts, [ + draft(M2, E3, GENERAL, P3), + draft(M2, E2, GENERAL, P2), + ]); + await expect(progress()).resolves.toMatchObject({ + [P1]: "not_started", + // Ada's, with Bea's draft on it. + [P2]: "taken_over", + // Bea's own. + [P3]: "grading", + }); + }); + + it("gives the assigned judge five minutes from the start before it flags them", async () => { + give(P2, M1); + startAt(P2, "14:56"); + await expect(progress()).resolves.toMatchObject({ [P2]: null }); + startAt(P2, "14:55"); + await expect(progress()).resolves.toMatchObject({ [P2]: "not_started" }); + }); + + it("does not count a draft left without its deadline once the presentation is over", async () => { + // P1's deadline was 14:00. Ada only opened it, so her empty draft was + // kept without its deadline and will never be submitted. + give(P1, M1); + store.set(judgingEvaluationDrafts, [ + { ...draft(M1, E1, GENERAL, P1, at("13:58")), deadlineAt: null }, + ]); + await expect(progress()).resolves.toMatchObject({ [P1]: "not_started" }); + + // Still carrying it: due, and about to be submitted for her. + store.set(judgingEvaluationDrafts, [ + { ...draft(M1, E1, GENERAL, P1, at("13:58")), deadlineAt: at("14:00") }, + ]); + await expect(progress()).resolves.toMatchObject({ [P1]: "grading" }); + }); + + it("matches evaluations and drafts to the presentation the way the board does", async () => { + startAt(P2, "14:10"); + startAt(P3, "14:20"); + give(P2, M1); + give(P3, M1); + store.set(judgingEvaluations, [ + // Names P2, whatever its challenge. + evaluation(M1, E2, ACME, P2), + // Names no slot: the project and a challenge under P3's root. + evaluation(M1, E3, GENERAL, null), + ]); + // Another project's evaluation, and a slot P3 was moved from. + store.set(judgingEvaluationDrafts, [ + draft(M2, E2, GENERAL, P3), + draft(M2, E1, GENERAL, null), + ]); + + await expect(progress()).resolves.toMatchObject({ + [P2]: "graded", + [P3]: "graded", + }); + }); + + it("leaves a presentation with no judge, or whose project is gone, blank", async () => { + startAt(P2, "14:10"); + startAt(P3, "14:20"); + give(P2, M1); + store.get(judgingAppointments)!.find((a) => a.id === P2)!.entryDeletedAt = + NOW; + store.set(judgingEvaluations, [evaluation(M2, E1, GENERAL, P1)]); + + const board = await admin().judging.schedule.get({ hackathonId: HACK }); + expect( + board.appointments.map((a) => [a.id, a.removed, a.judgeProgress]), + ).toEqual([ + // Graded by someone, but nobody was assigned. + [P1, false, null], + [P2, true, null], + [P3, false, null], + [P4, false, null], + ]); + }); + + it("reads a presentation's status from evaluations that hold every rating", async () => { + store.set(judgingEvaluations, [evaluation(M1, E1, GENERAL, P1, false)]); + const short = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(short.appointments[0]).toMatchObject({ + id: P1, + status: "incomplete", + evaluationCount: 1, + }); + + store.set(judgingEvaluations, [evaluation(M1, E1, GENERAL, P1, true)]); + const whole = await admin().judging.schedule.get({ hackathonId: HACK }); + expect(whole.appointments[0]).toMatchObject({ id: P1, status: "complete" }); + }); +}); + // --------------------------------------------------------------------------- describe("judging.schedule.assignJudges", () => { const run = () => diff --git a/packages/api/src/routers/judging/admin.ts b/packages/api/src/routers/judging/admin.ts index 45b089c1..925b10c5 100644 --- a/packages/api/src/routers/judging/admin.ts +++ b/packages/api/src/routers/judging/admin.ts @@ -93,6 +93,7 @@ import { stateChangeProblem, } from "../../services/judging/checklist"; import { notifyItineraryPosted } from "../../services/judging/notify"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; import { checkInFlagsForEntries, flaggedEntryIds, @@ -377,7 +378,8 @@ export const judgingAdminRouter = createTRPCRouter({ db .select({ total: count(), - complete: sql`count(*) filter (where ${judgingEvaluations.isComplete})::int`, + // The ones placings count: every rating in, written response or not. + complete: sql`count(*) filter (where ${holdsEveryRating(id)})::int`, }) .from(judgingEvaluations) .where(eq(judgingEvaluations.hackathonId, id)), @@ -718,7 +720,7 @@ export const judgingAdminRouter = createTRPCRouter({ .where( and( inArray(judgingEvaluations.entryId, ids), - eq(judgingEvaluations.isComplete, true), + holdsEveryRating(input.hackathonId), ), ) .groupBy(judgingEvaluations.entryId), @@ -2545,8 +2547,12 @@ export const judgingAdminRouter = createTRPCRouter({ * ratings, responses and revisions go with the evaluation by cascade. Allowed * while judging is closed, as setPrizeEligibility is: closing freezes judges' * writes, not the organiser's decisions on a project. No judge can grade the - * project until judging is open again. The draft placings still hold the old - * grade until results are computed again. + * project until judging is open again. + * + * The draft placings were ranked with the grade in them, in its challenge + * and overall, and a judge's calibration moved with it, so every draft + * placing is deleted in the same transaction. Publishing needs a compute + * after the project is graded again; until then there is nothing to publish. * * Refused while results are published: the public list was ranked on it. */ @@ -2619,8 +2625,21 @@ export const judgingAdminRouter = createTRPCRouter({ await tx .delete(judgingEvaluations) .where(eq(judgingEvaluations.id, input.evaluationId)); + const placings = await tx + .delete(judgingPlacements) + .where( + and( + eq(judgingPlacements.hackathonId, input.hackathonId), + isNull(judgingPlacements.publishedAt), + ), + ) + .returning({ id: judgingPlacements.id }); - return { ...evaluation, drafts: drafts.length }; + return { + ...evaluation, + drafts: drafts.length, + placings: placings.length, + }; }); await recordAdminAction(db, { @@ -2636,9 +2655,15 @@ export const judgingAdminRouter = createTRPCRouter({ revision: reopened.revision, wasComplete: reopened.isComplete, draftsDeleted: reopened.drafts, + placingsDeleted: reopened.placings, }, }); - return { reopened: true, drafts: reopened.drafts }; + return { + reopened: true, + drafts: reopened.drafts, + /** Draft placings cleared with it: compute again once it is regraded. */ + placings: reopened.placings, + }; }), // --------------------------------------------------------------------- diff --git a/packages/api/src/routers/judging/hacker.ts b/packages/api/src/routers/judging/hacker.ts index 8c0f2d54..458dcba2 100644 --- a/packages/api/src/routers/judging/hacker.ts +++ b/packages/api/src/routers/judging/hacker.ts @@ -60,6 +60,7 @@ import type { Itinerary, VisibleChallenge, } from "../../services/judging/itinerary"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; const hackathonInput = z.object({ hackathonId: z.string().uuid("Invalid hackathon ID"), @@ -642,7 +643,7 @@ export const judgingHackerRouter = createTRPCRouter({ judgeId: judgingEvaluations.judgeId, challengeId: judgingEvaluations.challengeId, appointmentId: judgingEvaluations.appointmentId, - isComplete: judgingEvaluations.isComplete, + ratingsComplete: holdsEveryRating(input.hackathonId), }) .from(judgingEvaluations) .where( @@ -694,9 +695,9 @@ export const judgingHackerRouter = createTRPCRouter({ return { ...state, entry, challenges: visible, ...itinerary }; }), - // Written feedback on the caller's project: public responses from member - // judges' complete evaluations. No judge names and no scores. Hidden in - // emergency mode. + // Written feedback on the caller's project: public responses that exist, from + // member judges' evaluations that hold every rating (the ones placings + // count). No judge names and no scores. Hidden in emergency mode. feedback: protectedProcedure .input(hackathonInput) .query(async ({ ctx, input }) => { @@ -725,7 +726,7 @@ export const judgingHackerRouter = createTRPCRouter({ and( eq(judgingEvaluations.hackathonId, input.hackathonId), eq(judgingEvaluations.entryId, held.entryId), - eq(judgingEvaluations.isComplete, true), + holdsEveryRating(input.hackathonId), eq(judgingJudges.kind, "member"), ), ); diff --git a/packages/api/src/routers/judging/results.ts b/packages/api/src/routers/judging/results.ts index 15dabca6..78b9bab1 100644 --- a/packages/api/src/routers/judging/results.ts +++ b/packages/api/src/routers/judging/results.ts @@ -400,7 +400,7 @@ export const judgingResultsRouter = createTRPCRouter({ throw new TRPCError({ code: "PRECONDITION_FAILED", message: - "There is nothing to publish yet. Compute results first, once judges have submitted.", + "There is nothing to publish yet. Compute results first, once judges have submitted. Reopening a grade clears the draft placings, so compute again after one.", }); } const rows = await tx diff --git a/packages/api/src/routers/judging/schedule.ts b/packages/api/src/routers/judging/schedule.ts index dcac59dd..dd707178 100644 --- a/packages/api/src/routers/judging/schedule.ts +++ b/packages/api/src/routers/judging/schedule.ts @@ -72,11 +72,14 @@ import { runCpsat, } from "../../services/judging/schedule/cpsat-runner"; import { reconcileDueDrafts } from "../../services/judging/evaluation-store"; +import { holdsEveryRating } from "../../services/judging/scoring-sql"; import { moveChoices } from "../../services/judging/schedule/moves"; import { staffedRoomIds, withStatus, } from "../../services/judging/schedule/board"; +import { judgeProgressById } from "../../services/judging/schedule/judge-progress"; +import type { JudgeProgress } from "../../services/judging/schedule/judge-progress"; import { appointmentTimes, firstSlotFrom, @@ -541,7 +544,7 @@ async function loadBoard( entryId: judgingEvaluations.entryId, challengeId: judgingEvaluations.challengeId, appointmentId: judgingEvaluations.appointmentId, - isComplete: judgingEvaluations.isComplete, + ratingsComplete: holdsEveryRating(hackathonId), }) .from(judgingEvaluations) .where(eq(judgingEvaluations.hackathonId, hackathonId)) @@ -564,18 +567,15 @@ async function loadBoard( rootOf(inventory.challenges, c.id)?.id ?? null, ]), ); + const rootIdOf = (id: string) => roots.get(id) ?? null; return { schedule, inventory, plan: selectPlan(inventory), names: namesOf(inventory), - appointments: withStatus( - booked, - evaluations, - presence, - (id) => roots.get(id) ?? null, - now, - ), + evaluations, + rootIdOf, + appointments: withStatus(booked, evaluations, presence, rootIdOf, now), staffed: staffedRoomIds(presence, now), }; } @@ -594,9 +594,12 @@ async function loadJudging(db: JudgingDb, board: Board, schedule: SavedSchedule) readRoster(db, hackathonId, board.inventory), db .select({ + judgeId: judgingEvaluationDrafts.judgeId, entryId: judgingEvaluationDrafts.entryId, challengeId: judgingEvaluationDrafts.challengeId, appointmentId: judgingEvaluationDrafts.appointmentId, + updatedAt: judgingEvaluationDrafts.updatedAt, + deadlineAt: judgingEvaluationDrafts.deadlineAt, }) .from(judgingEvaluationDrafts) .where(eq(judgingEvaluationDrafts.hackathonId, hackathonId)), @@ -639,6 +642,7 @@ async function loadJudging(db: JudgingDb, board: Board, schedule: SavedSchedule) return { roster, saved, + drafts, plan: planOfSaved(board, timing, saved, roster), timing, }; @@ -1258,6 +1262,19 @@ export const judgingScheduleRouter = createTRPCRouter({ const assignable = new Set( judging?.saved.filter((s) => s.open && !s.removed).map((s) => s.id), ); + // Deleted, or no longer in this challenge, since the schedule was saved. + const isRemoved = (a: Board["appointments"][number]) => + !!a.entryDeletedAt || !planned.has(taskKey(a.entryId, a.challengeId)); + // Where each assigned judge has got on a presentation that has started. + const progress = judging + ? judgeProgressById( + board.appointments.filter((a) => !isRemoved(a)), + board.evaluations, + judging.drafts, + board.rootIdOf, + now, + ) + : new Map(); return { serverNow: now, @@ -1290,9 +1307,7 @@ export const judgingScheduleRouter = createTRPCRouter({ id: a.id, entryId: a.entryId, entryTitle: a.entryTitle, - // Deleted, or no longer in this challenge, since the schedule was saved. - removed: - !!a.entryDeletedAt || !planned.has(taskKey(a.entryId, a.challengeId)), + removed: isRemoved(a), challengeId: a.challengeId, challengeLabel: names.label(a.challengeId), roomId: a.roomId, @@ -1309,6 +1324,7 @@ export const judgingScheduleRouter = createTRPCRouter({ ? (judging?.roster.names.get(a.assignedJudgeId) ?? "Removed judge") : null, canAssignJudge: assignable.has(a.id), + judgeProgress: progress.get(a.id) ?? null, })), judging: judging ? describeBoardJudging(judging, board, now) : null, unscheduled: schedule diff --git a/packages/api/src/services/judging/appointment-status.test.ts b/packages/api/src/services/judging/appointment-status.test.ts index 8ae78bb6..29f8415a 100644 --- a/packages/api/src/services/judging/appointment-status.test.ts +++ b/packages/api/src/services/judging/appointment-status.test.ts @@ -15,13 +15,13 @@ describe("appointmentStatus", () => { expect(appointmentStatus(slot, after)).toBe("missed"); }); - it("is complete when every submitted evaluation is complete", () => { + it("is complete when every submitted evaluation holds every rating", () => { const slot = { startsAt, endsAt, evaluations: [ - { judgeId: "a", isComplete: true }, - { judgeId: "b", isComplete: true }, + { judgeId: "a", ratingsComplete: true }, + { judgeId: "b", ratingsComplete: true }, ], presentJudgeIds: ["a", "b"], }; @@ -29,13 +29,13 @@ describe("appointmentStatus", () => { expect(appointmentStatus(slot, after)).toBe("complete"); }); - it("is incomplete when an evaluation was auto-submitted unfinished", () => { + it("is incomplete when an evaluation was auto-submitted short of a rating", () => { const slot = { startsAt, endsAt, evaluations: [ - { judgeId: "a", isComplete: true }, - { judgeId: "b", isComplete: false }, + { judgeId: "a", ratingsComplete: true }, + { judgeId: "b", ratingsComplete: false }, ], }; expect(appointmentStatus(slot, after)).toBe("incomplete"); @@ -45,7 +45,7 @@ describe("appointmentStatus", () => { const slot = { startsAt, endsAt, - evaluations: [{ judgeId: "a", isComplete: true }], + evaluations: [{ judgeId: "a", ratingsComplete: true }], presentJudgeIds: ["a", "b"], }; expect(appointmentStatus(slot, during)).toBe("incomplete"); @@ -55,7 +55,7 @@ describe("appointmentStatus", () => { const slot = { startsAt, endsAt, - evaluations: [{ judgeId: "a", isComplete: true }], + evaluations: [{ judgeId: "a", ratingsComplete: true }], }; expect(appointmentStatus(slot, before)).toBe("complete"); }); diff --git a/packages/api/src/services/judging/appointment-status.ts b/packages/api/src/services/judging/appointment-status.ts index 46cffa53..b63f02c3 100644 --- a/packages/api/src/services/judging/appointment-status.ts +++ b/packages/api/src/services/judging/appointment-status.ts @@ -2,7 +2,8 @@ // hacker itinerary alike. Evaluations decide it once any exist; until then the // clock does. // -// complete every judge who was in the room has a complete evaluation +// complete every judge who was in the room has an evaluation that holds +// every rating, the same rule placings count by // incomplete some evaluation exists, but not all of the above // missed the slot is over and nobody evaluated it // pending the slot has started, nothing submitted yet @@ -19,7 +20,7 @@ export type AppointmentStatusInput = { startsAt: Date; endsAt: Date; /** Evaluations of this appointment's project for its challenge. */ - evaluations: readonly { judgeId: string; isComplete: boolean }[]; + evaluations: readonly { judgeId: string; ratingsComplete: boolean }[]; /** Judges whose presence in the room overlapped the slot. */ presentJudgeIds?: readonly string[]; }; @@ -30,9 +31,11 @@ export function appointmentStatus( ): AppointmentStatus { if (input.evaluations.length > 0) { const completeBy = new Set( - input.evaluations.filter((e) => e.isComplete).map((e) => e.judgeId), + input.evaluations.filter((e) => e.ratingsComplete).map((e) => e.judgeId), + ); + const allSubmittedComplete = input.evaluations.every( + (e) => e.ratingsComplete, ); - const allSubmittedComplete = input.evaluations.every((e) => e.isComplete); const everyPresentJudged = (input.presentJudgeIds ?? []).every((id) => completeBy.has(id), ); diff --git a/packages/api/src/services/judging/itinerary.test.ts b/packages/api/src/services/judging/itinerary.test.ts index 6fb6b531..48b05f22 100644 --- a/packages/api/src/services/judging/itinerary.test.ts +++ b/packages/api/src/services/judging/itinerary.test.ts @@ -118,25 +118,25 @@ describe("evaluationsForAppointment", () => { judgeId: "a", challengeId: "general", appointmentId: "s1", - isComplete: true, + ratingsComplete: true, }, { judgeId: "b", challengeId: "general", appointmentId: null, - isComplete: true, + ratingsComplete: true, }, { judgeId: "c", challengeId: "general", appointmentId: "s9", - isComplete: true, + ratingsComplete: true, }, { judgeId: "d", challengeId: "sponsor", appointmentId: null, - isComplete: true, + ratingsComplete: true, }, ]; expect( @@ -269,7 +269,7 @@ describe("buildItinerary", () => { judgeId: "j1", challengeId: "general", appointmentId: "s1", - isComplete: true, + ratingsComplete: true, }, ], presence: [ @@ -295,7 +295,7 @@ describe("buildItinerary", () => { judgeId: "j1", challengeId: "general", appointmentId: "s1", - isComplete: true, + ratingsComplete: true, }, ], presence: [ @@ -319,6 +319,22 @@ describe("buildItinerary", () => { expect(itinerary.appointments[0]!.status).toBe("incomplete"); }); + it("keeps a slot incomplete while an evaluation lacks a rating", () => { + const itinerary = buildItinerary({ + ...base, + evaluations: [ + { + judgeId: "j1", + challengeId: "general", + appointmentId: "s1", + ratingsComplete: false, + }, + ], + now: at(12), + }); + expect(itinerary.appointments[0]!.status).toBe("incomplete"); + }); + it("lists unscheduled challenges with their live rooms and whether judged", () => { const pending = buildItinerary({ ...base, now: at(5) }); expect(pending.unscheduled).toEqual([ @@ -341,12 +357,26 @@ describe("buildItinerary", () => { judgeId: "j1", challengeId: "mlh", appointmentId: null, - isComplete: true, + ratingsComplete: true, }, ], now: at(5), }); expect(judged.unscheduled[0]!.judged).toBe(true); + + const unrated = buildItinerary({ + ...base, + evaluations: [ + { + judgeId: "j1", + challengeId: "mlh", + appointmentId: null, + ratingsComplete: false, + }, + ], + now: at(5), + }); + expect(unrated.unscheduled[0]!.judged).toBe(false); }); }); diff --git a/packages/api/src/services/judging/itinerary.ts b/packages/api/src/services/judging/itinerary.ts index 0899078a..02f8c87b 100644 --- a/packages/api/src/services/judging/itinerary.ts +++ b/packages/api/src/services/judging/itinerary.ts @@ -39,7 +39,8 @@ export type ItineraryEvaluation = { judgeId: string; challengeId: string; appointmentId: string | null; - isComplete: boolean; + /** Holds every rating (scoring.hasEveryRating), so placings count it. */ + ratingsComplete: boolean; }; export type ItineraryPresence = { @@ -159,7 +160,7 @@ export type UnscheduledChallengeView = { challenge: string; children: string[]; rooms: { name: string; kind: JudgingRoomKind; building: string | null }[]; - /** A judge has finished an evaluation of the project in this challenge. */ + /** A judge has rated the project in this challenge, every rating in. */ judged: boolean; }; @@ -243,7 +244,7 @@ export function buildItinerary(input: { })), judged: evaluations.some( (evaluation) => - evaluation.isComplete && + evaluation.ratingsComplete && rootId(evaluation.challengeId) === challenge.id, ), })); @@ -290,8 +291,8 @@ export type FeedbackItem = { /** * Written feedback, one item per evaluation that has any to share, in * challenge order then the order it arrived. Carries no judge and no score: - * the router passes only complete member evaluations and their public - * responses. + * the router passes only member evaluations that hold every rating, and their + * public responses. */ export function groupFeedback( evaluations: readonly FeedbackEvaluation[], diff --git a/packages/api/src/services/judging/schedule/board.test.ts b/packages/api/src/services/judging/schedule/board.test.ts index cdb1c1eb..4020ee2e 100644 --- a/packages/api/src/services/judging/schedule/board.test.ts +++ b/packages/api/src/services/judging/schedule/board.test.ts @@ -27,7 +27,7 @@ const evaluation = (over: Partial): BoardEvaluation => ({ entryId: "a", challengeId: "mlh", appointmentId: null, - isComplete: true, + ratingsComplete: true, ...over, }); diff --git a/packages/api/src/services/judging/schedule/board.ts b/packages/api/src/services/judging/schedule/board.ts index 1f756314..dee5cd8b 100644 --- a/packages/api/src/services/judging/schedule/board.ts +++ b/packages/api/src/services/judging/schedule/board.ts @@ -12,7 +12,8 @@ export type BoardEvaluation = { entryId: string; challengeId: string; appointmentId: string | null; - isComplete: boolean; + /** Holds every rating (scoring.hasEveryRating), so placings count it. */ + ratingsComplete: boolean; }; export type BoardPresence = { @@ -32,6 +33,21 @@ type Booked = { endsAt: Date; }; +/** + * Whether an evaluation or draft of the appointment's project belongs to the + * appointment: it names it, or, when it names none, its challenge sits under + * the appointment's. The caller has matched the project. + */ +export function belongsTo( + appointment: { id: string; challengeId: string }, + row: { challengeId: string; appointmentId: string | null }, + rootIdOf: (challengeId: string) => string | null, +): boolean { + return row.appointmentId + ? row.appointmentId === appointment.id + : rootIdOf(row.challengeId) === appointment.challengeId; +} + /** Rooms with a judge still in them and heard from in the last 15 minutes. */ export function staffedRoomIds( presence: readonly BoardPresence[], @@ -78,9 +94,7 @@ export function withStatus( return appointments.map((appointment) => { const mine = (byEntry.get(appointment.entryId) ?? []).filter((e) => - e.appointmentId - ? e.appointmentId === appointment.id - : rootIdOf(e.challengeId) === appointment.challengeId, + belongsTo(appointment, e, rootIdOf), ); // Seen in the room at some point during the slot. An open row's last // heartbeat stands in for when the judge left. diff --git a/packages/api/src/services/judging/schedule/judge-progress.test.ts b/packages/api/src/services/judging/schedule/judge-progress.test.ts new file mode 100644 index 00000000..aa27a690 --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-progress.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest"; +import { DEADLINE_GRACE_MS, takeoverAt } from "../timing"; +import { judgeProgress, judgeProgressById } from "./judge-progress"; + +const at = (time: string) => new Date(`2027-02-08T${time}:00Z`); +const startsAt = at("15:00"); +const deadlineAt = at("15:20"); +const JUDGE = "ada"; +const OTHER = "bea"; + +const input = (over: Partial[0]> = {}) => ({ + startsAt, + deadlineAt, + assignedJudgeId: JUDGE, + evaluations: [], + drafts: [], + ...over, +}); +// The desk stamps a presentation's draft with its deadline. +const draft = ( + judgeId: string, + time = "15:02", + draftDeadline: Date | null = deadlineAt, +) => ({ + judgeId, + updatedAt: at(time), + deadlineAt: draftDeadline, +}); + +describe("judgeProgress", () => { + it("says nothing without an assigned judge", () => { + expect( + judgeProgress( + input({ assignedJudgeId: null, evaluations: [{ judgeId: OTHER }] }), + at("16:00"), + ), + ).toBeNull(); + }); + + it("says nothing before the presentation starts", () => { + expect(judgeProgress(input(), at("14:59"))).toBeNull(); + // A judge cannot have begun, so nothing could show anyway. + expect( + judgeProgress(input({ drafts: [draft(JUDGE)] }), at("14:59")), + ).toBeNull(); + }); + + it("says nothing while the assigned judge still has the presentation to themselves", () => { + expect(judgeProgress(input(), at("15:00"))).toBeNull(); + expect(judgeProgress(input(), at("15:04"))).toBeNull(); + }); + + it("flags an assigned judge who has not begun once the takeover time passes", () => { + // The time the judge desk opens the presentation to everyone else. + const opens = takeoverAt(startsAt); + expect(judgeProgress(input(), new Date(opens.getTime() - 1))).toBeNull(); + expect(judgeProgress(input(), opens)).toBe("not_started"); + expect(judgeProgress(input(), at("15:05"))).toBe("not_started"); + // And stays so once the slot is over. + expect(judgeProgress(input(), at("18:00"))).toBe("not_started"); + }); + + it("is grading while the assigned judge has a draft open", () => { + expect(judgeProgress(input({ drafts: [draft(JUDGE)] }), at("15:01"))).toBe( + "grading", + ); + // Past the takeover time too: they began, so nothing was missed. + expect(judgeProgress(input({ drafts: [draft(JUDGE)] }), at("15:30"))).toBe( + "grading", + ); + }); + + it("is graded once the assigned judge has submitted", () => { + expect( + judgeProgress(input({ evaluations: [{ judgeId: JUDGE }] }), at("15:08")), + ).toBe("graded"); + // An evaluation outranks a draft left beside it. + expect( + judgeProgress( + input({ evaluations: [{ judgeId: JUDGE }], drafts: [draft(JUDGE)] }), + at("15:08"), + ), + ).toBe("graded"); + }); + + it("is taken over when another judge has submitted or has a draft", () => { + expect( + judgeProgress(input({ evaluations: [{ judgeId: OTHER }] }), at("15:09")), + ).toBe("taken_over"); + expect(judgeProgress(input({ drafts: [draft(OTHER)] }), at("15:09"))).toBe( + "taken_over", + ); + }); + + it("reads the judge who saved a draft last as the one who has it", () => { + // Ada's draft was left alone and Bea took over; both rows remain. + expect( + judgeProgress( + input({ drafts: [draft(JUDGE, "15:01"), draft(OTHER, "15:12")] }), + at("15:13"), + ), + ).toBe("taken_over"); + // Or Ada came back and saved after Bea. + expect( + judgeProgress( + input({ drafts: [draft(OTHER, "15:12"), draft(JUDGE, "15:20")] }), + at("15:21"), + ), + ).toBe("grading"); + }); + + it("stops counting a draft that lost its deadline once judging for the presentation is over", () => { + // The deadline and the grace a last write gets. + const over = new Date(deadlineAt.getTime() + DEADLINE_GRACE_MS); + const before = new Date(over.getTime() - 1); + // Opened and left empty: it is kept without its deadline and never + // submitted. + const empty = input({ drafts: [draft(JUDGE, "15:02", null)] }); + expect(judgeProgress(empty, before)).toBe("grading"); + expect(judgeProgress(empty, over)).toBe("not_started"); + // Another judge's, likewise: nobody took it over. + expect( + judgeProgress(input({ drafts: [draft(OTHER, "15:12", null)] }), over), + ).toBe("not_started"); + // One still carrying its deadline is about to be submitted for its judge. + expect(judgeProgress(input({ drafts: [draft(JUDGE)] }), over)).toBe( + "grading", + ); + expect( + judgeProgress(input({ drafts: [draft(OTHER, "15:12")] }), over), + ).toBe("taken_over"); + }); +}); + +describe("judgeProgressById", () => { + const roots: Record = { + general: "general", + "general-ai": "general", + acme: "acme", + }; + const rootIdOf = (id: string) => roots[id] ?? null; + const appointment = (over: Record = {}) => ({ + id: "apt", + entryId: "e1", + challengeId: "general", + startsAt, + deadlineAt, + assignedJudgeId: JUDGE, + ...over, + }); + const row = (over: Record = {}) => ({ + judgeId: JUDGE, + entryId: "e1", + challengeId: "general", + appointmentId: null as string | null, + ...over, + }); + const run = ( + appointments: ReturnType[], + evaluations: ReturnType[], + drafts: (ReturnType & { + updatedAt: Date; + deadlineAt: Date | null; + })[], + now = at("15:30"), + ) => judgeProgressById(appointments, evaluations, drafts, rootIdOf, now); + + it("matches by appointment, or by project and challenge root when none is named", () => { + const progress = run( + [appointment()], + [ + // Another judge's, on a challenge under the appointment's root. + row({ judgeId: OTHER, challengeId: "general-ai" }), + ], + [], + ); + expect(progress.get("apt")).toBe("taken_over"); + + // Written against another appointment, another project, another root. + const unrelated = run( + [appointment()], + [ + row({ appointmentId: "other" }), + row({ entryId: "e2" }), + row({ challengeId: "acme" }), + ], + [], + ); + expect(unrelated.get("apt")).toBe("not_started"); + + const named = run( + [appointment()], + [row({ appointmentId: "apt", challengeId: "acme" })], + [], + ); + expect(named.get("apt")).toBe("graded"); + }); + + it("reads each appointment on its own rows, and leaves out the ones with nothing to say", () => { + const progress = run( + [ + appointment({ id: "a1", entryId: "e1" }), + appointment({ id: "a2", entryId: "e2", assignedJudgeId: OTHER }), + appointment({ id: "a3", entryId: "e3", assignedJudgeId: null }), + appointment({ id: "a4", entryId: "e4", startsAt: at("16:00") }), + ], + [row({ entryId: "e1" })], + [ + { + ...row({ entryId: "e2", judgeId: OTHER }), + updatedAt: at("15:10"), + deadlineAt, + }, + ], + ); + expect(Object.fromEntries(progress)).toEqual({ + a1: "graded", + a2: "grading", + }); + }); +}); diff --git a/packages/api/src/services/judging/schedule/judge-progress.ts b/packages/api/src/services/judging/schedule/judge-progress.ts new file mode 100644 index 00000000..d6533eaf --- /dev/null +++ b/packages/api/src/services/judging/schedule/judge-progress.ts @@ -0,0 +1,130 @@ +import { DEADLINE_GRACE_MS, takeoverAt } from "../timing"; +import { belongsTo } from "./board"; + +// How far the judge assigned to a presentation has got, for the organiser's +// live board. Pure: the router loads the rows, this decides. Organisers see +// judge names on the board already, but this says only whether the assigned +// judge or someone else is on the presentation, not who. +// +// From the presentation's start its assigned judge has until its takeover time +// (timing.ts takeoverAt) to begin. After that any judge who may grade it can +// take it over, which the judge desk allows; the assignment itself is never +// rewritten. An evaluation or a draft is the sign a judge has begun. +// +// Once the deadline and the grace a last write gets have passed, a draft still +// carrying its deadline is about to be submitted for its judge +// (evaluation-store reconcileDueDrafts). One that lost it never will be: it +// was empty, or could not be submitted, so it no longer counts as a start. + +/** + * Where a started presentation's grading stands against its assigned judge: + * "grading" (their draft is open), "graded" (they submitted), "taken_over" + * (another judge has a draft or an evaluation), "not_started" (nobody has + * begun and the takeover time has passed). + */ +export type JudgeProgress = "grading" | "graded" | "taken_over" | "not_started"; + +type Started = { judgeId: string }; +type Draft = Started & { updatedAt: Date; deadlineAt: Date | null }; + +/** + * Null when there is nothing to say yet: no assigned judge, a presentation + * that has not started, or one nobody has begun within the takeover window. + */ +export function judgeProgress( + input: { + startsAt: Date; + /** The presentation's deadline: judging for it ends here. */ + deadlineAt: Date; + assignedJudgeId: string | null; + /** Judges who submitted an evaluation of the presentation. */ + evaluations: readonly Started[]; + /** Judges with a draft of it open. */ + drafts: readonly Draft[]; + }, + now: Date, +): JudgeProgress | null { + const { assignedJudgeId } = input; + if (!assignedJudgeId || now < input.startsAt) return null; + + if (input.evaluations.length > 0) { + return input.evaluations.some((e) => e.judgeId === assignedJudgeId) + ? "graded" + : "taken_over"; + } + const ended = now.getTime() >= input.deadlineAt.getTime() + DEADLINE_GRACE_MS; + const drafts = ended + ? input.drafts.filter((draft) => draft.deadlineAt !== null) + : input.drafts; + // Drafts a judge left behind when another took over stay in the table, so + // the one saved last is the judge who has it. + const holder = drafts.reduce( + (latest, draft) => + !latest || draft.updatedAt > latest.updatedAt ? draft : latest, + null, + ); + if (holder) + return holder.judgeId === assignedJudgeId ? "grading" : "taken_over"; + + return now >= takeoverAt(input.startsAt) ? "not_started" : null; +} + +type Booked = { + id: string; + entryId: string; + challengeId: string; + startsAt: Date; + deadlineAt: Date; + assignedJudgeId: string | null; +}; + +type Row = { + entryId: string; + challengeId: string; + appointmentId: string | null; +}; + +/** + * The progress of every appointment that has something to say, by id. One pass + * over the edition's evaluations and drafts, matched to appointments the way + * the board matches them (board.ts belongsTo). + */ +export function judgeProgressById( + appointments: readonly Booked[], + evaluations: readonly (Row & Started)[], + drafts: readonly (Row & Draft)[], + rootIdOf: (challengeId: string) => string | null, + now: Date, +): Map { + const byEntry = (rows: readonly T[]) => { + const map = new Map(); + for (const row of rows) { + const list = map.get(row.entryId); + if (list) list.push(row); + else map.set(row.entryId, [row]); + } + return map; + }; + const evaluationsOf = byEntry(evaluations); + const draftsOf = byEntry(drafts); + + const result = new Map(); + for (const appointment of appointments) { + const mine = (rows: Map) => + (rows.get(appointment.entryId) ?? []).filter((row) => + belongsTo(appointment, row, rootIdOf), + ); + const progress = judgeProgress( + { + startsAt: appointment.startsAt, + deadlineAt: appointment.deadlineAt, + assignedJudgeId: appointment.assignedJudgeId, + evaluations: mine(evaluationsOf), + drafts: mine(draftsOf), + }, + now, + ); + if (progress) result.set(appointment.id, progress); + } + return result; +} diff --git a/sites/mainweb/components/admin/judging/ChecklistSection.tsx b/sites/mainweb/components/admin/judging/ChecklistSection.tsx index 47cda90f..727942d0 100644 --- a/sites/mainweb/components/admin/judging/ChecklistSection.tsx +++ b/sites/mainweb/components/admin/judging/ChecklistSection.tsx @@ -91,7 +91,7 @@ export function ChecklistSection({ { label: "Evaluations", value: counts.evaluations.complete, - detail: `complete of ${counts.evaluations.total} · ${plural(counts.evaluations.drafts, "draft")}`, + detail: `counted of ${counts.evaluations.total} · ${plural(counts.evaluations.drafts, "draft")}`, }, ]; diff --git a/sites/mainweb/components/admin/judging/EntriesList.tsx b/sites/mainweb/components/admin/judging/EntriesList.tsx index d40387d9..fc35823f 100644 --- a/sites/mainweb/components/admin/judging/EntriesList.tsx +++ b/sites/mainweb/components/admin/judging/EntriesList.tsx @@ -397,7 +397,7 @@ function EntryRow({ {(entry.appointments > 0 || entry.completeEvaluations > 0) && (

    {plural(entry.appointments, "presentation")} scheduled ·{" "} - {plural(entry.completeEvaluations, "evaluation")} complete + {plural(entry.completeEvaluations, "evaluation")} counted

    )} {entry.checkInFlags.length > 0 && ( diff --git a/sites/mainweb/components/admin/judging/results/ResultsTab.tsx b/sites/mainweb/components/admin/judging/results/ResultsTab.tsx index e7f99d3e..7c80d6e2 100644 --- a/sites/mainweb/components/admin/judging/results/ResultsTab.tsx +++ b/sites/mainweb/components/admin/judging/results/ResultsTab.tsx @@ -590,10 +590,12 @@ function EvaluationAudit({ // Deleting the grade changes the project list and the judges' queues as // well as this one, so refresh all of judging. const reopen = trpc.judging.admin.reopenGrade.useMutation({ - onSuccess: () => { + onSuccess: (result) => { setError(null); setNotice( - "The grade is deleted and another judge can grade the project. Compute results again to update the placings.", + result.placings > 0 + ? "The grade is deleted and another judge can grade the project. The draft placings were ranked with it, so they are cleared. Compute results again once the project is graded." + : "The grade is deleted and another judge can grade the project. Compute results again once it is graded.", ); void refreshJudging(); }, @@ -685,7 +687,7 @@ function EvaluationAudit({ onReopen={() => { if ( !window.confirm( - `Reopen ${row.title} for grading? ${row.judgeName}'s grade is deleted, with its ratings, responses and history, and another judge can grade the project. This cannot be undone.`, + `Reopen ${row.title} for grading? ${row.judgeName}'s grade is deleted, with its ratings, responses and history, and another judge can grade the project. The draft placings are cleared until you compute results again. This cannot be undone.`, ) ) return; diff --git a/sites/mainweb/components/admin/judging/schedule/LiveBoard.tsx b/sites/mainweb/components/admin/judging/schedule/LiveBoard.tsx index 91c2fafa..08c41fb2 100644 --- a/sites/mainweb/components/admin/judging/schedule/LiveBoard.tsx +++ b/sites/mainweb/components/admin/judging/schedule/LiveBoard.tsx @@ -21,6 +21,7 @@ import { ErrorLine, plural } from "@/components/admin/judging/shared"; import { clock, fieldsFromTiming, + JUDGE_PROGRESS, STATUS, STATUS_ORDER, windowLabel, @@ -352,6 +353,13 @@ export function LiveBoard({ Judge: {a.assignedJudgeName ?? "none"} + {a.judgeProgress && ( + + {JUDGE_PROGRESS[a.judgeProgress].label} + + )} {a.canAssignJudge && (