diff --git a/changelog.d/1196.fixed.md b/changelog.d/1196.fixed.md new file mode 100644 index 00000000..91218c18 --- /dev/null +++ b/changelog.d/1196.fixed.md @@ -0,0 +1 @@ +`npm run db:backup` now takes an exclusive lock on `BACKUP_DIR` and refuses with `backup: another backup run holds (pid )` instead of pruning beside a backup that is already running, so a cron tick and a manual run can no longer delete each other's copy or crash on it; a stale lock from a killed run is taken over by the next run. The writer also treats only regular files as copies, so a directory named like a copy in `BACKUP_DIR` is skipped instead of crashing the run with `ERR_FS_EISDIR`. diff --git a/docs/deploy.md b/docs/deploy.md index 06bc36cc..73b490ff 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -1153,6 +1153,35 @@ npm run db:backup # outside the backup directory; BACKUP_WORK_DIR moves it (default: the temp dir) ``` +### One writer per backup directory + +A run takes the directory for its own length: `npm run db:backup` creates +`${BACKUP_DIR}/.fledge-backup.lock` exclusively before it writes anything and +removes it however the run ends. A second run started while the first holds the +directory — a cron tick meeting a manual backup — refuses instead of pruning +beside it: + +``` +backup: another backup run holds /data/backups (pid 4242); refusing to share the backup directory +``` + +That is exit 1, one line, and no copy is written or deleted. The lock is not a +wait: if the holder is gone (a `SIGKILL`, a host reboot) the next run takes the +lock over and proceeds, but only once the lock's own mtime is older than ten +seconds — a lock created milliseconds ago may belong to a run that is one +instruction from writing its pid, or to one whose exit handler is about to +release it. The takeover itself is serialised through +`.fledge-backup.lock.takeover`, an exclusive marker held only across the remove +and the recreate: several runs that judge the same dead lock stale can therefore +not delete each other's fresh lock and all write in the directory — the +concurrent-run defect the lock exists to stop. A lock file that exists but cannot +be read is only taken over once it is a minute old, so a run that is mid-creation +is never robbed. The +directory is also read as a directory of files: an entry that is not a regular +file (a directory or a symlink named like a copy) is never treated as a copy and +is never deleted — the writer and the dry run now use the same classifier, so a +copy the console counts is a copy the writer may prune. + ### Backup retention and the dry run The write-time counts are per class, and they are the counts the console reports diff --git a/scripts/backup.mjs b/scripts/backup.mjs index 3b22429c..8be5d026 100644 --- a/scripts/backup.mjs +++ b/scripts/backup.mjs @@ -2,7 +2,7 @@ // Online SQLite backup; safe to run while Fledge is serving. Every copy is // verified with integrity_check before it is reported, then old copies are // pruned to the retention count. -import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs' +import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs' import { createRequire } from 'node:module' import { join, resolve } from 'node:path' import { encryptBackupFile, loadBackupKey } from './lib/backup-crypto.mjs' @@ -12,6 +12,7 @@ import { copyKind, databaseFacts, manifestPath, + statOrNull, writeManifest, } from './lib/backup-manifest.mjs' import { INTERMEDIATE_GRACE_MS, backupWorkRoot, createPlaintextWorkDir, isPlaintextIntermediate, removeWorkDir } from './lib/backup-plaintext.mjs' @@ -89,7 +90,14 @@ function prune(dir, keep, target) { for (const entry of entries) { const kind = copyKind(entry.name) if (kind === null) continue - copies[kind].push({ path: entry.path, mtime: statSync(entry.path).mtimeMs }) + // #1196: an entry is a copy only when it is a regular file. A directory (or a + // symlink) named like a copy used to sort into the class and reach `rmSync`, + // which threw EISDIR out of the process; an entry that vanished between + // `readdir` and `stat` threw ENOENT there. Both are `null` here and are left + // alone — this writer only deletes files it is allowed to delete. + const stat = statOrNull(entry.path) + if (stat === null) continue + copies[kind].push({ path: entry.path, mtime: stat.mtimeMs }) } let pruned = 0 for (const kind of ['plaintext', 'encrypted']) { @@ -113,6 +121,147 @@ function prune(dir, keep, target) { return { pruned, intermediates } } +// One run per backup directory (#1196). A cron tick and a manual +// `npm run db:backup` used to prune each other's copies: whichever pruned second +// deleted a copy the other had just written and reported. The lock is the shape +// `deploy/fledge-deploy-tag.sh` uses — an exclusive lock file, never a wait — +// except that node has no `flock`, so the file carries the holder's pid and a +// lock whose holder is gone is taken over instead of blocking every later run. +const LOCK_NAME = '.fledge-backup.lock' +// Serialises takeovers of a lock whose holder is gone; see `acquireLock`. It is +// not the lock itself: it is only held across the remove-and-recreate, so two runs +// can never both delete and both create. +const LOCK_TAKEOVER_NAME = '.fledge-backup.lock.takeover' +// A lock file created but not yet written (its pid lands a moment later) looks +// the same as one left by a run that died inside those two calls — and judging a +// dead pid is not the same as owning the delete. Two guards keep a fresh lock out +// of another run's takeover: +// +// - a lock is takeable only once its own mtime is older than this grace, so a +// file created milliseconds ago is never a candidate: the run that made it is +// between `O_EXCL` and its pid write, or its exit handler is about to remove +// it. 10 s is short enough that a crashed run does not block the next cron +// tick for long and long enough that no live run is mistaken for a dead one. +// - the entry is read and stat'ed again immediately before the delete, and the +// delete happens only when pid, mtime and inode are still the ones judged +// dead. Otherwise another run already took the directory over. +const LOCK_TAKEOVER_GRACE_MS = 10_000 +// A lock that exists but cannot be read has no pid to judge, so it waits longer: +// only a file this old can be the remains of a run killed between create and +// write rather than a run in the middle of those two calls. +const LOCK_UNREADABLE_GRACE_MS = 60_000 + +function readLockPid(lockPath) { + try { + const pid = Number(readFileSync(lockPath, 'utf8').trim()) + return Number.isInteger(pid) && pid > 0 ? pid : null + } catch { + return null + } +} + +// One read of the entry: the pid it names and the identity of the file itself. +// The identity is what tells "the same dead lock I judged" from "a lock another +// run created since". +function lockSnapshot(lockPath) { + const stat = statOrNull(lockPath) + if (stat === null) return null + return { pid: readLockPid(lockPath), mtimeMs: stat.mtimeMs, ino: stat.ino, dev: stat.dev } +} + +function sameLock(left, right) { + return ( + left !== null && + right !== null && + left.pid === right.pid && + left.mtimeMs === right.mtimeMs && + left.ino === right.ino && + left.dev === right.dev + ) +} + +function lockIsStale(snapshot) { + if (snapshot === null) return false + const age = Date.now() - snapshot.mtimeMs + if (snapshot.pid === null) return age > LOCK_UNREADABLE_GRACE_MS + // A young lock is never stale, dead pid or not: its holder may be one + // instruction from writing its pid, and its exit handler releases it. + if (age <= LOCK_TAKEOVER_GRACE_MS) return false + try { + process.kill(snapshot.pid, 0) + return false + } catch (error) { + // ESRCH: no such process, so the lock outlived its holder. EPERM: the process + // exists under another account, which is still a holder. + return error?.code === 'ESRCH' + } +} + +function acquireLock(dir) { + const lockPath = join(dir, LOCK_NAME) + const takeoverPath = join(dir, LOCK_TAKEOVER_NAME) + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + // O_CREAT|O_EXCL: the first run to create the entry holds the directory. + writeFileSync(lockPath, String(process.pid), { flag: 'wx', mode: 0o600 }) + return lockPath + } catch (error) { + if (error?.code !== 'EEXIST') fail(2, `cannot lock the backup directory: ${error instanceof Error ? error.message : String(error)}`) + const judged = lockSnapshot(lockPath) + // The holder released between the failed create and this read: the next + // attempt can create the lock, and nothing is being taken over. + if (judged === null) continue + if (!lockIsStale(judged)) { + fail(1, `another backup run holds ${dir} (pid ${judged.pid === null ? 'unknown' : String(judged.pid)}); refusing to share the backup directory`) + } + // #1196 review: the delete is serialised by a second `O_EXCL` entry, so two + // runs that judge the same dead lock cannot both reach it. Re-reading and + // comparing is not enough on its own: `rmSync` removes whatever is at the + // path by the time it runs, so a fresh lock created between the comparison + // and the delete would be removed by a run that judged a different file. + // The marker makes the remove-then-create a critical section; a run that + // cannot take it lets the holder do the takeover and comes back to see the + // lock it created. + try { + writeFileSync(takeoverPath, String(process.pid), { flag: 'wx', mode: 0o600 }) + } catch (error) { + if (error?.code !== 'EEXIST') fail(2, `cannot lock the backup directory: ${error instanceof Error ? error.message : String(error)}`) + // The marker is held only across the two calls below, so an old one is a + // run that was killed inside them: sweep it and try the takeover again. + const abandoned = statOrNull(takeoverPath) + if (abandoned !== null && Date.now() - abandoned.mtimeMs > LOCK_TAKEOVER_GRACE_MS) { + rmSync(takeoverPath, { force: true }) + continue + } + // Another run is taking the dead lock over right now. Waiting is the one + // thing this lock never does, so this run refuses and names it: the run + // that holds the marker is about to create the lock this one would have + // taken. Retrying instead is what turned four runs on one dead lock into + // `cannot take the backup lock` for all of them. + const holder = readLockPid(takeoverPath) + fail(1, `another backup run holds ${dir} (pid ${holder === null ? 'unknown' : String(holder)}); refusing to share the backup directory`) + } + let takeoverError = null + try { + // Under the marker the entry can only change by this run's own delete: + // the only other writer of it is a takeover, and takeovers are serialised + // here. The comparison still runs, because a takeover that finished + // before this marker was taken has already replaced the dead lock. + const current = lockSnapshot(lockPath) + if (current !== null && sameLock(judged, current) && lockIsStale(current)) rmSync(lockPath, { force: true }) + } catch (error) { + takeoverError = error + } finally { + rmSync(takeoverPath, { force: true }) + } + if (takeoverError !== null) { + fail(2, `cannot take the backup lock ${lockPath}: ${takeoverError instanceof Error ? takeoverError.message : String(takeoverError)}`) + } + } + } + fail(1, `cannot take the backup lock ${lockPath}`) +} + const args = parseArgs(process.argv.slice(2)) const dbPath = resolve(args.db ?? process.env.DB_PATH ?? join(process.env.DATA_DIR ?? './data', 'fledge.db')) const outDir = resolve(args.backupDir ?? process.env.BACKUP_DIR ?? join(process.env.DATA_DIR ?? './data', 'backups')) @@ -148,6 +297,20 @@ const encrypt = key !== null mkdirSync(outDir, { recursive: true, mode: 0o700 }) chmodSync(outDir, 0o700) +// The lock is taken before anything is written, so the rest of the run is the +// only writer in this directory: the copy it writes cannot be pruned by another +// run, and the copy it reports cannot be deleted under it. +const lockPath = acquireLock(outDir) +// Released however the process ends, including every `fail()` below; a run that +// is killed outright leaves the file, and the next run takes it over by pid. +process.on('exit', () => { + try { + if (readLockPid(lockPath) === process.pid) rmSync(lockPath, { force: true }) + } catch { + // A lock that cannot be released must not change this run's exit code; the + // next run recognises the stale pid. + } +}) const stamp = new Date().toISOString().replace(/[:.]/g, '-') const target = join(outDir, `fledge-${stamp}.db${encrypt ? '.enc' : ''}`) // While encrypting, the plaintext copy is a second copy of everything the diff --git a/scripts/lib/backup-manifest.mjs b/scripts/lib/backup-manifest.mjs index 8f0a13b3..edf0bc64 100644 --- a/scripts/lib/backup-manifest.mjs +++ b/scripts/lib/backup-manifest.mjs @@ -109,7 +109,11 @@ export function copyKind(name) { return match[1] === undefined ? 'plaintext' : 'encrypted' } -function statOrNull(path) { +// One classifier for "this entry is a copy file", shared by the reader +// (`planBackupExpiry`, the drill) and the writer's prune. A directory named like +// a copy, a sidecar, a symlink or an entry that vanished between `readdir` and +// `stat` is `null`; the writer may never delete what this returns `null` for. +export function statOrNull(path) { try { const stat = statSync(path) return stat.isFile() ? stat : null diff --git a/test/backup.test.ts b/test/backup.test.ts index 483f1f69..5ed3c788 100644 --- a/test/backup.test.ts +++ b/test/backup.test.ts @@ -1,7 +1,7 @@ import test from 'node:test' import assert from 'node:assert/strict' import { execFileSync } from 'node:child_process' -import { existsSync, mkdtempSync, readdirSync, statSync, truncateSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readdirSync, statSync, truncateSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { fileURLToPath } from 'node:url' @@ -113,3 +113,30 @@ test('a rejected retention count reports the value that was rejected', () => { assert.equal(existsSync(join(root, 'flag')), false, 'a refused run must not create the backup directory') assert.equal(existsSync(join(root, 'env')), false, 'a refused run must not create the backup directory') }) + +// #1196: `prune` built the copy list from `readdirSync` + mtime and never asked +// whether an entry is a file, so a directory with a copy-shaped name entered the +// list and `rmSync` threw EISDIR out of the process — exit 1 with a stack, not a +// `backup: ` refusal. The reader side already skips non-files (`statOrNull`), so +// the two sides of the same directory disagreed. +test('an entry that is not a regular file is never treated as a copy', () => { + const root = mkdtempSync(join(tmpdir(), 'fledge-backup-')) + const dbPath = join(root, 'live', 'fledge.db') + const backupDir = join(root, 'backups') + seed(dbPath) + const decoy = join(backupDir, 'fledge-2026-01-01T00-00-00-000Z.db') + mkdirSync(decoy, { recursive: true }) + + const result = runBackup(['--db', dbPath, '--backup-dir', backupDir, '--keep', '1']) + + // Skipping it or refusing with its name are both honest; crashing is not. + if (result.code !== 0) { + assert.match(result.stderr, /^backup: /m, `a refusal is one backup: line, got: ${result.stderr}`) + assert.doesNotMatch(result.stderr, /\n\s+at /, `a refusal must not carry an uncaught stack: ${result.stderr}`) + } else { + assert.match(result.stdout, /^backup=/m) + assert.match(result.stdout, /^backup: OK$/m) + } + // Whatever it decided, it may not delete a directory it did not create. + assert.equal(statSync(decoy).isDirectory(), true, 'the writer deleted an entry that is not a copy') +}) diff --git a/test/backupPlaintext.test.ts b/test/backupPlaintext.test.ts index 131d3986..da08cee7 100644 --- a/test/backupPlaintext.test.ts +++ b/test/backupPlaintext.test.ts @@ -1,10 +1,11 @@ import test from 'node:test' import assert from 'node:assert/strict' import { spawn, spawnSync } from 'node:child_process' -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync, utimesSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' +import { DatabaseSync } from 'node:sqlite' import { Store } from '../src/db.js' import { makeUser } from './helpers.js' @@ -48,6 +49,23 @@ function seed(dbPath: string): void { } } +// A backup of a seeded database finishes in a couple of milliseconds, which lets +// a losing run start again after the winner released its lock and look like a +// second writer. A round of this test seeds a database with this many megabytes of +// rows so the holder keeps the directory long enough for every other run in the +// round to reach the lock while it is held. +function grow(dbPath: string, megabytes: number): void { + const db = new DatabaseSync(dbPath) + try { + db.exec('CREATE TABLE IF NOT EXISTS filler (id INTEGER PRIMARY KEY, blob BLOB)') + const insert = db.prepare('INSERT INTO filler (blob) VALUES (?)') + const chunk = Buffer.alloc(1024 * 1024, 7) + for (let index = 0; index < megabytes; index += 1) insert.run(chunk) + } finally { + db.close() + } +} + function names(dir: string): string[] { return existsSync(dir) ? readdirSync(dir) : [] } @@ -242,3 +260,156 @@ test('the drill --fresh path keeps the plaintext copy out of the backup director assert.deepEqual(observed, [], `the drill held a plaintext intermediate in the backup directory at ${observed.join(', ')}`) assert.deepEqual(sqliteNames(backupDir), [], 'the drill must not leave a readable database in the backup directory') }) + +// #1196 symptom A: two runs in one backup directory. The writer assumed it was +// the only writer, so each run pruned the other's copy and each could die on a +// `statSync`, `chmodSync` or `writeManifest` against a copy the other had just +// deleted. A run that exits 0 must leave the file it printed; a run that cannot +// must refuse on one `backup: ` line — never an uncaught stack. +interface CheckedRun extends ChildResult { + reported: string | null + existedAtExit: boolean | null +} + +function runWriter(args: string[], env: Record, onClose?: (path: string) => void): Promise { + const child = spawn(process.execPath, [backupScript, ...args], { stdio: ['ignore', 'pipe', 'pipe'], env: childEnv(env) }) + let stdout = '' + let stderr = '' + child.stdout.setEncoding('utf8').on('data', (chunk: string) => { stdout += chunk }) + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { stderr += chunk }) + return new Promise((resolve) => { + child.on('close', (code, signal) => { + const reported = /^backup=(.+)$/m.exec(stdout)?.[1] ?? null + // Read at the moment this run ends, not after both end: a later run + // pruning an older copy is the retention policy working, not this defect. + const existedAtExit = reported === null ? null : existsSync(reported) + if (reported !== null) onClose?.(reported) + resolve({ code, signal, stdout, stderr, reported, existedAtExit }) + }) + }) +} + +// The barrier the other suites use: the first writer exists once its copy is on +// disk, which is the point from which the second one can prune it. +async function waitForCopy(dir: string, timeoutMs = 20000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (names(dir).some((name) => name.startsWith('fledge-') && name.endsWith('.db'))) return + await new Promise((resolve) => setTimeout(resolve, 2)) + } + throw new Error('the first writer never put a copy in the directory') +} + +function assertHonestRun(label: string, result: CheckedRun): void { + assert.doesNotMatch(result.stderr, /\n\s+at /, `${label} died with an uncaught stack instead of a backup: line: ${result.stderr}`) + if (result.code === 0) { + assert.ok(result.reported, `${label} exited 0 without a backup= line: ${result.stdout}`) + assert.equal(result.existedAtExit, true, `${label} reported a copy that no longer exists: ${result.reported ?? ''}`) + } else { + assert.match(result.stderr, /^backup: /m, `${label} exited ${String(result.code)} without a backup: line: ${result.stderr}`) + assert.ok(result.code === 1 || result.code === 2, `${label} exited ${String(result.code)}; a refusal is exit 1 or 2`) + } +} + +test('a backup refuses to share the directory with a run that holds it', () => { + const root = mkdtempSync(join(tmpdir(), 'fledge-plaintext-')) + const dbPath = join(root, 'live', 'fledge.db') + const backupDir = join(root, 'backups') + seed(dbPath) + mkdirSync(backupDir, { recursive: true, mode: 0o700 }) + // This process is alive, so its lock is live: a second run must refuse instead + // of pruning beside it. + writeFileSync(join(backupDir, '.fledge-backup.lock'), String(process.pid), { mode: 0o600 }) + + const run = runToCompletionSync(backupScript, ['--db', dbPath, '--backup-dir', backupDir, '--keep', '1'], {}) + + assert.equal(run.code, 1, `a locked directory must refuse, got exit ${String(run.code)}: ${run.stdout}`) + assert.match(run.stderr, /^backup: another backup run holds /m, `the refusal must name the reason, got: ${run.stderr}`) + assert.doesNotMatch(run.stderr, /\n\s+at /, `a refusal must not carry an uncaught stack: ${run.stderr}`) + assert.deepEqual(names(backupDir).filter((name) => name.startsWith('fledge-')), [], 'a refused run must not write a copy') + assert.equal(existsSync(join(backupDir, '.fledge-backup.lock')), true, 'a refused run must not remove the holder\'s lock') +}) + +test('a lock left by a run that is gone does not block the next backup', () => { + const root = mkdtempSync(join(tmpdir(), 'fledge-plaintext-')) + const dbPath = join(root, 'live', 'fledge.db') + const backupDir = join(root, 'backups') + seed(dbPath) + mkdirSync(backupDir, { recursive: true, mode: 0o700 }) + // 2147483647 is outside the kernel's pid range on Linux, so it is a pid that + // cannot be alive: the shape a SIGKILLed run leaves. The file is also aged: + // a lock is takeable only once its mtime is older than the takeover grace, so + // a file created milliseconds ago is never mistaken for this one (#1196 + // review). + writeFileSync(join(backupDir, '.fledge-backup.lock'), '2147483647', { mode: 0o600 }) + const longAgo = Date.now() / 1000 - 60 + utimesSync(join(backupDir, '.fledge-backup.lock'), longAgo, longAgo) + + const run = runToCompletionSync(backupScript, ['--db', dbPath, '--backup-dir', backupDir, '--keep', '1'], {}) + + assert.equal(run.code, 0, run.stderr) + assert.match(run.stdout, /^backup=/m) + assert.equal(names(backupDir).includes('.fledge-backup.lock'), false, 'the run must release the lock it took over') +}) + +// #1196 review: the takeover is guarded, and this is the case that proves it. +// Every run that judges one dead lock stale used to delete it and create its own, +// so a second run could delete the fresh `O_EXCL` lock the first had just +// created and both wrote in the directory. Four runs on one planted dead lock, +// several rounds: exactly one may report a copy, and every other run must refuse +// on one `backup: ` line. +test('runs that judge the same dead lock produce exactly one writer', async () => { + const root = mkdtempSync(join(tmpdir(), 'fledge-plaintext-')) + const dbPath = join(root, 'live', 'fledge.db') + const backupDir = join(root, 'backups') + seed(dbPath) + grow(dbPath, 40) + mkdirSync(backupDir, { recursive: true, mode: 0o700 }) + const args = ['--db', dbPath, '--backup-dir', backupDir, '--keep', '10'] + const longAgo = Date.now() / 1000 - 60 + + for (let round = 1; round <= 6; round += 1) { + // Plant the lock inside the round: the first round's winner released its own. + const lockPath = join(backupDir, '.fledge-backup.lock') + writeFileSync(lockPath, '2147483647', { mode: 0o600 }) + utimesSync(lockPath, longAgo, longAgo) + + const runs = await Promise.all([1, 2, 3, 4].map(() => runWriter(args, {}))) + const writers = runs.filter((run) => run.reported !== null) + assert.equal( + writers.length, + 1, + `round ${round}: ${String(writers.length)} runs wrote a copy into one directory: ${runs + .map((run) => `exit ${String(run.code)} ${run.reported ?? run.stderr.trim().split('\n')[0] ?? ''}`) + .join(' | ')}`, + ) + for (const run of runs) assertHonestRun(`round ${round}`, run) + for (const run of runs.filter((candidate) => candidate.reported === null)) { + assert.equal(run.code, 1, `round ${round}: a non-writer is a refusal, got exit ${String(run.code)}: ${run.stderr}`) + assert.match(run.stderr, /^backup: another backup run holds /m, `round ${round}: the refusal names the holder: ${run.stderr}`) + } + + // Reset the directory for the next round: only the lock and the copies. + rmSync(lockPath, { force: true }) + for (const name of names(backupDir).filter((candidate) => candidate.startsWith('fledge-'))) { + rmSync(join(backupDir, name), { force: true }) + } + } +}) + +test('two overlapping writers never report a copy that is gone and never crash', async () => { + const root = mkdtempSync(join(tmpdir(), 'fledge-plaintext-')) + const dbPath = join(root, 'live', 'fledge.db') + const backupDir = join(root, 'backups') + seed(dbPath) + const args = ['--db', dbPath, '--backup-dir', backupDir, '--keep', '1'] + + for (let round = 1; round <= 10; round += 1) { + const first = runWriter(args, {}) + await waitForCopy(backupDir) + const second = runWriter(args, {}) + const [a, b] = await Promise.all([first, second]) + assertHonestRun(`round ${round}: the first writer`, a) + assertHonestRun(`round ${round}: the second writer`, b) + } +})