Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/3286.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A Workflow now creates its run Bots four at a time instead of one after another, so a six-node line starts in about 20 seconds instead of about 66 and no longer misses the 60-second batch deadline a slow host used to hit ([#3286](https://github.com/ThinkFlowLab/FlyBot/issues/3286)). If the host refuses one node, the line stops asking for new Bots, waits for the ones already in flight, and releases every Bot it did create; the run still maps the graph's nodes to their Bots in the graph's own order.
4 changes: 2 additions & 2 deletions docs/subsystems/workflow.md

Large diffs are not rendered by default.

58 changes: 46 additions & 12 deletions plugins/workflow/src/ephemeral.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,58 @@ export function runCaps(callerBotId, authorBotId) {
return [...new Set(ids)]
}

export async function spawnAll(nodes, roster, spawn, workflowModel = '', caps = {}) {
const made = []
// #3286: a host answers a spawn in about ten seconds, so a six-node lane created one
// Bot at a time needs ~66s and cannot fit the 60s batch deadline around a start. Four
// at a time finishes the same lane in two waves (~20s); a caller may pass a narrower
// or wider width as the last argument (tests use it to hold the waves still).
export const SPAWN_CONCURRENCY = 4

export async function spawnAll(nodes, roster, spawn, workflowModel = '', caps = {}, concurrency = SPAWN_CONCURRENCY) {
const list = Array.isArray(nodes) ? nodes : []
const width = Number.isInteger(concurrency) && concurrency > 0 ? concurrency : SPAWN_CONCURRENCY
const capping = runCaps(caps.callerBotId, caps.authorBotId)
try {
const ephemeral = {}
for (const node of nodes) {
// `made` is the order the host answered in: it is what `error.spawned` reports, so a
// failed start releases in the order the Bots came up, not the order they were asked for.
const made = []
const byIndex = new Map()
let next = 0
let failure = null

const create = async () => {
for (;;) {
// Checked before taking the next index, so a refusal stops new spawns at once; the
// ones already in flight are still awaited below and still land in `made`.
if (failure !== null) return
const index = next
next += 1
if (index >= list.length) return
const node = list[index]
const spec = nodeConfig(node, roster, workflowModel)
if (capping.length > 0) spec.cappedBy = capping
const id = await spawn(spec)
if (typeof id !== 'string' || id === '') throw new Error('The run could not start a Bot')
let id
try {
id = await spawn(spec)
if (typeof id !== 'string' || id === '') throw new Error('The run could not start a Bot')
} catch (error) {
if (failure === null) failure = error
return
}
made.push(id)
ephemeral[node.id] = id
byIndex.set(index, id)
}
return ephemeral
} catch (error) {
error.spawned = made
throw error
}

const workers = []
for (let i = 0; i < Math.min(width, list.length); i += 1) workers.push(create())
await Promise.all(workers)
if (failure !== null) {
failure.spawned = made.slice()
throw failure
}
// Keys follow the graph's nodes, whatever order the host answered in.
const ephemeral = {}
for (let i = 0; i < list.length; i += 1) ephemeral[list[i].id] = byIndex.get(i)
return ephemeral
}

// Returns the ids whose release failed: that Bot is still there, so the caller
Expand Down
26 changes: 16 additions & 10 deletions plugins/workflow/src/service.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import { randomBytes } from 'node:crypto'
import { copyFileSync, lstatSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { releaseAll, spawnAll } from './ephemeral.js'
import { SPAWN_CONCURRENCY, releaseAll, spawnAll } from './ephemeral.js'
import {
chainLabel, cleanLine, cleanPorts, cleanPrompt, cleanTools, completeNode, cutCodePoints, doneCount, edgeOutputs, failNode, freshStatus,
layerMarks, firstBranchValue, missingProviders, missingVerdicts, nodeById, normalizeGraph, orderedIds, readStepResult, resolveInputs, retryNode,
Expand Down Expand Up @@ -376,13 +376,13 @@
}

function clearRunTimers(runId) {
for (const [key, entry] of [...timers]) {

Check warning on line 379 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-spread)

Using a spread operator here creates a new array unnecessarily.
if (entry.runId === runId) clearTimerAt(key)
}
for (const key of [...warned]) {

Check warning on line 382 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-spread)

Using a spread operator here creates a new array unnecessarily.
if (key.endsWith(`:${runId}`)) warned.delete(key)
}
for (const key of [...handOffTries.keys()]) {

Check warning on line 385 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-spread)

Using a spread operator here creates a new array unnecessarily.
if (key.startsWith(`${runId}:`)) handOffTries.delete(key)
}
}
Expand Down Expand Up @@ -425,7 +425,7 @@
// keep its deadline: without it the run holds its lane slot with nothing left
// to free it until the owner presses Stop again (P-560).
const stopRetry = run.status === 'running' && run.paused === true && run.releasePending === true
for (const [key, entry] of [...timers]) {

Check warning on line 428 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-spread)

Using a spread operator here creates a new array unnecessarily.
if (entry.runId !== run.id) continue
if (run.status !== 'running' || (run.paused === true && !stopRetry) || !running.has(entry.nodeId)) clearTimerAt(key)
}
Expand Down Expand Up @@ -503,12 +503,13 @@
const waited = current.startedAt > 0 ? now() - current.startedAt : 0
rest.push(`node ${current.label} still had no Bot${waited > 0 ? ` after ${formatMs(waited)}` : ''}`)
}
// The average turns "slow" into a number the owner can act on: six nodes at eleven seconds
// each cannot fit a sixty-second deadline, whatever the settings say.
// The average turns "slow" into a number the owner can act on: the lane needs as many
// waves of SPAWN_CONCURRENCY as its node count requires (#3286).
const average = created.length > 0 ? created.reduce((sum, one) => sum + (one.ms > 0 ? one.ms : 0), 0) / created.length : 0
if (average > 0 && total > 0) {
const needed = Math.round((average * total) / 1000)
if (needed > seconds) rest.push(`a lane of ${total} nodes needs about ${needed}s on this host at ${formatMs(average)} per Bot, past the ${seconds}s deadline`)
const waves = Math.ceil(total / SPAWN_CONCURRENCY)
const needed = Math.round((average * waves) / 1000)
if (needed > seconds) rest.push(`a lane of ${total} nodes needs about ${needed}s on this host at ${formatMs(average)} per Bot with ${SPAWN_CONCURRENCY} spawned at a time, past the ${seconds}s deadline`)
}
return rest.length === 0
? `the host did not create the run Bots within ${seconds}s`
Expand Down Expand Up @@ -841,7 +842,7 @@
const missing = unmetRequired(item, id, run.produced, run.task)
if (missing.length === 0) { started.push(id); continue }
const preds = item.edges.filter(edge => edge.to === id && edge.loop !== true).map(edge => edge.from)
const stuck = preds.length === 0 || preds.every(pred => status[pred] === 'done' || status[pred] === 'failed' || status[pred] === 'skipped')

Check warning on line 845 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-length-check)

Found a useless array length check
if (stuck) {
status[id] = 'failed'
blocked.push({ id, missing })
Expand All @@ -854,7 +855,7 @@
function applyOp(item, run, op) {
let working = run
if (op.type === 'complete' && op.outputs && typeof op.outputs === 'object') {
working = { ...run, produced: { ...(run.produced ?? {}), [op.nodeId]: op.outputs } }

Check warning on line 858 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-fallback-in-spread)

Empty fallbacks in spreads are unnecessary
}
let result
if (op.type === 'start') result = { ...startFrontier(item, working.nodeStatus, working.fired), joined: false, layerSize: 0 }
Expand Down Expand Up @@ -1065,7 +1066,7 @@
break
}
handOffTries.delete(`${current.id}:${nodeId}`)
current = { ...current, handed: { ...current.handed, [nodeId]: true }, inputs: { ...(current.inputs ?? {}), [nodeId]: body.inputs } }

Check warning on line 1069 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-fallback-in-spread)

Empty fallbacks in spreads are unnecessary
storeRun(item.id, current)
}
syncTimers(item, current)
Expand Down Expand Up @@ -1411,23 +1412,28 @@
// this run may use; the host intersects them with each node's `tools`.
// #3282: the loop reports the node it is on and the Bots already made, so the deadline
// below can name the node that never answered instead of one bare sentence.
const progress = { total: item.nodes.length, created: [], current: null }
// #3286: with several spawns in flight, "the node the host is on" is a set, not one
// slot. The message names the node started last that the host has not answered yet.
const progress = { total: item.nodes.length, created: [], current: null, pending: new Map() }
const spawnNode = async (spec) => {
const startedAt = now()
const nodeId = String(spec?.nodeId ?? '')
const nodeName = String(spec?.name ?? '')
progress.current = { nodeId, nodeName, label: nodeLabel({ nodeId, nodeName }), startedAt }
const label = nodeLabel({ nodeId, nodeName })
progress.pending.set(nodeId, { nodeId, nodeName, label, startedAt })
progress.current = progress.pending.get(nodeId)
try {
const id = await spawn(spec)
// A host that answers an empty id is a refusal too, and it has to name the node the
// way a thrown one does: spawnAll() would otherwise refuse it without a node context.
if (typeof id !== 'string' || id === '') throw new Error('the host returned no Bot id')
progress.created.push({ nodeId, nodeName, label: progress.current.label, ms: Math.max(0, now() - startedAt) })
progress.current = null
progress.created.push({ nodeId, nodeName, label, ms: Math.max(0, now() - startedAt) })
return id
} catch (error) {
progress.current = null
throw spawnFailure(error, spec, item.id)
} finally {
progress.pending.delete(nodeId)
progress.current = [...progress.pending.values()].at(-1) ?? null
}
}
ephemeral = await boundedSpawn(spawnAll(item.nodes, bots() ?? [], spawnNode, item.model ?? '', {
Expand Down Expand Up @@ -2544,7 +2550,7 @@
return pending
},
dispose() {
for (const key of [...timers.keys()]) clearTimerAt(key)

Check warning on line 2553 in plugins/workflow/src/service.js

View workflow job for this annotation

GitHub Actions / plugins (workflow)

unicorn(no-useless-spread)

Using a spread operator here creates a new array unnecessarily.
// #2704: a window that is only now taking the store over has no run of its own to
// end — the runs on disk belong to the writer that died — so the takeover reap is
// the cleanup. Nothing here would be needed on top of it.
Expand Down
125 changes: 125 additions & 0 deletions plugins/workflow/test/parallel-spawn.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// #3286: a run creates its node Bots with bounded concurrency. A host answers a
// spawn in about ten seconds, so a six-node lane built one Bot at a time needs
// ~66s and cannot fit the 60s batch deadline the service puts around a start.
// Four at a time finishes the same lane in two waves (~20s).
//
// The host is a fake clock: a spawn resolves only when the test moves time to its
// due moment, so the timing assertions are about how many waves `spawnAll` needs,
// not about how fast this machine is.
import assert from 'node:assert/strict'
import { test } from 'node:test'
import * as ephemeralModule from '../src/ephemeral.js'

const NODES = Array.from({ length: 6 }, (_, index) => ({
id: `node-${index + 1}`,
name: `Node ${index + 1}`,
role: 'dev',
prompt: `Do step ${index + 1}.`,
}))
const DELAY = 10_000

const tick = () => new Promise(done => setImmediate(done))

function fakeHost(delayFor = () => DELAY) {
let now = 0
let active = 0
let maxActive = 0
const calls = []
const queue = []
const spawn = (spec) => {
active += 1
maxActive = Math.max(maxActive, active)
const entry = { spec, id: `eph-${spec.nodeId}`, due: now + delayFor(spec) }
calls.push(entry.spec.nodeId)
return new Promise((resolve) => {
entry.finish = () => {
active -= 1
resolve(entry.id)
}
queue.push(entry)
})
}
async function advance() {
const due = Math.min(...queue.map(entry => entry.due))
now = due
const ready = queue.filter(entry => entry.due <= due)
for (const entry of ready) queue.splice(queue.indexOf(entry), 1)
for (const entry of ready) entry.finish()
await tick()
}
return {
spawn,
calls,
advance,
get now() { return now },
get maxActive() { return maxActive },
get pending() { return queue.length },
}
}

// Drives the fake clock until `spawnAll` settles, failing loudly if it waits on
// a spawn that the test never released.
async function run(batch, host) {
let done = false
batch.then(() => { done = true }, () => { done = true })
while (!done) {
await tick()
if (done) break
if (host.pending === 0) throw new Error('spawnAll waits with no spawn in flight')
await host.advance()
}
return batch
}

test('#3286: spawns run four at a time and a six-node lane fits the 60s batch deadline', async () => {
const host = fakeHost()
const batch = ephemeralModule.spawnAll(NODES, [], host.spawn)
const ephemeral = await run(batch, host)

assert.equal(host.maxActive, 4, `never more than four spawns in flight: ${host.maxActive}`)
assert.equal(Object.keys(ephemeral).length, 6, 'every node got a Bot')
// Six nodes at 10s each: serial would be 60s (the old failure) and four at a
// time is 20s. The bound keeps a slow machine from passing by luck.
assert.ok(host.now < 60_000, `the whole batch took ${host.now}ms of host time, past the 60s deadline`)
assert.ok(host.now <= 3 * DELAY, `two waves at ${DELAY}ms were expected, the batch took ${host.now}ms`)
assert.equal(ephemeralModule.SPAWN_CONCURRENCY, 4, 'the default width is four')
})

test('#3286: a failed spawn stops new starts, lets the ones in flight land, and reports what was made', async () => {
const calls = []
const pending = []
const spawn = (spec) => {
calls.push(spec.nodeId)
return new Promise((resolve, reject) => pending.push({ resolve, reject }))
}
const batch = ephemeralModule.spawnAll(NODES, [], spawn, '', {}, 2)
const failed = batch.then(() => null, error => error)
await tick()
assert.deepEqual(calls, ['node-1', 'node-2'], 'two spawns are in flight')

pending[0].resolve('eph-node-1')
pending[1].resolve('eph-node-2')
await tick()
assert.deepEqual(calls, ['node-1', 'node-2', 'node-3', 'node-4'], 'both workers took the next nodes')

// The third node the host answers is a refusal. The fourth is still in flight:
// it must land, and no fifth or sixth spawn may start.
pending[2].reject(new Error('the host could not create the coder'))
await tick()
assert.deepEqual(calls, ['node-1', 'node-2', 'node-3', 'node-4'], 'no new spawn starts after the refusal')
pending[3].resolve('eph-node-4')

const error = await failed
assert.equal(error?.message, 'the host could not create the coder', 'the host reason is carried')
assert.deepEqual(error.spawned, ['eph-node-1', 'eph-node-2', 'eph-node-4'], 'exactly the Bots that were made, in the order they landed')
assert.deepEqual(calls, ['node-1', 'node-2', 'node-3', 'node-4'], 'the other two nodes were never asked for')
})

test('#3286: the result keeps the graph order even when the Bots land out of order', async () => {
const delays = [50_000, 10_000, 20_000, 30_000, 40_000, 5_000]
const host = fakeHost(spec => delays[Number(spec.nodeId.slice('node-'.length)) - 1])
const ephemeral = await run(ephemeralModule.spawnAll(NODES, [], host.spawn), host)

assert.deepEqual(Object.keys(ephemeral), NODES.map(node => node.id), 'the keys follow the nodes, not the answers')
for (const node of NODES) assert.equal(ephemeral[node.id], `eph-${node.id}`)
})
Loading