From 86a447dd4432a66b5d7235ea3ff318b3ec61257d Mon Sep 17 00:00:00 2001 From: Chris Park Date: Thu, 24 Sep 2026 11:38:24 +0930 Subject: [PATCH 1/2] ADC-141: append testing environment information --- .github/workflows/vercel-performance.yml | 101 ++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index 206fb75..d11f684 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -393,6 +393,56 @@ jobs: } }; + const formatNumber = (value) => + typeof value === 'number' + ? value.toLocaleString('en-US', { maximumFractionDigits: 2 }) + : 'n/a'; + + // Worded as the Lighthouse report header words it, so the comment + // reads the same as a report opened locally. Values are taken from + // the settings the run recorded, never assumed from a preset. + const describeThrottling = (settings) => { + const throttling = settings?.throttling ?? {}; + const cpu = `${formatNumber(throttling.cpuSlowdownMultiplier)}x slowdown`; + switch (settings?.throttlingMethod) { + case 'simulate': + return { + cpu: `${cpu} (Simulated)`, + network: + `${formatNumber(throttling.rttMs)} ms TCP RTT, ` + + `${formatNumber(throttling.throughputKbps)} kb/s throughput (Simulated)`, + }; + case 'devtools': + return { + cpu: `${cpu} (DevTools)`, + network: + `${formatNumber(throttling.requestLatencyMs)} ms HTTP RTT, ` + + `${formatNumber(throttling.downloadThroughputKbps)} kb/s down, ` + + `${formatNumber(throttling.uploadThroughputKbps)} kb/s up (DevTools)`, + }; + case 'provided': + return { cpu: 'Provided by environment', network: 'Provided by environment' }; + default: + return { cpu: 'Unknown', network: 'Unknown' }; + } + }; + + const describeScreen = (screen) => { + if (!screen) return 'n/a'; + if (screen.disabled) return 'No emulation'; + return `${screen.width}x${screen.height}, DPR ${screen.deviceScaleFactor}`; + }; + + // Settings come from the config, so they are the same for every run + // and are read from the first report. The benchmark index measures + // the runner itself and varies between runs, so it is collected + // from every representative report. + const describeEnvironment = (report) => ({ + ...describeThrottling(report.configSettings), + screen: describeScreen(report.configSettings?.screenEmulation), + benchmarkIndex: [], + }); + const resultsPath = process.env.RESULTS_PATH; const formFactor = process.env.FORM_FACTOR || 'desktop'; const baselinePath = process.env.BASELINE_PATH || ''; @@ -414,8 +464,14 @@ jobs: // Collect the medians into a plain object, both for the comparison below and // so the run on the default branch can persist it as the next baseline. const measured = {}; + let environment = null; for (const entry of representative) { const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); + environment = environment ?? describeEnvironment(report); + const benchmarkIndex = report.environment?.benchmarkIndex; + if (typeof benchmarkIndex === 'number') { + environment.benchmarkIndex.push(benchmarkIndex); + } const url = toLabel(entry.url); measured[url] = { score: entry.summary?.performance ?? null }; for (const id of METRIC_IDS) { @@ -498,7 +554,16 @@ jobs: writeFileSync( process.env.SUMMARY_OUT || 'summary.json', JSON.stringify( - { formFactor, urls, runs, measured, baseline, warningsByUrl, budgets }, + { + formFactor, + urls, + runs, + measured, + baseline, + warningsByUrl, + budgets, + environment, + }, null, 2 ) @@ -743,6 +808,40 @@ jobs: }); lines.push(`_Median per URL — ${runNotes.join(', ')}._`); + // The benchmark index varies between runs, so show its spread + // rather than picking one run's value. + const formatBenchmark = (values) => { + if (!Array.isArray(values) || values.length === 0) return 'n/a'; + const low = Math.round(Math.min(...values)); + const high = Math.round(Math.max(...values)); + return low === high ? `${low}` : `${low}–${high}`; + }; + + // The conditions each form factor ran under, as the Lighthouse + // report header shows them. A runner with a low benchmark index + // explains slow numbers that no code change caused. + const ENVIRONMENT_ROWS = [ + ['Unthrottled CPU/Memory Power', (env) => formatBenchmark(env.benchmarkIndex)], + ['CPU throttling', (env) => env.cpu], + ['Network throttling', (env) => env.network], + ['Screen emulation', (env) => env.screen], + ]; + const withEnvironment = present.filter((ff) => summaries[ff].environment); + if (withEnvironment.length > 0) { + lines.push( + '', + '
Test environment', + '', + `| | ${withEnvironment.map((ff) => LABELS[ff]).join(' | ')} |`, + `| --- | ${withEnvironment.map(() => '---').join(' | ')} |` + ); + for (const [label, read] of ENVIRONMENT_ROWS) { + const cells = withEnvironment.map((ff) => read(summaries[ff].environment) ?? 'n/a'); + lines.push(`| ${label} | ${cells.join(' | ')} |`); + } + lines.push('', '
'); + } + const missing = FORM_FACTORS.filter((ff) => !summaries[ff]); if (missing.length > 0) { lines.push( From 3af7a902a14f390d2e0e243f5d8414e0a44b681a Mon Sep 17 00:00:00 2001 From: Chris Park Date: Fri, 25 Sep 2026 11:03:41 +0930 Subject: [PATCH 2/2] ADC-141: clarify variable name to avoid confusion --- .github/workflows/vercel-performance.yml | 52 ++++++++++++------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/.github/workflows/vercel-performance.yml b/.github/workflows/vercel-performance.yml index d11f684..bebad1a 100644 --- a/.github/workflows/vercel-performance.yml +++ b/.github/workflows/vercel-performance.yml @@ -433,14 +433,9 @@ jobs: return `${screen.width}x${screen.height}, DPR ${screen.deviceScaleFactor}`; }; - // Settings come from the config, so they are the same for every run - // and are read from the first report. The benchmark index measures - // the runner itself and varies between runs, so it is collected - // from every representative report. - const describeEnvironment = (report) => ({ - ...describeThrottling(report.configSettings), - screen: describeScreen(report.configSettings?.screenEmulation), - benchmarkIndex: [], + const describeSettings = (settings) => ({ + ...describeThrottling(settings), + screen: describeScreen(settings?.screenEmulation), }); const resultsPath = process.env.RESULTS_PATH; @@ -463,15 +458,20 @@ jobs: // Collect the medians into a plain object, both for the comparison below and // so the run on the default branch can persist it as the next baseline. + // + // The benchmark index measures the runner rather than the page and + // varies between runs, so it is collected from every representative + // report. const measured = {}; - let environment = null; + const benchmarkIndexes = []; + let settings = null; for (const entry of representative) { const report = JSON.parse(readFileSync(entry.jsonPath, 'utf8')); - environment = environment ?? describeEnvironment(report); + // Every run in this job uses the same Lighthouse config, so the + // first report's throttling and screen emulation describe them all. + if (settings === null) settings = describeSettings(report.configSettings); const benchmarkIndex = report.environment?.benchmarkIndex; - if (typeof benchmarkIndex === 'number') { - environment.benchmarkIndex.push(benchmarkIndex); - } + if (typeof benchmarkIndex === 'number') benchmarkIndexes.push(benchmarkIndex); const url = toLabel(entry.url); measured[url] = { score: entry.summary?.performance ?? null }; for (const id of METRIC_IDS) { @@ -562,7 +562,7 @@ jobs: baseline, warningsByUrl, budgets, - environment, + testConditions: { ...settings, benchmarkIndexes }, }, null, 2 @@ -820,23 +820,25 @@ jobs: // The conditions each form factor ran under, as the Lighthouse // report header shows them. A runner with a low benchmark index // explains slow numbers that no code change caused. - const ENVIRONMENT_ROWS = [ - ['Unthrottled CPU/Memory Power', (env) => formatBenchmark(env.benchmarkIndex)], - ['CPU throttling', (env) => env.cpu], - ['Network throttling', (env) => env.network], - ['Screen emulation', (env) => env.screen], + const TEST_CONDITION_ROWS = [ + ['Unthrottled CPU/Memory Power', (conditions) => formatBenchmark(conditions.benchmarkIndexes)], + ['CPU throttling', (conditions) => conditions.cpu], + ['Network throttling', (conditions) => conditions.network], + ['Screen emulation', (conditions) => conditions.screen], ]; - const withEnvironment = present.filter((ff) => summaries[ff].environment); - if (withEnvironment.length > 0) { + const withConditions = present.filter((ff) => summaries[ff].testConditions); + if (withConditions.length > 0) { lines.push( '', '
Test environment', '', - `| | ${withEnvironment.map((ff) => LABELS[ff]).join(' | ')} |`, - `| --- | ${withEnvironment.map(() => '---').join(' | ')} |` + `| | ${withConditions.map((ff) => LABELS[ff]).join(' | ')} |`, + `| --- | ${withConditions.map(() => '---').join(' | ')} |` ); - for (const [label, read] of ENVIRONMENT_ROWS) { - const cells = withEnvironment.map((ff) => read(summaries[ff].environment) ?? 'n/a'); + for (const [label, read] of TEST_CONDITION_ROWS) { + const cells = withConditions.map( + (ff) => read(summaries[ff].testConditions) ?? 'n/a' + ); lines.push(`| ${label} | ${cells.join(' | ')} |`); } lines.push('', '
');